initial commit (07787d2c) · Commits · to be continuous... / tools / tracking

.gitignore

0 → 100644

+4 −0

Original line number	Diff line number	Diff line
		.idea/
		*.iml
		bin/
		reports/

.gitlab-ci.yml

0 → 100644

+28 −0

Original line number	Diff line number	Diff line
		include:
		- project: 'to-be-continuous/docker'
		ref: '1.2.0'
		file: '/templates/gitlab-ci-docker.yml'
		- project: 'to-be-continuous/golang'
		ref: '1.2.0'
		file: '/templates/gitlab-ci-golang.yml'

		variables:
		GO_BUILD_ARGS: "install -tags netgo ./..."

		DOCKER_BUILD_ARGS: "--build-arg CI_PROJECT_URL --build-arg TRACKING_CONFIGURATION"
		DOCKER_HEALTHCHECK_CONTAINER_ARGS: "--service --skip_tracking my-template-service my-template-version"

		stages:
		- build
		- test
		- package-build
		- package-test
		- acceptance
		- publish

		# override base jobs not to depend on tracking image
		.go-base:
		services: []

		.docker-base:
		services: []

Dockerfile

0 → 100644

+28 −0

Original line number	Diff line number	Diff line
		FROM busybox:1.31.0 AS busybox

		FROM scratch

		ARG CI_PROJECT_URL
		ARG TRACKING_CONFIGURATION=""
		ENV TRACKING_CONFIGURATION="${TRACKING_CONFIGURATION}"

		# hadolint ignore=DL3048
		LABEL name="tracking" \
		description="Image for tracking 'to be continuous' templates usage"\
		url=$CI_PROJECT_URL \
		maintainer="tbc-dev@googlegroups.com"


		COPY bin/tracking_service /tracking_service
		COPY --from=busybox /bin/wget /wget


		EXPOSE 80

		HEALTHCHECK CMD ["/wget", "-Y", "off", "-O", "-", "http://localhost/health" ]

		ENTRYPOINT [ "/tracking_service" ]

		CMD [""]

README.md

0 → 100644

+113 −0

Original line number	Diff line number	Diff line
		# Tracking Probe Service

		This project builds a Docker image able to send GitLab CI job information to any Elasticsearch server.

		It is aimed at being used in GitLab CI as a [service container](https://docs.gitlab.com/ee/ci/services/)
		in order to track _to be continuous_ usage and compute statistics.

		## Which information is tracked ?

		The _Tracking Probe Service_, whenever added as a [service container](https://docs.gitlab.com/ee/ci/services/)
		of a GitLab CI job, will send the following JSON payload to one or several Elasticsearch servers:

		```json
		{
		"@type": "ci-job",
		"@timestamp": "2020-10-19T07:34:08Z",
		"template": {
		"name": "maven",
		"version": "1.0.0"
		},
		"ci": {
		"user": {
		"id": 6097,
		"login": "pismy"
		},
		"project": {
		"id": 23568,
		"path": "https://gitlab.com/to-be-continuous/samples/maven-library"
		},
		"build": {
		"ref": "master"
		},
		"job": {
		"id": 8585338,
		"name": "mvn-build",
		"stage": "package-publish",
		"url": "https://gitlab.com/to-be-continuous/samples/maven-library/-/jobs/8585338"
		},
		"pipeline": {
		"id": 1835260,
		"url": "https://gitlab.com/to-be-continuous/samples/maven-library/-/pipelines/1835260"
		},
		"runner": {
		"id": 44949,
		"description": "shared-runners-manager-4.gitlab.com",
		"tags": "gce, east-c, shared, docker, linux, ruby, mysql, postgres, mongo, git-annex",
		"version": "13.12.0-rc1"
		}
		}
		}
		```

		Each of those information are retrieved from [GitLab CI predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html).

		From this, you can build any valuable statistics, visualization or so.

		## How is it used in GitLab CI ?

		The Docker image can be used as a [service container](https://docs.gitlab.com/ee/ci/services/)
		in any GitLab CI file as follows:

		```yaml
		.some-base-job:
		services:
		- name: "$CI_REGISTRY/to-be-continuous/tools/tracking:master"
		command: ["--service", "some-template", "1.0.0"]

		some-job-build:
		stage: build
		extends: .some-base-job
		script:
		- echo "build"

		some-job-test:
		stage: test
		extends: .some-base-job
		script:
		- echo "test"
		```

		The 2 arguments passed to the service container are the template name and version that will be sent with
		the JSON payload (the only 2 information that can't be retrieved from [GitLab CI predefined variables](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)).

		:bulb: this is configured in every _to be continuous_ template.

		## How to configure the Elasticsearch servers to send to ?

		The configuration is defined in JSON and is built with the Docker image.

		You shall define it as the `TRACKING_CONFIGURATION` CI/CD variable of this project as follows:

		```JSON
		{
		"clients": [
		{
		"url":"https://elasticsearch-host",
		"authentication": {
		"username":"tbc-tracking",
		"password":"mYp@55w0rd"
		},
		"timeout":5,
		"indexPrefix":"tbc-",
		"esMajorVersion":7,
		"skipSslVerification":true
		}
		]
		}
		```

		:bulb: Notice that you may configure more than one Elasticsearch server.

		Then manually start a pipeline on the `master` branch: this will (re)generate a new Docker image with your
		configuration that will now be used by every template job.

cmd/tracking_service/internal/client.go

0 → 100644

+110 −0

Original line number	Diff line number	Diff line
		/*
		* Copyright (C) 2021 Orange & contributors
		*
		* This program is free software; you can redistribute it and/or modify it under the terms
		*
		* of the GNU Lesser General Public License as published by the Free Software Foundation;
		* either version 3 of the License, or (at your option) any later version.
		*
		* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
		* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
		* See the GNU Lesser General Public License for more details.
		*
		* You should have received a copy of the GNU Lesser General Public License along with this
		* program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth
		* Floor, Boston, MA 02110-1301, USA.
		*/

		package internal

		import (
		"bytes"
		"crypto/tls"
		"encoding/json"
		"fmt"
		"log"
		"net/http"
		"os"
		"time"
		)

		type Authentication struct {
		Username string `json:"username"`
		Password string `json:"password"`
		}

		type ClientTracking struct {
		Url string `json:"url"`
		IndexPrefix string `json:"indexPrefix"`
		ESMajorVersion int `json:"esMajorVersion"`
		Authentication *Authentication `json:"authentication"`
		Timeout time.Duration `json:"timeout"`
		SkipSslVerification bool `json:"skipSslVerification"`
		}

		func handleProxy() {
		log.Printf("'http_proxy' set to '%s'\n", os.Getenv("http_proxy"))
		log.Printf("'https_proxy' set to '%s'\n", os.Getenv("https_proxy"))
		log.Printf("'no_proxy' set to '%s'\n", os.Getenv("no_proxy"))
		}

		func init() {
		handleProxy()
		}

		func (client ClientTracking) buildRequest(message Message) (error, *http.Request) {
		var index, url string
		if client.IndexPrefix != "" {
		index = fmt.Sprintf("%s%d%02d", client.IndexPrefix, time.Now().Year(), time.Now().Month())
		} else {
		index = fmt.Sprintf("p19032_data_%d%02d", time.Now().Year(), time.Now().Month())
		}
		if client.ESMajorVersion >= 7 {
		url = fmt.Sprintf("%s/%s/_doc/", client.Url, index)
		} else {
		url = fmt.Sprintf("%s/%s/event", client.Url, index)
		}

		if payload, err := json.Marshal(*message); err != nil {
		return err, nil
		} else if request, err := http.NewRequest("POST", url, bytes.NewBuffer(payload)); err != nil {
		return err, nil
		} else {
		if client.Authentication != nil {
		request.SetBasicAuth(client.Authentication.Username, client.Authentication.Password)
		}
		request.Header.Set("Content-Type", "application/json")
		return err, request
		}
		}

		func (client ClientTracking) sendRequest(request http.Request) error {
		httpClient := &http.Client{}
		if client.Timeout != 0 {
		httpClient.Timeout = client.Timeout * time.Second
		}
		if response, err := httpClient.Do(request); err != nil {
		return err
		} else if (response.StatusCode / 100) != 2 {
		return fmt.Errorf("invalid status code %d", response.StatusCode)
		} else {
		return nil
		}
		}

		func (client ClientTracking) SendTracking(message Message) error {
		if client.SkipSslVerification {
		// nolint
		http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
		}
		if err, request := client.buildRequest(message); err != nil {
		log.Printf("Error while building request %+v\n", *message)
		return err
		} else if err := client.sendRequest(request); err != nil {
		log.Printf("Error while sending %+v\n", *message)
		return err
		} else {
		log.Printf("Message sent: %+v\n", *message)
		return nil
		}
		}