Update .gitlab-ci.yml (c9b81c4c) · Commits · mirror-bot / Hub

.gitlab-ci.yml

+205 −0

Original line number	Diff line number	Diff line
		@@ -4,6 +4,40 @@ stages:
		- build
		- deploy

		include:
		- remote: 'https://jobs.r2devops.io/links_checker.yml'
		- remote: 'https://jobs.r2devops.io/1.4.0/mkdocs.yml?scope=0.yml'
		- remote: 'https://jobs.r2devops.io/0.3.0/pages.yml?scope=0.yml'

		ci_linter:
		stage: static_tests
		image:
		name: alpine/httpie:2.4.0
		entrypoint: [""]
		variables:
		GITLAB_CI_LINT_URL: "https://gitlab.com/api/v4/ci/lint?include_merged_yaml=true"
		JOB_FILES_EXTENSION: ".yml"
		script:
		- apk add --no-cache jq
		- mkdir ci_linter_reports
		- for JOB_PATH in jobs/*; do
		- echo "Start checking ${JOB_PATH}/${JOB} using Gitlab CI linter API"
		- JOB=$(basename ${JOB_PATH})
		- jq --null-input --arg yaml "$(cat ${JOB_PATH}/${JOB}${JOB_FILES_EXTENSION})" '.content=$yaml' \| http --print 'b' --check-status "$GITLAB_CI_LINT_URL" Content-Type:"application/json" PRIVATE-TOKEN:${API_TOKEN} > ci_linter_reports/${JOB}.json
		- if [ $(jq -r '.status' ci_linter_reports/${JOB}.json) == "valid" ]; then
		- echo "👍 job ${JOB_PATH}/${JOB} is valid"
		- else
		- echo "👎 job ${JOB_PATH}/${JOB} is invalid"
		- jq '.' ci_linter_reports/${JOB}.json
		- exit 1
		- fi
		- done
		artifacts:
		expose_as: "job_ci_lint"
		paths:
		- ci_linter_reports/
		when: always

		job_structure:
		image: python:3.9.1-alpine
		stage: static_tests
		@@ -20,3 +54,174 @@ job_structure:
		paths:
		- ${JOB_LOGFILE}
		when: always

		job_customs:
		image: python:3.9.1-alpine
		stage: static_tests
		variables:
		PIPENV_PIPFILE: tools/job_customs/Pipfile
		JOB_LOGFILE: "job_customs.log"
		before_script:
		- pip install --ignore-installed distlib pipenv
		- pipenv install
		script:
		- pipenv run python3 tools/job_customs/job_customs.py
		artifacts:
		expose_as: "jobs_customs"
		paths:
		- ${JOB_LOGFILE}

		job_image_scan:
		image: docker:19.03
		stage: static_tests
		services:
		- name: docker:19.03-dind
		entrypoint: ["env", "-u", "DOCKER_HOST"]
		command: ["dockerd-entrypoint.sh"]
		variables:
		PIPENV_PIPFILE: tools/job_image/Pipfile
		JOB_LOGFILE: "job_image.log"
		OUTPUT_DIR: "scan_output"

		TRIVY_EXIT_ON_SEVERITY: ""
		TRIVY_SEVERITY: "LOW,MEDIUM,HIGH,CRITICAL"
		TRIVY_EXIT_CODE: 0
		TRIVY_VULN_TYPE: "os,library"
		TRIVY_NO_PROGRESS: "false"
		TRIVY_OUTPUT: "junit-report.xml"
		TRIVY_IGNOREFILE: .trivyignore
		TRIVY_CACHE_DIR: .trivycache/
		TRIVY_FORMAT: "template"
		TEMPLATE_NAME: "junit.tpl"
		TRIVY_CLEAR_CACHE: "false"
		TRIVY_IGNORE_UNFIXED: "false"
		TRIVY_DEBUG: "false"

		DOCKER_HOST: tcp://docker:2375
		DOCKER_DRIVER: overlay2
		DOCKER_TLS_CERTDIR: ""
		TRIVY_VERSION: "0.9.2"
		TRIVY_REMOTE: ""
		TRIVY_TIMEOUT: ""
		TRIVY_LIGHT: "false"
		TRIVY_DOWNLOAD_DB_ONLY: "false"
		TRIVY_TOKEN: ""
		TRIVY_QUIET: "false"
		TRIVY_SKIP_UPDATE: "false"

		before_script:
		- apk add --no-cache python3 py3-pip
		- pip install --ignore-installed distlib pipenv
		- pipenv install

		- wget https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz
		- tar zxvf trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz
		- wget -O $TEMPLATE_NAME https://github.com/aquasecurity/trivy/raw/v${TRIVY_VERSION}/contrib/junit.tpl

		- mkdir ${OUTPUT_DIR}
		script:
		- for JOB in $(ls -A jobs); do
		- IMAGE=$(pipenv run python3 tools/job_image/job_image.py ${JOB})
		- if [ ! -z ${IMAGE} ]; then
		- NAME=$(basename ${IMAGE})
		- ./trivy --template "@${TEMPLATE_NAME}" -o ${OUTPUT_DIR}/${NAME}.${TRIVY_OUTPUT} ${IMAGE}
		- if [ ! -z ${TRIVY_EXIT_ON_SEVERITY} ]; then
		- ./trivy --template "@${TEMPLATE_NAME}" --exit-code 1 --severity ${TRIVY_EXIT_ON_SEVERITY} -o ${OUTPUT_DIR}/${NAME}-failed-${TRIVY_OUTPUT} ${IMAGE}
		- fi
		- fi
		- IMAGE=""
		- done
		cache:
		paths:
		- "$TRIVY_CACHE_DIR"
		artifacts:
		expose_as: "job_image_scan"
		paths:
		- "${JOB_LOGFILE}"
		- "${OUTPUT_DIR}"
		reports:
		junit:
		- "${OUTPUT_DIR}/*.xml"
		expire_in: 30 days
		when: always


		# See https://docs.gitlab.com/ee/api/labels.html
		job_gitlab_labels:
		image: python:3.9.1-alpine
		stage: project_setup
		variables:
		PIPENV_PIPFILE: tools/job_gitlab_labels/Pipfile
		JOB_LOGFILE: "job_gitlab_labels.log"
		before_script:
		- pip install --ignore-installed distlib pipenv
		- pipenv install
		script:
		- pipenv run python3 tools/job_gitlab_labels/job_gitlab_labels.py
		artifacts:
		expose_as: "job_gitlab_labels"
		paths:
		- ${JOB_LOGFILE}
		expire_in: 30 days
		when: always
		rules:
		- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops/hub"'


		mkdocs:
		variables:
		PIPENV_PIPFILE: tools/builder/Pipfile
		before_script:
		- pip install pipenv && pipenv install
		- pipenv run python tools/builder/builder.py


		# See https://docs.gitlab.com/ee/api/releases/
		# We can only control the link to the hub, the release is still storing the source code
		release:
		stage: deploy
		image:
		name: alpine/httpie:2.2.0
		entrypoint: [""]
		variables:
		PIPENV_PIPFILE: tools/notify/Pipfile
		before_script:
		- apk update && apk add --no-cache bash
		- pip install --ignore-installed distlib pipenv
		- pipenv install

		script: >
		PROJECT_ENCODED=$(/bin/bash -c "$(http --ignore-stdin --body https://gist.githubusercontent.com/cdown/1163649/raw/8a35c36fdd24b373788a7057ed483a5bcd8cd43e/gistfile1.sh) && _encode '$CI_PROJECT_PATH'");

		for JOB in jobs/*; do
		JOB=$(basename ${JOB})
		for VERSION in jobs/${JOB}/versions/*; do
		VERSION=$(basename ${VERSION} .md)
		result=$(http --ignore-stdin POST https://gitlab.com/api/v4/projects/$PROJECT_ENCODED/releases \
		"JOB-TOKEN: ${CI_JOB_TOKEN}" \
		tag_name=${JOB}-${VERSION} \
		ref=${CI_COMMIT_SHA} \
		"description=$(cat jobs/${JOB}/versions/${VERSION}.md)" \
		'assets:={"links": [{"name": "Hub link", "url": "https://jobs.r2devops.io/'"${VERSION}/${JOB}"'.yml"}]}')

		if [ $(echo $result \| grep "Release already exists\\|${JOB}-${VERSION}" \| wc -l) -eq 0 ]; then
		echo "[ERROR] Problem when attempting to create release ${JOB}-${VERSION}"
		echo "[ERROR] ${result}"
		exit 1;
		else
		if [ $(echo ${result} \| grep "Release already exists" \| wc -l) -eq 0 ]; then
		echo "New version detected for $JOB, sending notification to discord"
		pipenv run python tools/notify/discord_release_notify.py -n ${JOB} -v ${VERSION}
		fi

		echo "Processed ${JOB}-${VERSION} : ${result}";
		fi

		done
		done
		rules:
		- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

		pages:
		rules:
		- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'