Merge branch '634-fix-job_image_scan-trivy-fails-initialize-a-scanner' into 'latest' (2dc52d6a) · Commits · mirror-bot / Hub

.gitlab-ci.yml

+21 −18

Original line number	Diff line number	Diff line
		@@ -2,14 +2,16 @@ stages:
		- static_tests
		- merge_tests
		- project_setup
		- build
		- dynamic_tests
		- deploy

		include:
		- remote: 'https://api.r2devops.io/job/r/r2devops-bot/links_checker/0.2.0.yml?ignore=true.yml'
		- remote: 'https://api.r2devops.io/job/r/r2devops-bot/spell_check/0.2.1.yml?ignore=true.yml'

		variables:
		IMAGE_TAG_PYTHON_ALPINE: "alpine3.16"
		IMAGE_TAG_HTTPIE_ALPINE: "3.2.1"

		workflow:
		rules:
		- if: $CI_MERGE_REQUEST_IID
		@@ -19,7 +21,7 @@ workflow:
		ci_linter:
		stage: static_tests
		image:
		name: alpine/httpie:2.4.0
		name: alpine/httpie:${IMAGE_TAG_HTTPIE_ALPINE}
		entrypoint: [""]
		variables:
		GITLAB_CI_LINT_URL: "https://gitlab.gnome.org/api/v4/ci/lint?include_merged_yaml=true"
		@@ -46,7 +48,7 @@ ci_linter:
		when: always

		job_structure:
		image: python:3.9.1-alpine
		image: python:${IMAGE_TAG_PYTHON_ALPINE}
		stage: static_tests
		variables:
		PIPENV_PIPFILE: tools/job_structure/Pipfile
		@@ -64,7 +66,7 @@ job_structure:
		when: always

		job_customs:
		image: python:3.9.1-alpine
		image: python:${IMAGE_TAG_PYTHON_ALPINE}
		stage: static_tests
		variables:
		PIPENV_PIPFILE: tools/job_customs/Pipfile
		@@ -81,10 +83,10 @@ job_customs:
		- ${JOB_LOGFILE}

		job_image_scan:
		image: docker:19.03
		image: docker:20.10
		stage: static_tests
		services:
		- name: docker:19.03-dind
		- name: docker:20.10-dind
		entrypoint: ["env", "-u", "DOCKER_HOST"]
		command: ["dockerd-entrypoint.sh"]
		variables:
		@@ -94,8 +96,8 @@ job_image_scan:
		PYTHONPATH: "./:$PYTHONPATH"

		TRIVY_EXIT_ON_SEVERITY: ""
		TRIVY_SEVERITY: "LOW,MEDIUM,HIGH,CRITICAL"
		TRIVY_EXIT_CODE: 0
		TRIVY_SEVERITY: "HIGH,CRITICAL"
		TRIVY_EXIT_CODE: "0"
		TRIVY_VULN_TYPE: "os,library"
		TRIVY_NO_PROGRESS: "false"
		TRIVY_OUTPUT: "junit-report.xml"
		@@ -110,7 +112,7 @@ job_image_scan:
		DOCKER_HOST: tcp://docker:2375
		DOCKER_DRIVER: overlay2
		DOCKER_TLS_CERTDIR: ""
		TRIVY_VERSION: "0.9.2"
		TRIVY_VERSION: "0.31.3"
		TRIVY_REMOTE: ""
		TRIVY_TIMEOUT: ""
		TRIVY_LIGHT: "false"
		@@ -136,15 +138,15 @@ job_image_scan:
		- IMAGE=$(pipenv run python3 tools/job_image/job_image.py ${JOB})
		- if [ ! -z ${IMAGE} ]; then
		- NAME=$(basename ${IMAGE})
		- ./trivy --skip-update --template "@${TEMPLATE_NAME}" --cache-dir ${TRIVY_CACHE_DIR} -o ${OUTPUT_DIR}/${NAME}.${TRIVY_OUTPUT} ${IMAGE}
		- ./trivy image --template "@${TEMPLATE_NAME}" --security-checks vuln --vuln-type ${TRIVY_VULN_TYPE} --cache-dir ${TRIVY_CACHE_DIR} -o ${OUTPUT_DIR}/${NAME}.${TRIVY_OUTPUT} ${IMAGE}
		- if [ ! -z ${TRIVY_EXIT_ON_SEVERITY} ]; then
		- ./trivy --skip-update --template "@${TEMPLATE_NAME}" --cache-dir ${TRIVY_CACHE_DIR} --exit-code 1 --severity ${TRIVY_EXIT_ON_SEVERITY} -o ${OUTPUT_DIR}/${NAME}-failed-${TRIVY_OUTPUT} ${IMAGE}
		- ./trivy image --template "@${TEMPLATE_NAME}" --security-checks vuln --vuln-type ${TRIVY_VULN_TYPE} --cache-dir ${TRIVY_CACHE_DIR} --exit-code 1 --severity ${TRIVY_SEVERITY} -o ${OUTPUT_DIR}/${NAME}-failed-${TRIVY_OUTPUT} ${IMAGE}
		- fi
		- fi
		- IMAGE=""
		- done
		rules:
		- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'
		# rules:
		# - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'
		cache:
		paths:
		- "$TRIVY_CACHE_DIR"
		@@ -170,12 +172,13 @@ links_checker:

		# See https://docs.gitlab.com/ee/api/labels.html
		job_gitlab_labels:
		image: python:3.9.1-alpine
		image: python:${IMAGE_TAG_PYTHON_ALPINE}
		stage: project_setup
		variables:
		PIPENV_PIPFILE: tools/job_gitlab_labels/Pipfile
		JOB_LOGFILE: "job_gitlab_labels.log"
		PYTHONPATH: "./:$PYTHONPATH"
		API_TOKEN: "kTLrRuV7yyrKXC95sSRd"
		before_script:
		- pip install --ignore-installed distlib pipenv
		- pipenv install
		@@ -200,7 +203,7 @@ spell_check:
		release:
		stage: deploy
		image:
		name: alpine/httpie:2.2.0
		name: alpine/httpie:${IMAGE_TAG_HTTPIE_ALPINE}
		entrypoint: [""]
		variables:
		PIPENV_PIPFILE: tools/notify/Pipfile
		@@ -248,7 +251,7 @@ release:
		- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'

		refresh_job_av_database:
		image: python:3.9.1-alpine
		image: python:${IMAGE_TAG_PYTHON_ALPINE}
		stage: static_tests
		variables:
		PIPENV_PIPFILE: tools/job_av/Pipfile
		@@ -274,7 +277,7 @@ refresh_job_av_database:
		- if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_CLAMAV_PIPELINE && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'

		generate_job_av:
		image: python:3.9.1-alpine
		image: python:${IMAGE_TAG_PYTHON_ALPINE}
		stage: static_tests
		dependencies:
		- refresh_job_av_database

README.md

+56 −0

Original line number	Diff line number	Diff line
		@@ -72,6 +72,62 @@ pipenv run mkdocs serve

		4. See your update in live at [https://localhost:8000](https://localhost:8000)

		### CI/CD Pipeline

		This file aims to explain all jobs used on the CI/CD pipeline.

		#### Jobs & stages

		There are several jobs used on the CI/CD pipeline. The following list shows all jobs and their purpose. The jobs are executed in the order they are listed.

		##### Static_tests


		1. `ci_linter`
		This jobs use the [CI lint API](https://docs.gitlab.com/ee/api/lint.html) to validate the configuration of each jobs.yaml file.

		2. `job_structure`
		This job written in Python ensures every files respect the structure we want. It checks that every file has the right name, the right path and the right content.

		3. `job_customs`
		This job written in Python ensures every script of the jobs doesn't made modifications on the repository. It checks that every script doesn't use `git commit` or `git push`.

		4. `job_image_scan`
		Runs only on the default branch. And uses some cache for the images.
		This job uses [trivy](https://aquasecurity.github.io/trivy/) to scan all images used in the jobs. It checks that the image doesn't have any vulnerability.

		5. `code_spell`
		This job uses codespell to check the spelling of the code. It checks that the code doesn't have any spelling mistake.

		6. `links_checker`
		This job ensures all links are valid in the documentation.

		#### merge_test & scheduled pipeline

		A scheduled pipeline is triggered at 8 pm each day to launch a full antivirus scan on each jobs.
		This pipeline triggers 3 jobs :

		1. `refresh_job_av_database`
		Refresh antivirus definition's with `freshclam` command. See the [english documentation(https://help.ubuntu.com/community/ClamAV)(english) or [french documentation](https://doc.ubuntu-fr.org/clamav) for moreinformation.
		2. `generate_job_av`
		This job is only trigger when a branch is being merged or on a schedule pipeline. Iterates over the jobs to get their image and write a .gitlab-ci.yml that can run a child pipeline in order to use ClamAV for virus detection. The generated .gitlab-ci.yml is launched in the next job.
		3. `child_job_av`
		It is launched by the previous job and scan the docker image and warn if they are know virus listed in the database.

		#### project_setup

		1.`job_gitlab_labels`
		This job retrieve all labels in the project and see if each job has it's own label. If not, it creates it and assign it to the job.


		#### deploy

		1. `release`
		This job is like a swiss knife ⚒️ and performs many action.
		First, it creates a new release within GitLab and print the `CHANGELOG` of the created/updated job in the release description.
		Then, it sends a discord notification to the `#updates` channel.


		### How to update hub tools

		#### Guidelines

tools/job_av/job_av.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -26,7 +26,7 @@ JOB_DIR = "job_av"
		SCANNED_IMAGES_FILE = os.getenv("SCANNED_IMAGES_FILE")
		SCANNED_IMAGES = []

		BLACKLIST = ["github/super-linter:v3.14.3", "shiftleft/sast-scan:v1.9.29", "nvuillam/mega-linter:v4", "github/super-linter:v4.9.0"]
		BLACKLIST = ["github/super-linter:v3.14.3", "shiftleft/sast-scan:v1.9.29", "oxsecurity/megalinter:v6.8.0", "github/super-linter:v4.9.0"]


		def argparse_setup():