Commit 7c9ed0e8 authored by Thomas Boni's avatar Thomas Boni
Browse files

Merge branch '681-clean-the-ci-and-remove-duplicate-jobs' into 'latest' 

Resolve "Clean the CI and remove duplicate jobs"

Closes #681

See merge request r2devops/hub!431
parents 41f4f33b ee61f4b9

.gitlab-ci.yml

+73 −19
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ include: 
variables:

  IMAGE_TAG_PYTHON_ALPINE: "alpine3.16"

  IMAGE_TAG_HTTPIE_ALPINE: "3.2.1"

  JOB_FILES_EXTENSION: ".yml"



workflow:

  rules:
@@ -23,28 +24,43 @@ ci_linter: 
    name: alpine/httpie:${IMAGE_TAG_HTTPIE_ALPINE}

    entrypoint: [""]

  variables:

    GITLAB_CI_LINT_URL: "https://gitlab.gnome.org/api/v4/ci/lint?include_merged_yaml=true"

    JOB_FILES_EXTENSION: ".yml"

    GITLAB_CI_LINT_URLS: "https://gitlab.gnome.org https://git.pleroma.social https://0xacab.org https://git.silence.dev"

    GITLAB_API_POSTFIX: "/api/v4/ci/lint?include_merged_yaml=true"

  script:

    - apk add --no-cache jq

    - apk add --no-cache jq git

    - mkdir ci_linter_reports

    - for JOB_PATH in jobs/*; do

    -   echo "Start checking ${JOB_PATH}/${JOB} using Gitlab CI linter API"

    # Get all changed files that match templates yaml in the current branch

    - git fetch

    - JOBS_TEMPLATE_CHANGED=$(git diff-tree --no-commit-id --name-only -r origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -r $CI_COMMIT_SHA | tr ' ' '\n' | grep "jobs/.*${JOB_FILES_EXTENSION}$" | tr '\n' ' ')

    - for JOB_PATH in $JOBS_TEMPLATE_CHANGED; do

    -   echo "Start checking ${JOB_PATH} using Gitlab CI linter API"

    -   JOB=$(basename ${JOB_PATH})

    -   jq --null-input --arg yaml "$(cat ${JOB_PATH}/${JOB}${JOB_FILES_EXTENSION})" '.content=$yaml' | http --print 'b' --check-status "$GITLAB_CI_LINT_URL" Content-Type:"application/json" > ci_linter_reports/${JOB}.json

      # Loop over all GitLab CI linter API URLs until one of them is working

    -   for CI_URL in ${GITLAB_CI_LINT_URLS}; do

    -     GITLAB_CI_LINT_URL="${CI_URL}${GITLAB_API_POSTFIX}"

    -     if jq --null-input --arg yaml "$(cat ${JOB_PATH})" '.content=$yaml' | http --print 'b' --check-status "$GITLAB_CI_LINT_URL" Content-Type:"application/json" > ci_linter_reports/${JOB}.json; then

    -       if [ $(jq -r '.status' ci_linter_reports/${JOB}.json) == "valid" ]; then

    -     echo "👍 job ${JOB_PATH}/${JOB} is valid"

    -         echo "👍 job ${JOB_PATH} is valid"

    -       else

    -     echo "👎 job ${JOB_PATH}/${JOB} is invalid"

    -         echo "👎 job ${JOB_PATH} is invalid"

    -         jq '.' ci_linter_reports/${JOB}.json

    -         exit 1

    -       fi

          # If the API URL is working, stop the loop

    -       break

    -     fi

    -   done

    - done

  artifacts:

    expose_as: "job_ci_lint"

    paths:

      - ci_linter_reports/

    when: always

  rules:

    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

      when: never

    - changes: 

      - "jobs/**/*${JOB_FILES_EXTENSION}"



job_structure:

  image: python:${IMAGE_TAG_PYTHON_ALPINE}
@@ -63,6 +79,9 @@ job_structure: 
    paths:

      - ${JOB_LOGFILE}

    when: always

  rules:

    - changes: 

      - jobs/**/*



job_customs:

  image: python:${IMAGE_TAG_PYTHON_ALPINE}
@@ -80,8 +99,11 @@ job_customs: 
    expose_as: "jobs_customs"

    paths:

      - ${JOB_LOGFILE}

  rules:

    - changes: 

      - jobs/**/*



job_image_scan:

.job_image_scan:

  image: docker:20.10

  stage: static_tests

  services:
@@ -111,7 +133,7 @@ job_image_scan: 
    DOCKER_HOST: tcp://docker:2375

    DOCKER_DRIVER: overlay2

    DOCKER_TLS_CERTDIR: ""

    TRIVY_VERSION: "0.31.3"

    TRIVY_VERSION: "0.36.1"

    TRIVY_REMOTE: ""

    TRIVY_TIMEOUT: ""

    TRIVY_LIGHT: "false"
@@ -120,8 +142,9 @@ job_image_scan: 
    TRIVY_QUIET: "false"

    TRIVY_SKIP_UPDATE: "false"



    SCAN_ALL_IMAGES: "false"

  before_script:

    - apk add --no-cache python3 py3-pip

    - apk add --no-cache python3 py3-pip git

    - pip install --ignore-installed distlib pipenv

    - pipenv install
@@ -130,10 +153,18 @@ job_image_scan: 
    - wget -O $TEMPLATE_NAME https://github.com/aquasecurity/trivy/raw/v${TRIVY_VERSION}/contrib/junit.tpl



    - mkdir ${OUTPUT_DIR}



  script:

    # We force database download once so we don't do much GitHub Api calls

    - ./trivy --cache-dir ${TRIVY_CACHE_DIR} image --download-db-only

    - for JOB in $(ls -A jobs); do

    - if [ "${SCAN_ALL_IMAGES}" = "true" ]; then

    -   JOBS=$(ls -A jobs)

    - else

    # Retrieve the list of jobs that have been modified in the current branch and keep only the job name 

    -   git fetch

    -   JOBS=$(git diff-tree --no-commit-id --name-only -r origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -r $CI_COMMIT_SHA | tr ' ' '\n' | grep "jobs/.*${JOB_FILES_EXTENSION}$" | tr '\n' ' ' | cut -d'/' -f2)

    - fi

    - for JOB in $JOBS; do

    -   IMAGE=$(pipenv run python3 tools/job_image/job_image.py ${JOB})

    -   if [ ! -z ${IMAGE} ]; then

    -     NAME=$(basename ${IMAGE})
@@ -144,8 +175,6 @@ job_image_scan: 
    -   fi

    -   IMAGE=""

    - done

  # rules:

  #   - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'

  cache:

    paths:

      - "$TRIVY_CACHE_DIR"
@@ -160,6 +189,31 @@ job_image_scan: 
    expire_in: 30 days

    when: always





# Extends the job_image_scan job to scan all images

# Is triggered only by the schedule

job_image_scan_schedule:

  extends: .job_image_scan

  variables:

    SCAN_ALL_IMAGES: "true"

  rules:

    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'

    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_CLAMAV_PIPELINE && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'



# Scan only the images of the jobs that have changed on merge requests

job_image_scan:

  extends: .job_image_scan

  variables:

    SCAN_ALL_IMAGES: "false"

  rules:

    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "merge_train"'

      when: never

    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_CLAMAV_PIPELINE'

      when: never

    - changes: 

      - "jobs/**/*${JOB_FILES_EXTENSION}"

      - "tools/job_image/**/*"



links_checker:

  variables:

    # The following hard excluded link is due that this

jobs/aws_s3_sync/README.md

+4 −0
Original line number Diff line number Diff line 
## Objective



!!! error "This job is deprecated 🚨"

    This job has been moved to [`s3_deploy`](https://r2devops.io/_/r2devops-bot/s3_deploy) and must be use instead.

    The job is no more maintained and is now deprecated. Despites it still exists to keep working on pipelines.



This job uses the latest AWS CLI version (v2 for now) in order to sync files between a directory and a target S3 bucket. It's compatible with all s3 object storage (not only AWS).



[AWS CLI](https://aws.amazon.com/cli/){:target="_blank"} is a unified tool to manage your AWS services. With just one tool to download and configure, you can control multiple AWS services from the command line and automate them through scripts.

jobs/deploy_s3_terraform/README.md

+4 −0
Original line number Diff line number Diff line 
## Objective



!!! error "This job is deprecated 🚨"

    This job has been moved to [`s3_deploy`](https://r2devops.io/_/r2devops-bot/s3_deploy) and must be use instead.

    The job is no more maintained and is now deprecated. Despites it still exists to keep working on pipelines.



Deploy a static website on a S3 bucket provided by a Terraform state. It is a fusion of [gitlab-terraform_apply](https://r2devops.io/_/r2devops-bot/gitlab-terraform_apply) and  [aws_s3_sync](https://r2devops.io/_/r2devops-bot/aws_s3_sync).



## How to use it

jobs/super_linter/README.md

+1 −1
Original line number Diff line number Diff line
@@ -95,7 +95,7 @@ customize its behavior. 
| **MARKDOWN_CONFIG_FILE**          | Filename for [Markdownlint configuration](https://github.com/DavidAnson/markdownlint#optionsconfig){:target="_blank"} (ex: `.markdown-lint.yml`, `.markdownlint.json`, `.markdownlint.yaml`) | `.markdown-lint.yml` |

| **PYTHON_PYLINT_CONFIG_FILE**     | Filename for [pylint configuration](https://pylint.pycqa.org/en/latest/user_guide/run.html?highlight=rcfile#command-line-options){:target="_blank"} (ex: `.python-lint`, `.pylintrc`)        | `.python-lint`       |

| **PYTHON_FLAKE8_CONFIG_FILE**     | Filename for [flake8 configuration](https://flake8.pycqa.org/en/latest/user/configuration.html){:target="_blank"} (ex: `.flake8`, `tox.ini`)                                                 | `.flake8`            |

| **PYTHON_BLACK_CONFIG_FILE**      | Filename for [black configuration](https://github.com/psf/black/blob/master/docs/compatible_configs.md){:target="_blank"} (ex: `.isort.cfg`, `pyproject.toml`)                               | `.python-black`      |

| **PYTHON_BLACK_CONFIG_FILE**      | Filename for [black configuration](https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md#black-compatible-configurations){:target="_blank"} (ex: `.isort.cfg`, `pyproject.toml`)                               | `.python-black`      |

| **RUBY_CONFIG_FILE**              | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html){:target="_blank"} (ex: `.ruby-lint.yml`, `.rubocop.yml`)                                           | `.ruby-lint.yml`     |

| **SNAKEMAKE_SNAKEFMT_CONFIG_FILE**| Filename for [Snakemake configuration](https://github.com/snakemake/snakefmt#configuration){:target="_blank"} (ex: `pyproject.toml`, `.snakefmt.toml`)                                       | `.snakefmt.toml`     |

| **TYPESCRIPT_ES_CONFIG_FILE**     | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats){:target="_blank"} (ex: `.eslintrc.yml`, `.eslintrc.json`)                     | `.eslintrc.yml`      |