Commit 46f5297d authored by GridexX's avatar GridexX
Browse files

ci: fix broken links, improve jobs 



Signed-off-by: default avatarGridexX <arsene582@gmail.com>
parent a12a23c9

.gitlab-ci.yml

+73 −19
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ include: 
variables:

  IMAGE_TAG_PYTHON_ALPINE: "alpine3.16"

  IMAGE_TAG_HTTPIE_ALPINE: "3.2.1"

  JOB_FILES_EXTENSION: ".yml"



workflow:

  rules:
@@ -23,28 +24,43 @@ ci_linter: 
    name: alpine/httpie:${IMAGE_TAG_HTTPIE_ALPINE}

    entrypoint: [""]

  variables:

    GITLAB_CI_LINT_URL: "https://gitlab.gnome.org/api/v4/ci/lint?include_merged_yaml=true"

    JOB_FILES_EXTENSION: ".yml"

    GITLAB_CI_LINT_URLS: "https://gitlab.gnome.org https://git.pleroma.social https://0xacab.org https://git.silence.dev"

    GITLAB_API_POSTFIX: "/api/v4/ci/lint?include_merged_yaml=true"

  script:

    - apk add --no-cache jq

    - apk add --no-cache jq git

    - mkdir ci_linter_reports

    - for JOB_PATH in jobs/*; do

    -   echo "Start checking ${JOB_PATH}/${JOB} using Gitlab CI linter API"

    # Get all changed files that match templates yaml in the current branch

    - git fetch

    - JOBS_TEMPLATE_CHANGED=$(git diff-tree --no-commit-id --name-only -r origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -r $CI_COMMIT_SHA | tr ' ' '\n' | grep "jobs/.*${JOB_FILES_EXTENSION}$" | tr '\n' ' ')

    - for JOB_PATH in $JOBS_TEMPLATE_CHANGED; do

    -   echo "Start checking ${JOB_PATH} using Gitlab CI linter API"

    -   JOB=$(basename ${JOB_PATH})

    -   jq --null-input --arg yaml "$(cat ${JOB_PATH}/${JOB}${JOB_FILES_EXTENSION})" '.content=$yaml' | http --print 'b' --check-status "$GITLAB_CI_LINT_URL" Content-Type:"application/json" > ci_linter_reports/${JOB}.json

      # Loop over all GitLab CI linter API URLs until one of them is working

    -   for CI_URL in ${GITLAB_CI_LINT_URLS}; do

    -     GITLAB_CI_LINT_URL="${CI_URL}${GITLAB_API_POSTFIX}"

    -     if jq --null-input --arg yaml "$(cat ${JOB_PATH})" '.content=$yaml' | http --print 'b' --check-status "$GITLAB_CI_LINT_URL" Content-Type:"application/json" > ci_linter_reports/${JOB}.json; then

    -       if [ $(jq -r '.status' ci_linter_reports/${JOB}.json) == "valid" ]; then

    -     echo "👍 job ${JOB_PATH}/${JOB} is valid"

    -         echo "👍 job ${JOB_PATH} is valid"

    -       else

    -     echo "👎 job ${JOB_PATH}/${JOB} is invalid"

    -         echo "👎 job ${JOB_PATH} is invalid"

    -         jq '.' ci_linter_reports/${JOB}.json

    -         exit 1

    -       fi

          # If the API URL is working, stop the loop

    -       break

    -     fi

    -   done

    - done

  artifacts:

    expose_as: "job_ci_lint"

    paths:

      - ci_linter_reports/

    when: always

  rules:

    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

      when: never

    - changes: 

      - "jobs/**/*${JOB_FILES_EXTENSION}"



job_structure:

  image: python:${IMAGE_TAG_PYTHON_ALPINE}
@@ -63,6 +79,9 @@ job_structure: 
    paths:

      - ${JOB_LOGFILE}

    when: always

  rules:

    - changes: 

      - jobs/**/*



job_customs:

  image: python:${IMAGE_TAG_PYTHON_ALPINE}
@@ -80,8 +99,11 @@ job_customs: 
    expose_as: "jobs_customs"

    paths:

      - ${JOB_LOGFILE}

  rules:

    - changes: 

      - jobs/**/*



job_image_scan:

.job_image_scan:

  image: docker:20.10

  stage: static_tests

  services:
@@ -111,7 +133,7 @@ job_image_scan: 
    DOCKER_HOST: tcp://docker:2375

    DOCKER_DRIVER: overlay2

    DOCKER_TLS_CERTDIR: ""

    TRIVY_VERSION: "0.31.3"

    TRIVY_VERSION: "0.36.1"

    TRIVY_REMOTE: ""

    TRIVY_TIMEOUT: ""

    TRIVY_LIGHT: "false"
@@ -120,8 +142,9 @@ job_image_scan: 
    TRIVY_QUIET: "false"

    TRIVY_SKIP_UPDATE: "false"



    SCAN_ALL_IMAGES: "false"

  before_script:

    - apk add --no-cache python3 py3-pip

    - apk add --no-cache python3 py3-pip git

    - pip install --ignore-installed distlib pipenv

    - pipenv install
@@ -130,10 +153,18 @@ job_image_scan: 
    - wget -O $TEMPLATE_NAME https://github.com/aquasecurity/trivy/raw/v${TRIVY_VERSION}/contrib/junit.tpl



    - mkdir ${OUTPUT_DIR}



  script:

    # We force database download once so we don't do much GitHub Api calls

    - ./trivy --cache-dir ${TRIVY_CACHE_DIR} image --download-db-only

    - for JOB in $(ls -A jobs); do

    - if [ "${SCAN_ALL_IMAGES}" = "true" ]; then

    -   JOBS=$(ls -A jobs)

    - else

    # Retrieve the list of jobs that have been modified in the current branch and keep only the job name 

    -   git fetch

    -   JOBS=$(git diff-tree --no-commit-id --name-only -r origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -r $CI_COMMIT_SHA | tr ' ' '\n' | grep "jobs/.*${JOB_FILES_EXTENSION}$" | tr '\n' ' ' | cut -d'/' -f2)

    - fi

    - for JOB in $JOBS; do

    -   IMAGE=$(pipenv run python3 tools/job_image/job_image.py ${JOB})

    -   if [ ! -z ${IMAGE} ]; then

    -     NAME=$(basename ${IMAGE})
@@ -144,8 +175,6 @@ job_image_scan: 
    -   fi

    -   IMAGE=""

    - done

  # rules:

  #   - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'

  cache:

    paths:

      - "$TRIVY_CACHE_DIR"
@@ -160,6 +189,31 @@ job_image_scan: 
    expire_in: 30 days

    when: always





# Extends the job_image_scan job to scan all images

# Is triggered only by the schedule

job_image_scan_schedule:

  extends: .job_image_scan

  variables:

    SCAN_ALL_IMAGES: "true"

  rules:

    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'

    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_CLAMAV_PIPELINE && $CI_PROJECT_PATH_SLUG == "r2devops-hub"'



# Scan only the images of the jobs that have changed on merge requests

job_image_scan:

  extends: .job_image_scan

  variables:

    SCAN_ALL_IMAGES: "false"

  rules:

    - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "merge_train"'

      when: never

    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_CLAMAV_PIPELINE'

      when: never

    - changes: 

      - "jobs/**/*${JOB_FILES_EXTENSION}"

      - "tools/job_image/**/*"



links_checker:

  variables:

    # The following hard excluded link is due that this

jobs/aws_s3_sync/README.md

+4 −0
Original line number Diff line number Diff line 
## Objective



!!! error "This job is deprecated 🚨"

    This job has been moved to [`s3_deploy`](https://r2devops.io/_/r2devops-bot/s3_deploy) and must be use instead.

    The job is no more maintained and is now deprecated. Despites it still exists to keep working on pipelines.



This job uses the latest AWS CLI version (v2 for now) in order to sync files between a directory and a target S3 bucket. It's compatible with all s3 object storage (not only AWS).



[AWS CLI](https://aws.amazon.com/cli/){:target="_blank"} is a unified tool to manage your AWS services. With just one tool to download and configure, you can control multiple AWS services from the command line and automate them through scripts.

jobs/deploy_s3_terraform/README.md

+4 −0
Original line number Diff line number Diff line 
## Objective



!!! error "This job is deprecated 🚨"

    This job has been moved to [`s3_deploy`](https://r2devops.io/_/r2devops-bot/s3_deploy) and must be use instead.

    The job is no more maintained and is now deprecated. Despites it still exists to keep working on pipelines.



Deploy a static website on a S3 bucket provided by a Terraform state. It is a fusion of [gitlab-terraform_apply](https://r2devops.io/_/r2devops-bot/gitlab-terraform_apply) and  [aws_s3_sync](https://r2devops.io/_/r2devops-bot/aws_s3_sync).



## How to use it

jobs/semantic_release/README.md

+1 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ version of the software is needed, and the content of your release note. Learn m 
| Name | Description | Default |

| ---- | ----------- | ------- |

| `SEMANTIC_CONF_PATH` | Direction where `semantic-release` config is available | `.releaserc` |

| `SEMANTIC_DRY_RUN` | Run the tool in [dry run](https://en.wikipedia.org/wiki/Dry_run_(testing)){:target="_blank"} | `false` |

| `SEMANTIC_DRY_RUN` | Run the tool in [dry run](https://en.wikipedia.org/wiki/Dry_run_(testing\)){:target="_blank"} | `false` |

| `GITLAB_TOKEN` | Authentication token to create the release (do not write in in clear, add it as project [CI/CD variable](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project))  | ` ` |

| `SEMANTIC_ADDITIONAL_OPTIONS` | Additional options to run on `semantic-release` command | ` ` |

| `SEMANTIC_ADDITIONAL_PACKAGES` | Additional packages to install needed to match your config (this job already installs all [official plugins](https://github.com/semantic-release/semantic-release/blob/master/docs/extending/plugins-list.md#plugins-list){:target="_blank"}) | ` ` |

jobs/super_linter/README.md

+1 −1
Original line number Diff line number Diff line
@@ -95,7 +95,7 @@ customize its behavior. 
| **MARKDOWN_CONFIG_FILE**          | Filename for [Markdownlint configuration](https://github.com/DavidAnson/markdownlint#optionsconfig){:target="_blank"} (ex: `.markdown-lint.yml`, `.markdownlint.json`, `.markdownlint.yaml`) | `.markdown-lint.yml` |

| **PYTHON_PYLINT_CONFIG_FILE**     | Filename for [pylint configuration](https://pylint.pycqa.org/en/latest/user_guide/run.html?highlight=rcfile#command-line-options){:target="_blank"} (ex: `.python-lint`, `.pylintrc`)        | `.python-lint`       |

| **PYTHON_FLAKE8_CONFIG_FILE**     | Filename for [flake8 configuration](https://flake8.pycqa.org/en/latest/user/configuration.html){:target="_blank"} (ex: `.flake8`, `tox.ini`)                                                 | `.flake8`            |

| **PYTHON_BLACK_CONFIG_FILE**      | Filename for [black configuration](https://github.com/psf/black/blob/master/docs/compatible_configs.md){:target="_blank"} (ex: `.isort.cfg`, `pyproject.toml`)                               | `.python-black`      |

| **PYTHON_BLACK_CONFIG_FILE**      | Filename for [black configuration](https://github.com/psf/black/blob/main/docs/guides/using_black_with_other_tools.md#black-compatible-configurations){:target="_blank"} (ex: `.isort.cfg`, `pyproject.toml`)                               | `.python-black`      |

| **RUBY_CONFIG_FILE**              | Filename for [rubocop configuration](https://docs.rubocop.org/rubocop/configuration.html){:target="_blank"} (ex: `.ruby-lint.yml`, `.rubocop.yml`)                                           | `.ruby-lint.yml`     |

| **SNAKEMAKE_SNAKEFMT_CONFIG_FILE**| Filename for [Snakemake configuration](https://github.com/snakemake/snakefmt#configuration){:target="_blank"} (ex: `pyproject.toml`, `.snakefmt.toml`)                                       | `.snakefmt.toml`     |

| **TYPESCRIPT_ES_CONFIG_FILE**     | Filename for [eslint configuration](https://eslint.org/docs/user-guide/configuring#configuration-file-formats){:target="_blank"} (ex: `.eslintrc.yml`, `.eslintrc.json`)                     | `.eslintrc.yml`      |