Commit 649c55f9 authored by Ruben ten Hove's avatar Ruben ten Hove
Browse files

feat: performance improvements and job enhancements 

- Pipeline time is down by 50% due to only installing npm packages when they don't yet exist
- Change base images to new just-ci paths
- Revamped badge job to only rely on job artifacts. Also simplified using badges
- First show all failures in gitlab recommended settings before failing
- Add pages test job
- Removed caching for mypy and pytest
parent cb5fd92a

.gitlab-ci.yml

+21 −6
Original line number Diff line number Diff line 
---

# CI of ci/templates itself



stages:

  - test

  - pre-release



variables:

  SCHEDULE_PIPELINE_PROJECT_IDS: "30771297 30771293 30771284 30771254 30771239"
@@ -13,13 +9,32 @@ include: 
  - local: pipelines/workflow.yml

  - local: yaml/yamllint.yml

  - local: project-automation/pipeline-scheduler.yml

  - local: project-automation/pages-hugo.yml



gitlab:recommended:

  variables:

    GITLAB_RECOMMENDED_AUTO_FIX: "true"



tbump:

  stage: pre-release

badge:test:

  extends: .badge

  stage: test

  variables:

    NAME: "branch"

    VALUE: "${CI_COMMIT_REF_NAME}"

    COLOR: "fuchsia"

  rules:

    - if: $GL_TOKEN

      when: never

    - when: always



pages:

  stage: test

  before_script:

    - cd tests/mockup_projects/pages

    - git clone https://github.com/alexandrevicenzi/soho.git themes/soho

    - mkdir content

    - cp ${CI_PROJECT_DIR}/README.md content/index.md

  needs: []



# child pipeline to unit tests all our jobs

docs/report-pandoc.yml

+1 −1
Original line number Diff line number Diff line 
---

docs:pandoc-from-template:

  image:

    name: registry.gitlab.com/notno/pandoc

    name: registry.gitlab.com/just-ci/images/pandoc:latest

    entrypoint: [""]

  variables:

    FORMATS: pdf docx html

pipelines/pages.yml

0 → 100644
+12 −0
Original line number Diff line number Diff line 
---

include:

  - local: "project-automation/badge.yml"

  - local: "project-automation/pages-hugo.yml"



badge:pages:

  extends: .badge

  variables:

    NAME: "GitLab"

    VALUE: "Pages"

    COLOR: "teal"

    URL: "${CI_PAGES_URL}"

pipelines/project-automation.yml

+4 −6
Original line number Diff line number Diff line
@@ -8,14 +8,12 @@ include: 
  - local: "python/tbump.yml"



tbump:

  before_script:

    - NEXT_RELEASE_VERSION=`cat .VERSION` || true

    - echo $NEXT_RELEASE_VERSION

  rules:

    - exists:

        - 'tbump.toml'

      # tbump does not work easily with versions not compliant with PEP440

      # e.g. v1.0.1-beta.1 is not PEP440 compliant

      # but it's the default used by semantic-release

      # as a temporary workaround we run tbump only on default branch

      if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

      exists:

        - tbump.toml

        - pyproject.toml

project-automation/badge.yml

+54 −36
Original line number Diff line number Diff line 
---

# This job will create or update any badges set through other jobs

# This job will create or update a badge when extended using the following template.

# It requires a Project Access Token or Personal Access Token with api scope set

# as a CI/CD variable called GL_TOKEN.

# To create your own automatic badges, add the following to your .gitlab-ci.yml:



# It requires:

# - A Project Access Token set as a CI/CD variable called GL_TOKEN

# - Protected tags set with a wildcard: "v*"

# - A variable called I_WANT_BADGES set to a truthy value



# To create your own automatic badges, add the following global variables to a job:

# badge:your_custom_job:

#   extends: .badge

#   variables:

#   BADGE_job_name: "label-message-color"  # See https://shields.io/

#   BADGE_job_name_URL: "https://some_url/${WITH_VARIABLES}"

# Make sure job_name is replaced and unique, and only contains ASCII or underscores

#     NAME: "version"

#     VALUE: "1.2.3"

#     COLOR: "red"

#     URL: "${CI_PROJECT_URL}/tags/${CI_COMMIT_TAG}"



# Make sure your_custom_job is replaced and unique, and you've changed the variables

# according to your preferences.



# Choose a color from this list: https://github.com/jongracecox/anybadge#colors



# Make sure your rules at least include the requirement of $GL_TOKEN



badge:

  image: registry.gitlab.com/notno/tool/bash:latest

.badge:

  image: registry.gitlab.com/just-ci/images/anybadge:latest

  stage: .post

  variables:

    GIT_STRATEGY: "none"

  script:

    - IMAGE_URL=${CI_PROJECT_URL}/-/jobs/artifacts/${CI_COMMIT_REF_SLUG}/raw/ci_badges/${NAME}.svg?job=${CI_JOB_NAME}

    - mkdir -p ci_badges

    - anybadge -l "${NAME}" -v "${VALUE}" -c "${COLOR}" -f "ci_badges/${NAME}.svg"

    - |

      IFS='-'

      for BADGE_VARIABLE in ${!BADGE_@}; do

        case "${BADGE_VARIABLE}" in *_URL) continue;; esac



        read -ra BADGE_ELEMENTS <<< "${!BADGE_VARIABLE}"

        BADGE_NAME="${BADGE_ELEMENTS[0]}"

        BADGE_URL_VARIABLE=${BADGE_VARIABLE}_URL

        BADGE_URL=${!BADGE_URL_VARIABLE}



        if [ -z "${BADGE_URL}" ]; then

          BADGE_URL="${CI_PROJECT_URL}"

        fi

      echo -e "[+] Created ci_badges/${NAME}.svg:\n- Label: ${NAME}\n- Value: ${VALUE}\n- Color: ${COLOR}\n- URL: ${URL}"



        echo "[*] Creating/updating badge '${BADGE_NAME}'..."

        BADGE_ID=$(curl --silent --fail --header "PRIVATE-TOKEN: ${GL_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/badges" | jq --arg BADGE_NAME "${BADGE_NAME}" '.[] | select(.name==$BADGE_NAME) | .id') || true

    - |

      if [ -z "${GL_TOKEN}" ]; then

        echo "[!] GL_TOKEN not set. You can check the badge in the artifacts. It will not be automatically installed/updated in your project."

        echo "[?] You can set it manually. Create/update a badge named '${NAME}' with the image URL ${IMAGE_URL} and as the link URL ${URL:-$CI_PROJECT_URL}."

        echo "[?] You can also set this badge anywhere else. Just reference the image ${IMAGE_URL}."

        exit 1

      else

        echo "[*] Creating/updating badge..."

        BADGE_ID=$(curl -SsLf -H "PRIVATE-TOKEN: ${GL_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/badges" | jq --arg NAME "${NAME}" '.[] | select(.name==$NAME) | .id') || true

        if [ -z "${BADGE_ID}" ]; then

          echo "[-] Unable to get badge id. It probably doesn't exist yet, so we'll create a new one."

          echo "[-] Badge doesn't exist yet. We'll create a new one..."

          STATUS_CODE=$(curl -SsL -w "%{http_code}" -X POST -H "PRIVATE-TOKEN: ${GL_TOKEN}" -d "image_url=${IMAGE_URL}&link_url=${URL:-$CI_PROJECT_URL}&name=${NAME}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/badges" -o /dev/null)

        else

          echo "Badge '${BADGE_NAME}' id is ${BADGE_ID}."

          echo "[*] Updating badge ID ${BADGE_ID}..."

          STATUS_CODE=$(curl -SsL -w "%{http_code}" -X PUT -H "PRIVATE-TOKEN: ${GL_TOKEN}" -d "image_url=${IMAGE_URL}&link_url=${URL:-$CI_PROJECT_URL}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/badges/${BADGE_ID}" -o /dev/null)

        fi



        IMAGE_URL="https://img.shields.io/badge/${!BADGE_VARIABLE}"



        case ${BADGE_ID} in

          ''|*[!0-9]*) echo "[*] Creating badge..." && curl --silent --fail --request POST --header "PRIVATE-TOKEN: ${GL_TOKEN}" --data "image_url=${IMAGE_URL}&link_url=${BADGE_URL}&name=${BADGE_NAME}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/badges" >/dev/null && echo "[+] Badge '${BADGE_NAME}' created successfully." || echo "[!] Badge '${BADGE_NAME}' creation failed" ;;

          *) echo "[*] Updating badge..." && curl --silent --fail --request PUT --header "PRIVATE-TOKEN: ${GL_TOKEN}" --data "image_url=${IMAGE_URL}&link_url=${BADGE_URL}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/badges/${BADGE_ID}" >/dev/null && echo "[+] Badge '${BADGE_NAME}' updated successfully." || echo "[!] Badge '${BADGE_NAME}' update failed" ;;

        esac

      done

        if [ "${STATUS_CODE:0:1}" = "2" ]; then

          echo "[+] Badge set successfully."

        elif [ "${STATUS_CODE}" = "403" ]; then

          echo "[!] Access Forbidden. Does GL_TOKEN have access to badge creation? A Personal Access Token requires at least a Maintainer user."

          exit 1

        else

          echo "[!] Something went wrong. We received the status code ${STATUS_CODE}."

          exit 1

        fi

      fi

  artifacts:

    paths:

      - ci_badges/*.svg

    when: always

  allow_failure: true

  rules:

    - if: ($CI_COMMIT_TAG =~ /^v[0-9]+(\.[0-9]+){2}$/ && $GL_TOKEN && $I_WANT_BADGES)
 
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'