Commit 22003d19 authored by Bertrand Goareguer's avatar Bertrand Goareguer Committed by Pierre Smeyers
Browse files

feat: add support for slim and alpine Python images + change default base image 

BREAKING CHANGE: the default base image has been changed to python:debian-slim
parent 8b778f44

README.md

+5 −4
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ include: 
  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1

    # 2: set/override component inputs

    inputs:

      image: registry.hub.docker.com/library/python:3.10

      image: registry.hub.docker.com/library/python:3.12-slim

      pytest-enabled: true

```
@@ -34,7 +34,7 @@ include: 


variables:

  # 2: set/override template variables

  PYTHON_IMAGE: registry.hub.docker.com/library/python:3.10

  PYTHON_IMAGE: registry.hub.docker.com/library/python:3.12-slim

  PYTEST_ENABLED: "true"

```
@@ -44,7 +44,7 @@ The Python template uses some global configuration used throughout all jobs. 


| Input / Variable     | Description                                                                           | Default value      |

| -------------------- | ------------------------------------------------------------------------------------- | ------------------ |

| `image` / `PYTHON_IMAGE` | The Docker image used to run Python <br/>:warning: **set the version required by your project** | `registry.hub.docker.com/library/python:3` |

| `image` / `PYTHON_IMAGE` | The Docker image used to run Python <br/>:warning: **set the version required by your project** | `registry.hub.docker.com/library/python:3-slim` |

| `project-dir` / `PYTHON_PROJECT_DIR` | Python project root directory                                                         | `.`                |

| `build-system` / `PYTHON_BUILD_SYSTEM`| Python build-system to use to install dependencies, build and package the project (see below) | `auto` (auto-detect) |

| `PIP_INDEX_URL`      | Python repository url                                                                 | _none_             |
@@ -243,6 +243,7 @@ It is bound to the `test` stage, and uses the following variables: 
| Input / Variable | Description                                                             | Default value     |

| ---------------- | ----------------------------------------------------------------------- | ----------------- |

| `trivy-enabled` / `PYTHON_TRIVY_ENABLED` | Set to `true` to enable Trivy job                                 | _none_ (disabled) |

| `trivy-dist-url` / `PYTHON_TRIVY_DIST_URL` | Url to the `tar.gz` package for `linux_amd64` of Trivy to use (ex: `https://github.com/aquasecurity/trivy/releases/download/v0.51.1/trivy_0.51.1_Linux-64bit.tar.gz`)<br/>_When unset, the latest version will be used_ | _none_ |

| `trivy-args` / `PYTHON_TRIVY_ARGS`       | Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/) | `--vuln-type library`   |



In addition to a textual report in the console, this job produces the following reports, kept for one day:
@@ -561,7 +562,7 @@ include: 
  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1

    # 2: set/override component inputs

    inputs:

      image: registry.hub.docker.com/library/python:3.10

      image: registry.hub.docker.com/library/python:3.12-slim

      pytest-enabled: true



  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.11.1

kicker.json

+3 −4
Original line number Diff line number Diff line
@@ -10,7 +10,7 @@ 
    {

      "name": "PYTHON_IMAGE",

      "description": "The Docker image used to run Python - **set the version required by your project**",

      "default": "registry.hub.docker.com/library/python:3"

      "default": "registry.hub.docker.com/library/python:3-slim"

    },

    {

      "name": "PYTHON_PROJECT_DIR",
@@ -138,9 +138,8 @@ 
      "enable_with": "PYTHON_TRIVY_ENABLED",

      "variables": [

        {

          "name": "PYTHON_TRIVY_IMAGE",

          "description": "The Docker image used to run Trivy",

          "default": "registry.hub.docker.com/aquasec/trivy:latest",

          "name": "PYTHON_TRIVY_DIST_URL",

          "description": "Url to the `tar.gz` package for `linux_amd64` of Trivy to use\n\n_When unset, the latest version will be used_",

          "advanced": true

        },

        {

templates/gitlab-ci-python.yml

+55 −16
Original line number Diff line number Diff line
@@ -17,7 +17,7 @@ spec: 
  inputs:

    image:

      description: The Docker image used to run Python - **set the version required by your project**

      default: registry.hub.docker.com/library/python:3

      default: registry.hub.docker.com/library/python:3-slim

    project-dir:

      description: Python project root directory

      default: .
@@ -100,9 +100,12 @@ spec: 
      description: Enable Trivy

      type: boolean

      default: false

    trivy-image:

      description: The Docker image used to run Trivy

      default: registry.hub.docker.com/aquasec/trivy:latest

    trivy-dist-url:

      description: |-

        Url to the `tar.gz` package for `linux_amd64` of Trivy to use



        _When unset, the latest version will be used_

      default: ''

    trivy-args:

      description: Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/)

      default: --vuln-type library
@@ -246,7 +249,7 @@ variables: 


  # Trivy tool

  PYTHON_TRIVY_ENABLED: $[[ inputs.trivy-enabled ]]

  PYTHON_TRIVY_IMAGE: $[[ inputs.trivy-image ]]

  PYTHON_TRIVY_DIST_URL: $[[ inputs.trivy-dist-url ]]

  PYTHON_TRIVY_ARGS: $[[ inputs.trivy-args ]]



  PYTHON_SBOM_NAME: $[[ inputs.sbom-name ]]
@@ -508,6 +511,18 @@ variables: 
        else

          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"

        fi

      elif command -v python3 > /dev/null

      then

        decoded=$(mktemp)

        errors=$(mktemp)

        # shellcheck disable=SC2086

        if python3 -c "import urllib.request ; urllib.request.urlretrieve(\"$url\",\"${decoded}\")" > "${errors}" 2>&1 

        then

          export ${name}="$(cat ${decoded})" 

          log_info "Successfully fetched secret \\e[33;1m${name}\\e[0m"

        else

          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"

        fi

      else

        log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"

      fi
@@ -530,7 +545,7 @@ variables: 
      if ! dpkg --status "$@" > /dev/null

      then

        apt-get update

        apt-get install --yes --quiet "$@"

        apt-get install --no-install-recommends --yes --quiet "$@"

      fi

    elif command -v apk > /dev/null

    then
@@ -610,7 +625,7 @@ variables: 
  }



  function maybe_install_poetry() {

    if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] && ! command -v poetry > /dev/null

    if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry.* ]] && ! command -v poetry > /dev/null

    then

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM"
@@ -1136,13 +1151,27 @@ py-trivy: 
  # force no dependencies

  dependencies: []

  script:

    - maybe_install_packages wget apt-transport-https gnupg lsb-release

    - mkdir -p -m 777 reports

    - install_requirements

    - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add -

    - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | tee -a /etc/apt/sources.list.d/trivy.list

    - apt-get update

    - apt-get install trivy

    - |

      if [[ -z "$PYTHON_TRIVY_DIST_URL" ]]

      then

        log_info "Trivy version unset: retrieve latest version..."

        trivy_version=$(python3 -c 'import urllib.request;url="https://github.com/aquasecurity/trivy/releases/latest";opener=urllib.request.build_opener(type("NoRedirection", (urllib.request.HTTPErrorProcessor,), {"http_response": lambda self, req, resp: resp, "https_response": lambda self, req, resp: resp})());req=urllib.request.Request(url, method="HEAD");print(opener.open(req).headers.get("Location").split("/tag/v")[1])')

        PYTHON_TRIVY_DIST_URL="https://github.com/aquasecurity/trivy/releases/download/v${trivy_version}/trivy_${trivy_version}_Linux-64bit.tar.gz"

        log_info "... use latest Trivy version: \\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m"

      fi

      python_trivy="$XDG_CACHE_HOME/trivy-$(echo "$PYTHON_TRIVY_DIST_URL" | md5sum | cut -d" " -f1)"

      if [[ -f $python_trivy ]]

      then

        log_info "Trivy found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): reuse"

      else

        log_info "Trivy not found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): download"

        python3 -c 'import urllib.request;urllib.request.urlretrieve("'$PYTHON_TRIVY_DIST_URL'","trivy.tar.gz")'

        tar zxf trivy.tar.gz trivy

        mkdir -p $XDG_CACHE_HOME

        mv ./trivy $python_trivy

      fi  

    - |

      if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]

      then
@@ -1162,15 +1191,15 @@ py-trivy: 
          log_warn "The ./requirements.txt file does not match the ./reports/requirements.txt file generated via pip freeze. Make sure to include all dependencies with pinned versions in ./requirements.txt and re-commit the file."

        fi

      fi

      if [ $(trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/ | grep -c "Number of language-specific files: 0") -eq 1 ]; then

      if [ $($python_trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/ | grep -c "Number of language-specific files: 0") -eq 1 ]; then

        log_error "Could not find a file listing all dependencies with their versions."

        exit 1

      fi

      if [[ "$DEFECTDOJO_TRIVY_REPORTS" ]]

      then

        trivy fs ${PYTHON_TRIVY_ARGS} --exit-code 0 --list-all-pkgs --format json --output reports/py-trivy.trivy.json ./reports/

        $python_trivy fs ${PYTHON_TRIVY_ARGS} --exit-code 0 --list-all-pkgs --format json --output reports/py-trivy.trivy.json ./reports/

      fi

      trivy fs ${PYTHON_TRIVY_ARGS} --format table ./reports/

      $python_trivy fs ${PYTHON_TRIVY_ARGS} --format table ./reports/

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"

    expire_in: 1 day
@@ -1203,7 +1232,7 @@ py-sbom: 
      if [[ -z "$PYTHON_SBOM_SYFT_URL" ]]

      then

        log_info "Syft version unset: retrieve latest version..."

        syft_version=$(curl -Ls -o /dev/null -w %{url_effective} https://github.com/anchore/syft/releases/latest | grep -o '[^/v]*$')

        syft_version=$(python3 -c 'import urllib.request;print(urllib.request.urlopen("https://github.com/anchore/syft/releases/latest").url)' | grep -o '[^/v]*$')

        PYTHON_SBOM_SYFT_URL="https://github.com/anchore/syft/releases/download/v${syft_version}/syft_${syft_version}_linux_amd64.tar.gz"

        log_info "... use latest Syft version: \\e[32m$PYTHON_SBOM_SYFT_URL\\e[0m"

      fi
@@ -1235,7 +1264,13 @@ py-sbom: 
py-release:

  extends: .python-base

  stage: publish

  before_script:

    - !reference [.python-base, before_script]

    # install git and OpenSSH

    - maybe_install_packages git openssh-client

  script:

    - apt-get update

    - apt-get install -y git openssh-client

    - git config --global user.email "$GITLAB_USER_EMAIL"

    - git config --global user.name "$GITLAB_USER_LOGIN"

    - git checkout -B $CI_COMMIT_REF_NAME
@@ -1257,6 +1292,10 @@ py-release: 
py-publish:

  extends: .python-base

  stage: publish

  before_script:

    - !reference [.python-base, before_script]

    # install curl (to decode @url@ variables)

    - maybe_install_packages curl

  script:

    - py_publish

  artifacts: