Commit bd5ed0c5 authored by Gaëtan Montury's avatar Gaëtan Montury
Browse files

feat: trivy and syft can use local installation

parent 367c1ad1
Loading
Loading
Loading
Loading
+4 −1
Original line number Diff line number Diff line
@@ -326,7 +326,10 @@ In addition to a textual report in the console, this job produces the following

| Report         | Format                                                                       | Usage             |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-trivy.trivy.json` | [JSON](https://trivy.dev/docs/latest/configuration/reporting/#json) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/trivy/)<br/>_This report is generated only if DefectDojo template is detected_ |
| `$PYTHON_PROJECT_DIR/reports/py-trivy.native.json` | [JSON](https://trivy.dev/docs/latest/configuration/reporting/#json) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/trivy/)<br/>_This report is generated only if DefectDojo template is detected_ |
| `$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-codequality.json` | [Trivy report format for GitLab Code Quality](https://trivy.dev/docs/latest/tutorials/integrations/gitlab-ci/) format       | [GitLab Code Quality](https://docs.gitlab.com/ci/yaml/artifacts_reports/#artifactsreportscodequality)  |
| `$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-sast.json` | [Trivy report format for GitLab SAST](https://trivy.dev/docs/latest/tutorials/integrations/gitlab-ci/) format       | [GitLab SAST](https://docs.gitlab.com/ci/yaml/artifacts_reports/#artifactsreportssast)  |


### `py-sbom` job

+111 −34
Original line number Diff line number Diff line
@@ -379,6 +379,12 @@ variables:
  # BEGSCRIPT
  set -e

  function log_debug() {
    if [ -n "$TRACE" ]; then
      echo -e "[\\e[1;96mDEBUG\\e[0m] 🔰 $*"
    fi
  }

  function log_info() {
      echo -e "[\\e[1;94mINFO\\e[0m] $*"
  }
@@ -1267,17 +1273,44 @@ variables:
  }

  function github_get_latest_version() {
    repo="$1"

    if command -v curl &> /dev/null
    then
      curl -sSf -I "https://github.com/$1/releases/latest" | awk -F '/' -v RS='\r\n' '/location:/ {print $NF}'
      curl -sSf -I "https://github.com/${repo}/releases/latest" | awk -F '/' -v RS='\r\n' '/location:/ {print $NF}'
    elif command -v python3 &> /dev/null
    then
      python3 -c "import urllib.request;url='https://github.com/$1/releases/latest';opener=urllib.request.build_opener(type('NoRedirection', (urllib.request.HTTPErrorProcessor,), {'http_response': lambda self, req, resp: resp, 'https_response': lambda self, req, resp: resp})());req=urllib.request.Request(url, method='HEAD');print(opener.open(req).headers.get('Location').split('/')[-1])"
      python3 -c "import urllib.request;url='https://github.com/${repo}/releases/latest';opener=urllib.request.build_opener(type('NoRedirection', (urllib.request.HTTPErrorProcessor,), {'http_response': lambda self, req, resp: resp, 'https_response': lambda self, req, resp: resp})());req=urllib.request.Request(url, method='HEAD');print(opener.open(req).headers.get('Location').split('/')[-1])"
    else
      fail "curl or python3 required"
    fi
  }

  function tbc_cache_url_path() {
    cache_dir="$XDG_CACHE_HOME"
    prefix="$1"
    url="$2"
    filename="$3"

    cache_hash=$(echo "$url" | md5sum | cut -d" " -f1)
    cache_path="$cache_dir/${prefix}-${cache_hash}${filename:+.${filename}}"
    echo "$cache_path"
  }

  function tbc_cache_put() {
    source="$1"
    cache_path="$2"

    mkdir -p "$(dirname "$cache_path")"
    mv "$source" "$cache_path"
  }

  function download_file() {
    url="$1"
    output="$2"
    python3 -c 'import urllib.request;urllib.request.urlretrieve("'"$url"'","'"$output"'")'
  }

  unscope_variables
  eval_all_secrets

@@ -1645,6 +1678,11 @@ py-trivy:
  script:
    - mkdir -p -m 777 reports
    - |
      if command -v trivy > /dev/null
      then
        log_info "Trivy use local installation"
        python_trivy=trivy
      else
        if [[ -z "$PYTHON_TRIVY_DIST_URL" ]]
        then
          log_info "Trivy version unset: retrieve latest version..."
@@ -1652,18 +1690,23 @@ py-trivy:
          PYTHON_TRIVY_DIST_URL="https://github.com/aquasecurity/trivy/releases/download/${trivy_version}/trivy_${trivy_version:1}_Linux-64bit.tar.gz"
          log_info "... use latest Trivy version: \\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m"
        fi
      python_trivy="$XDG_CACHE_HOME/trivy-$(echo "$PYTHON_TRIVY_DIST_URL" | md5sum | cut -d" " -f1)"
      python_trivy_codequality_tpl="$python_trivy.gitlab-codequality.tpl"
      if [[ -f $python_trivy ]] && [[ -f $python_trivy_codequality_tpl ]]

        python_trivy=$(tbc_cache_url_path "trivy" "$PYTHON_TRIVY_DIST_URL" "trivy")
        python_trivy_codequality_tpl=$(tbc_cache_url_path "trivy" "$PYTHON_TRIVY_DIST_URL" "gitlab-codequality.tpl")
        python_trivy_gitlab_sast_tpl=$(tbc_cache_url_path "trivy" "$PYTHON_TRIVY_DIST_URL" "gitlab.tpl")

        if [[ -f "$python_trivy" ]] && [[ -f "$python_trivy_codequality_tpl" ]] && [[ -f "$python_trivy_gitlab_sast_tpl" ]]
        then
          log_info "Trivy found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): reuse"
        else
          log_info "Trivy not found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): download"
        python3 -c 'import urllib.request;urllib.request.urlretrieve("'$PYTHON_TRIVY_DIST_URL'","trivy.tar.gz")'
        tar zxf trivy.tar.gz trivy contrib/gitlab-codequality.tpl
        mkdir -p $XDG_CACHE_HOME
        mv ./trivy $python_trivy
        mv ./contrib/gitlab-codequality.tpl $python_trivy_codequality_tpl
          download_file "$PYTHON_TRIVY_DIST_URL" "trivy.tar.gz"
          tar zxf trivy.tar.gz trivy contrib/gitlab-codequality.tpl contrib/gitlab.tpl
          tbc_cache_put "./trivy" "$python_trivy"
          tbc_cache_put "./contrib/gitlab-codequality.tpl" "$python_trivy_codequality_tpl"
          tbc_cache_put "./contrib/gitlab.tpl" "$python_trivy_gitlab_sast_tpl"
          rm -f trivy.tar.gz
        fi
      fi
    - |
      case "$PYTHON_BUILD_SYSTEM" in
@@ -1698,7 +1741,7 @@ py-trivy:
      fi

      # Generate the native JSON report that can later be converted to other formats
      $python_trivy fs ${PYTHON_TRIVY_ARGS} --format json --list-all-pkgs --output reports/py-trivy.trivy.json --exit-code 1 ./reports/ > ./reports/trivy.log 2>&1 || exit_code=$?
      $python_trivy fs ${PYTHON_TRIVY_ARGS} --format json --list-all-pkgs --output reports/py-trivy.native.json --exit-code 1 ./reports/ > ./reports/trivy.log 2>&1 || exit_code=$?
      cat ./reports/trivy.log
      if [ $(grep -ic "Number of language-specific files[^0-9]*0$" ./reports/trivy.log) -eq 1 ]; then
        log_error "Could not find a file listing all dependencies with their versions."
@@ -1706,11 +1749,38 @@ py-trivy:
      fi
      rm ./reports/trivy.log

      find_template_in_locations() {
        local filename="$1"
        for path in "/usr/local/share/trivy/templates/$filename" "/usr/share/trivy/templates/$filename" "/contrib/$filename"; do
          if [ -f "$path" ]; then
            echo "$path"
            return
          fi
        done
      }

      # Generate a report in the Code Quality format
      $python_trivy convert --format template --template "@$python_trivy_codequality_tpl" --output reports/py-trivy.gitlab-codequality.json reports/py-trivy.trivy.json
      if [ -z "$python_trivy_codequality_tpl" ]; then
        python_trivy_codequality_tpl=$(find_template_in_locations "gitlab-codequality.tpl")
      fi
      if [ -f "$python_trivy_codequality_tpl" ]; then
        $python_trivy convert --format template --template "@$python_trivy_codequality_tpl" --output reports/py-trivy.gitlab-codequality.json reports/py-trivy.native.json
      else
        log_warn "Trivy gitlab-codequality.tpl template not found: skip gitlab-codequality report generation"
      fi

      # Generate a report in the Gitlab SAST format
      if [ -z "$python_trivy_gitlab_sast_tpl" ]; then
        python_trivy_gitlab_sast_tpl=$(find_template_in_locations "gitlab.tpl")
      fi
      if [ -f "$python_trivy_gitlab_sast_tpl" ]; then
        $python_trivy convert --format template --template "@$python_trivy_gitlab_sast_tpl" --output reports/py-trivy.gitlab-sast.json reports/py-trivy.native.json
      else
        log_warn "Trivy gitlab.tpl template not found: skip gitlab-sast report generation"
      fi

      # console output
      $python_trivy convert --format table reports/py-trivy.trivy.json
      $python_trivy convert --format table reports/py-trivy.native.json
      exit $exit_code
  artifacts:
    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
@@ -1722,6 +1792,7 @@ py-trivy:
      - "$PYTHON_PROJECT_DIR/reports/requirements.txt"
    reports:
      codequality: "$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-codequality.json"
      sast: "$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-sast.json"
  rules:
    # exclude if $PYTHON_TRIVY_ENABLED not set
    - if: '$PYTHON_TRIVY_DISABLED == "true"'
@@ -1763,6 +1834,11 @@ py-sbom:
          ;;
      esac
    - |
      if command -v syft > /dev/null
      then
        log_info "Syft use local installation"
        python_sbom_syft=syft
      else
        if [[ -z "$PYTHON_SBOM_SYFT_URL" ]]
        then
          log_info "Syft version unset: retrieve latest version..."
@@ -1777,6 +1853,7 @@ py-sbom:
          mkdir -p $PIP_CACHE_DIR
          mv ./syft $python_sbom_syft
        fi
      fi
    - $python_sbom_syft dir:. ${TRACE+-vv} --source-name $PYTHON_SBOM_NAME $PYTHON_SBOM_OPTS -o cyclonedx-json > reports/py-sbom.cyclonedx.json
    - chmod a+r reports/py-sbom.cyclonedx.json
  artifacts: