Commit 89295945 authored by Gaëtan Montury's avatar Gaëtan Montury
Browse files

Merge branch 'feat/cmd_go' into 'master' 

feat: trivy and syft can use local installation

Closes #129 and #130

See merge request to-be-continuous/python!180
parents 367c1ad1 bd5ed0c5

README.md

+4 −1
Original line number Diff line number Diff line
@@ -326,7 +326,10 @@ In addition to a textual report in the console, this job produces the following 


| Report         | Format                                                                       | Usage             |

| -------------- | ---------------------------------------------------------------------------- | ----------------- |

| `$PYTHON_PROJECT_DIR/reports/py-trivy.trivy.json` | [JSON](https://trivy.dev/docs/latest/configuration/reporting/#json) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/trivy/)<br/>_This report is generated only if DefectDojo template is detected_ |

| `$PYTHON_PROJECT_DIR/reports/py-trivy.native.json` | [JSON](https://trivy.dev/docs/latest/configuration/reporting/#json) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/trivy/)<br/>_This report is generated only if DefectDojo template is detected_ |

| `$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-codequality.json` | [Trivy report format for GitLab Code Quality](https://trivy.dev/docs/latest/tutorials/integrations/gitlab-ci/) format       | [GitLab Code Quality](https://docs.gitlab.com/ci/yaml/artifacts_reports/#artifactsreportscodequality)  |

| `$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-sast.json` | [Trivy report format for GitLab SAST](https://trivy.dev/docs/latest/tutorials/integrations/gitlab-ci/) format       | [GitLab SAST](https://docs.gitlab.com/ci/yaml/artifacts_reports/#artifactsreportssast)  |





### `py-sbom` job

templates/gitlab-ci-python.yml

+111 −34
Original line number Diff line number Diff line
@@ -379,6 +379,12 @@ variables: 
  # BEGSCRIPT

  set -e



  function log_debug() {

    if [ -n "$TRACE" ]; then

      echo -e "[\\e[1;96mDEBUG\\e[0m] 🔰 $*"

    fi

  }



  function log_info() {

      echo -e "[\\e[1;94mINFO\\e[0m] $*"

  }
@@ -1267,17 +1273,44 @@ variables: 
  }



  function github_get_latest_version() {

    repo="$1"



    if command -v curl &> /dev/null

    then

      curl -sSf -I "https://github.com/$1/releases/latest" | awk -F '/' -v RS='\r\n' '/location:/ {print $NF}'

      curl -sSf -I "https://github.com/${repo}/releases/latest" | awk -F '/' -v RS='\r\n' '/location:/ {print $NF}'

    elif command -v python3 &> /dev/null

    then

      python3 -c "import urllib.request;url='https://github.com/$1/releases/latest';opener=urllib.request.build_opener(type('NoRedirection', (urllib.request.HTTPErrorProcessor,), {'http_response': lambda self, req, resp: resp, 'https_response': lambda self, req, resp: resp})());req=urllib.request.Request(url, method='HEAD');print(opener.open(req).headers.get('Location').split('/')[-1])"

      python3 -c "import urllib.request;url='https://github.com/${repo}/releases/latest';opener=urllib.request.build_opener(type('NoRedirection', (urllib.request.HTTPErrorProcessor,), {'http_response': lambda self, req, resp: resp, 'https_response': lambda self, req, resp: resp})());req=urllib.request.Request(url, method='HEAD');print(opener.open(req).headers.get('Location').split('/')[-1])"

    else

      fail "curl or python3 required"

    fi

  }



  function tbc_cache_url_path() {

    cache_dir="$XDG_CACHE_HOME"

    prefix="$1"

    url="$2"

    filename="$3"



    cache_hash=$(echo "$url" | md5sum | cut -d" " -f1)

    cache_path="$cache_dir/${prefix}-${cache_hash}${filename:+.${filename}}"

    echo "$cache_path"

  }



  function tbc_cache_put() {

    source="$1"

    cache_path="$2"



    mkdir -p "$(dirname "$cache_path")"

    mv "$source" "$cache_path"

  }



  function download_file() {

    url="$1"

    output="$2"

    python3 -c 'import urllib.request;urllib.request.urlretrieve("'"$url"'","'"$output"'")'

  }



  unscope_variables

  eval_all_secrets
@@ -1645,6 +1678,11 @@ py-trivy: 
  script:

    - mkdir -p -m 777 reports

    - |

      if command -v trivy > /dev/null

      then

        log_info "Trivy use local installation"

        python_trivy=trivy

      else

        if [[ -z "$PYTHON_TRIVY_DIST_URL" ]]

        then

          log_info "Trivy version unset: retrieve latest version..."
@@ -1652,18 +1690,23 @@ py-trivy: 
          PYTHON_TRIVY_DIST_URL="https://github.com/aquasecurity/trivy/releases/download/${trivy_version}/trivy_${trivy_version:1}_Linux-64bit.tar.gz"

          log_info "... use latest Trivy version: \\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m"

        fi

      python_trivy="$XDG_CACHE_HOME/trivy-$(echo "$PYTHON_TRIVY_DIST_URL" | md5sum | cut -d" " -f1)"

      python_trivy_codequality_tpl="$python_trivy.gitlab-codequality.tpl"

      if [[ -f $python_trivy ]] && [[ -f $python_trivy_codequality_tpl ]]



        python_trivy=$(tbc_cache_url_path "trivy" "$PYTHON_TRIVY_DIST_URL" "trivy")

        python_trivy_codequality_tpl=$(tbc_cache_url_path "trivy" "$PYTHON_TRIVY_DIST_URL" "gitlab-codequality.tpl")

        python_trivy_gitlab_sast_tpl=$(tbc_cache_url_path "trivy" "$PYTHON_TRIVY_DIST_URL" "gitlab.tpl")



        if [[ -f "$python_trivy" ]] && [[ -f "$python_trivy_codequality_tpl" ]] && [[ -f "$python_trivy_gitlab_sast_tpl" ]]

        then

          log_info "Trivy found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): reuse"

        else

          log_info "Trivy not found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): download"

        python3 -c 'import urllib.request;urllib.request.urlretrieve("'$PYTHON_TRIVY_DIST_URL'","trivy.tar.gz")'

        tar zxf trivy.tar.gz trivy contrib/gitlab-codequality.tpl

        mkdir -p $XDG_CACHE_HOME

        mv ./trivy $python_trivy

        mv ./contrib/gitlab-codequality.tpl $python_trivy_codequality_tpl

          download_file "$PYTHON_TRIVY_DIST_URL" "trivy.tar.gz"

          tar zxf trivy.tar.gz trivy contrib/gitlab-codequality.tpl contrib/gitlab.tpl

          tbc_cache_put "./trivy" "$python_trivy"

          tbc_cache_put "./contrib/gitlab-codequality.tpl" "$python_trivy_codequality_tpl"

          tbc_cache_put "./contrib/gitlab.tpl" "$python_trivy_gitlab_sast_tpl"

          rm -f trivy.tar.gz

        fi

      fi

    - |

      case "$PYTHON_BUILD_SYSTEM" in
@@ -1698,7 +1741,7 @@ py-trivy: 
      fi



      # Generate the native JSON report that can later be converted to other formats

      $python_trivy fs ${PYTHON_TRIVY_ARGS} --format json --list-all-pkgs --output reports/py-trivy.trivy.json --exit-code 1 ./reports/ > ./reports/trivy.log 2>&1 || exit_code=$?

      $python_trivy fs ${PYTHON_TRIVY_ARGS} --format json --list-all-pkgs --output reports/py-trivy.native.json --exit-code 1 ./reports/ > ./reports/trivy.log 2>&1 || exit_code=$?

      cat ./reports/trivy.log

      if [ $(grep -ic "Number of language-specific files[^0-9]*0$" ./reports/trivy.log) -eq 1 ]; then

        log_error "Could not find a file listing all dependencies with their versions."
@@ -1706,11 +1749,38 @@ py-trivy: 
      fi

      rm ./reports/trivy.log



      find_template_in_locations() {

        local filename="$1"

        for path in "/usr/local/share/trivy/templates/$filename" "/usr/share/trivy/templates/$filename" "/contrib/$filename"; do

          if [ -f "$path" ]; then

            echo "$path"

            return

          fi

        done

      }



      # Generate a report in the Code Quality format

      $python_trivy convert --format template --template "@$python_trivy_codequality_tpl" --output reports/py-trivy.gitlab-codequality.json reports/py-trivy.trivy.json

      if [ -z "$python_trivy_codequality_tpl" ]; then

        python_trivy_codequality_tpl=$(find_template_in_locations "gitlab-codequality.tpl")

      fi

      if [ -f "$python_trivy_codequality_tpl" ]; then

        $python_trivy convert --format template --template "@$python_trivy_codequality_tpl" --output reports/py-trivy.gitlab-codequality.json reports/py-trivy.native.json

      else

        log_warn "Trivy gitlab-codequality.tpl template not found: skip gitlab-codequality report generation"

      fi



      # Generate a report in the Gitlab SAST format

      if [ -z "$python_trivy_gitlab_sast_tpl" ]; then

        python_trivy_gitlab_sast_tpl=$(find_template_in_locations "gitlab.tpl")

      fi

      if [ -f "$python_trivy_gitlab_sast_tpl" ]; then

        $python_trivy convert --format template --template "@$python_trivy_gitlab_sast_tpl" --output reports/py-trivy.gitlab-sast.json reports/py-trivy.native.json

      else

        log_warn "Trivy gitlab.tpl template not found: skip gitlab-sast report generation"

      fi



      # console output

      $python_trivy convert --format table reports/py-trivy.trivy.json

      $python_trivy convert --format table reports/py-trivy.native.json

      exit $exit_code

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
@@ -1722,6 +1792,7 @@ py-trivy: 
      - "$PYTHON_PROJECT_DIR/reports/requirements.txt"

    reports:

      codequality: "$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-codequality.json"

      sast: "$PYTHON_PROJECT_DIR/reports/py-trivy.gitlab-sast.json"

  rules:

    # exclude if $PYTHON_TRIVY_ENABLED not set

    - if: '$PYTHON_TRIVY_DISABLED == "true"'
@@ -1763,6 +1834,11 @@ py-sbom: 
          ;;

      esac

    - |

      if command -v syft > /dev/null

      then

        log_info "Syft use local installation"

        python_sbom_syft=syft

      else

        if [[ -z "$PYTHON_SBOM_SYFT_URL" ]]

        then

          log_info "Syft version unset: retrieve latest version..."
@@ -1777,6 +1853,7 @@ py-sbom: 
          mkdir -p $PIP_CACHE_DIR

          mv ./syft $python_sbom_syft

        fi

      fi

    - $python_sbom_syft dir:. ${TRACE+-vv} --source-name $PYTHON_SBOM_NAME $PYTHON_SBOM_OPTS -o cyclonedx-json > reports/py-sbom.cyclonedx.json

    - chmod a+r reports/py-sbom.cyclonedx.json

  artifacts: