Commit 591e979f authored by Federico Falconieri's avatar Federico Falconieri
Browse files

Merge branch '16-fix-enhance-grype-job' into 'beta' 

feat!: improves container pipelines, kaniko and grype in monorepo and polirepo repositories

- rules are back to the kaniko job
- kaniko and grype jobs hidden by default, but include default polirepo job
- kaniko and grype now provide hidden monorepo job, to be extended by user in pipelines
- tests are changed to reflect the above

BREAKING CHANGE: use of the word docker was replaced by container where appropriate

See merge request just-ci/templates!30
parents 451e7574 df16a074

.gitlab-ci.yml

+15 −8
Original line number Diff line number Diff line
@@ -45,11 +45,11 @@ jobs:c: 
      - local: tests/jobs/c.yml

    strategy: depend



jobs:docker:

jobs:container:

  stage: test

  trigger:

    include:

      - local: tests/jobs/docker.yml

      - local: tests/jobs/container.yml

    strategy: depend



jobs:docs:
@@ -74,23 +74,30 @@ pipelines:c: 
      - local: tests/pipelines/c.yml

    strategy: depend



pipelines:docker:

pipelines:python:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/docker.yml

      - local: tests/pipelines/python.yml

    strategy: depend



pipelines:python:

pipelines:container:polirepo:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/python.yml

      - local: tests/pipelines/container/polirepo.yml

    strategy: depend



pipelines:container:monorepo:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/container/monorepo.yml

    strategy: depend



pipelines:python&docker:

pipelines:container:python:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/python-docker.yml

      - local: tests/pipelines/container/python.yml

    strategy: depend

README.md

+14 −25
Original line number Diff line number Diff line
@@ -47,7 +47,7 @@ include: 


The above is a template for Python projects. Other templates can be found in `templates/`, for example:



* Python projects with a `Dockerfile`: `templates/python-docker.yml`. This template will use your `Dockerfile` to create an image in which we run tests.

* Python projects with a `Dockerfile`: `templates/container/python.yml`. This template will use your `Dockerfile` to create an image in which we run tests.

* C projects: `templates/c.yml`.



## Disabling specific jobs
@@ -81,46 +81,35 @@ include: 
    ref: 'v3.19.2'

```



# Repositories with multiple `Dockerfile`s

# monorepos vs polirepos



Use [child pipelines](https://docs.gitlab.com/ee/ci/pipelines/parent_child_pipelines.html) to work with multiple dockerfiles. An example is provided, based on a mockup project used within the tests of this repository.

We support both! Read [this](pipelines/container/readme.md) for more information on how to configure pipelines for monorepos and polirepos.



> `.gitlab-ci.yml`

### example python polirepo



Polirepos will likely work out of the box without extra configuration needed.



```yaml

---

include:

  - project: 'just-ci/templates'

    templates: 'templates/docker.yml'

    ref: 'v4.1.0'

    # root image will build fine without any further change needed



# child pipeline for second docker image

second-image:

  stage: build

  inherit:

    variables: false

  trigger:

    include:

      - local: second-image/.gitlab-ci.yml

    strategy: depend

    file: 'templates/container/python.yml'

    ref: v5.0.0-beta.1

```



> `second-image/.gitlab-ci.yml`

### example monorepo



In monorepos you can start by including the following, but you will need to write your own kaniko and grype job. Again, read [this](pipelines/container/readme.md)



```yaml

---

include:

  - project: 'just-ci/templates'

    file: 'templates/docker.yml'

    ref: 'v4.1.0'



variables:

  KANIKO_REGISTRY_IMAGE: ${CI_REGISTRY_IMAGE}/second-image

  KANIKO_CONTEXT: ${CI_PROJECT_DIR}/second-image

  KANIKO_DOCKERFILE: ${CI_PROJECT_DIR}/second-image/Dockerfile

    file: 'templates/container/monorepo.yml'

    ref: v5.0.0-beta.1

```





## Contributing



See [`CONTRIBUTING.md`](CONTRIBUTING.md)

container/grype.yml

0 → 100644
+48 −0
Original line number Diff line number Diff line 
variables:

  SYFT_OUTPUT_FILE: ${CI_COMMIT_SHORT_SHA}.json

  SYFT_REGISTRY_AUTH_AUTHORITY: ${CI_REGISTRY}

  SYFT_REGISTRY_AUTH_USERNAME: ${CI_REGISTRY_USER}

  SYFT_REGISTRY_AUTH_PASSWORD: ${CI_REGISTRY_PASSWORD}

  GRYPE_IMAGE: ${CI_REGISTRY_IMAGE}:dev-${CI_COMMIT_SHORT_SHA}

  GRYPE_OUTPUT_FILE: ${CI_COMMIT_SHORT_SHA}.txt

  GRYPE_FAIL_ON_THRESHOLD: "critical"

  GRYPE_EXTRA_ARGS: ""

  GRYPE_DEFAULT_ARGS: "--only-fixed"

  GRYPE_CVE_BLACKLIST_REGEX: "CVE-xxx"



.grype:

  # TODO: replace alpine and installation with our custom image

  image: alpine:3

  stage: test

  script:

    - apk add --no-cache curl

    # versions are pinned to these because of a bug in grype v0.36.0

    - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin v0.35.1

    - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin v0.31.1

    - echo ${GRYPE_IMAGE}

    # 0) get the SBOM from syft

    - syft packages ${GRYPE_IMAGE} -o json > ${SYFT_OUTPUT_FILE}

    # 1) run grype on syft SBOM report to obtain a full grype report with all vulnerabilities

    - grype sbom:${SYFT_OUTPUT_FILE} --output=table --file ${GRYPE_OUTPUT_FILE}

    # 2) fail job if any of blacklisted vulnerabilities is in grype output. Grype does not provide blacklisting natively.

    - cat ${GRYPE_OUTPUT_FILE} | grep -E ${GRYPE_CVE_BLACKLIST_REGEX} && exit 1 || exit 0

    # 3) fail job if vulnerabilities at or above GRYPE_FAIL_ON_THRESHOLD

    - grype sbom:${SYFT_OUTPUT_FILE} --output=table --file ${GRYPE_OUTPUT_FILE} --fail-on ${GRYPE_FAIL_ON_THRESHOLD} ${GRYPE_DEFAULT_ARGS} ${GRYPE_EXTRA_ARGS}

  artifacts:

    paths:

      - ${SYFT_OUTPUT_FILE}

      - ${GRYPE_OUTPUT_FILE}

  allow_failure: true



# default for polirepos

grype:

  extends: .grype



# hidden job for monorepos

.grype:monorepo:

  extends: .grype

  variables:

    GRYPE_CONTEXT: changeme

    GRYPE_IMAGE: ${CI_REGISTRY_IMAGE}/${GRYPE_CONTEXT}:dev-${CI_COMMIT_SHORT_SHA}

    SYFT_OUTPUT_FILE: ${GRYPE_CONTEXT}-${CI_COMMIT_SHORT_SHA}.json

    GRYPE_OUTPUT_FILE: ${GRYPE_CONTEXT}-${CI_COMMIT_SHORT_SHA}.txt

docker/kaniko.yml→container/kaniko.yml

+27 −1
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ variables: 
  KANIKO_EXTRA_ARGS: ""  # See https://github.com/GoogleContainerTools/kaniko#additional-flags

  KANIKO_CACHE: "true"



kaniko:

.kaniko:

  stage: build

  image:

    name: gcr.io/kaniko-project/executor:debug
@@ -22,3 +22,29 @@ kaniko: 
  script:

    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json

    - /kaniko/executor --cache=${KANIKO_CACHE} --context=${KANIKO_CONTEXT} --dockerfile=${KANIKO_DOCKERFILE} --destination=${KANIKO_REGISTRY_IMAGE}:${KANIKO_DEV_TAG} --destination=${KANIKO_REGISTRY_IMAGE}:${KANIKO_TAG} ${LABELS} ${KANIKO_EXTRA_ARGS}

  rules:

    # master/main

    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH

      variables:

        KANIKO_TAG: "latest"

    # tags

    - if: $CI_COMMIT_TAG

      variables:

        KANIKO_TAG: $CI_COMMIT_TAG

    # branches

    - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH

      variables:

        KANIKO_TAG: ${CI_COMMIT_REF_SLUG}



# default for polirepos

kaniko:

  extends: .kaniko



# monorepos: user should extend this hidden job (one job for each container)

.kaniko:monorepo:

  extends: .kaniko

  variables:

    CONTEXT_PATH: changeme

    KANIKO_REGISTRY_IMAGE: ${CI_REGISTRY_IMAGE}/${CONTEXT_PATH}

    KANIKO_CONTEXT: ${CI_PROJECT_DIR}/${CONTEXT_PATH}

    KANIKO_DOCKERFILE: ${CI_PROJECT_DIR}/${CONTEXT_PATH}/Dockerfile