Commit ce72a706 authored by Federico Falconieri's avatar Federico Falconieri
Browse files

Merge branch 'molyrepo' into 'beta' 

feat: single template for image building

See merge request just-ci/templates!73
parents c5b33e35 4b5ab751

.gitlab-ci.yml

+3 −3
Original line number Diff line number Diff line
@@ -62,7 +62,7 @@ jobs:python: 
      - local: tests/jobs/python.yml

    strategy: depend



# child pipelines to test our off-the-shelves pipelines

# child pipelines to test our off-the-shelf pipelines

pipelines:c:

  stage: test

  trigger:
@@ -88,12 +88,12 @@ pipelines:kaniko:monorepo: 
  stage: test

  trigger:

    include:

      - local: tests/pipelines/kaniko/monorepo.yml

      - local: tests/pipelines/container/monorepo.yml

    strategy: depend



pipelines:kaniko:python:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/kaniko/python.yml

      - local: tests/pipelines/container/python.yml

    strategy: depend

README.md

+11 −37
Original line number Diff line number Diff line
@@ -36,8 +36,8 @@ templates. 


- **Templates** are collections of pipelines. Users should import templates

  rather than dealing with pipelines or jobs directly. They are in the directory

  `templates`. For example the `templates/python-docker.yml` provides the user

  with pipelines for python and docker jobs, our default workflow rules and

  `templates`. For example the `templates/container/python.yml` provides the

  user with pipelines for python and docker jobs, our default workflow rules and

  project-automation pipeline.



# How to use
@@ -100,9 +100,9 @@ you want you can always disable specific jobs with custom rules. 
```yaml

---

include:

  - project: "just-ci/templates"

    file: "templates/python.yml"

    ref: "v5.4.0"

  - project: just-ci/templates

    file: templates/python.yml

    ref: v5.4.0



python:pytest:

  rules:
@@ -122,41 +122,15 @@ there has been a breaking change in `master`, use an older tag like this: 
```yaml

---

include:

  - project: "just-ci/templates"

    file: "python/pylint.yml"

    ref: "v3.19.2"

```



# monorepos vs polyrepos



We support both! Read [this](pipelines/container/readme.md) for more information

on how to configure pipelines for monorepos and polyrepos.



### example python polyrepo



polyrepos will likely work out of the box without extra configuration needed.



```yaml

---

include:

  - project: "just-ci/templates"

    file: "templates/container/python.yml"

    ref: v5.4.0-beta.1

  - project: just-ci/templates

    file: python/pylint.yml

    ref: v3.19.2

```



### example monorepo



In monorepos you can start by including the following, but you will need to

write your own kaniko and grype job. Again, read

[this](pipelines/container/readme.md)

# Monorepo (multiple Dockerfiles) or polyrepo (one Dockerfile)



```yaml

---

include:

  - project: "just-ci/templates"

    file: "templates/container/monorepo.yml"

    ref: v5.4.0-beta.1

```

We support both! Read [this](pipelines/container.md) for more information on how

to configure pipelines for monorepos and polyrepos.



## Contributing

container/buildah.yml

+12 −21
Original line number Diff line number Diff line
@@ -8,34 +8,25 @@ variables: 


.buildah:

  stage: build

  extends: .container:rules

  extends: [".image:rules", ".image:name"]

  image: quay.io/buildah/stable:latest

  script:

    - echo "[*] Building and pushing '${IMAGE_CONTAINERFILE}' in context

      '${IMAGE_CONTEXT}' to '${IMAGE_NAME}:${IMAGE_DEV_TAG}' and

      '${IMAGE_NAME}:${IMAGE_TAG}'."

    - if [ "${BUILDAH_EXTRA_ARGS}" != "" ]; then echo "[*] Applying the extra

      arguments '${BUILDAH_EXTRA_ARGS}'."; fi

    - !reference [".image:name", script]

    - |

      if [ "${BUILDAH_EXTRA_ARGS}" != "" ]; then

        echo "[*] Applying the extra arguments '${BUILDAH_EXTRA_ARGS}'."

      fi

    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n

      ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" >

      /tmp/auth.json

    - buildah build --authfile /tmp/auth.json -f ${IMAGE_CONTAINERFILE}

    - |

      if [ "${IMAGE_CONTAINERFILE}" != "" ]; then

        CONTAINERFILE_ARG="-f ${IMAGE_CONTAINERFILE}"

      fi

    - buildah build --authfile /tmp/auth.json ${CONTAINERFILE_ARG}

      --tag=${IMAGE_NAME}:${IMAGE_DEV_TAG} --tag=${IMAGE_NAME}:${IMAGE_TAG}

      --layers=${IMAGE_CACHE} ${BUILDAH_EXTRA_ARGS}

      --layers=${IMAGE_CACHE} ${BUILDAH_EXTRA_ARGS} ${IMAGE_CONTEXT}

    - |

      for IMAGE in "${IMAGE_NAME}:${IMAGE_TAG}" "${IMAGE_NAME}:${IMAGE_DEV_TAG}"; do

        buildah push --authfile /tmp/auth.json ${IMAGE}

      done
 


# default for polyrepos

buildah:

  extends: .buildah



# monorepos: user should extend this hidden job (one job for each container)

.buildah:monorepo:

  extends: .buildah

  variables:

    CONTEXT_PATH: changeme

    IMAGE_NAME: ${CI_REGISTRY_IMAGE}/${CONTEXT_PATH}

    IMAGE_CONTEXT: ${CI_PROJECT_DIR}/${CONTEXT_PATH}

    IMAGE_CONTAINERFILE: ${CI_PROJECT_DIR}/${CONTEXT_PATH}/Dockerfile

container/grype.yml

+20 −33
Original line number Diff line number Diff line 
---

include:

  - local: container/image.yml



variables:

  SYFT_OUTPUT_FILE: ${CI_COMMIT_SHORT_SHA}.json

  SYFT_REGISTRY_AUTH_AUTHORITY: ${CI_REGISTRY}

  SYFT_REGISTRY_AUTH_USERNAME: ${CI_REGISTRY_USER}

  SYFT_REGISTRY_AUTH_PASSWORD: ${CI_REGISTRY_PASSWORD}

  GRYPE_IMAGE: ${CI_REGISTRY_IMAGE}:dev-${CI_COMMIT_SHORT_SHA}

  GRYPE_OUTPUT_FILE: ${CI_COMMIT_SHORT_SHA}.txt

  GRYPE_FAIL_ON_THRESHOLD: "critical"

  GRYPE_FAIL_ON_THRESHOLD: critical

  GRYPE_EXTRA_ARGS: ""

  GRYPE_DEFAULT_ARGS: "--only-fixed"

  GRYPE_CVE_BLACKLIST_REGEX: "CVE-xxx"

  GRYPE_DEFAULT_ARGS: --only-fixed

  GRYPE_CVE_BLACKLIST_REGEX: CVE-xxx



.grype:

  # TODO: replace alpine and installation with our custom image

  image: docker.io/alpine:3

  stage: test

  script:

    - apk add --no-cache curl

    # versions are pinned to these because of a bug in grype v0.36.0

    - curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh

      | sh -s -- -b /usr/local/bin v0.35.1

    - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh

      | sh -s -- -b /usr/local/bin v0.31.1

    - echo ${GRYPE_IMAGE}

    - !reference [".image:name", script]

    - wget -O- https://raw.githubusercontent.com/anchore/syft/main/install.sh |

      sh -s -- -b /usr/local/bin

    - wget -O- https://raw.githubusercontent.com/anchore/grype/main/install.sh |

      sh -s -- -b /usr/local/bin

    - echo "${IMAGE_NAME}:${IMAGE_DEV_TAG}"

    # 0) get the SBOM from syft

    - syft packages ${GRYPE_IMAGE} -o json > ${SYFT_OUTPUT_FILE}

    - syft packages ${IMAGE_NAME}:${IMAGE_DEV_TAG} -o json > syft.json

    # 1) run grype on syft SBOM report to obtain a full grype report with all vulnerabilities

    - grype sbom:${SYFT_OUTPUT_FILE} --output=table --file ${GRYPE_OUTPUT_FILE}

    - grype sbom:syft.json --output=table --file grype.txt

    # 2) fail job if any of blacklisted vulnerabilities is in grype output. Grype does not provide blacklisting natively.

    - cat ${GRYPE_OUTPUT_FILE} | grep -E ${GRYPE_CVE_BLACKLIST_REGEX} && exit 1

      || exit 0

    - cat grype.txt | grep -E ${GRYPE_CVE_BLACKLIST_REGEX} && exit 1 || exit 0

    # 3) fail job if vulnerabilities at or above GRYPE_FAIL_ON_THRESHOLD

    - grype sbom:${SYFT_OUTPUT_FILE} --output=table --file ${GRYPE_OUTPUT_FILE}

      --fail-on ${GRYPE_FAIL_ON_THRESHOLD} ${GRYPE_DEFAULT_ARGS}

      ${GRYPE_EXTRA_ARGS}

    - grype sbom:syft.json --output=table --file grype.txt --fail-on

      ${GRYPE_FAIL_ON_THRESHOLD} ${GRYPE_DEFAULT_ARGS} ${GRYPE_EXTRA_ARGS}

  artifacts:

    paths:

      - ${SYFT_OUTPUT_FILE}

      - ${GRYPE_OUTPUT_FILE}

      - syft.json

      - grype.txt

    when: always

  allow_failure: true



# default for polyrepos

grype:

  extends: .grype
 


# hidden job for monorepos

.grype:monorepo:

  extends: .grype

  variables:

    GRYPE_CONTEXT: changeme

    GRYPE_IMAGE: ${CI_REGISTRY_IMAGE}/${GRYPE_CONTEXT}:dev-${CI_COMMIT_SHORT_SHA}

    SYFT_OUTPUT_FILE: ${GRYPE_CONTEXT}-${CI_COMMIT_SHORT_SHA}.json

    GRYPE_OUTPUT_FILE: ${GRYPE_CONTEXT}-${CI_COMMIT_SHORT_SHA}.txt

container/image.yml

+42 −12
Original line number Diff line number Diff line 
---

# IMAGE_DEV_TAG: to always be used for follow-up jobs



variables:

  IMAGE_NAME: ${CI_REGISTRY_IMAGE}

  IMAGE_TAG: ${CI_COMMIT_SHORT_SHA}

  IMAGE_DEV_TAG: dev-${CI_COMMIT_SHORT_SHA} # allows registry cleanup policy

  IMAGE_DEV_TAG: dev-${CI_COMMIT_SHORT_SHA} # to always be used for follow-up jobs

  IMAGE_CONTEXT: ${CI_PROJECT_DIR}

  IMAGE_CONTAINERFILE: "Dockerfile" # Can be a path

  IMAGE_CACHE: "true"



.container:rules:

.image:rules:

  rules:

    # default branch

    - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH

    - if: "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $IMAGE_TAG == null"

      variables:

        IMAGE_TAG: "latest"

        IMAGE_TAG: latest

    # tags

    - if: $CI_COMMIT_TAG

    - if: "$CI_COMMIT_TAG && $IMAGE_TAG == null"

      variables:

        IMAGE_TAG: $CI_COMMIT_TAG

        IMAGE_TAG: ${CI_COMMIT_TAG}

    # non-default branches

    - if: $CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH

    - if: "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH && $IMAGE_TAG == null"

      variables:

        IMAGE_TAG: ${CI_COMMIT_REF_SLUG}

    # Always run if IMAGE_TAG is set by user

    - if: $IMAGE_TAG



.image:name:

  script:

    - IMAGE_CONTEXT=${IMAGE_CONTEXT/${CI_PROJECT_DIR}\/} # Remove CI_PROJECT_DIR prefix path

    - |

      if [ "${IMAGE_NAME}" = "" ]; then

        if [ "$(realpath ${IMAGE_CONTEXT})" != "$(realpath ${CI_PROJECT_DIR})" ]; then

          if [ "$(echo ${IMAGE_CONTEXT} | tr -cd '/' | wc -c)" -gt 2 ]; then

            echo "[!] Your image path is too deep. The limit is 2. Please set IMAGE_NAME to '\${CI_REGISTRY_IMAGE}/your_name/up_to_here'."

            exit 1

          fi

          IMAGE_NAME=$(echo ${CI_REGISTRY_IMAGE}/${IMAGE_CONTEXT} | tr [:upper:] [:lower:])  # Ensure lowercase

        else

          IMAGE_NAME=${CI_REGISTRY_IMAGE}

        fi

      fi

      echo "IMAGE_NAME=${IMAGE_NAME}" > ${CI_PROJECT_DIR}/image.env

    - |

      echo "[*] Build info:"

      echo "Containerfile: ${IMAGE_CONTAINERFILE:-(default builder value - usually Dockerfile)}"

      echo "Context: ${IMAGE_CONTEXT}"

      echo "Image name: ${IMAGE_NAME}"

      echo "Tags: ${IMAGE_TAG} ${IMAGE_DEV_TAG}"

  artifacts:

    reports:

      dotenv: image.env



# Allows for less problems with a monorepo setup. See pipelines/container.md.

.image:build:

  extends: .kaniko # Default to kaniko.



image:build:

  extends: .image:build