Commit 043e2620 authored by rob bot's avatar rob bot Committed by Federico Falconieri
Browse files

feat!: v4 

details: https://gitlab.com/just-ci/templates/-/merge_requests/1

BREAKING CHANGE: see link above
parent d9053991

.gitlab-ci.yml

+71 −117
Original line number Diff line number Diff line 
---

include:

  - local: "other/gitlab/project-settings.yml"



  - local: "other/badge.yml"



  - local: "docs/drawio.yml"



  - local: "docker/anchore/grype.yml"



  - local: "pipelines/poetry/docker-pro.yml"



  - local: "python/flake8.yml"

  - local: "python/pylic.yml"

  - local: "python/vulture.yml"

  - local: "python/pip-licenses.yml"



  - local: "c/cppcheck.yml"

  - local: "c/flawfinder.yml"

# CI of ci/templates itself



  - local: "yaml/yamllint.yml"

stages:

  - test

  - pre-release



docker:kaniko:

variables:

    CONTEXT: templates_tests/python



# We only check if it runs properly, not if our image has problems

docker:anchore:grype:

  variables:

    GRYPE_FAIL_ON: ""



python:poetry:check:

  before_script:

    - cd templates_tests/python



python:black:

  before_script:

    - cd templates_tests/python



python:bandit:

  before_script:

    - cd templates_tests/python



python:isort:

  before_script:

    - cd templates_tests/python



python:mypy:

  before_script:

    - cd templates_tests/python

  SCHEDULE_PIPELINE_PROJECT_IDS: "30771297 30771293 30771284 30771254 30771239"



python:pylint:

  before_script:

    - cd templates_tests/python



python:pytest:

  before_script:

    - cd templates_tests/python



python:safety:

  before_script:

    - cd templates_tests/python

include:

  - local: pipelines/project-automation.yml

  - local: pipelines/workflow.yml

  - local: yaml/yamllint.yml

  - local: project-automation/pipeline-scheduler.yml



python:flake8:

  before_script:

    - cd templates_tests/python

gitlab:recommended:

  variables:

    GITLAB_RECOMMENDED_AUTO_FIX: "true"



python:pylic:

  before_script:

    - cd templates_tests/python

tbump:

  stage: pre-release



python:vulture:

  before_script:

    - cd templates_tests/python

# child pipeline to unit tests all our jobs



python:build:

  before_script:

    - cd templates_tests/python

jobs:c:

  stage: test

  trigger:

    include:

      - local: tests/jobs/c.yml

    strategy: depend

  rules:

    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH

    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

  artifacts:

    paths:

      - templates_tests/python/dist



python:twine:

  before_script:

    - cd templates_tests/python

    - if: $CI_OPEN_MERGE_REQUESTS



python:pip-licenses:

  before_script:

    - cd templates_tests/python

  variables:

    ALLOWED: "MIT License;UNKNOWN"

jobs:docker:

  stage: test

  trigger:

    include:

      - local: tests/jobs/docker.yml

    strategy: depend

  rules:

    - if: $CI_OPEN_MERGE_REQUESTS



c:cppcheck:

  variables:

    ERROR_EXIT_CODE: 0

    CHECK_PATH: templates_tests/c

jobs:docs:

  stage: test

  trigger:

    include:

      - local: tests/jobs/docs.yml

    strategy: depend

  rules:

    - if: $CI_OPEN_MERGE_REQUESTS



c:flawfinder:

  variables:

    ERROR_LEVEL: "6"

    CHECK_PATH: templates_tests/c

jobs:python:

  stage: test

  trigger:

    include:

      - local: tests/jobs/python.yml

    strategy: depend

  rules:

    - if: $CI_OPEN_MERGE_REQUESTS



gitlab:project-settings:

  variables:

    AUTO_FIX: "true"

# child pipelines to test our off-the-shelves pipelines

pipelines:c:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/c.yml

    strategy: depend

  rules:

    - if: $CI_OPEN_MERGE_REQUESTS



# We keep this downstream, as we don't expect many MRs for this specific test

downstream:pages:

pipelines:docker:

  stage: test

  trigger:

    project: ci/tests/pages

    include:

      - local: tests/pipelines/docker.yml

    strategy: depend

  rules:

    - if: $CI_COMMIT_TAG

      when: never

    - changes:

        - other/pages-hugo.yml

    - if: $CI_OPEN_MERGE_REQUESTS



# We keep this downstream, as we don't expect many MRs for this specific test

downstream:pandoc:

pipelines:python:

  stage: test

  trigger:

    project: template/report

    include:

      - local: tests/pipelines/python.yml

    strategy: depend

  rules:

    - if: $CI_COMMIT_TAG

      when: never

    - changes:

        - docs/report-pandoc.yml

    - if: $CI_OPEN_MERGE_REQUESTS



semantic-release:dry-run:

  extends: semantic-release

  variables:

    EXTRA_ARGS: "--dry-run --branches ${CI_COMMIT_BRANCH}"

pipelines:python&docker:

  stage: test

  trigger:

    include:

      - local: tests/pipelines/python-docker.yml

    strategy: depend

  rules:

    - if: $CI_COMMIT_TAG

      when: never

    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

      when: never

    - when: always
 
    - if: $CI_OPEN_MERGE_REQUESTS

.releaserc

+21 −14
Original line number Diff line number Diff line 
---

plugins:

  - "@semantic-release/commit-analyzer"

  - "@semantic-release/release-notes-generator"

  - "@semantic-release/gitlab"

  - - "@semantic-release/git"

    - assets:

        - CHANGELOG.md

        - pyproject.toml

        - setup.py

      message: |-

        chore(release): ${nextRelease.version}



        ${nextRelease.notes}
 
{

  "plugins": [

    "@semantic-release/commit-analyzer",

    "@semantic-release/release-notes-generator",

    "@semantic-release/gitlab"

  ],

  "branches": [

    "+([0-9])?(.{+([0-9]),x}).x",

    "main",

    "next",

    "next-major",

    {

      "name": "beta",

      "prerelease": true

    },

    {

      "name": "alpha",

      "prerelease": true

    }

  ]

}

CONTRIBUTING.md

+8 −0
Original line number Diff line number Diff line
@@ -14,3 +14,11 @@ poetry run pre-commit install 
poetry run pre-commit install --hook-type commit-msg

poetry run pre-commit install --hook-type post-commit

```



### testing

We try to keep the code coverage as high as possible here, so if you contribute with a functionality please add a test for it.

How do we test things? We use Gitlab-CI to test our gitlab ci templates of jobs and pipelines! This way we know we don't break stuff in your downstream project.

`tests/jobs` is where we keep jobs tests, `tests/pipelines` is where we keep pipeline tests.

`tests/mockup_projects` is where we keep fake projects to test our jobs and pipelines on.

We use [child pipelines](https://docs.gitlab.com/ee/ci/pipelines/parent_child_pipelines.html) to keep things neat and separate logically separated jobs (different language/pipeline scenario).

For example if you contribute a python job, please add it to the `include` block in `tests/jobs/python.yml`.

README.md

+76 −65
Original line number Diff line number Diff line 
# CI templates

# CI templates v4.0.0-beta3



Maintainers:

* Federico Falconieri: @falconierif
@@ -6,108 +6,119 @@ Maintainers: 


Any questions, problems, suggestions? [Create an issue](issues/new)!



## Version 3 released :tada: 

This repository contains a collection of modular gitlab ci jobs, pipelines and templates.



This repository contains a collection of modular gitlab ci jobs and pipelines.

In order to use them in your project, you will need to have a [GitLab runner](https://docs.gitlab.com/ee/ci/runners/) configured.

## Setup



This is a example of how you import the pylint job from this repository in your 

own project's `.gitlab-ci.yml`:

In order to use our templates your project will need:



```yaml

include:

  - project: 'ci/templates'

    file: 'python/pylint.yml'

```

* a [GitLab runner](https://docs.gitlab.com/ee/ci/runners/) configured at project/group level.

* a `.gitlab-ci.yml` file in the root directory of your repository.



That's all!

and then for the repository automation jobs to work as intended you will need to:



> See the [gitlab docs](https://docs.gitlab.com/ee/ci/yaml/includes.html) for more details on how

> includes work.

* create a Project Access Token with 'api' scope access (settings -> Access Tokens), call it `GL_TOKEN`

* store the Project Access Token in a Gitlab CI/CD variable (settings -> CI/CD -> variables) called `GL_TOKEN`

* set the token to be `protected`

* enable push permission in your protected branches (probably `master` or `main`) of the fictitious user `GL_TOKEN`

* a `.releaserc` file for `semantic-release`. We use one here, you can copy it. If you don't have this file, semantic-release defaults will be used.

* a `tbump.toml` file for `tbump` or a `tbump` section in your `pyproject.toml`. If you don't have any of these files, the job will be skipped. 



## Semantic versioning and breaking changes

## Version 4



We tag changes automagically using [semantic-release](https://semantic-release.gitbook.io/semantic-release/). Every time a merge request is accepted, a semantic-release job produces arelease (and an associated tag).

* [Jobs](https://docs.gitlab.com/ee/ci/jobs/): are individual tasks/tests performed over your repository. You can find them in the directory where they logically belong to, for example the job for pytest is in `python/pytest.yml`. Jobs have the same name of the file they are in (`/` are replaced by `:`) so pytest gitlab job name is `python:pytest`. We try to keep the jobs as unopinionated/universal as possible. Most of our jobs are for python projects, but there are also jobs for c projects and for other automation tasks (badges, semantic-versioning, documentation).

* [Pipelines](https://docs.gitlab.com/ee/ci/pipelines/): are collections of jobs. Jobs can be grouped in [stages](https://docs.gitlab.com/ee/ci/pipelines/). Jobs define what to do, stages define when (and thus in which order) to run them. Our jobs are all configured to always use the default stages `.pre`, `build`, `test`, `deploy`, `.post`. We keep our pipelines in `pipelines`. Our pipelines files group logically connected jobs. For example `pipelines/python.yml` allows to easily import all our python jobs, and `pipelines/docker.yml` allows to easily import the docker jobs and set some other global variables accordingly. Pipelines are opinionated, this is where we put workflow rules and make decisions on which jobs should run together.

* Templates: are collections of pipelines. Users should import templates rather than dealing with pipelines or jobs directly. They are in the directory `templates`. For example the `templates/python-docker.yml` provides the user with pipelines for python and docker jobs, our default workflow rules and project-automation pipeline.



You can import jobs and pipelines from a specific tag or branch if you desire, for example if you

like the way things were before.

If you do not specify a tag or branch, you will import what is on `master`, i.e. the latest.

If your project does not work anymore because there has been a breaking change in `master`,

use an older tag like this:

### Usage



You can import our templates in your projects using gitlab [include](https://docs.gitlab.com/ee/ci/yaml/includes.html) functionality.

Here is an example `.gitlab-ci.yml` importing the python template:



```yaml

include:

  - project: 'ci/templates'

    file: 'python/pylint.yml'

    ref: 'v3.1.2'

    file: 'templates/python.yml'

```



## Customize kaniko builds

### Chosing a template



* python project: `templates/python.yml`

* python project with custom docker image (to be used in the required tests): `templates/python-docker`

* c project: `templates/c.yml`



### Project Automation jobs



In order for the jobs in `project-automation/` to work as intended you will need to:



* create a Project Access Token with 'api' scope access

* store the Project Access Token in a Gitlab CI/CD variable (settings -> CI/CD -> variables) called `GL_TOKEN`

* set the token to be `protected`



Kaniko is a special job here, as it builds Docker images for you without any special requirements,

but also prepares Docker images for specific tests which require an image.

### Disabling specific jobs



You can add variables to customize the Kaniko build. Use the following syntax to setup

a kaniko build job, and change variables where you'd like. See comments for their use.

None are required, you can just just the first 3 lines and get your builds done.

Templates and pipelines may come with jobs you don't want/need to run. We have carefuly selected our defaults to provide the best code quality possible, but if you want you can always disable specific jobs with custom rules.



```yaml

include:

  - project: 'ci/templates'

    file: 'docker/kaniko.yml'

    file: 'templates/python.yml'



docker:kaniko:

  variables:

    # Set to "false" to disable kaniko caching. See here: https://github.com/GoogleContainerTools/kaniko/blob/master/README.md#caching

    USE_CACHE: "true"

    # Will override all default destination with your custom tag

    DESTINATIONS: "--destination $CI_REGISTRY_IMAGE:my_awesome_tag"

    # Use these to give extra args to the kaniko executor. See here: https://github.com/GoogleContainerTools/kaniko/blob/master/README.md#additional-flags  

    EXTRA_ARGS: "--build-arg=ARG=VALUE"

    # Will add a $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA tag to your build, regardless of overriding $DESTINATIONS.

    DEV_BUILD: "true"

    DOCKERFILE: "mydockerfiles/myawesome.Dockerfile"  # Optional, defaults to just Dockerfile



another-build:

  extends: docker:kaniko  # This will inherit the above as well

  variables:

    DESTINATIONS: "--destination $CI_REGISTRY_IMAGE:my_custom_tag"

    DOCKERFILE: "mydockerfiles/mycustom.Dockerfile"

python:pytest:

  rules:

    when: never

```



The example above now has two jobs, one is called `kaniko`, the other `another-build`.



## Python out-of-the-box pipelines

## Semantic versioning and breaking changes



Do you want to use a basic set of jobs from this repository, but can't be bothered to 

import them one by one manually?

You can easily import one of three python pipelines with a focus on levels of quality like this:

We tag changes automagically using [semantic-release](https://semantic-release.gitbook.io/semantic-release/). Every time a merge request is accepted, a semantic-release job produces arelease (and an associated tag).

You can import jobs and pipelines from a specific tag or branch if you desire, for example if you like the way things were before.

If you do not specify a tag or branch, you will import what is on `master`, i.e. the latest.

If your project does not work anymore because there has been a breaking change in `master`, use an older tag like this:



#### Basic

```yaml

include:

  - project: 'ci/templates'

    file: 'pipelines/python-basic.yml'

    file: 'python/pylint.yml'

    ref: 'v3.19.2'

```



#### Advanced

## Monorepos with multiple Docker images



Use [child pipelines](https://docs.gitlab.com/ee/ci/pipelines/parent_child_pipelines.html) to work with multiple dockerfiles. An example is provided, based on a mockup project used within the tests of this repository.



```yaml

# .gitlab-ci.yml

---

include:

  - project: 'ci/templates'

    file: 'pipelines/python-docker-basic.yml'

    templates: 'templates/docker.yml'

    # root image will build fine without any further change needed



# child pipeline for second docker image

second-image:

  stage: build

  inherit:

    variables: false

  trigger:

    include:

      - local: second-image/.gitlab-ci.yml

    strategy: depend

```



#### Pro

```yaml

# second-image/.gitlab-ci.yml

---

include:

  - project: 'ci/templates'

    file: 'pipelines/python-docker-pro.yml'

    file: 'templates/docker.yml'



variables:

  KANIKO_REGISTRY_IMAGE: ${CI_REGISTRY_IMAGE}/second-image

  KANIKO_CONTEXT: ${CI_PROJECT_DIR}/second-image

  KANIKO_DOCKERFILE: ${CI_PROJECT_DIR}/second-image/Dockerfile

```



# Contributing

## Contributing



See [`CONTRIBUTING.md`](CONTRIBUTING.md)
 


# We even do drawio



![](/../-/jobs/artifacts/master/raw/templates_tests/other/drawio.png?job=docs%3Adrawio)

c/cppcheck.yml

+1 −1
Original line number Diff line number Diff line 
---

c:cppcheck:

  stage: test

  image: registry.gitlab.com/notno/test/c:latest

  image: registry.gitlab.com/just-ci/images/c:latest

  variables:

    CHECK_PATH: "."  # Can be a file

    DEFAULT_ARGS: "--report-progress --verbose"