Commit 14bdd00a authored by Mathieu Coupé's avatar Mathieu Coupé Committed by Pierre Smeyers
Browse files

feat: add pipelines cleanup feature

parent 19feeea3

.gitlab-ci.yml

+2 −3
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ include: 
  - component: "gitlab.com/$TBC_NAMESPACE/docker/gitlab-ci-docker@5"

    inputs:

      healthcheck-disabled: true

      sbom-disabled: true

      build-args: "--cache-ttl=6h"

      prod-publish-strategy: "auto"

      release-extra-tags: "latest \\g<major>.\\g<minor>\\g<build> \\g<major>\\g<build>"
@@ -13,7 +12,6 @@ include: 
    inputs:

      image: "registry.hub.docker.com/library/python:3.12"

      ruff-enabled: true

      sbom-disabled: true

      package-enabled: true

      release-enabled: true

  # semantic-release template
@@ -35,8 +33,9 @@ dry-run-test: 
    GITLAB_API: https://gitlab.com/api/v4

    GROUP_PATH: to-be-continuous

    EXCLUDE: samples

    KEEP_PIPELINES_PER_BRANCH: 'main:10 renovate/*:5 25'

  script:

    - gitlab-butler --dry-run

    -  gitlab-butler --dry-run --pipeline-deletion-limit 5 --verbose

  rules:

    # run only on original project on gitlab.com

    - if: '$CI_SERVER_HOST == "gitlab.com"'

README.md

+143 −12
Original line number Diff line number Diff line
@@ -16,9 +16,11 @@ gitlab-butler --help 
## Usage



```bash

usage: gitlab-butler [-h] [--api-url API_URL] [--token TOKEN] [--group-path GROUP_PATH] [--exclude EXCLUDE] [--insecure] [--dry-run] [--halt-on-error]

usage: gitlab-butler [-h] [--api-url API_URL] [--token TOKEN] [--group-path GROUP_PATH] [--skip-subgroups] [--exclude EXCLUDE] [--insecure] [--dry-run] [--verbose] [--halt-on-error] [--delay-between-api-calls DELAY_BETWEEN_API_CALLS]

                     [--pipeline-deletion-limit PIPELINE_DELETION_LIMIT] [--pipelines-keep-per-branch PIPELINES_KEEP_PER_BRANCH] [--pipelines-keep-per-tag PIPELINES_KEEP_PER_TAG] [--pipelines-keep-per-mr PIPELINES_KEEP_PER_MR]

                     [--pipelines-delete-older-than PIPELINES_DELETE_OLDER_THAN]



This tool recusively cleanups a GitLab group.

This tool recursively cleanups a GitLab group.



options:

  -h, --help            show this help message and exit
@@ -26,21 +28,150 @@ options: 
  --token TOKEN         GitLab token

  --group-path GROUP_PATH

                        GitLab root group path to cleanup

  --skip-subgroups      skip subgroups cleaning

  --exclude EXCLUDE     project/group path to exclude from processing (relative to --group-path)

  --insecure            skip SSL verification

  --dry-run             dry run (don't execute any write action)

  --verbose             verbose logs

  --halt-on-error       halt whenever an error occurs

  --delay-between-api-calls DELAY_BETWEEN_API_CALLS

                        delay between GitLab API calls, in seconds

  --pipeline-deletion-limit PIPELINE_DELETION_LIMIT

                        maximum number of pipelines deleted per project

  --pipelines-keep-per-branch PIPELINES_KEEP_PER_BRANCH

                        number of pipelines to keep per branch (⚠ branch MUST still exist)

  --pipelines-keep-per-tag PIPELINES_KEEP_PER_TAG

                        number of pipelines to keep per tag (⚠ tag MUST still exist)

  --pipelines-keep-per-mr PIPELINES_KEEP_PER_MR

                        number of pipelines to keep per merge request (⚠ MR MUST still exist)

  --pipelines-delete-older-than PIPELINES_DELETE_OLDER_THAN

                        max age (in days) after which pipelines are deleted (unless they are kept by a keep rule)

```



| CLI option                          | Env. Variable                  | Description                                                                                    |

| ----------------- | --------------- | ---------------------------------------------------------------------------------------------- |

| ----------------------------------- | ------------------------------ | ---------------------------------------------------------------------------------------------- |

| `--api-api`                         | `$GITLAB_API`                  | GitLab source API url (defaults to `https://gitlab.com/api/v4`)                                |

| `--insecure`                        | `$INSECURE`                    | skip SSL verification                                                                          |

| `--token`                           | `$GITLAB_TOKEN`                | GitLab source token                                                                            |

| `--group-path`                      | `$GROUP_PATH`                  | GitLab root group path to cleanup (**mandatory**)                                              |

| `--skip-subgroups`                  | _none_                         | skip subgroups cleaning                                                                        |

| `--exclude`                         | `$EXCLUDE`                     | project/group path(s) to exclude (multiple CLI option; env. variable is a coma separated list) |

| `--insecure`      | `$INSECURE`     | skip SSL verification                                                                          |

| `--dry-run`                         | _none_                         | dry run (don't execute any write action)                                                       |

| `--verbose`                         | _none_                         | verbose logs                                                                                   |

| `--halt-on-error`                   | _none_                         | halt when an error occurs                                                                      |

| `--delay-between-api-calls-in-secs` | _none_                         | delay between GitLab API calls, in seconds (default = 3 seconds)                               |

| `--pipeline-deletion-limit`         | `$PIPELINES_DELETION_LIMIT`    | maximum number of pipelines deleted per project (default = no limit)                           |

| `--pipelines-keep-per-branch`       | `$PIPELINES_KEEP_PER_BRANCH`   | number of pipelines to keep per branch (⚠ branch MUST still exist)                             |

| `--pipelines-keep-per-tag`             | `$PIPELINES_KEEP_PER_TAG`         | number of pipelines to keep per tag (⚠ tag MUST still exist)                                   |

| `--pipelines-keep-per-mr`           | `$PIPELINES_KEEP_PER_MR`       | number of pipelines to keep per merge request (⚠ MR MUST still exist)                          |

| `--pipelines-delete-older-than`     | `$PIPELINES_DELETE_OLDER_THAN` | max age (in days) after which pipelines are deleted (unless they are kept by a keep rule)      |



Cleanup is following the same principle as GitLab's cleanup policy : [GitLab's cleanup policy](https://docs.gitlab.com/ee/user/packages/container_registry/reduce_container_registry_storage.html#cleanup-policy)



The cleanup policy:



- Collects all pipelines from a given project,

- Orders pipelines by `created_date` (from oldest to newest),

- Excludes pipelines more recent than `--pipelines-delete-older-than`,

- Deletes pipelines when the source no longer exists (eg: a branch pipeline triggered by a branch that have been deleted since)

- For each pipeline source (branch, MR, tag), at most N pipelines are kept (N is chosen by user through CLI options or specific project configuration file - see below)



### Specify a different number of pipelines to keep setting per source



#### Branches



The `--pipelines-keep-per-branch` option (or variable `$PIPELINES_KEEP_PER_BRANCH`) allows to specify the number of pipelines kept for each branch.



Example: the following options will keep up to 20 pipelines from the `main` branch, 10 pipelines from the `develop` branch, 1 from each `feature/*`, `feat/*` or `fix/*` branch



```bash

gitlab-butler --group-path to-be-continuous \

    --pipelines-keep-per-branch main:20 \

    --pipelines-keep-per-branch develop:10 \

    --pipelines-keep-per-branch features/*:1 \

    --pipelines-keep-per-branch feat/*:1 \

    --pipelines-keep-per-branch fix/*:1

```



#### Tags



Using the parameter `--pipelines-keep-per-tag` (or variable `$PIPELINES_KEEP_PER_TAG`), user can specify the number of pipelines which are kept for each tag.

The `--pipelines-keep-per-tag` option (or variable `$PIPELINES_KEEP_PER_BRANCH`) allows to specify the number of pipelines kept for each branch.



#### Merge-Requests



Using the parameter `--pipelines-keep-per-mr` (or variable `$PIPELINES_KEEP_PER_MR`), user can specify the number of pipelines which are kept for each merge-request.

The `--pipelines-keep-per-branch` option (or variable `$PIPELINES_KEEP_PER_BRANCH`) allows to specify the number of pipelines kept for each branch.



:warning: This parameter affect the pipelines linked to the merge-request only, not the pipelines linked to the merge-request source branch.



### Configuration file



By adding a configuration file (named `.butlercfg.yaml` or `.butlercfg.yml`) inside a project repository, user can use dedicated settings for this project.



```yaml

# allows to disable globally GitLab Butler

enabled: true

pipelines:

  # allows to disable specifically pipelines cleanup

  enabled: true



  # defines the number of pipelines to keep for each branch or tag

  keep:

    # rules per branch (/!\ branches MUST exist)

    per_branch:

      # allows to disable specifically branch pipelines cleanup

      enabled: true



      # 3 ways to define limit (in order of descending priority)

      # limit based on branch name

      by_name:

        main: 10

        master: 10

        develop: 10



      # limit for default branch (ex: main or master)

      default_branch: 10



      # limit for protected branches

      protected_branches: 10



      # default limit

      any_branch: 1



    # rules per tag pipelines to keep (/!\ tags MUST exist)

    per_tag:

      # allows to disable specifically tags pipelines cleanup

      enabled: true



      # 2 ways to define limit (in order of descending priority)

      # limit based on tag name

      by_name:

        latest: 10

        stable: 10



      # default limit

      any_tag: 5



    # rules per MR pipelines to keep PER MR (/!\ MRs MUST exist)

    per_merge_request:

      # allows to disable specifically merge-request pipelines cleanup

      enabled: true



      # 2 ways to define limit (in order of descending priority)

      # limit based on merge-request status

      by_status:

        opened: 1

        closed: 0

        locked: 1

        merged: 0



      # default limit

      any_mr: 2



  # max age after which pipelines are deleted (unless they are kept by an above rule)

  delete_older_than: 30

```



## Developers

SECURITY.md

+2 −2
Original line number Diff line number Diff line
@@ -2,13 +2,13 @@ 


## Supported Versions



Security fixes and updates are only applied to the latest released version. So always try to be up to date.

Security fixes and updates are only applied to the latest released version. So always try to be up-to-date.



## Reporting a Vulnerability



In order to minimize risks of attack while investigating and fixing the issue, any vulnerability shall be reported by 

opening a [**confidential** issue on gitlab.com](https://gitlab.com/to-be-continuous/tools/gitlab-butler/-/issues/new).



Follow-up and fixing will be made on a _best effort_ basis.

Follow-up and fixing will be made on _best effort_ basis.



If you have doubts about a potential vulnerability, please reach out one of the maintainers on Discord.

gitlab_butler/butler.py

0 → 100644
+413 −0

File added.

Preview size limit exceeded, changes collapsed.

gitlab_butler/butlercfg.py

0 → 100644
+64 −0
Original line number Diff line number Diff line 
from pydantic import BaseModel, Field





class ButlerCfg(BaseModel):

    """Butler configuration."""



    class PipelinesCfg(BaseModel):

        """Pipelines configuration."""



        class KeepCfg(BaseModel):

            """Keep configuration."""



            class PerBranchCfg(BaseModel):

                """Per branch configuration."""

                enabled: bool = True

                default_branch: int = Field(alias="default-branch", default=10)

                protected_branches: int = Field(alias="protected-branches", default=10)

                any_branch: int = Field(alias="any-branch", default=1)

                by_name: dict[str, int] = Field(

                    alias="by-name",

                    default={

                        "main": 10,

                        "master": 10,

                        "develop": 10,

                        "renovate/*": 0,

                    },

                )



            class PerTagCfg(BaseModel):

                """Per branch configuration."""

                enabled: bool = True

                any_tag: int = Field(alias="any-tag", default=1)

                by_name: dict[str, int] = Field(

                    alias="by-name",

                    default={

                        "latest": 10,

                        "stable": 10,

                    },

                )



            class PerMergeRequestCfg(BaseModel):

                """Per merge-request configuration."""

                enabled: bool = True

                any_mr: int = Field(alias="any-mr", default=1)

                by_status: dict[str, int] = Field(

                    alias="by-status",

                    default={

                        "opened": 1,

                        "closed": 0,

                        "locked": 1,

                        "merged": 0,

                    },

                )



            per_branch: PerBranchCfg = Field(alias="per-branch", default_factory=PerBranchCfg)

            per_tag: PerTagCfg = Field(alias="per-tag", default_factory=PerTagCfg)

            per_merge_request: PerMergeRequestCfg = Field(alias="per-merge-request", default_factory=PerMergeRequestCfg)



        enabled: bool = True

        keep: KeepCfg = Field(default_factory=KeepCfg)

        delete_older_than: int = Field(alias="delete-older-than", default=30)



    enabled: bool = True

    pipelines: PipelinesCfg = Field(default_factory=PipelinesCfg)