Commit c112c9ee authored by Pierre Smeyers's avatar Pierre Smeyers
Browse files

Merge branch '8-allow-to-specify-tag-limit-with-command-line-parameter' into 'main' 

Resolve "Allow to specify tag limit with command line parameter"

Closes #8

See merge request to-be-continuous/tools/gitlab-butler!41
parents eaef2490 bfb8b1ff

README.md

+11 −3
Original line number Diff line number Diff line
@@ -95,13 +95,21 @@ gitlab-butler --group-path to-be-continuous \ 


#### Tags



Using the parameter `--pipelines-keep-per-tag` (or variable `$PIPELINES_KEEP_PER_TAG`), user can specify the number of pipelines which are kept for each tag.

The `--pipelines-keep-per-tag` option (or variable `$PIPELINES_KEEP_PER_BRANCH`) allows to specify the number of pipelines kept for each branch.

The `--pipelines-keep-per-tag` option (or variable `$PIPELINES_KEEP_PER_TAG`) allows to specify the number of pipelines kept for each tag.



Example: the following options will keep up to 20 pipelines from the `latest` tag, 10 pipelines from the `stable` tag, 1 from each `v1.*`, `v2.*` tags



```bash

gitlab-butler --group-path to-be-continuous \

    --pipelines-keep-per-tag latest:20 \

    --pipelines-keep-per-tag stable:10 \

    --pipelines-keep-per-tag v1.*:1 \

    --pipelines-keep-per-tag v2.*:1

```



#### Merge-Requests



Using the parameter `--pipelines-keep-per-mr` (or variable `$PIPELINES_KEEP_PER_MR`), user can specify the number of pipelines which are kept for each merge-request.

The `--pipelines-keep-per-branch` option (or variable `$PIPELINES_KEEP_PER_BRANCH`) allows to specify the number of pipelines kept for each branch.



:warning: This parameter affect the pipelines linked to the merge-request only, not the pipelines linked to the merge-request source branch.

gitlab_butler/butlercfg.py

+1 −1
Original line number Diff line number Diff line
@@ -29,7 +29,7 @@ class ButlerCfg(BaseModel): 
            class PerTagCfg(BaseModel):

                """Per branch configuration."""

                enabled: bool = True

                any_tag: int = Field(alias="any-tag", default=1)

                any_tag: int = Field(alias="any-tag", default=5)

                by_name: dict[str, int] = Field(

                    alias="by-name",

                    default={

gitlab_butler/main.py

+31 −10
Original line number Diff line number Diff line
@@ -17,6 +17,7 @@ LOGGER = Logger(__name__) 
def to_url(api_url: str) -> str:

    return api_url[0:-7] if api_url.endswith("/api/v4") else api_url





def generate_parser():

    # define command parser

    parser = argparse.ArgumentParser(
@@ -95,8 +96,11 @@ def generate_parser(): 
    )

    parser.add_argument(

        "--pipelines-keep-per-tag",

        type=int,

        default=int(os.getenv("PIPELINES_KEEP_PER_TAG", "5")),

        type=str,

        action="append",

        default=os.getenv("PIPELINES_KEEP_PER_TAG").split(" ")

        if os.getenv("PIPELINES_KEEP_PER_TAG")

        else [],

        help="number of pipelines to keep per tag (⚠ tag MUST still exist)",

    )

    parser.add_argument(
@@ -113,21 +117,40 @@ def generate_parser(): 
    )

    return parser





def generate_cfg(args):

    # build default config

    cfg = ButlerCfg()

    cfg.pipelines.delete_older_than = args.pipelines_delete_older_than

    cfg.pipelines.keep.per_tag.any_tag = args.pipelines_keep_per_tag

    cfg.pipelines.keep.per_merge_request.any_mr = args.pipelines_keep_per_mr



    # parse branch parameters

    if args.pipelines_keep_per_branch:

        cfg.pipelines.keep.per_branch.by_name = {

            branch: int(n)

            for (branch, n) in [b.split(":") for b in args.pipelines_keep_per_branch]

        }

        cfg.pipelines.keep.per_branch.by_name = {}

        for entry in args.pipelines_keep_per_branch:

            if len(entry.split(":")) == 2:

                (branch, n) = entry.split(":", maxsplit=1)

                cfg.pipelines.keep.per_branch.by_name[branch] = int(n)

            else:

                # default value

                cfg.pipelines.keep.per_branch.any_branch = int(entry)



    # parse tag parameters

    if args.pipelines_keep_per_tag:

        cfg.pipelines.keep.per_tag.by_name = {}

        for entry in args.pipelines_keep_per_tag:

            if len(entry.split(":")) == 2:

                (branch, n) = entry.split(":", maxsplit=1)

                cfg.pipelines.keep.per_tag.by_name[branch] = int(n)

            else:

                # default value

                cfg.pipelines.keep.per_tag.any_tag = int(entry)



    if args.verbose:

        print(f"Default configuration: {cfg}")

    return cfg





def run() -> None:

    # build parser

    parser = generate_parser()
@@ -178,7 +201,7 @@ def run() -> None: 
        f"- Number of pipelines to keep per branch (--pipelines-keep-per-branch)  : {AnsiColors.CYAN}{', '.join(args.pipelines_keep_per_branch)}{AnsiColors.RESET}"

    )

    print(

        f"- Number of pipelines to keep per tag    (--pipelines-keep-per-tag)     : {AnsiColors.CYAN}{args.pipelines_keep_per_tag}{AnsiColors.RESET}"

        f"- Number of pipelines to keep per tag    (--pipelines-keep-per-tag)     : {AnsiColors.CYAN}{', '.join(args.pipelines_keep_per_tag)}{AnsiColors.RESET}"

    )

    print(

        f"- Number of pipelines to keep per MR     (--pipelines-keep-per-mr)      : {AnsiColors.CYAN}{args.pipelines_keep_per_mr}{AnsiColors.RESET}"
@@ -241,5 +264,3 @@ def run() -> None: 


    if len(butler.errors) > 0:

        exit(128)

poetry.lock

+22 −22
Original line number Diff line number Diff line
@@ -311,13 +311,13 @@ files = [ 


[[package]]

name = "packaging"

version = "24.1"

version = "24.2"

description = "Core utilities for Python packages"

optional = false

python-versions = ">=3.8"

files = [

    {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"},

    {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"},

    {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"},

    {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},

]



[[package]]
@@ -669,29 +669,29 @@ tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asy 


[[package]]

name = "ruff"

version = "0.7.1"

version = "0.7.3"

description = "An extremely fast Python linter and code formatter, written in Rust."

optional = false

python-versions = ">=3.7"

files = [

    {file = "ruff-0.7.1-py3-none-linux_armv6l.whl", hash = "sha256:cb1bc5ed9403daa7da05475d615739cc0212e861b7306f314379d958592aaa89"},

    {file = "ruff-0.7.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:27c1c52a8d199a257ff1e5582d078eab7145129aa02721815ca8fa4f9612dc35"},

    {file = "ruff-0.7.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:588a34e1ef2ea55b4ddfec26bbe76bc866e92523d8c6cdec5e8aceefeff02d99"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94fc32f9cdf72dc75c451e5f072758b118ab8100727168a3df58502b43a599ca"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:985818742b833bffa543a84d1cc11b5e6871de1b4e0ac3060a59a2bae3969250"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:32f1e8a192e261366c702c5fb2ece9f68d26625f198a25c408861c16dc2dea9c"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:699085bf05819588551b11751eff33e9ca58b1b86a6843e1b082a7de40da1565"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:344cc2b0814047dc8c3a8ff2cd1f3d808bb23c6658db830d25147339d9bf9ea7"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4316bbf69d5a859cc937890c7ac7a6551252b6a01b1d2c97e8fc96e45a7c8b4a"},

    {file = "ruff-0.7.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79d3af9dca4c56043e738a4d6dd1e9444b6d6c10598ac52d146e331eb155a8ad"},

    {file = "ruff-0.7.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c5c121b46abde94a505175524e51891f829414e093cd8326d6e741ecfc0a9112"},

    {file = "ruff-0.7.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8422104078324ea250886954e48f1373a8fe7de59283d747c3a7eca050b4e378"},

    {file = "ruff-0.7.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:56aad830af8a9db644e80098fe4984a948e2b6fc2e73891538f43bbe478461b8"},

    {file = "ruff-0.7.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:658304f02f68d3a83c998ad8bf91f9b4f53e93e5412b8f2388359d55869727fd"},

    {file = "ruff-0.7.1-py3-none-win32.whl", hash = "sha256:b517a2011333eb7ce2d402652ecaa0ac1a30c114fbbd55c6b8ee466a7f600ee9"},

    {file = "ruff-0.7.1-py3-none-win_amd64.whl", hash = "sha256:f38c41fcde1728736b4eb2b18850f6d1e3eedd9678c914dede554a70d5241307"},

    {file = "ruff-0.7.1-py3-none-win_arm64.whl", hash = "sha256:19aa200ec824c0f36d0c9114c8ec0087082021732979a359d6f3c390a6ff2a37"},

    {file = "ruff-0.7.1.tar.gz", hash = "sha256:9d8a41d4aa2dad1575adb98a82870cf5db5f76b2938cf2206c22c940034a36f4"},

    {file = "ruff-0.7.3-py3-none-linux_armv6l.whl", hash = "sha256:34f2339dc22687ec7e7002792d1f50712bf84a13d5152e75712ac08be565d344"},

    {file = "ruff-0.7.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb397332a1879b9764a3455a0bb1087bda876c2db8aca3a3cbb67b3dbce8cda0"},

    {file = "ruff-0.7.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:37d0b619546103274e7f62643d14e1adcbccb242efda4e4bdb9544d7764782e9"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d59f0c3ee4d1a6787614e7135b72e21024875266101142a09a61439cb6e38a5"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:44eb93c2499a169d49fafd07bc62ac89b1bc800b197e50ff4633aed212569299"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d0242ce53f3a576c35ee32d907475a8d569944c0407f91d207c8af5be5dae4e"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:6b6224af8b5e09772c2ecb8dc9f3f344c1aa48201c7f07e7315367f6dd90ac29"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c50f95a82b94421c964fae4c27c0242890a20fe67d203d127e84fbb8013855f5"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f3eff9961b5d2644bcf1616c606e93baa2d6b349e8aa8b035f654df252c8c67"},

    {file = "ruff-0.7.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8963cab06d130c4df2fd52c84e9f10d297826d2e8169ae0c798b6221be1d1d2"},

    {file = "ruff-0.7.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:61b46049d6edc0e4317fb14b33bd693245281a3007288b68a3f5b74a22a0746d"},

    {file = "ruff-0.7.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:10ebce7696afe4644e8c1a23b3cf8c0f2193a310c18387c06e583ae9ef284de2"},

    {file = "ruff-0.7.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:3f36d56326b3aef8eeee150b700e519880d1aab92f471eefdef656fd57492aa2"},

    {file = "ruff-0.7.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5d024301109a0007b78d57ab0ba190087b43dce852e552734ebf0b0b85e4fb16"},

    {file = "ruff-0.7.3-py3-none-win32.whl", hash = "sha256:4ba81a5f0c5478aa61674c5a2194de8b02652f17addf8dfc40c8937e6e7d79fc"},

    {file = "ruff-0.7.3-py3-none-win_amd64.whl", hash = "sha256:588a9ff2fecf01025ed065fe28809cd5a53b43505f48b69a1ac7707b1b7e4088"},

    {file = "ruff-0.7.3-py3-none-win_arm64.whl", hash = "sha256:1713e2c5545863cdbfe2cbce21f69ffaf37b813bfd1fb3b90dc9a6f1963f5a8c"},

    {file = "ruff-0.7.3.tar.gz", hash = "sha256:e1d1ba2e40b6e71a61b063354d04be669ab0d39c352461f3d789cac68b54a313"},

]



[[package]]

tests/test_butlercfg.py

+114 −16
Original line number Diff line number Diff line 
import os



from gitlab_butler.butlercfg import ButlerCfg

from gitlab_butler.main import generate_cfg, generate_parser
@@ -34,6 +36,9 @@ class TestButlerCfg: 
        assert isinstance(cfg.pipelines.keep.per_merge_request.any_mr, int)

        assert isinstance(cfg.pipelines.keep.per_merge_request.by_status, dict)



        # to allow chaining

        return cfg



    def test_default_cfg(self):

            cfg = ButlerCfg()

            self.validate_cfg(cfg)
@@ -46,25 +51,118 @@ class TestButlerCfg: 
            self.validate_cfg(cfg)



    def test_build_cfg_from_default_args(self):

        cfg = generate_cfg(generate_parser().parse_args([]))

        self.validate_cfg(cfg)

        cfg = self.validate_cfg(generate_cfg(generate_parser().parse_args([])))



    def test_build_cfg_from_args_keep_pipelines_per_branch(self):

        cfg = generate_cfg(generate_parser().parse_args(['--pipelines-keep-per-branch', 'main:10']))

        self.validate_cfg(cfg)

        assert cfg.pipelines.keep.per_branch.by_name == {'main': 10}

    def test_build_cfg_from_args_keep_pipelines_per_mr(self):

        cfg = self.validate_cfg(generate_cfg(generate_parser().parse_args(['--pipelines-keep-per-mr', '5'])))

        assert cfg.pipelines.keep.per_merge_request.any_mr == 5



        cfg = generate_cfg(generate_parser().parse_args(['--pipelines-keep-per-branch', 'main:10', '--pipelines-keep-per-branch', 'master:5']))

        self.validate_cfg(cfg)

        assert cfg.pipelines.keep.per_branch.by_name == {'main': 10, 'master': 5}

    def test_basic_args(self):

        # --api-url

        assert generate_parser().parse_args([]).api_url == os.getenv("GITLAB_API", "https://gitlab.com/api/v4")

        assert generate_parser().parse_args(['--api-url', 'https://my-gitlab.com/api/v4']).api_url == 'https://my-gitlab.com/api/v4'



    def test_build_cfg_from_args_keep_pipelines_per_tag(self):

        cfg = generate_cfg(generate_parser().parse_args(['--pipelines-keep-per-tag', '5']))

        self.validate_cfg(cfg)

        # --token

        assert generate_parser().parse_args([]).token == os.getenv("GITLAB_TOKEN")

        assert generate_parser().parse_args(['--token', 'token']).token == 'token'



        # --group-path

        assert generate_parser().parse_args([]).group_path == os.getenv("GROUP_PATH")

        assert generate_parser().parse_args(['--group-path', 'group/sub-group/']).group_path == 'group/sub-group/'



        # --exclude

        assert generate_parser().parse_args([]).exclude == (os.getenv("EXCLUDE").split(",") if os.getenv("EXCLUDE") else [])

        assert generate_parser().parse_args(['--exclude', 'group1']).exclude == ['group1']

        assert generate_parser().parse_args(['--exclude', 'group1', '--exclude', 'group2', '--exclude', 'group3']).exclude == ['group1','group2','group3']



    def test_toggle_args(self):

        # dry-run

        assert generate_parser().parse_args([]).dry_run is False

        assert generate_parser().parse_args(['--dry-run']).dry_run is True



        # verbose

        assert generate_parser().parse_args([]).verbose is False

        assert generate_parser().parse_args(['--verbose']).verbose is True



        # halt-on-error

        assert generate_parser().parse_args([]).halt_on_error is False

        assert generate_parser().parse_args(['--halt-on-error']).halt_on_error is True



        # insecure

        assert generate_parser().parse_args([]).insecure is False

        assert generate_parser().parse_args(['--insecure']).insecure is True



        # --skip-subgroups

        assert generate_parser().parse_args([]).skip_subgroups is False

        assert generate_parser().parse_args(['--skip-subgroups']).skip_subgroups is True



        # pipeline-deletion-limit

        assert generate_parser().parse_args([]).pipeline_deletion_limit == 0

        assert generate_parser().parse_args(['--pipeline-deletion-limit', '10']).pipeline_deletion_limit == 10



        # delay-between-api-calls

        assert generate_parser().parse_args([]).delay_between_api_calls is None

        assert generate_parser().parse_args(['--delay-between-api-calls', '10']).delay_between_api_calls == 10



        # pipelines-delete-older-than

        assert generate_parser().parse_args([]).pipelines_delete_older_than == 30

        assert generate_parser().parse_args(['--pipelines-delete-older-than', '90']).pipelines_delete_older_than == 90



    def test_branch_limits(self):

        # no args, default value

        args = generate_parser().parse_args([])

        assert args.pipelines_keep_per_branch == []

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_branch.any_branch == 1

        assert cfg.pipelines.keep.per_branch.by_name == {'develop': 10, 'main': 10, 'master': 10, 'renovate/*': 0}



        # single value

        args = generate_parser().parse_args(['--pipelines-keep-per-branch', 'main:20'])

        assert args.pipelines_keep_per_branch == ['main:20']

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_branch.any_branch == 1

        assert cfg.pipelines.keep.per_branch.by_name == {'main': 20}



        # single value with default value

        args = generate_parser().parse_args(['--pipelines-keep-per-branch', 'main:20', '--pipelines-keep-per-branch', '7'])

        assert args.pipelines_keep_per_branch == ['main:20', '7']

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_branch.any_branch == 7

        assert cfg.pipelines.keep.per_branch.by_name == {'main': 20}



        # multiple values

        args = generate_parser().parse_args(['--pipelines-keep-per-branch', 'main:20', '--pipelines-keep-per-branch', 'develop:10', '--pipelines-keep-per-branch', 'other:5'])

        assert args.pipelines_keep_per_branch == ['main:20', 'develop:10', 'other:5']

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_branch.any_branch == 1

        assert cfg.pipelines.keep.per_branch.by_name == {'develop': 10, 'main': 20, 'other': 5}



    def test_tag_limits(self):

        # no args

        args = generate_parser().parse_args([])

        assert args.pipelines_keep_per_tag == []

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_tag.any_tag == 5

        assert cfg.pipelines.keep.per_tag.by_name == {'latest': 10, 'stable': 10}



    def test_build_cfg_from_args_keep_pipelines_per_mr(self):

        cfg = generate_cfg(generate_parser().parse_args(['--pipelines-keep-per-mr', '5']))

        self.validate_cfg(cfg)

        assert cfg.pipelines.keep.per_merge_request.any_mr == 5

        # single value

        args = generate_parser().parse_args(['--pipelines-keep-per-tag', 'latest:20'])

        assert args.pipelines_keep_per_tag == ['latest:20']

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_tag.any_tag == 5

        assert cfg.pipelines.keep.per_tag.by_name == {'latest': 20}



        # single value with default value

        args = generate_parser().parse_args(['--pipelines-keep-per-tag', 'latest:20', '--pipelines-keep-per-tag', '7'])

        assert args.pipelines_keep_per_tag == ['latest:20', '7']

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_tag.any_tag == 7

        assert cfg.pipelines.keep.per_tag.by_name == {'latest': 20}



        # multiple values

        args = generate_parser().parse_args(['--pipelines-keep-per-tag', 'latest:20', '--pipelines-keep-per-tag', 'v1.*:10', '--pipelines-keep-per-tag', 'v2.*:5'])

        assert args.pipelines_keep_per_tag == ['latest:20', 'v1.*:10', 'v2.*:5']

        cfg = self.validate_cfg(generate_cfg(args))

        assert cfg.pipelines.keep.per_tag.any_tag == 5

        assert cfg.pipelines.keep.per_tag.by_name == {'latest': 20, 'v1.*': 10, 'v2.*': 5}