Commit 301238d9 authored by Mathieu Coupé's avatar Mathieu Coupé
Browse files

Merge branch '12-some-pipelines-seems-to-be-triggered-by-mr-but-dont-have-mr-ref' into 'main' 

Handle pipelines from MR with invalid ref

See merge request to-be-continuous/tools/gitlab-butler!61
parents 4d8a2144 8166309b

gitlab_butler/butler.py

+24 −13
Original line number Diff line number Diff line
@@ -132,6 +132,24 @@ class PipelineSource: 
        return f'PipelineSource(existing={self.existing}, source_type={self.source_type}, protected={self.protected}, default={self.default}, state={self.state})'





def merge_request_iid_from_ref(ref: str) -> str | None:

    """

    Extract ID from merge request ref as 'refs/merge-requests/<n>/head'

    returns None if IID cannot be extracted or if provided ref is not a merge request ref.

    :param ref: pipeline ref

    :return: IID or None

    """

    #

    if ref.startswith('refs/merge-requests/') and ref.endswith('/head'):

        iid = re.search(r'refs/merge-requests/(\d+?)/head', ref)

        if iid is None:

            return None



        return iid.group(1)



    return None





class Butler:

    def __init__(

            self,
@@ -199,18 +217,8 @@ class Butler: 
            return ButlerCfg.model_validate(obj)

        return self.default_cfg



    def merge_request_iid_from_ref(self, ref: str) -> str | None:

        # extract ID from ref as 'refs/merge-requests/122/head'

        if ref.startswith('refs/merge-requests/') and ref.endswith('/head'):

            iid = re.search(r'refs/merge-requests/(\d+?)/head', ref)

            if iid is None:

                self.handle_error(ValueError(f'Unable to extract iid from "{ref}"'))

                return None



            return iid.group(1)



        self.handle_error(ValueError(f'Unknown reference format "{ref}"'))

        return None

    def pipeline_url(self, project: Project, pipeline: ProjectPipeline) -> str:

        return f'{self.client.url}/{project.path_with_namespace}/pipelines/{pipeline.id}'



    def get_max_pipelines_for_source(self, pipeline: ProjectPipeline, cfg: ButlerCfg, source: PipelineSource) -> int:

        """
@@ -321,10 +329,13 @@ class Butler: 
        Check if the source of the pipeline still exists

        """

        if pipeline.source in ['merge_request_event']:

            merge_request_iid = self.merge_request_iid_from_ref(pipeline.ref)

            merge_request_iid = merge_request_iid_from_ref(pipeline.ref)



            # if MR id cannot be computed (and user allowed processing after an error), skip this pipeline

            if merge_request_iid is None:

                print(

                    f'Pipeline {AnsiColors.BLUE}{pipeline.id}{AnsiColors.RESET} was triggered by a merge request event but \'{AnsiColors.BLUE}{pipeline.ref}{AnsiColors.RESET}\' is not a merge request reference. '

                    f'Consider deleting this pipeline manually : {self.pipeline_url(project, pipeline)}')

                return None



            # check if MR is already in the cache and get it from API if needed

tests/test_basic_functions.py

+10 −15
Original line number Diff line number Diff line
@@ -4,7 +4,7 @@ import pytest 
import responses

from gitlab import Gitlab



from gitlab_butler.butler import Butler

from gitlab_butler.butler import Butler, merge_request_iid_from_ref

from gitlab_butler.butlercfg import ButlerCfg

from gitlab_butler.main import to_url
@@ -130,18 +130,13 @@ class TestBasic: 
        assert to_url('http://gitlab.test/api/v4') == 'http://gitlab.test'



    def test_merge_request_iid_from_ref(self) -> None:

        butler = self.create_butler()

        butler.continue_on_error = True



        butler.errors = []

        assert butler.merge_request_iid_from_ref('') is None

        assert butler.merge_request_iid_from_ref('refs/merge-requests/42') is None

        assert butler.merge_request_iid_from_ref('ref/42/head') is None

        assert butler.merge_request_iid_from_ref('refs/merge-requests/head') is None

        assert butler.merge_request_iid_from_ref('refs/merge-requests/xxx/head') is None

        assert butler.merge_request_iid_from_ref('refs/merge-requests/42/42/head') is None

        assert butler.merge_request_iid_from_ref('refs/merge-requests/42/xxx/head') is None

        assert len(butler.errors) == 7



        assert butler.merge_request_iid_from_ref('refs/merge-requests/42/head') == '42'

        assert merge_request_iid_from_ref('') is None

        assert merge_request_iid_from_ref('refs/merge-requests/42') is None

        assert merge_request_iid_from_ref('ref/42/head') is None

        assert merge_request_iid_from_ref('refs/merge-requests/head') is None

        assert merge_request_iid_from_ref('refs/merge-requests/xxx/head') is None

        assert merge_request_iid_from_ref('refs/merge-requests/42/42/head') is None

        assert merge_request_iid_from_ref('refs/merge-requests/42/xxx/head') is None



        assert merge_request_iid_from_ref('refs/merge-requests/42/head') == '42'

tests/test_pipelines.py

0 → 100644
+221 −0
Original line number Diff line number Diff line 
from datetime import datetime, timedelta



import responses

from gitlab import Gitlab



from gitlab_butler.butler import Butler

from gitlab_butler.butlercfg import ButlerCfg

from gitlab_butler.main import to_url





def generate_pipelines(count: int, ref: str, source: str, first_id: int, first: datetime, last: datetime) -> list[dict[str, object]]:

    """

    build list of pipelines for the same ref with spread timestamps

    :return:

    """



    pipelines = []

    delta = (last - first) / count



    for n in range(count):

        pipelines.append({

            'id': first_id + n,

            'created_at': (first + n * delta).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),

            'updated_at': (first + n * delta + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),

            'source': source,

            'ref': ref.format(n=first_id + n)

        })



    return pipelines



# project details : https://docs.gitlab.com/ee/api/projects.html#get-single-project

def mock_empty_project(project_id: str, branches: list[object] | None = None, tags: list[object] | None = None, merge_requests: list[object] | None = None, pipelines: list[dict[str, object]] = []) -> None:

    permission = dict()

    permission['project_access'] = {'access_level': 30}

    permission['group_access'] = {'access_level': 30}



    default_branches = [

        {'name': 'main', 'protected': True, 'default': True}

    ]



    default_tags: list[str] = []



    # pipelines list is expected to be sorted

    sorted_pipelines = sorted(pipelines, key=lambda p: p['updated_at'] or 0) # type: ignore[arg-type, return-value]



    responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/{project_id}', status=200, json=

    {'id': project_id, 'path': f'project_{project_id}', 'path_with_namespace': f'path/to/group/project_{project_id}',

     'default_branch': 'main', 'archived': False, 'permissions': permission, 'jobs_enabled': True }

    )

    responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/{project_id}/repository/files/.butlercfg.yaml?ref=main', status=404)

    responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/{project_id}/repository/files/.butlercfg.yml?ref=main', status=404)

    responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/{project_id}/repository/branches?per_page=100', status=200, json=branches or default_branches)

    responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/{project_id}/repository/tags?per_page=100', status=200, json=tags or default_tags)

    responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/{project_id}/pipelines?sort=asc&order_by=updated_at&per_page=100', status=200, json=sorted_pipelines)



# goal : test processing of pipeline on one project

class TestPipelines:

    @staticmethod

    def create_butler(pipeline_deletion_limit: int = 0) -> Butler:

        # create butler

        return Butler(

            Gitlab(

                url=to_url('http://gitlab.test'),

                private_token='test_token',

                per_page=100

            ),

            'path/to/group',

            [],

            dry_run=False,

            verbose=True,

            debug=True,

            continue_on_error=False,

            skip_subgroups=False,

            delay_between_api_call=0,

            pipeline_deletion_limit=pipeline_deletion_limit,

            default_cfg=ButlerCfg(),

        )



    @responses.activate

    def test_clean_project_empty(self) -> None:

        butler = self.create_butler()



        # Setup empty project

        mock_empty_project('42')



        # get project

        project = butler.client.projects.get('42')

        assert project.id == '42'



        # process project

        butler.clean_project(project)

        assert butler.pipelines_deletions_attempts == 0

        assert butler.pipelines_count == 0



    @responses.activate

    def test_clean_project_with_branch_pipelines(self) -> None:

        butler = self.create_butler()



        total_pipelines = 25

        expected_deleted_pipelines = total_pipelines - butler.default_cfg.pipelines.keep.per_branch.protected_branches



        # Setup project with branch pipelines

        mock_empty_project('42', pipelines=generate_pipelines(count=total_pipelines, first_id=1, ref='main', source='push', first=datetime.now() - timedelta(days=150), last=datetime.now() - timedelta(days=100)))



        # setup mock for deletes

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.DELETE, f'http://gitlab.test/api/v4/projects/42/pipelines/{n}', status=200)



        # get project

        project = butler.client.projects.get('42')

        assert project.id == '42'



        # process project

        butler.clean_project(project)

        assert butler.pipelines_deletions_attempts == expected_deleted_pipelines

        assert butler.pipelines_count == expected_deleted_pipelines



    @responses.activate

    def test_clean_project_with_branch_pipelines_with_custom_limit(self) -> None:

        butler = self.create_butler()



        total_pipelines = 25

        expected_deleted_pipelines = total_pipelines - butler.default_cfg.pipelines.keep.per_branch.by_name['renovate/*']



        # Setup project with branch pipelines

        mock_empty_project('42', pipelines=generate_pipelines(count=total_pipelines, first_id=1, ref='renovate/xxx', source='push', first=datetime.now() - timedelta(days=150), last=datetime.now() - timedelta(days=100)))



        # setup mock for deletes

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.DELETE, f'http://gitlab.test/api/v4/projects/42/pipelines/{n}', status=200)



        # get project

        project = butler.client.projects.get('42')

        assert project.id == '42'



        # process project

        butler.clean_project(project)

        assert butler.pipelines_deletions_attempts == expected_deleted_pipelines

        assert butler.pipelines_count == expected_deleted_pipelines



    @responses.activate

    def test_clean_project_with_branch_pipelines_limited_deletion(self) -> None:

        butler = self.create_butler(5)



        expected_deleted_pipelines = 5

        total_pipelines = 25



        # Setup project with branch pipelines

        mock_empty_project('42', pipelines=generate_pipelines(count=total_pipelines, first_id=1, ref='main', source='push', first=datetime.now() - timedelta(days=150), last=datetime.now() - timedelta(days=100)))



        # setup mock for deletes

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.DELETE, f'http://gitlab.test/api/v4/projects/42/pipelines/{n}', status=200)



        # get project

        project = butler.client.projects.get('42')

        assert project.id == '42'



        # process project

        butler.clean_project(project)

        assert butler.pipelines_deletions_attempts == expected_deleted_pipelines

        assert butler.pipelines_count == expected_deleted_pipelines





    @responses.activate

    def test_clean_project_with_mr_pipelines(self) -> None:

        butler = self.create_butler()



        expected_deleted_pipelines = 25

        total_pipelines = 25



        # Setup project with branch pipelines

        mock_empty_project('42', pipelines=generate_pipelines(count=total_pipelines, first_id=1, ref='refs/merge-requests/{n}/head', source='merge_request_event', first=datetime.now() - timedelta(days=150), last=datetime.now() - timedelta(days=100)))



        # setup mock for merge requests

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/42/merge_requests/{n}', status=200, json={

                'id': n, 'iid': n, 'source_branch': f'source_branch_{n}', 'target_branch': 'main', 'state': 'merged'

            })



        # setup mock for deletes

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.DELETE, f'http://gitlab.test/api/v4/projects/42/pipelines/{n}', status=200)



        # get project

        project = butler.client.projects.get('42')

        assert project.id == '42'



        # process project

        butler.clean_project(project)

        assert butler.pipelines_deletions_attempts == expected_deleted_pipelines

        assert butler.pipelines_count == expected_deleted_pipelines



    @responses.activate

    def test_clean_project_with_mr_pipelines_with_invalid_ref(self) -> None:

        butler = self.create_butler()



        expected_deleted_pipelines = 25

        total_pipelines = 25



        # Setup project with branch pipelines

        mock_empty_project('42', pipelines=generate_pipelines(count=total_pipelines, first_id=1, ref='some_branch', source='merge_request_event', first=datetime.now() - timedelta(days=150), last=datetime.now() - timedelta(days=100)))



        # setup mock for merge requests

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.GET, f'http://gitlab.test/api/v4/projects/42/merge_requests/{n}', status=200, json={

                'id': n, 'iid': n, 'source_branch': f'source_branch_{n}', 'target_branch': 'main', 'state': 'merged'

            })



        # setup mock for deletes

        for n in range(1, expected_deleted_pipelines + 1):

            responses.add(responses.DELETE, f'http://gitlab.test/api/v4/projects/42/pipelines/{n}', status=200)



        # get project

        project = butler.client.projects.get('42')

        assert project.id == '42'



        # process project

        butler.clean_project(project)

        assert butler.pipelines_deletions_attempts == 0

        assert butler.pipelines_count == 0