Merge branch 'improve-output' into 'main' (b4d903a1) · Commits · to be continuous... / tools / gitlab-butler

gitlab_butler/butler.py

+37 −24

Original line number	Diff line number	Diff line
		@@ -72,6 +72,7 @@ class Butler:
		self.groups_count = 0
		self.projects_count = 0
		self.pipelines_count = 0
		self.pipelines_deletions_attempts = 0
		self.pipeline_deletion_limit = pipeline_deletion_limit
		self.skip_subgroups = skip_subgroups
		self.delay_between_api_call = delay_between_api_call
		@@ -293,15 +294,18 @@ class Butler:

		return PipelineSource(existing=False)

		def delete_pipeline(self, project: Project, pipeline: ProjectPipeline) -> None:
		def delete_pipeline(self, project: Project, pipeline: ProjectPipeline) -> bool:
		"""
		Delete a pipeline
		:param pipeline:
		:return: None
		:return: boolean indicating if the pipeline has been deleted
		"""
		deleted = False
		try:
		if not self.dry_run:
		project.pipelines.delete(pipeline.id)
		deleted = True

		if self.verbose:
		print(
		f" - Pipeline {AnsiColors.BLUE}{pipeline.id}{AnsiColors.RESET} (ref: {AnsiColors.BLUE}{pipeline.ref}{AnsiColors.RESET}) deleted"
		@@ -309,8 +313,6 @@ class Butler:
		if self.debug:
		print(pipeline)

		self.pipelines_count += 1

		# wait before executing the next request
		if self.delay_between_api_call:
		time.sleep(self.delay_between_api_call)
		@@ -332,6 +334,9 @@ class Butler:
		f"{AnsiColors.BLUE}Warning:{AnsiColors.RESET} failed to delete pipeline {pipeline.id} (ref: {AnsiColors.BLUE}{pipeline.ref}{AnsiColors.RESET}) : {e}"
		)

		# return True if pipeline was actually deleted
		return deleted

		# Cleanup a GitLab project
		# $1: project to be cleaned
		def clean_project(self, project: Project) -> None:
		@@ -347,8 +352,9 @@ class Butler:
		# # generate cutoff date
		updated_limit = datetime.today() - timedelta(days=cfg.pipelines.delete_older_than)

		# deletion budget
		pipeline_deletion_limit = self.pipeline_deletion_limit
		# count deletion attempts, and actual deletions
		pipelines_deletions_attempts = 0
		pipelines_deletions_count = 0

		# build cache
		# list of existing elements may be long, it could be better to fetch the elements on the fly
		@@ -361,6 +367,12 @@ class Butler:

		# iterate over project pipelines (starting with oldest)
		for pipeline in project.pipelines.list(iterator=True, sort="asc", order_by="updated_at"):
		# stop right there if enough pipelines deletions have been attempted already
		if self.pipeline_deletion_limit and (pipelines_deletions_attempts >= self.pipeline_deletion_limit):
		if self.verbose:
		print('Pipeline deletion limit reached, stopping processing for this project.')
		break

		# list() methods in python-gitlab are not narrowly typed, so let's help mypy
		assert isinstance(pipeline, ProjectPipeline)

		@@ -387,12 +399,9 @@ class Butler:
		if not pipeline_source.existing:
		if self.verbose and self.debug:
		print(f'Pipeline {pipeline.id} source ({pipeline_source}) no longer exists -> delete')
		self.delete_pipeline(project, pipeline)
		pipeline_deletion_limit = pipeline_deletion_limit - 1
		if pipeline_deletion_limit == 0:
		if self.verbose:
		print('Pipeline deletion limit reached, stopping processing for this project.')
		return
		if self.delete_pipeline(project, pipeline):
		pipelines_deletions_count += 1
		pipelines_deletions_attempts += 1
		continue

		# if source still exists, store the pipeline
		@@ -401,12 +410,9 @@ class Butler:
		if max_pipelines_for_source == 0:
		if self.verbose and self.debug:
		print(f'Pipeline {pipeline.id} source ({pipeline_source}) configured not to keep any -> delete')
		self.delete_pipeline(project, pipeline)
		pipeline_deletion_limit = pipeline_deletion_limit - 1
		if pipeline_deletion_limit == 0:
		if self.verbose:
		print('Pipeline deletion limit reached, stopping processing for this project.')
		return
		if self.delete_pipeline(project, pipeline):
		pipelines_deletions_count += 1
		pipelines_deletions_attempts += 1
		continue

		# create/update list of kept pipelines for this source
		@@ -419,12 +425,19 @@ class Butler:
		pipeline = kept_pipelines_for_ref.pop()
		if self.verbose and self.debug:
		print(f'Pipeline list ({len(kept_pipelines_for_ref)+1}) over limit ({max_pipelines_for_source}) -> delete pipeline')
		self.delete_pipeline(project, pipeline)
		pipeline_deletion_limit = pipeline_deletion_limit - 1
		if pipeline_deletion_limit == 0:
		if self.verbose:
		print('Pipeline deletion limit reached, stopping processing for this project.')
		return
		if self.delete_pipeline(project, pipeline):
		pipelines_deletions_count += 1
		pipelines_deletions_attempts += 1

		# update global statistics
		self.pipelines_deletions_attempts += pipelines_deletions_attempts
		self.pipelines_count += pipelines_deletions_count

		# display project statistics
		if self.dry_run and pipelines_deletions_attempts > 0:
		print(f" ==> number of pipelines which would have been deleted: {AnsiColors.HPURPLE}{pipelines_deletions_attempts}{AnsiColors.RESET}")
		elif not self.dry_run and pipelines_deletions_count > 0:
		print(f" ==> number of pipelines which have been deleted: {AnsiColors.HPURPLE}{pipelines_deletions_count}{AnsiColors.RESET}")

		# Cleanup recursively a GitLab group
		# $1: group to be cleaned

gitlab_butler/main.py

+16 −3

Original line number	Diff line number	Diff line
		@@ -256,8 +256,21 @@ def run() -> None:
		print(
		"----------------------------------------------------------------------------------------------"
		)
		if butler.dry_run:
		print(
		f"Summary: {butler.groups_count} groups and {butler.projects_count} projects cleaned; {butler.pipelines_count} pipelines deleted; {len(butler.warnings)} warnings; {len(butler.errors)} errors (see logs)"
		f"Summary: {butler.groups_count} groups and {butler.projects_count} projects cleaned;"
		f" {AnsiColors.HPURPLE}{butler.pipelines_deletions_attempts}{AnsiColors.RESET} pipelines would be deleted;"
		f" {AnsiColors.RED}{len(butler.warnings)}{AnsiColors.RESET} warnings;"
		f" {AnsiColors.RED}{len(butler.errors)}{AnsiColors.RESET} errors"
		f"{" (see logs)" if butler.warnings or butler.errors else ""}"
		)
		else:
		print(
		f"Summary: {butler.groups_count} groups and {butler.projects_count} projects cleaned;"
		f" {AnsiColors.HPURPLE}{butler.pipelines_count}{AnsiColors.RESET} pipelines deleted;"
		f" {AnsiColors.RED}{len(butler.warnings)}{AnsiColors.RESET} warnings;"
		f" {AnsiColors.RED}{len(butler.errors)}{AnsiColors.RESET} errors"
		f"{" (see logs)" if butler.warnings or butler.errors else ""}"
		)

		if len(butler.errors) > 0: