Merge branch 'feat/tag-branch-14-05-2025' into 'main' (15d2a30a) · Commits · to be continuous... / tools / gitlab-cp

README.md

+11 −4

Original line number	Diff line number	Diff line
		@@ -16,10 +16,11 @@ gitlab-cp --help
		## Usage

		```bash
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}]
		[--skip-visibility] [--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure] [--update-release] [--update-avatar] [--no-group-description]
		[--no-project-description] [--new-group-options NEW_GROUP_OPTIONS] [--new-group-options-from NEW_GROUP_OPTIONS_FROM] [--new-project-options NEW_PROJECT_OPTIONS]
		[--new-project-options-from NEW_PROJECT_OPTIONS_FROM] [--use-src-issue-tracker] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH]
		[--max-visibility {public,internal,private}] [--skip-visibility] [--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure]
		[--include-branch INCLUDE_BRANCH] [--exclude-branch EXCLUDE_BRANCH] [--update-release] [--update-avatar] [--no-group-description] [--no-project-description] [--new-group-options NEW_GROUP_OPTIONS]
		[--new-group-options-from NEW_GROUP_OPTIONS_FROM] [--new-project-options NEW_PROJECT_OPTIONS] [--new-project-options-from NEW_PROJECT_OPTIONS_FROM] [--dry-run] [--halt-on-error]
		[--cache-dir CACHE_DIR]

		This tool recursively copies/synchronizes a GitLab group from one GitLab server to another.

		@@ -47,6 +48,10 @@ options:
		--include-from INCLUDE_FROM
		a file which lists paths to include (one per line); incompatible with --include
		--insecure skip SSL verification
		--include-branch INCLUDE_BRANCH
		branch to include for git sync, comma-separated and supporting globbing; '!default' is the default value and translated to the default branch of the source project
		--exclude-branch EXCLUDE_BRANCH
		branch to exclude for git sync, comma-separated and supporting globbing; '!default' is translated to the default branch of the source project; Empty by default
		--update-release force the update of the latest release
		--update-avatar force update the avatar images even when they exist and look the same
		--no-group-description
		@@ -84,6 +89,8 @@ options:
		\| `--include` \| `$INCLUDE` \| project/group path(s) to include (multiple CLI option; env. var. is coma separated; all paths included by default) \|
		\| `--include-from` \| `$INCLUDE_FROM` \| a file which lists paths to include (one per line); incompatible with `--include` / `$INCLUDE` \|
		\| `--insecure` \| `$INSECURE` \| skip SSL verification \|
		\| `--include-branch` \| `$INCLUDE_BRANCH` \| branch(s) to include for git sync, comma-separated and supporting globbing; `!default` is the default value and translated to the default branch of the source project; Examples: `!default`, `develop`, `feat-`* \|
		\| `--exclude-branch` \| `$EXCLUDE_BRANCH` \| branch(s) to exclude for git sync, comma-separated and supporting globbing; `!default` is translated to the default branch of the source project; Empty by default; Take precedence over `--include-branch`; Examples: `!default`, `develop`, `feat-`* \|
		\| `--update-release` \| `$UPDATE_RELEASE` \| set to force the update of the latest release (in order to trigger GitLab CI/CD catalog publication) \|
		\| `--update-avatar` \| `$UPDATE_AVATAR` \| force update the avatar images even when they exist and look the same \|
		\| `--no-group-description` \| `$GROUP_DESCRIPTION_DISABLED` \| don't synchronize group description \|

gitlab_cp/sync.py

+155 −37

Original line number	Diff line number	Diff line
		@@ -4,7 +4,7 @@ import os
		import shutil
		from enum import Enum
		from logging import Logger
		from pathlib import Path
		from pathlib import Path, PurePath
		from typing import Optional, Union

		import requests
		@@ -17,7 +17,7 @@ from gitlab import (
		GitlabListError,
		GitlabUpdateError,
		)
		from gitlab.v4.objects import Group, Project, ProjectCommit
		from gitlab.v4.objects import Group, Project, ProjectBranch, ProjectTag

		LOGGER = Logger(__name__)

		@@ -69,6 +69,8 @@ class Synchronizer:
		skip_visibility=False,
		exclude: Optional[list[str]] = None,
		include: Optional[list[str]] = None,
		include_branches: Optional[list[str]] = None,
		exclude_branches: Optional[list[str]] = None,
		update_release=False,
		group_description=True,
		project_description=True,
		@@ -89,6 +91,8 @@ class Synchronizer:
		self.skip_visibility = skip_visibility
		self.exclude = exclude or []
		self.include = include or []
		self.include_branches = include_branches or []
		self.exclude_branches = exclude_branches or []
		self.force_update_latest_release = update_release
		self.group_description = group_description
		self.project_description = project_description
		@@ -151,6 +155,28 @@ class Synchronizer:
		else self.dest_sync_path + "/" + self.rel_path(src_path)
		)

		def is_tag_equal(self, tag1: ProjectTag, tag2: ProjectTag) -> bool:
		# Compare tag by its 'name', 'target' and 'message'
		return all([getattr(tag1, attr) == getattr(tag2, attr) for attr in ['name', 'target', 'message']])

		def is_branch_equal(self, branch1: ProjectBranch, branch2: ProjectBranch) -> bool:
		# Compare tag by its 'name', 'protected' and 'commit.id'
		return (
		branch1.name == branch2.name
		and branch1.protected == branch2.protected
		and branch1.commit['id'] == branch2.commit['id']
		)

		def is_branch_selected(self, branch: ProjectBranch, default_branch: str) -> bool:
		# Check if branch is selected by include/exclude patterns
		return any(
		PurePath(branch.name).match(pattern.replace("!default", default_branch))
		for pattern in self.include_branches
		) and not any(
		PurePath(branch.name).match(pattern.replace("!default", default_branch))
		for pattern in self.exclude_branches
		)

		# Synchronize a project release
		def sync_releases(
		self,
		@@ -223,48 +249,67 @@ class Synchronizer:
		)

		def sync_git_repo(self, src_project: Project, dest_project: Project) -> None:
		src_default_branch = src_project.default_branch
		## Prepare 1: get remote branches
		try:
		src_latest_commit: ProjectCommit = next(
		src_project.commits.list(ref_name=src_default_branch, iterator=True),
		None,
		)
		src_branches = {branch: branch for branch in src_project.branches.list(iterator=True) if self.is_branch_selected(branch, src_project.default_branch)}
		except GitlabListError as ge:
		print(
		f" - latest commit from source repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		f" - branches from source repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)
		return

		try:
		dest_latest_commit = next(
		dest_project.commits.list(ref_name=src_default_branch, iterator=True),
		None,
		dest_branches = {branch: branch for branch in dest_project.branches.list(iterator=True)}
		except GitlabListError as ge:
		print(
		f" - branches from dest repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)

		# Prepare 2: get remote tags
		try:
		src_tags = {tag: tag for tag in src_project.tags.list(iterator=True)}
		except GitlabListError as ge:
		print(
		f" - latest commit from dest repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		f" - tags from source repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)
		return

		if src_latest_commit is None:
		try:
		dest_tags = {tag: tag for tag in dest_project.tags.list(iterator=True)}
		dict()
		except GitlabListError as ge:
		print(
		f" - tags from dest repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)

		# 3: Case of project with no branch detected
		if not src_branches:
		# Git sync is not required: skip
		print(f" - git repository: {AnsiColors.HGRAY}empty{AnsiColors.RESET}")
		return

		# 4: Exit if already up-to-date
		if (
		dest_latest_commit
		and src_latest_commit.get_id() == dest_latest_commit.get_id()
		set(src_tags) <= set(dest_tags) # All source tags are in dest
		and set(src_branches) <= set(dest_branches) # All source branches are in dest
		and all(self.is_tag_equal(src_tag, dest_tags[src_tag]) for src_tag in src_tags) # All source tags are equal to dest tags
		and all(self.is_branch_equal(src_branch, dest_branches[src_branch]) for src_branch in src_branches) # All source branches are equal to dest branches
		):
		# Git sync is not required: skip
		print(
		f" - git repository: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({src_default_branch} on #{src_latest_commit.get_id()})"
		f" - git repository: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({len(src_branches)} branches and {len(src_tags)} tags)"
		)
		return

		# Git sync is required
		# 5: Git sync is required
		repo_dir = self.work_dir / self.rel_path(src_project.path_with_namespace)
		shutil.rmtree(path=repo_dir, ignore_errors=True)

		@@ -294,18 +339,21 @@ class Synchronizer:

		if self.dry_run:
		print(
		f" - git repository: {AnsiColors.HYELLOW}update needed{AnsiColors.RESET} ({src_default_branch} on #{src_latest_commit.get_id()})"
		f" - git repository: {AnsiColors.HYELLOW}update needed{AnsiColors.RESET} ({len(src_branches)} branches and {len(src_tags)} tags)"
		)
		return

		# if project already exists: unprotect default branch first
		if dest_latest_commit:
		# if project already exists: unprotect branches first
		if dest_branches:
		try:
		dest_project.protectedbranches.delete(src_default_branch)
		for branch in dest_branches.keys() & src_branches.keys():
		# Only if branch divergence detected
		if not self.is_branch_equal(src_branches[branch], dest_branches[branch]):
		dest_project.protectedbranches.delete(branch.name)
		except GitlabDeleteError as ge:
		if ge.response_code != 404:
		print(
		f" - unprotect default branch ({src_default_branch}) in dest project: {AnsiColors.HRED}failed{AnsiColors.RESET}",
		f" - unprotect branches in dest project: {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)
		@@ -322,11 +370,11 @@ class Synchronizer:
		"://", f"://token:{self.dest_client.private_token}@"
		)

		# git push --force "$dest_repo_url" --tags "$src_default_branch"
		# git push --force "$dest_repo_url" --tags "src_branches[0],src_branches[1]..."
		try:
		dest = repo.create_remote("dest", dest_repo_url)
		dest.push(
		refspec=src_default_branch,
		refspec=[branch.name for branch in src_branches], #List of branches to push
		force=True,
		tags=True,
		allow_unsafe_options=True,
		@@ -339,19 +387,21 @@ class Synchronizer:
		)
		self.handle_error(ge)

		# protect default branch
		# protect branches
		for branch in src_branches:
		if branch.protected and (branch not in dest_branches or not self.is_branch_equal(branch, dest_branches[branch])):
		try:
		dest_project.protectedbranches.create({"name": src_default_branch})
		dest_project.protectedbranches.create({"name": branch.name})
		except GitlabCreateError as ge:
		if ge.response_code != 409 and ge.response_code != 422:
		print(
		f" - protect default branch ({src_default_branch}) in dest project: {AnsiColors.HRED}failed{AnsiColors.RESET}",
		f" - protect '{branch.name}' branch in dest project: {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)

		print(
		f" - git repository: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_default_branch} on #{src_latest_commit.get_id() if src_latest_commit else 'n/a'})"
		f" - git repository: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({len(src_branches)} branches and {len(src_tags)} tags)"
		)

		# Synchronize a project/group avatar
		@@ -465,6 +515,51 @@ class Synchronizer:
		self.handle_warn(ge)
		return

		def sync_default_branch(self, src_project: Project, dest_project: Project) -> None:
		try:
		src_latest_commit = next(
		src_project.commits.list(iterator=True),
		None,
		)
		except GitlabListError as ge:
		print(
		f" - latest commit from source repo: get {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)
		return

		# Ignore default branch sync if project not initialized
		if src_latest_commit is None:
		return

		if src_project.default_branch == dest_project.default_branch:
		print(
		f" - default branch: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({src_project.default_branch})"
		)
		return

		if self.dry_run:
		print(
		f" - default branch: {AnsiColors.HYELLOW}update needed{AnsiColors.RESET} ({src_project.default_branch} to {dest_project.default_branch})"
		)
		return

		try:
		dest_project.default_branch = src_project.default_branch
		dest_project.save()
		except GitlabUpdateError as ge:
		print(
		f" - default branch: {AnsiColors.HRED}failed{AnsiColors.RESET}",
		ge,
		)
		self.handle_error(ge)
		return

		print(
		f" - default branch: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_project.default_branch})"
		)

		# Synchronizes a GitLab project
		# $1: source project JSON
		# $2: destination parent group ID (number)
		@@ -589,7 +684,10 @@ class Synchronizer:
		# 3: sync Git repository
		self.sync_git_repo(src_project, dest_project)

		# 4: sync Releases
		# 4: sync Default branch
		self.sync_default_branch(src_project, dest_project)

		# 5: sync Releases
		self.sync_releases(src_project, dest_project)

		# Synchronizes recursively a GitLab group
		@@ -876,6 +974,18 @@ def run() -> None:
		default=trueish_env_var("INSECURE"),
		help="skip SSL verification",
		)
		parser.add_argument(
		"--include-branch",
		action="append",
		default=x.split(",") if (x := os.getenv("INCLUDE_BRANCH", "!default")) else [],
		help="branch to include for git sync, comma-separated and supporting globbing; '!default' is the default value and translated to the default branch of the source project",
		)
		parser.add_argument(
		"--exclude-branch",
		action="append",
		default=x.split(",") if (x := os.getenv("EXCLUDE_BRANCH")) else [],
		help="branch to exclude for git sync, comma-separated and supporting globbing; '!default' is translated to the default branch of the source project; Empty by default",
		)
		parser.add_argument(
		"--update-release",
		default=trueish_env_var("UPDATE_RELEASE"),
		@@ -1072,6 +1182,12 @@ def run() -> None:
		print(
		f"- cache dir (--cache-dir) : {AnsiColors.CYAN}{args.cache_dir}{AnsiColors.RESET}"
		)
		print(
		f"- include-branch (--include-branch) : {AnsiColors.CYAN}{', '.join(args.include_branch)}{AnsiColors.RESET}"
		)
		print(
		f"- exclude-branch (--exclude-branch) : {AnsiColors.CYAN}{', '.join(args.exclude_branch)}{AnsiColors.RESET}"
		)
		print(
		f"- new group options (--new-group-options) : {AnsiColors.CYAN}{json.dumps(new_group_options)}{AnsiColors.RESET}"
		)
		@@ -1117,6 +1233,8 @@ def run() -> None:
		skip_visibility=args.skip_visibility,
		exclude=exclude_list,
		include=include_list,
		include_branches=args.include_branch,
		exclude_branches=args.exclude_branch,
		update_release=args.update_release,
		group_description=(not args.no_group_description),
		project_description=(not args.no_project_description),