feat: add --include option (sync only some projects and subgroups) (3266e812) · Commits · to be continuous... / tools / gitlab-cp

README.md

+3 −1

Original line number	Diff line number	Diff line
		@@ -16,7 +16,7 @@ gitlab-cp --help
		## Usage

		```bash
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}] [--exclude EXCLUDE] [--insecure]
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}] [--exclude EXCLUDE] [--include INCLUDE] [--insecure]
		[--update-release] [--update-avatar] [--no-group-description] [--no-project-description] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]

		This tool recursively copies/synchronizes a GitLab group from one GitLab server to another.
		@@ -36,6 +36,7 @@ options:
		--max-visibility {public,internal,private}
		maximum visibility of projects in destination group
		--exclude EXCLUDE project/group path to exclude from processing (relative to --src-sync-path)
		--include INCLUDE project/group path to include for processing (relative to --src-sync-path); all paths are included by default
		--insecure skip SSL verification
		--update-release force the update of the latest release
		--update-avatar force update the avatar images even when they exist and look the same
		@@ -59,6 +60,7 @@ options:
		\| `--dest-sync-path` \| `$DEST_SYNC_PATH` \| GitLab destination root group path to synchronize (defaults to `--src-sync-path`) \|
		\| `--max-visibility` \| `$MAX_VISIBILITY` \| maximum visibility of projects in destination group (defaults to `public`) \|
		\| `--exclude` \| `$EXCLUDE` \| project/group path(s) to exclude (multiple CLI option; env. variable is a coma separated list) \|
		\| `--include` \| `$INCLUDE` \| project/group path(s) to include (multiple CLI option; env. var. is coma separated; all paths included by default) \|
		\| `--insecure` \| `$INSECURE` \| skip SSL verification \|
		\| `--update-release` \| `$UPDATE_RELEASE` \| set to force the update of the latest release (in order to trigger GitLab CI/CD catalog publication) \|
		\| `--update-avatar` \| `$UPDATE_AVATAR` \| force update the avatar images even when they exist and look the same \|

gitlab_cp/sync.py

+34 −2

Original line number	Diff line number	Diff line
		@@ -72,6 +72,7 @@ class Synchronizer:
		work_dir: Path,
		max_visibility=GlVisibility.public,
		exclude: Optional[list[str]] = None,
		include: Optional[list[str]] = None,
		update_release=False,
		group_description=True,
		project_description=True,
		@@ -86,6 +87,7 @@ class Synchronizer:
		self.work_dir = work_dir
		self.max_visibility = max_visibility
		self.exclude = exclude or []
		self.include = include or []
		self.force_update_latest_release = update_release
		self.group_description = group_description
		self.project_description = project_description
		@@ -123,6 +125,18 @@ class Synchronizer:
		def is_excluded(self, src_path: str) -> bool:
		return self.rel_path(src_path) in self.exclude

		def is_included(self, src_path: str) -> bool:
		# true if:
		# - there's no include list (any group/project is included by default)
		# - realtive path exactly matches an include item
		# - realtive path is an ascendant of an included item (parent groups must be considered to reach the item)
		# - realtive path is a descendant of an included item (inclusion of a group is recursive)
		def are_related_paths(p1, p2):
		return (p1 == p2) or p2.startswith(p1 + "/") or p1.startswith(p2 + "/")
		rel_path = self.rel_path(src_path)
		return (not self.include) \
		or True in (are_related_paths(rel_path, x) for x in self.include)

		def to_dest_path(self, src_path: str) -> str:
		return (
		self.dest_sync_path
		@@ -666,7 +680,11 @@ class Synchronizer:
		subprojects = src_group.projects.list(all=True)
		print(f"- sync {len(subprojects)} sub projects...")
		for src_project in subprojects:
		if self.is_excluded(src_project.path_with_namespace):
		if not self.is_included(src_project.path_with_namespace):
		print(
		f" - 🏠 Project {AnsiColors.BLUE}{src_project.path_with_namespace}{AnsiColors.RESET} does not match includes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		elif self.is_excluded(src_project.path_with_namespace):
		print(
		f" - 🏠 Project {AnsiColors.BLUE}{src_project.path_with_namespace}{AnsiColors.RESET} matches excludes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		@@ -679,7 +697,11 @@ class Synchronizer:
		subgroups = src_group.subgroups.list(all=True)
		print(f"- sync {len(subgroups)} sub groups...")
		for src_subgroup in subgroups:
		if self.is_excluded(src_subgroup.full_path):
		if not self.is_included(src_subgroup.full_path):
		print(
		f"🏢 Group {AnsiColors.BLUE}{src_subgroup.full_path}{AnsiColors.RESET} does not match includes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		elif self.is_excluded(src_subgroup.full_path):
		print(
		f"🏢 Group {AnsiColors.BLUE}{src_subgroup.full_path}{AnsiColors.RESET} matches excludes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		@@ -750,6 +772,12 @@ def run() -> None:
		default=os.getenv("EXCLUDE").split(",") if os.getenv("EXCLUDE") else [],
		help="project/group path to exclude from processing (relative to --src-sync-path)",
		)
		parser.add_argument(
		"--include",
		action="append",
		default=os.getenv("INCLUDE").split(",") if os.getenv("INCLUDE") else [],
		help="project/group path to include for processing (relative to --src-sync-path); all paths are included by default",
		)
		parser.add_argument(
		"--insecure",
		action="store_true",
		@@ -834,6 +862,9 @@ def run() -> None:
		print(
		f"- exclude (--exclude) : {AnsiColors.CYAN}{', '.join(args.exclude)}{AnsiColors.RESET}"
		)
		print(
		f"- include (--include) : {AnsiColors.CYAN}{', '.join(args.include)}{AnsiColors.RESET}"
		)
		print(
		f"- insecure (--insecure) : {AnsiColors.CYAN}{args.insecure}{AnsiColors.RESET}"
		)
		@@ -886,6 +917,7 @@ def run() -> None:
		work_dir,
		args.max_visibility,
		args.exclude,
		args.include,
		args.update_release,
		not args.no_group_description,
		not args.no_project_description,