Merge branch 'more-options' into 'main' (286364e0) · Commits · to be continuous... / tools / gitlab-cp

README.md

+53 −21

Original line number	Diff line number	Diff line
		@@ -16,8 +16,10 @@ gitlab-cp --help
		## Usage

		```bash
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}] [--exclude EXCLUDE] [--insecure]
		[--update-release] [--update-avatar] [--no-group-description] [--no-project-description] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}]
		[--skip-visibility] [--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure] [--update-release] [--update-avatar] [--no-group-description]
		[--no-project-description] [--new-group-options NEW_GROUP_OPTIONS] [--new-group-options-from NEW_GROUP_OPTIONS_FROM] [--new-project-options NEW_PROJECT_OPTIONS]
		[--new-project-options-from NEW_PROJECT_OPTIONS_FROM] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]

		This tool recursively copies/synchronizes a GitLab group from one GitLab server to another.

		@@ -34,8 +36,14 @@ options:
		--dest-sync-path DEST_SYNC_PATH
		GitLab destination root group path to synchronize (defaults to --src-sync-path)
		--max-visibility {public,internal,private}
		maximum visibility of projects in destination group
		maximum visibility of groups and projects in destination group
		--skip-visibility skip updating the destination group or project visibility (when it exists already)
		--exclude EXCLUDE project/group path to exclude from processing (relative to --src-sync-path)
		--exclude-from EXCLUDE_FROM
		a file which lists paths to exclude (one per line); incompatible with --exclude
		--include INCLUDE project/group path to include for processing (relative to --src-sync-path); all paths are included by default
		--include-from INCLUDE_FROM
		a file which lists paths to include (one per line); incompatible with --include
		--insecure skip SSL verification
		--update-release force the update of the latest release
		--update-avatar force update the avatar images even when they exist and look the same
		@@ -43,6 +51,14 @@ options:
		don't synchronize group description
		--no-project-description
		don't synchronize project description
		--new-group-options NEW_GROUP_OPTIONS
		a JSON string with extra options for groups creation
		--new-group-options-from NEW_GROUP_OPTIONS_FROM
		a JSON file with extra options for groups creation; incompatible with --new-group-options
		--new-project-options NEW_PROJECT_OPTIONS
		a JSON string with extra options for projects creation
		--new-project-options-from NEW_PROJECT_OPTIONS_FROM
		a JSON file with extra options for projects creation; incompatible with --new-project-options
		--dry-run dry run (don't execute any write action)
		--halt-on-error halt synchronizing whenever an error occurs
		--cache-dir CACHE_DIR
		@@ -50,7 +66,7 @@ options:
		```

		\| CLI option \| Env. Variable \| Description \|
		\| -------------------------- \| ------------------------------- \| --------------------------------------------------------------------------------------------------------------------- \|
		\| ---------------------------- \| ------------------------------- \| --------------------------------------------------------------------------------------------------------------------- \|
		\| `--src-api` \| `$SRC_GITLAB_API` \| GitLab source API url (mandatory) \|
		\| `--src-token` \| `$SRC_TOKEN` \| GitLab source token (_optional_ if source GitLab group and sub projects have `public` visibility) \|
		\| `--src-sync-path` \| `$SRC_SYNC_PATH` \| GitLab source root group path to synchronize (mandatory) \|
		@@ -58,16 +74,32 @@ options:
		\| `--dest-token` \| `$DEST_TOKEN` \| GitLab destination token with at least scopes `api,read_repository,write_repository` and `Owner` role (mandatory) \|
		\| `--dest-sync-path` \| `$DEST_SYNC_PATH` \| GitLab destination root group path to synchronize (defaults to `--src-sync-path`) \|
		\| `--max-visibility` \| `$MAX_VISIBILITY` \| maximum visibility of projects in destination group (defaults to `public`) \|
		\| `--exclude` \| `$EXCLUDE` \| project/group path(s) to exclude (multiple CLI option; env. variable is a coma separated list) \|
		\| `--skip-visibility` \| `$SKIP_VISIBILITY` \| skip updating the destination group or project visibility (when it exists already) \|
		\| `--exclude` \| `$EXCLUDE` \| project/group path(s) to exclude (multiple CLI option; env. variable is a coma separated list; wins over `--include`) \|
		\| `--exclude-from` \| `$EXCLUDE_FROM` \| a file which lists paths to exclude (one per line); incompatible with `--exclude` / `$EXCLUDE` \|
		\| `--include` \| `$INCLUDE` \| project/group path(s) to include (multiple CLI option; env. var. is coma separated; all paths included by default) \|
		\| `--include-from` \| `$INCLUDE_FROM` \| a file which lists paths to include (one per line); incompatible with `--include` / `$INCLUDE` \|
		\| `--insecure` \| `$INSECURE` \| skip SSL verification \|
		\| `--update-release` \| `$UPDATE_RELEASE` \| set to force the update of the latest release (in order to trigger GitLab CI/CD catalog publication) \|
		\| `--update-avatar` \| `$UPDATE_AVATAR` \| force update the avatar images even when they exist and look the same \|
		\| `--no-group-description` \| `$GROUP_DESCRIPTION_DISABLED` \| don't synchronize group description \|
		\| `--no-project-description` \| `$PROJECT_DESCRIPTION_DISABLED` \| don't synchronize project description \|
		\| `--dry-run` \| _none_ \| dry run (don't execute any write action) \|
		\| `--halt-on-error` \| _none_ \| halt synchronizing when an error occurs \|
		\| `--new-group-options` \| `$NEW_GROUP_OPTIONS` \| a JSON string with extra options for groups creation (no default value) \|
		\| `--new-group-options-from` \| `$NEW_GROUP_OPTIONS_FROM` \| a JSON file with extra options for groups creation; incompatible with --new-group-options \|
		\| `--new-project-options` \| `$NEW_PROJECT_OPTIONS` \| a JSON string with extra options for projects creation (default value disables issues and MR, see below) \|
		\| `--new-project-options-from` \| `$NEW_PROJECT_OPTIONS_FROM` \| a JSON file with extra options for projects creation; incompatible with --new-project-options \|
		\| `--dry-run` \| `$DRY_RUN` \| dry run (don't execute any write action) \|
		\| `--halt-on-error` \| `$HALT_ON_ERROR` \| halt synchronizing when an error occurs \|
		\| `--cache-dir` \| `$CACHE_DIR` \| cache directory (used to download resources such as images and Git repositories) (defaults to `.work`) \|

		Default options for creating new projects:
		```json
		{
		"issues_access_level": "disabled",
		"merge_requests_access_level": "disabled"
		}
		```

		## Developers

		`gitlab-cp` is implemented in Python and relies on [Poetry](https://python-poetry.org/) for its packaging and dependency management.

gitlab_cp/sync.py

+194 −21

Original line number	Diff line number	Diff line
		import argparse
		import json
		import os
		import shutil
		from enum import Enum
		from logging import Logger
		from pathlib import Path
		from typing import Optional
		from typing import Optional, Union

		import requests
		from git import GitError, Repo
		@@ -65,7 +66,9 @@ class Synchronizer:
		dest_sync_path: str,
		work_dir: Path,
		max_visibility=GlVisibility.public,
		skip_visibility=False,
		exclude: Optional[list[str]] = None,
		include: Optional[list[str]] = None,
		update_release=False,
		group_description=True,
		project_description=True,
		@@ -73,6 +76,8 @@ class Synchronizer:
		continue_on_error=False,
		update_avatar=False,
		ssl_verify=True,
		new_project_options={},
		new_group_options={},
		):
		self.src_client = src_client
		self.src_sync_path = src_sync_path
		@@ -80,7 +85,9 @@ class Synchronizer:
		self.dest_sync_path = dest_sync_path
		self.work_dir = work_dir
		self.max_visibility = max_visibility
		self.skip_visibility = skip_visibility
		self.exclude = exclude or []
		self.include = include or []
		self.force_update_latest_release = update_release
		self.group_description = group_description
		self.project_description = project_description
		@@ -92,6 +99,8 @@ class Synchronizer:
		self.warnings: list[Exception] = []
		self.groups_count = 0
		self.projects_count = 0
		self.new_project_options = new_project_options
		self.new_group_options = new_group_options

		def handle_error(self, err: Exception) -> None:
		self.errors.append(err)
		@@ -119,6 +128,18 @@ class Synchronizer:
		def is_excluded(self, src_path: str) -> bool:
		return self.rel_path(src_path) in self.exclude

		def is_included(self, src_path: str) -> bool:
		# true if:
		# - there's no include list (any group/project is included by default)
		# - realtive path exactly matches an include item
		# - realtive path is an ascendant of an included item (parent groups must be considered to reach the item)
		# - realtive path is a descendant of an included item (inclusion of a group is recursive)
		def are_related_paths(p1, p2):
		return (p1 == p2) or p2.startswith(p1 + "/") or p1.startswith(p2 + "/")
		rel_path = self.rel_path(src_path)
		return (not self.include) \
		or True in (are_related_paths(rel_path, x) for x in self.include)

		def to_dest_path(self, src_path: str) -> str:
		return (
		self.dest_sync_path
		@@ -387,6 +408,11 @@ class Synchronizer:
		dest_project.name = src_project.name
		changed_attr.append("name")
		if dest_project.visibility != dest_visibility:
		if self.skip_visibility:
		print(
		f" - visibility: {AnsiColors.HYELLOW}skipping update{AnsiColors.RESET} ({dest_project.visibility} -> {dest_visibility}) ({dest_project.get_id()})"
		)
		else:
		dest_project.visibility = dest_visibility
		changed_attr.append("visibility")
		if (
		@@ -442,8 +468,7 @@ class Synchronizer:
		"description": src_project.description,
		# ??? "default_branch": src_default_branch,
		"namespace_id": dest_parent_group.get_id(),
		"issues_access_level": "disabled",
		"merge_requests_access_level": "disabled",
		** self.new_project_options,
		}
		)
		print(
		@@ -528,6 +553,11 @@ class Synchronizer:
		dest_group.name = src_group.name
		changed_attr.append("name")
		if dest_group.visibility != dest_visibility:
		if self.skip_visibility:
		print(
		f"- visibility: {AnsiColors.HYELLOW}skipping update{AnsiColors.RESET} ({dest_group.visibility} -> {dest_visibility}) ({dest_group.get_id()})"
		)
		else:
		dest_group.visibility = dest_visibility
		changed_attr.append("visibility")
		if (
		@@ -578,6 +608,7 @@ class Synchronizer:
		"visibility": dest_visibility,
		"description": src_group.description,
		"parent_id": dest_parent_group.get_id(),
		** self.new_group_options,
		}
		)
		print(
		@@ -666,7 +697,11 @@ class Synchronizer:
		subprojects = src_group.projects.list(all=True)
		print(f"- sync {len(subprojects)} sub projects...")
		for src_project in subprojects:
		if self.is_excluded(src_project.path_with_namespace):
		if not self.is_included(src_project.path_with_namespace):
		print(
		f" - 🏠 Project {AnsiColors.BLUE}{src_project.path_with_namespace}{AnsiColors.RESET} does not match includes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		elif self.is_excluded(src_project.path_with_namespace):
		print(
		f" - 🏠 Project {AnsiColors.BLUE}{src_project.path_with_namespace}{AnsiColors.RESET} matches excludes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		@@ -679,7 +714,11 @@ class Synchronizer:
		subgroups = src_group.subgroups.list(all=True)
		print(f"- sync {len(subgroups)} sub groups...")
		for src_subgroup in subgroups:
		if self.is_excluded(src_subgroup.full_path):
		if not self.is_included(src_subgroup.full_path):
		print(
		f"🏢 Group {AnsiColors.BLUE}{src_subgroup.full_path}{AnsiColors.RESET} does not match includes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		elif self.is_excluded(src_subgroup.full_path):
		print(
		f"🏢 Group {AnsiColors.BLUE}{src_subgroup.full_path}{AnsiColors.RESET} matches excludes: {AnsiColors.HGRAY}skip{AnsiColors.RESET}"
		)
		@@ -700,6 +739,27 @@ def dirname(path: Optional[str]) -> Optional[str]:
		def to_url(api_url: str) -> str:
		return api_url[0:-7] if api_url.endswith("/api/v4") else api_url

		def read_paths_list_from_file(file_path: Path) -> list[str]:
		# read a list of paths from a file (one per line), ignoring comments,
		# blank lines and leading/trailing spaces
		with open(file_path, "r") as f:
		lines = (line.partition('#')[0] for line in f)
		return [line.strip() for line in lines if line.strip()]

		def simple_json_to_dict(json_str: str, source: Union[str,Path]) -> dict[str,Union[int,str]]:
		options = json.loads(json_str)
		assert (
		isinstance(options, (dict))
		), f"Invalid type of JSON in {source}, expected an object"
		for k, v in options.items():
		assert (
		isinstance(v, (int, str))
		), f"Invalid value type for key {k} in {source}, expected only strings, integers and booleans"
		return options

		def trueish_env_var(env_var_name: str) -> bool:
		value = os.getenv(env_var_name)
		return bool(value) and (value.lower() not in [ "0", "false", "no" ])

		def run() -> None:
		# define command parser
		@@ -742,7 +802,13 @@ def run() -> None:
		default=os.getenv("MAX_VISIBILITY") or GlVisibility.public,
		type=GlVisibility,
		choices=list(GlVisibility),
		help="maximum visibility of projects in destination group",
		help="maximum visibility of groups and projects in destination group",
		)
		parser.add_argument(
		"--skip-visibility",
		action="store_true",
		default=trueish_env_var("SKIP_VISIBILITY"),
		help="skip updating the destination group or project visibility (when it exists already)",
		)
		parser.add_argument(
		"--exclude",
		@@ -750,43 +816,85 @@ def run() -> None:
		default=os.getenv("EXCLUDE").split(",") if os.getenv("EXCLUDE") else [],
		help="project/group path to exclude from processing (relative to --src-sync-path)",
		)
		parser.add_argument(
		"--exclude-from",
		default=os.getenv("EXCLUDE_FROM"),
		type=Path,
		help="a file which lists paths to exclude (one per line); incompatible with --exclude",
		)
		parser.add_argument(
		"--include",
		action="append",
		default=os.getenv("INCLUDE").split(",") if os.getenv("INCLUDE") else [],
		help="project/group path to include for processing (relative to --src-sync-path); all paths are included by default",
		)
		parser.add_argument(
		"--include-from",
		default=os.getenv("INCLUDE_FROM"),
		type=Path,
		help="a file which lists paths to include (one per line); incompatible with --include",
		)
		parser.add_argument(
		"--insecure",
		action="store_true",
		default=os.getenv("INSECURE") is not None,
		default=trueish_env_var("INSECURE"),
		help="skip SSL verification",
		)
		parser.add_argument(
		"--update-release",
		default=os.getenv("UPDATE_RELEASE") is not None,
		default=trueish_env_var("UPDATE_RELEASE"),
		action="store_true",
		help="force the update of the latest release",
		)
		parser.add_argument(
		"--update-avatar",
		default=os.getenv("UPDATE_AVATAR") is not None,
		default=trueish_env_var("UPDATE_AVATAR"),
		action="store_true",
		help="force update the avatar images even when they exist and look the same",
		)
		parser.add_argument(
		"--no-group-description",
		default=os.getenv("GROUP_DESCRIPTION_DISABLED") is not None,
		default=trueish_env_var("GROUP_DESCRIPTION_DISABLED"),
		action="store_true",
		help="don't synchronize group description",
		)
		parser.add_argument(
		"--no-project-description",
		default=os.getenv("PROJECT_DESCRIPTION_DISABLED") is not None,
		default=trueish_env_var("PROJECT_DESCRIPTION_DISABLED"),
		action="store_true",
		help="don't synchronize project description",
		)
		parser.add_argument(
		"--new-group-options",
		default=os.getenv("NEW_GROUP_OPTIONS"),
		help="a JSON string with extra options for groups creation",
		)
		parser.add_argument(
		"--new-group-options-from",
		default=os.getenv("NEW_GROUP_OPTIONS_FROM"),
		type=Path,
		help="a JSON file with extra options for groups creation; incompatible with --new-group-options",
		)
		parser.add_argument(
		"--new-project-options",
		default=os.getenv("NEW_PROJECT_OPTIONS"),
		help="a JSON string with extra options for projects creation",
		)
		parser.add_argument(
		"--new-project-options-from",
		default=os.getenv("NEW_PROJECT_OPTIONS_FROM"),
		type=Path,
		help="a JSON file with extra options for projects creation; incompatible with --new-project-options",
		)
		parser.add_argument(
		"--dry-run",
		default=trueish_env_var("DRY_RUN"),
		action="store_true",
		help="dry run (don't execute any write action)",
		)
		parser.add_argument(
		"--halt-on-error",
		default=trueish_env_var("HALT_ON_ERROR"),
		action="store_true",
		help="halt synchronizing whenever an error occurs",
		)
		@@ -814,6 +922,55 @@ def run() -> None:
		assert (
		args.dest_token
		), "Dest token is required (use --dest-token CLI option or DEST_TOKEN variable)"
		assert (
		not args.exclude or not args.exclude_from
		), "Cannot use both --exclude and --exclude-from"
		assert (
		not args.exclude_from or args.exclude_from.is_file()
		), f"No such file: {args.exclude_from}"
		assert (
		not args.include or not args.include_from
		), "Cannot use both --include and --include-from"
		assert (
		not args.include_from or args.include_from.is_file()
		), f"No such file: {args.include_from}"
		assert (
		not args.new_group_options or not args.new_group_options_from
		), "Cannot use both --new-group-options and --new-group-options-from"
		assert (
		not args.new_group_options_from or args.new_group_options_from.is_file()
		), f"No such file: {args.new_group_options_from}"
		assert (
		not args.new_project_options or not args.new_project_options_from
		), "Cannot use both --new-project-options and --new-project-options-from"
		assert (
		not args.new_project_options_from or args.new_project_options_from.is_file()
		), f"No such file: {args.new_project_options_from}"

		new_group_options: dict[str, Union[int,str]] = {}
		if args.new_group_options:
		new_group_options = simple_json_to_dict(args.new_group_options, "--new-group-options")
		elif args.new_group_options_from:
		new_group_options = simple_json_to_dict(args.new_group_options_from.read_text(), str(args.new_group_options_from))

		# default options for new project: disable issues and MR
		new_project_options: dict[str, Union[int,str]] = { "issues_access_level": "disabled", "merge_requests_access_level": "disabled" }
		if args.new_project_options:
		new_project_options = simple_json_to_dict(args.new_project_options, "--new-project-options")
		elif args.new_project_options_from:
		new_project_options = simple_json_to_dict(args.new_project_options_from.read_text(), args.new_project_options_from)

		exclude_list = []
		if args.exclude:
		exclude_list = args.exclude
		elif args.exclude_from:
		exclude_list = read_paths_list_from_file(args.exclude_from)

		include_list = []
		if args.include:
		include_list = args.include
		elif args.include_from:
		include_list = read_paths_list_from_file(args.include_from)

		print("Synchronizing GitLab group")
		print(
		@@ -832,7 +989,13 @@ def run() -> None:
		f"- max visi. (--max-visibility) : {AnsiColors.CYAN}{args.max_visibility}{AnsiColors.RESET}"
		)
		print(
		f"- exclude (--exclude) : {AnsiColors.CYAN}{', '.join(args.exclude)}{AnsiColors.RESET}"
		f"- skip visi (--skip-visibility) : {AnsiColors.CYAN}{args.skip_visibility}{AnsiColors.RESET}"
		)
		print(
		f"- exclude (--exclude(-from)) : {AnsiColors.CYAN}{', '.join(exclude_list)}{AnsiColors.RESET}"
		)
		print(
		f"- include (--include(-from)) : {AnsiColors.CYAN}{', '.join(include_list)}{AnsiColors.RESET}"
		)
		print(
		f"- insecure (--insecure) : {AnsiColors.CYAN}{args.insecure}{AnsiColors.RESET}"
		@@ -852,6 +1015,12 @@ def run() -> None:
		print(
		f"- cache dir (--cache-dir) : {AnsiColors.CYAN}{args.cache_dir}{AnsiColors.RESET}"
		)
		print(
		f"- new group options (--new-group-options) : {AnsiColors.CYAN}{json.dumps(new_group_options)}{AnsiColors.RESET}"
		)
		print(
		f"- new project options (--new-project-options) : {AnsiColors.CYAN}{json.dumps(new_project_options)}{AnsiColors.RESET}"
		)
		print(
		f"- skip group desc. (--no-group-description) : {AnsiColors.CYAN}{args.no_group_description}{AnsiColors.RESET}"
		)
		@@ -884,15 +1053,19 @@ def run() -> None:
		),
		dest_sync_path,
		work_dir,
		args.max_visibility,
		args.exclude,
		args.update_release,
		not args.no_group_description,
		not args.no_project_description,
		max_visibility=args.max_visibility,
		skip_visibility=args.skip_visibility,
		exclude=exclude_list,
		include=include_list,
		update_release=args.update_release,
		group_description=(not args.no_group_description),
		project_description=(not args.no_project_description),
		dry_run=args.dry_run,
		continue_on_error=not args.halt_on_error,
		update_avatar=args.update_avatar,
		ssl_verify=not args.insecure,
		new_group_options=new_group_options,
		new_project_options=new_project_options,
		)
		# retrieve src root group
		try: