feat: add --use-src-tracker option (42329a98) · Commits · to be continuous... / tools / gitlab-cp

README.md

+4 −1

Original line number	Diff line number	Diff line
		@@ -19,7 +19,7 @@ gitlab-cp --help
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}]
		[--skip-visibility] [--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure] [--update-release] [--update-avatar] [--no-group-description]
		[--no-project-description] [--new-group-options NEW_GROUP_OPTIONS] [--new-group-options-from NEW_GROUP_OPTIONS_FROM] [--new-project-options NEW_PROJECT_OPTIONS]
		[--new-project-options-from NEW_PROJECT_OPTIONS_FROM] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]
		[--new-project-options-from NEW_PROJECT_OPTIONS_FROM] [--use-src-issue-tracker] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]

		This tool recursively copies/synchronizes a GitLab group from one GitLab server to another.

		@@ -59,6 +59,8 @@ options:
		a JSON string with extra options for projects creation
		--new-project-options-from NEW_PROJECT_OPTIONS_FROM
		a JSON file with extra options for projects creation; incompatible with --new-project-options
		--use-src-issue-tracker
		make destination projects to use their source project issue tracker
		--dry-run dry run (don't execute any write action)
		--halt-on-error halt synchronizing whenever an error occurs
		--cache-dir CACHE_DIR
		@@ -88,6 +90,7 @@ options:
		\| `--new-group-options-from` \| `$NEW_GROUP_OPTIONS_FROM` \| a JSON file with [extra options for groups creation](https://docs.gitlab.com/api/groups/#create-a-group); incompatible with `--new-group-options` \|
		\| `--new-project-options` \| `$NEW_PROJECT_OPTIONS` \| a JSON string with [extra options for projects creation](https://docs.gitlab.com/api/projects/#create-a-project) (default value disables issues and MR, see below) \|
		\| `--new-project-options-from` \| `$NEW_PROJECT_OPTIONS_FROM` \| a JSON file with [extra options for projects creation](https://docs.gitlab.com/api/projects/#create-a-project); incompatible with `--new-project-options` \|
		\| `----use-src-issue-tracker` \| `$USE_SRC_ISSUE_TRACKER` \| make destination projects to use their source project issue tracker \|
		\| `--dry-run` \| `$DRY_RUN` \| dry run (don't execute any write action) \|
		\| `--halt-on-error` \| `$HALT_ON_ERROR` \| halt synchronizing when an error occurs \|
		\| `--cache-dir` \| `$CACHE_DIR` \| cache directory (used to download resources such as images and Git repositories) (defaults to `.work`) \|

gitlab_cp/sync.py

+163 −76

Original line number	Diff line number	Diff line
		@@ -78,6 +78,7 @@ class Synchronizer:
		ssl_verify=True,
		new_project_options={},
		new_group_options={},
		use_src_issue_tracker=False,
		):
		self.src_client = src_client
		self.src_sync_path = src_sync_path
		@@ -101,6 +102,7 @@ class Synchronizer:
		self.projects_count = 0
		self.new_project_options = new_project_options
		self.new_group_options = new_group_options
		self.use_src_issue_tracker = use_src_issue_tracker

		def handle_error(self, err: Exception) -> None:
		self.errors.append(err)
		@@ -136,9 +138,11 @@ class Synchronizer:
		# - realtive path is a descendant of an included item (inclusion of a group is recursive)
		def are_related_paths(p1, p2):
		return (p1 == p2) or p2.startswith(p1 + "/") or p1.startswith(p2 + "/")

		rel_path = self.rel_path(src_path)
		return (not self.include) \
		or True in (are_related_paths(rel_path, x) for x in self.include)
		return (not self.include) or True in (
		are_related_paths(rel_path, x) for x in self.include
		)

		def to_dest_path(self, src_path: str) -> str:
		return (
		@@ -245,11 +249,9 @@ class Synchronizer:
		)
		self.handle_error(ge)

		if ( src_latest_commit is None ):
		if src_latest_commit is None:
		# Git sync is not required: skip
		print(
		f" - git repository: {AnsiColors.HGRAY}empty{AnsiColors.RESET}"
		)
		print(f" - git repository: {AnsiColors.HGRAY}empty{AnsiColors.RESET}")
		return

		if (
		@@ -358,20 +360,39 @@ class Synchronizer:
		src_avatar: Project \| Group,
		dest_avatar: Project \| Group,
		) -> None:
		assert all(any(isinstance(resource, type) for type in [Project, Group]) for resource in [src_avatar, dest_avatar])
		assert all(
		any(isinstance(resource, type) for type in [Project, Group])
		for resource in [src_avatar, dest_avatar]
		)

		# Python lib `python-gitlab` doesn't support avatar endpoint (02/12/2024):
		# > We write our own requests calls.
		src_web_url = src_avatar.web_url
		src_avatar_url = src_avatar.avatar_url
		src_avatar_resource = "projects" if isinstance(src_avatar, Project) else "groups"
		src_avatar_api = f"{self.src_client.api_url}/{src_avatar_resource}/{src_avatar.id}/avatar"
		src_avatar_api_headers = {"PRIVATE-TOKEN": self.src_client.private_token} if self.src_client.private_token is not None else {}
		src_avatar_resource = (
		"projects" if isinstance(src_avatar, Project) else "groups"
		)
		src_avatar_api = (
		f"{self.src_client.api_url}/{src_avatar_resource}/{src_avatar.id}/avatar"
		)
		src_avatar_api_headers = (
		{"PRIVATE-TOKEN": self.src_client.private_token}
		if self.src_client.private_token is not None
		else {}
		)
		dest_web_url = dest_avatar.web_url
		dest_avatar_url = dest_avatar.avatar_url
		dest_avatar_resource = "projects" if isinstance(dest_avatar, Project) else "groups"
		dest_avatar_api = f"{self.dest_client.api_url}/{dest_avatar_resource}/{dest_avatar.id}/avatar"
		dest_avatar_api_headers = {"PRIVATE-TOKEN": self.dest_client.private_token} if self.dest_client.private_token is not None else {}
		dest_avatar_resource = (
		"projects" if isinstance(dest_avatar, Project) else "groups"
		)
		dest_avatar_api = (
		f"{self.dest_client.api_url}/{dest_avatar_resource}/{dest_avatar.id}/avatar"
		)
		dest_avatar_api_headers = (
		{"PRIVATE-TOKEN": self.dest_client.private_token}
		if self.dest_client.private_token is not None
		else {}
		)

		# Up-to-date if no source avatar url or implicit avatar url ('/path/to/proj/-/avatar')
		if src_avatar_url is None or src_avatar_url == f"{src_web_url}/-/avatar":
		@@ -385,10 +406,24 @@ class Synchronizer:
		if dest_avatar_url is not None and not self.force_update_avatar:
		# fetch avatar size
		try:
		src_avatar_response = requests.head(url=src_avatar_api, headers=src_avatar_api_headers, verify=self.ssl_verify)
		_, src_avatar_size = src_avatar_response.raise_for_status(), src_avatar_response.headers['Content-Length']
		dest_avatar_response = requests.head(url=dest_avatar_api, headers=dest_avatar_api_headers, verify=self.ssl_verify)
		_, dest_avatar_size = dest_avatar_response.raise_for_status(), dest_avatar_response.headers['Content-Length']
		src_avatar_response = requests.head(
		url=src_avatar_api,
		headers=src_avatar_api_headers,
		verify=self.ssl_verify,
		)
		_, src_avatar_size = (
		src_avatar_response.raise_for_status(),
		src_avatar_response.headers["Content-Length"],
		)
		dest_avatar_response = requests.head(
		url=dest_avatar_api,
		headers=dest_avatar_api_headers,
		verify=self.ssl_verify,
		)
		_, dest_avatar_size = (
		dest_avatar_response.raise_for_status(),
		dest_avatar_response.headers["Content-Length"],
		)
		# compare avatar size
		if src_avatar_size == dest_avatar_size:
		# image up-to-date
		@@ -413,7 +448,11 @@ class Synchronizer:

		# Update avatar
		try:
		dest_avatar.avatar = requests.get(url=src_avatar_api, headers=src_avatar_api_headers, verify=self.ssl_verify).content
		dest_avatar.avatar = requests.get(
		url=src_avatar_api,
		headers=src_avatar_api_headers,
		verify=self.ssl_verify,
		).content
		dest_avatar.save()
		print(
		f" - avatar image: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_avatar_url})"
		@@ -527,6 +566,23 @@ class Synchronizer:
		self.handle_error(ge)
		return

		# maybe link to src issue tracker
		if self.use_src_issue_tracker and not self.dry_run:
		# see: https://docs.gitlab.com/api/project_integrations/#custom-issue-tracker
		# https://python-gitlab.readthedocs.io/en/stable/gl_objects/projects.html#project-integrations
		citi = dest_project.integrations.get("custom-issue-tracker", lazy=True)
		citi.project_url = f"{src_project.web_url}/-/issues"
		citi.new_issue_url = f"{src_project.web_url}/-/issues/new"
		citi.issues_url = f"{src_project.web_url}/-/issues/:id"
		try:
		citi.save()
		except GitlabUpdateError as ge:
		print(
		f" - project: set custom issue tracker {AnsiColors.HRED}failed{AnsiColors.RESET} ({dest_project_path})",
		ge,
		)
		self.handle_error(ge)

		# 2: sync Avatar
		self.sync_avatar(src_project, dest_project)

		@@ -711,28 +767,34 @@ def dirname(path: Optional[str]) -> Optional[str]:
		def to_url(api_url: str) -> str:
		return api_url[0:-7] if api_url.endswith("/api/v4") else api_url


		def read_paths_list_from_file(file_path: Path) -> list[str]:
		# read a list of paths from a file (one per line), ignoring comments,
		# blank lines and leading/trailing spaces
		with open(file_path, "r") as f:
		lines = (line.partition('#')[0] for line in f)
		lines = (line.partition("#")[0] for line in f)
		return [line.strip() for line in lines if line.strip()]

		def simple_json_to_dict(json_str: str, source: Union[str,Path]) -> dict[str,Union[int,str]]:

		def simple_json_to_dict(
		json_str: str, source: Union[str, Path]
		) -> dict[str, Union[int, str]]:
		options = json.loads(json_str)
		assert (
		isinstance(options, (dict))
		), f"Invalid type of JSON in {source}, expected an object"
		assert isinstance(options, (dict)), (
		f"Invalid type of JSON in {source}, expected an object"
		)
		for k, v in options.items():
		assert (
		isinstance(v, (int, str))
		), f"Invalid value type for key {k} in {source}, expected only strings, integers and booleans"
		assert isinstance(v, (int, str)), (
		f"Invalid value type for key {k} in {source}, expected only strings, integers and booleans"
		)
		return options


		def trueish_env_var(env_var_name: str) -> bool:
		value = os.getenv(env_var_name)
		return bool(value) and (value.lower() not in ["0", "false", "no"])


		def run() -> None:
		# define command parser
		parser = argparse.ArgumentParser(
		@@ -858,6 +920,12 @@ def run() -> None:
		type=Path,
		help="a JSON file with extra options for projects creation; incompatible with --new-project-options",
		)
		parser.add_argument(
		"--use-src-issue-tracker",
		action="store_true",
		default=trueish_env_var("USE_SRC_ISSUE_TRACKER"),
		help="make destination projects to use their source project issue tracker",
		)
		parser.add_argument(
		"--dry-run",
		default=trueish_env_var("DRY_RUN"),
		@@ -879,58 +947,73 @@ def run() -> None:
		# parse command and args
		args = parser.parse_args()

		assert (
		args.src_api
		), "Source API is required (use --src-api CLI option or SRC_GITLAB_API variable)"
		assert args.src_sync_path, "Source sync path is required (use --src-sync-path CLI option or SRC_SYNC_PATH variable)"
		assert (
		args.dest_api
		), "Dest API is required (use --dest-api CLI option or DEST_GITLAB_API variable)"
		assert args.src_api, (
		"Source API is required (use --src-api CLI option or SRC_GITLAB_API variable)"
		)
		assert args.src_sync_path, (
		"Source sync path is required (use --src-sync-path CLI option or SRC_SYNC_PATH variable)"
		)
		assert args.dest_api, (
		"Dest API is required (use --dest-api CLI option or DEST_GITLAB_API variable)"
		)
		dest_sync_path = args.dest_sync_path or args.src_sync_path
		assert dest_sync_path, "Dest sync path is required (use --dest-sync-path CLI option or DEST_SYNC_PATH variable)"
		assert (
		args.src_api != args.dest_api or args.src_sync_path != dest_sync_path
		), "Cannot have same source and destination"
		assert (
		args.dest_token
		), "Dest token is required (use --dest-token CLI option or DEST_TOKEN variable)"
		assert (
		not args.exclude or not args.exclude_from
		), "Cannot use both --exclude and --exclude-from"
		assert (
		not args.exclude_from or args.exclude_from.is_file()
		), f"No such file: {args.exclude_from}"
		assert (
		not args.include or not args.include_from
		), "Cannot use both --include and --include-from"
		assert (
		not args.include_from or args.include_from.is_file()
		), f"No such file: {args.include_from}"
		assert (
		not args.new_group_options or not args.new_group_options_from
		), "Cannot use both --new-group-options and --new-group-options-from"
		assert (
		not args.new_group_options_from or args.new_group_options_from.is_file()
		), f"No such file: {args.new_group_options_from}"
		assert (
		not args.new_project_options or not args.new_project_options_from
		), "Cannot use both --new-project-options and --new-project-options-from"
		assert dest_sync_path, (
		"Dest sync path is required (use --dest-sync-path CLI option or DEST_SYNC_PATH variable)"
		)
		assert args.src_api != args.dest_api or args.src_sync_path != dest_sync_path, (
		"Cannot have same source and destination"
		)
		assert args.dest_token, (
		"Dest token is required (use --dest-token CLI option or DEST_TOKEN variable)"
		)
		assert not args.exclude or not args.exclude_from, (
		"Cannot use both --exclude and --exclude-from"
		)
		assert not args.exclude_from or args.exclude_from.is_file(), (
		f"No such file: {args.exclude_from}"
		)
		assert not args.include or not args.include_from, (
		"Cannot use both --include and --include-from"
		)
		assert not args.include_from or args.include_from.is_file(), (
		f"No such file: {args.include_from}"
		)
		assert not args.new_group_options or not args.new_group_options_from, (
		"Cannot use both --new-group-options and --new-group-options-from"
		)
		assert not args.new_group_options_from or args.new_group_options_from.is_file(), (
		f"No such file: {args.new_group_options_from}"
		)
		assert not args.new_project_options or not args.new_project_options_from, (
		"Cannot use both --new-project-options and --new-project-options-from"
		)
		assert (
		not args.new_project_options_from or args.new_project_options_from.is_file()
		), f"No such file: {args.new_project_options_from}"

		new_group_options: dict[str, Union[int, str]] = {}
		if args.new_group_options:
		new_group_options = simple_json_to_dict(args.new_group_options, "--new-group-options")
		new_group_options = simple_json_to_dict(
		args.new_group_options, "--new-group-options"
		)
		elif args.new_group_options_from:
		new_group_options = simple_json_to_dict(args.new_group_options_from.read_text(), str(args.new_group_options_from))
		new_group_options = simple_json_to_dict(
		args.new_group_options_from.read_text(), str(args.new_group_options_from)
		)

		# default options for new project: disable issues and MR
		new_project_options: dict[str, Union[int,str]] = { "issues_access_level": "disabled", "merge_requests_access_level": "disabled" }
		new_project_options: dict[str, Union[int, str]] = {
		"issues_access_level": "disabled",
		"merge_requests_access_level": "disabled",
		}
		if args.new_project_options:
		new_project_options = simple_json_to_dict(args.new_project_options, "--new-project-options")
		new_project_options = simple_json_to_dict(
		args.new_project_options, "--new-project-options"
		)
		elif args.new_project_options_from:
		new_project_options = simple_json_to_dict(args.new_project_options_from.read_text(), args.new_project_options_from)
		new_project_options = simple_json_to_dict(
		args.new_project_options_from.read_text(), args.new_project_options_from
		)

		exclude_list = []
		if args.exclude:
		@@ -999,6 +1082,9 @@ def run() -> None:
		print(
		f"- skip project desc. (--no-project-description) : {AnsiColors.CYAN}{args.no_project_description}{AnsiColors.RESET}"
		)
		print(
		f"- use src iss. tracker (--use-src-issue-tracker): {AnsiColors.CYAN}{args.use_src_issue_tracker}{AnsiColors.RESET}"
		)
		print()

		# TODO: configurable
		@@ -1038,6 +1124,7 @@ def run() -> None:
		ssl_verify=not args.insecure,
		new_group_options=new_group_options,
		new_project_options=new_project_options,
		use_src_issue_tracker=args.use_src_issue_tracker,
		)
		# retrieve src root group
		try: