feat: add --new-{group,project}-option(-from) options (JSON options for group/project creation) (105fcee3) · Commits · to be continuous... / tools / gitlab-cp

README.md

+45 −23

Original line number	Diff line number	Diff line
		@@ -16,8 +16,10 @@ gitlab-cp --help
		## Usage

		```bash
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}] [--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure]
		[--update-release] [--update-avatar] [--no-group-description] [--no-project-description] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]
		usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}]
		[--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure] [--update-release] [--update-avatar] [--no-group-description] [--no-project-description]
		[--new-group-options NEW_GROUP_OPTIONS] [--new-group-options-from NEW_GROUP_OPTIONS_FROM] [--new-project-options NEW_PROJECT_OPTIONS] [--new-project-options-from NEW_PROJECT_OPTIONS_FROM] [--dry-run]
		[--halt-on-error] [--cache-dir CACHE_DIR]

		This tool recursively copies/synchronizes a GitLab group from one GitLab server to another.

		@@ -48,6 +50,14 @@ options:
		don't synchronize group description
		--no-project-description
		don't synchronize project description
		--new-group-options NEW_GROUP_OPTIONS
		a JSON string with extra options for groups creation
		--new-group-options-from NEW_GROUP_OPTIONS_FROM
		a JSON file with extra options for groups creation; incompatible with --new-group-options
		--new-project-options NEW_PROJECT_OPTIONS
		a JSON string with extra options for projects creation
		--new-project-options-from NEW_PROJECT_OPTIONS_FROM
		a JSON file with extra options for projects creation; incompatible with --new-project-options
		--dry-run dry run (don't execute any write action)
		--halt-on-error halt synchronizing whenever an error occurs
		--cache-dir CACHE_DIR
		@@ -55,7 +65,7 @@ options:
		```

		\| CLI option \| Env. Variable \| Description \|
		\| -------------------------- \| ------------------------------- \| --------------------------------------------------------------------------------------------------------------------- \|
		\| ---------------------------- \| ------------------------------- \| --------------------------------------------------------------------------------------------------------------------- \|
		\| `--src-api` \| `$SRC_GITLAB_API` \| GitLab source API url (mandatory) \|
		\| `--src-token` \| `$SRC_TOKEN` \| GitLab source token (_optional_ if source GitLab group and sub projects have `public` visibility) \|
		\| `--src-sync-path` \| `$SRC_SYNC_PATH` \| GitLab source root group path to synchronize (mandatory) \|
		@@ -72,10 +82,22 @@ options:
		\| `--update-avatar` \| `$UPDATE_AVATAR` \| force update the avatar images even when they exist and look the same \|
		\| `--no-group-description` \| `$GROUP_DESCRIPTION_DISABLED` \| don't synchronize group description \|
		\| `--no-project-description` \| `$PROJECT_DESCRIPTION_DISABLED` \| don't synchronize project description \|
		\| `--new-group-options` \| `$NEW_GROUP_OPTIONS` \| a JSON string with extra options for groups creation (no default value) \|
		\| `--new-group-options-from` \| `$NEW_GROUP_OPTIONS_FROM` \| a JSON file with extra options for groups creation; incompatible with --new-group-options \|
		\| `--new-project-options` \| `$NEW_PROJECT_OPTIONS` \| a JSON string with extra options for projects creation (default value disables issues and MR, see below) \|
		\| `--new-project-options-from` \| `$NEW_PROJECT_OPTIONS_FROM` \| a JSON file with extra options for projects creation; incompatible with --new-project-options \|
		\| `--dry-run` \| _none_ \| dry run (don't execute any write action) \|
		\| `--halt-on-error` \| _none_ \| halt synchronizing when an error occurs \|
		\| `--cache-dir` \| `$CACHE_DIR` \| cache directory (used to download resources such as images and Git repositories) (defaults to `.work`) \|

		Default options for creating new projects:
		```json
		{
		"issues_access_level": "disabled",
		"merge_requests_access_level": "disabled"
		}
		```

		## Developers

		`gitlab-cp` is implemented in Python and relies on [Poetry](https://python-poetry.org/) for its packaging and dependency management.

gitlab_cp/sync.py

+72 −3

Original line number	Diff line number	Diff line
		import argparse
		import json
		import os
		import shutil
		import ssl
		from enum import Enum
		from logging import Logger
		from pathlib import Path
		from typing import Optional
		from typing import Optional, Union
		from urllib.request import urlretrieve

		from git import GitError, Repo
		@@ -79,6 +80,8 @@ class Synchronizer:
		dry_run=True,
		continue_on_error=False,
		update_avatar=False,
		new_project_options={},
		new_group_options={},
		):
		self.src_client = src_client
		self.src_sync_path = src_sync_path
		@@ -98,6 +101,8 @@ class Synchronizer:
		self.warnings: list[Exception] = []
		self.groups_count = 0
		self.projects_count = 0
		self.new_project_options = new_project_options
		self.new_group_options = new_group_options

		def handle_error(self, err: Exception) -> None:
		self.errors.append(err)
		@@ -456,8 +461,7 @@ class Synchronizer:
		"description": src_project.description,
		# ??? "default_branch": src_default_branch,
		"namespace_id": dest_parent_group.get_id(),
		"issues_access_level": "disabled",
		"merge_requests_access_level": "disabled",
		** self.new_project_options,
		}
		)
		print(
		@@ -592,6 +596,7 @@ class Synchronizer:
		"visibility": dest_visibility,
		"description": src_group.description,
		"parent_id": dest_parent_group.get_id(),
		** self.new_group_options,
		}
		)
		print(
		@@ -730,6 +735,17 @@ def read_paths_list_from_file(file_path: str) -> list[str]:
		lines = [line.strip() for line in lines if line.strip()]
		return lines

		def simple_json_to_dict(json_str: str, source: str) -> dict[str,Union[int,str]]:
		options = json.loads(json_str)
		assert (
		isinstance(options, (dict))
		), f"Invalid type of JSON in {source}, expected an object"
		for k, v in options.items():
		assert (
		isinstance(v, (int, str))
		), f"Invalid value type for key {k} in {source}, expected only strings, integers and booleans"
		return options

		def run() -> None:
		# define command parser
		parser = argparse.ArgumentParser(
		@@ -825,6 +841,26 @@ def run() -> None:
		action="store_true",
		help="don't synchronize project description",
		)
		parser.add_argument(
		"--new-group-options",
		default=os.getenv("NEW_GROUP_OPTIONS"),
		help="a JSON string with extra options for groups creation",
		)
		parser.add_argument(
		"--new-group-options-from",
		default=os.getenv("NEW_GROUP_OPTIONS_FROM"),
		help="a JSON file with extra options for groups creation; incompatible with --new-group-options",
		)
		parser.add_argument(
		"--new-project-options",
		default=os.getenv("NEW_PROJECT_OPTIONS"),
		help="a JSON string with extra options for projects creation",
		)
		parser.add_argument(
		"--new-project-options-from",
		default=os.getenv("NEW_PROJECT_OPTIONS_FROM"),
		help="a JSON file with extra options for projects creation; incompatible with --new-project-options",
		)
		parser.add_argument(
		"--dry-run",
		action="store_true",
		@@ -871,6 +907,31 @@ def run() -> None:
		assert (
		not args.include_from or Path(args.include_from).is_file()
		), "No such file: " + args.include_from
		assert (
		not args.new_group_options or not args.new_group_options_from
		), "Cannot use both --new-group-options and --new-group-options-from"
		assert (
		not args.new_group_options_from or Path(args.new_group_options_from).is_file()
		), "No such file: " + args.new_group_options_from
		assert (
		not args.new_project_options or not args.new_project_options_from
		), "Cannot use both --new-project-options and --new-project-options-from"
		assert (
		not args.new_project_options_from or Path(args.new_project_options_from).is_file()
		), "No such file: " + args.new_project_options_from

		new_group_options = {}
		if args.new_group_options:
		new_group_options = simple_json_to_dict(args.new_group_options, "--new-group-options")
		elif args.new_group_options_from:
		new_group_options = simple_json_to_dict(Path(args.new_group_options_from).read_text(), args.new_group_options_from)

		# default options for new project: disable issues and MR
		new_project_options = { "issues_access_level": "disabled", "merge_requests_access_level": "disabled" }
		if args.new_project_options:
		new_project_options = simple_json_to_dict(args.new_project_options, "--new-project-options")
		elif args.new_project_options_from:
		new_project_options = simple_json_to_dict(Path(args.new_project_options_from).read_text(), args.new_project_options_from)

		exclude_list = []
		if args.exclude:
		@@ -924,6 +985,12 @@ def run() -> None:
		print(
		f"- cache dir (--cache-dir) : {AnsiColors.CYAN}{args.cache_dir}{AnsiColors.RESET}"
		)
		print(
		f"- new group options (--new-group-options) : {AnsiColors.CYAN}{json.dumps(new_group_options)}{AnsiColors.RESET}"
		)
		print(
		f"- new project options (--new-project-options) : {AnsiColors.CYAN}{json.dumps(new_project_options)}{AnsiColors.RESET}"
		)
		print(
		f"- skip group desc. (--no-group-description) : {AnsiColors.CYAN}{args.no_group_description}{AnsiColors.RESET}"
		)
		@@ -965,6 +1032,8 @@ def run() -> None:
		dry_run=args.dry_run,
		continue_on_error=not args.halt_on_error,
		update_avatar=args.update_avatar,
		new_group_options=new_group_options,
		new_project_options=new_project_options,
		)
		# retrieve src root group
		try: