Merge branch 'main' into 'renovate/pytest-cov-6.x' (00103900) · Commits · to be continuous... / tools / gitlab-cp

CHANGELOG.md

+19 −0

Original line number	Diff line number	Diff line
		## [1.3.1](https://gitlab.com/to-be-continuous/tools/gitlab-cp/compare/1.3.0...1.3.1) (2024-12-06)


		### Bug Fixes

		* avatar: avatar comparison - check HEAD requests return status ([685671f](https://gitlab.com/to-be-continuous/tools/gitlab-cp/commit/685671f3415a1ee5dceb1ae25e6789a0b5b3c63e))

		# [1.3.0](https://gitlab.com/to-be-continuous/tools/gitlab-cp/compare/1.2.4...1.3.0) (2024-12-05)


		### Bug Fixes

		* skip Git sync on empty repositories ([46d11b1](https://gitlab.com/to-be-continuous/tools/gitlab-cp/commit/46d11b1af35becf345921ecfa4a752f8f9acc8de))


		### Features

		* avatar: use GitLab API to download avatar on private repositories ([edc67c1](https://gitlab.com/to-be-continuous/tools/gitlab-cp/commit/edc67c10108cd83e553e2932954a22c41558118a))

		## [1.2.4](https://gitlab.com/to-be-continuous/tools/gitlab-cp/compare/1.2.3...1.2.4) (2024-10-11)

gitlab_cp/sync.py

+89 −117

Original line number	Diff line number	Diff line
		@@ -147,42 +147,6 @@ class Synchronizer:
		else self.dest_sync_path + "/" + self.rel_path(src_path)
		)

		# can't be implemented: there is no simple way to GET the avatar image from a non-public project
		# def look_same_resources(self, url1: str, url2: str) -> bool:
		# resp1: HTTPResponse = urlopen(
		# Request(
		# url1,
		# headers=(
		# {
		# "Authorization": f"Basic: {base64.b64encode(bytes('token:'+self.src_client.private_token, 'utf-8')).decode()}"
		# }
		# if self.src_client.private_token
		# else {}
		# ),
		# method="HEAD",
		# ),
		# context=None if self.src_client.ssl_verify else INSECURE_SSL_CTX,
		# )
		# resp2: HTTPResponse = urlopen(
		# Request(
		# url2,
		# headers=(
		# {
		# "Authorization": f"Basic: {base64.b64encode(bytes('token:'+self.dest_client.private_token, 'utf-8')).decode()}"
		# }
		# if self.dest_client.private_token
		# else {}
		# ),
		# method="HEAD",
		# ),
		# context=None if self.dest_client.ssl_verify else INSECURE_SSL_CTX,
		# )
		# return (
		# resp1.headers.get_content_type() == resp2.headers.get_content_type()
		# and resp1.headers.get("content-length")
		# == resp2.headers.get("content-length")
		# )

		# Synchronize a project release
		def sync_releases(
		self,
		@@ -281,6 +245,13 @@ class Synchronizer:
		)
		self.handle_error(ge)

		if ( src_latest_commit is None ):
		# Git sync is not required: skip
		print(
		f" - git repository: {AnsiColors.HGRAY}empty{AnsiColors.RESET}"
		)
		return

		if (
		dest_latest_commit
		and src_latest_commit.get_id() == dest_latest_commit.get_id()
		@@ -381,6 +352,80 @@ class Synchronizer:
		f" - git repository: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_default_branch} on #{src_latest_commit.get_id() if src_latest_commit else 'n/a'})"
		)

		# Synchronize a project/group avatar
		def sync_avatar(
		self,
		src_avatar: Project \| Group,
		dest_avatar: Project \| Group,
		) -> None:
		assert all(any(isinstance(resource, type) for type in [Project, Group]) for resource in [src_avatar, dest_avatar])

		# Python lib `python-gitlab` doesn't support avatar endpoint (02/12/2024):
		# > We write our own requests calls.
		src_web_url = src_avatar.web_url
		src_avatar_url = src_avatar.avatar_url
		src_avatar_resource = "projects" if isinstance(src_avatar, Project) else "groups"
		src_avatar_api = f"{self.src_client.api_url}/{src_avatar_resource}/{src_avatar.id}/avatar"
		src_avatar_api_headers = {"PRIVATE-TOKEN": self.src_client.private_token} if self.src_client.private_token is not None else {}
		dest_web_url = dest_avatar.web_url
		dest_avatar_url = dest_avatar.avatar_url
		dest_avatar_resource = "projects" if isinstance(dest_avatar, Project) else "groups"
		dest_avatar_api = f"{self.dest_client.api_url}/{dest_avatar_resource}/{dest_avatar.id}/avatar"
		dest_avatar_api_headers = {"PRIVATE-TOKEN": self.dest_client.private_token} if self.dest_client.private_token is not None else {}

		# Up-to-date if no source avatar url or implicit avatar url ('/path/to/proj/-/avatar')
		if src_avatar_url is None or src_avatar_url == f"{src_web_url}/-/avatar":
		# image up-to-date
		print(
		f" - avatar image: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url})"
		)
		return

		# Compare avatar size if destination avatar exist and not forced by user
		if dest_avatar_url is not None and not self.force_update_avatar:
		# fetch avatar size
		try:
		src_avatar_response = requests.head(url=src_avatar_api, headers=src_avatar_api_headers, verify=self.ssl_verify)
		_, src_avatar_size = src_avatar_response.raise_for_status(), src_avatar_response.headers['Content-Length']
		dest_avatar_response = requests.head(url=dest_avatar_api, headers=dest_avatar_api_headers, verify=self.ssl_verify)
		_, dest_avatar_size = dest_avatar_response.raise_for_status(), dest_avatar_response.headers['Content-Length']
		# compare avatar size
		if src_avatar_size == dest_avatar_size:
		# image up-to-date
		print(
		f" - avatar image: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url}) size: {src_avatar_size}"
		)
		return
		except requests.RequestException as ge:
		print(
		f" - avatar image: compare {AnsiColors.HYELLOW}failed{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url}) - assume update needed:",
		ge,
		)
		# don't rethrow - continue anyway
		self.handle_warn(ge)

		# Update needed but dry run
		if self.dry_run:
		print(
		f" - avatar image: {AnsiColors.HYELLOW}update needed{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url})"
		)
		return

		# Update avatar
		try:
		dest_avatar.avatar = requests.get(url=src_avatar_api, headers=dest_avatar_api_headers, verify=self.ssl_verify).content
		dest_avatar.save()
		print(
		f" - avatar image: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_avatar_url})"
		)
		except GitlabUpdateError as ge:
		print(
		f" - avatar image: update {AnsiColors.HYELLOW}failed{AnsiColors.RESET} ({src_avatar_url})",
		ge,
		)
		self.handle_warn(ge)
		return

		# Synchronizes a GitLab project
		# $1: source project JSON
		# $2: destination parent group ID (number)
		@@ -482,53 +527,13 @@ class Synchronizer:
		self.handle_error(ge)
		return

		# set/update avatar url
		src_avatar_url = src_project.avatar_url
		src_web_url = src_project.web_url
		dest_avatar_url = dest_project.avatar_url
		dest_web_url = dest_project.web_url
		if (src_avatar_url is None or src_avatar_url == f"{src_web_url}/-/avatar") or (
		dest_avatar_url is not None
		and dest_avatar_url != f"{dest_web_url}/-/avatar"
		and not self.force_update_avatar
		):
		# image up-to-date
		print(
		f" - avatar image: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url})"
		)
		elif self.dry_run:
		print(
		f" - avatar image: {AnsiColors.HYELLOW}update needed{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url})"
		)
		elif (
		src_project.attributes.get("visibility", GlVisibility.public)
		!= GlVisibility.public
		):
		# no way to download the avatar image of a non-public project
		print(
		f"- avatar image: {AnsiColors.HGRAY}update needed{AnsiColors.HRED} skipped due to src project not public ({src_avatar_url} / {dest_avatar_url})"
		)
		else:
		try:
		dest_project.avatar = requests.get(
		src_avatar_url, verify=self.ssl_verify
		).content
		dest_project.save()
		print(
		f" - avatar image: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_avatar_url})"
		)
		except GitlabUpdateError as ge:
		print(
		f" - avatar image: update {AnsiColors.HYELLOW}failed{AnsiColors.RESET} ({src_avatar_url})",
		ge,
		)
		self.handle_warn(ge)
		# don't rethrow - continue anyway
		# 2: sync Avatar
		self.sync_avatar(src_project, dest_project)

		# 2: sync Git repository
		# 3: sync Git repository
		self.sync_git_repo(src_project, dest_project)

		# 3: sync Releases
		# 4: sync Releases
		self.sync_releases(src_project, dest_project)

		# Synchronizes recursively a GitLab group
		@@ -657,43 +662,10 @@ class Synchronizer:
		self.handle_warn(ge)
		# continue anyway

		# set/update avatar url
		src_avatar_url = src_group.avatar_url
		dest_avatar_url = dest_group.avatar_url
		if src_avatar_url is None or (
		dest_avatar_url is not None and not self.force_update_avatar
		):
		# image up-to-date
		print(
		f"- avatar image: {AnsiColors.HGRAY}up-to-date{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url})"
		)
		elif self.dry_run:
		print(
		f"- avatar image: {AnsiColors.HYELLOW}update needed{AnsiColors.RESET} ({src_avatar_url} / {dest_avatar_url})"
		)
		elif src_group.visibility != GlVisibility.public:
		# no way to download the avatar image of a non-public group
		print(
		f"- avatar image: {AnsiColors.HRED}update needed{AnsiColors.RESET} skipped due to src group not public ({src_avatar_url} / {dest_avatar_url})"
		)
		else:
		try:
		dest_group.avatar = requests.get(
		src_avatar_url, verify=self.ssl_verify
		).content
		dest_group.save()
		print(
		f"- avatar image: {AnsiColors.HYELLOW}updated{AnsiColors.RESET} ({src_avatar_url})"
		)
		except GitlabUpdateError as ge:
		print(
		f"- avatar image: update {AnsiColors.HYELLOW}failed{AnsiColors.RESET} ({src_avatar_url})",
		ge,
		)
		# don't rethrow - continue anyway
		self.handle_warn(ge)
		# 2: sync Avatar
		self.sync_avatar(src_group, dest_group)

		# 2: sync sub-projects
		# 3: sync sub-projects
		subprojects = src_group.projects.list(all=True)
		print(f"- sync {len(subprojects)} sub projects...")
		for src_project in subprojects:
		@@ -710,7 +682,7 @@ class Synchronizer:
		self.src_client.projects.get(src_project.get_id()), dest_group
		)

		# 3: sync sub-groups
		# 4: sync sub-groups
		subgroups = src_group.subgroups.list(all=True)
		print(f"- sync {len(subgroups)} sub groups...")
		for src_subgroup in subgroups:

poetry.lock

+210 −182

File changed.

Preview size limit exceeded, changes collapsed.

pyproject.toml

+2 −2

Original line number	Diff line number	Diff line
		@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

		[tool.poetry]
		name = "gitlab-cp"
		version = "1.2.4"
		version = "1.3.1"
		description = "GitLab Copy CLI"
		authors = ["Pierre Smeyers <pierre.smeyers@gmail.com>"]
		license = "LGPL"
		@@ -26,7 +26,7 @@ pytest = "^8.0.0"
		pytest-cov = "^6.0.0"
		pytest-env = "^1.0.0"
		mypy = "^1.0.0"
		ruff = "^0.7.0"
		ruff = "^0.8.0"

		[tool.pytest.ini_options]
		# this project uses "pytest" for unit testing