Commit 06dd1c13 authored by Thomas de Grenier de Latour's avatar Thomas de Grenier de Latour
Browse files

feat: add --exclude-from and --include-from options (flat file alternatives to --exclude/--include)

parent 3266e812

README.md

+8 −2
Original line number Diff line number Diff line
@@ -16,7 +16,7 @@ gitlab-cp --help 
## Usage



```bash

usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}] [--exclude EXCLUDE] [--include INCLUDE] [--insecure]

usage: gitlab-cp [-h] [--src-api SRC_API] [--src-token SRC_TOKEN] [--src-sync-path SRC_SYNC_PATH] [--dest-api DEST_API] [--dest-token DEST_TOKEN] [--dest-sync-path DEST_SYNC_PATH] [--max-visibility {public,internal,private}] [--exclude EXCLUDE] [--exclude-from EXCLUDE_FROM] [--include INCLUDE] [--include-from INCLUDE_FROM] [--insecure]

                 [--update-release] [--update-avatar] [--no-group-description] [--no-project-description] [--dry-run] [--halt-on-error] [--cache-dir CACHE_DIR]



This tool recursively copies/synchronizes a GitLab group from one GitLab server to another.
@@ -36,7 +36,11 @@ options: 
  --max-visibility {public,internal,private}

                        maximum visibility of projects in destination group

  --exclude EXCLUDE     project/group path to exclude from processing (relative to --src-sync-path)

  --exclude-from EXCLUDE_FROM

                        a file which lists paths to exclude (one per line); incompatible with --exclude

  --include INCLUDE     project/group path to include for processing (relative to --src-sync-path); all paths are included by default

  --include-from INCLUDE_FROM

                        a file which lists paths to include (one per line); incompatible with --include

  --insecure            skip SSL verification

  --update-release      force the update of the latest release

  --update-avatar       force update the avatar images even when they exist and look the same
@@ -59,8 +63,10 @@ options: 
| `--dest-token`             | `$DEST_TOKEN`                   | GitLab destination token with at least scopes `api,read_repository,write_repository` and `Owner` role (**mandatory**) |

| `--dest-sync-path`         | `$DEST_SYNC_PATH`               | GitLab destination root group path to synchronize (defaults to `--src-sync-path`)                                     |

| `--max-visibility`         | `$MAX_VISIBILITY`               | maximum visibility of projects in destination group (defaults to `public`)                                            |

| `--exclude`                | `$EXCLUDE`                      | project/group path(s) to exclude (multiple CLI option; env. variable is a coma separated list)                        |

| `--exclude`                | `$EXCLUDE`                      | project/group path(s) to exclude (multiple CLI option; env. variable is a coma separated list; wins over `--include`) |

| `--exclude-from`           | `$EXCLUDE_FROM`                 | a file which lists paths to exclude (one per line); incompatible with `--exclude` / `$EXCLUDE`                        |

| `--include`                | `$INCLUDE`                      | project/group path(s) to include (multiple CLI option; env. var. is coma separated; all paths included by default)    |

| `--include-from`           | `$INCLUDE_FROM`                 | a file which lists paths to include (one per line); incompatible with `--include` / `$INCLUDE`                        |

| `--insecure`               | `$INSECURE`                     | skip SSL verification                                                                                                 |

| `--update-release`         | `$UPDATE_RELEASE`               | set to force the update of the latest release (in order to trigger GitLab CI/CD catalog publication)                  |

| `--update-avatar`          | `$UPDATE_AVATAR`                | force update the avatar images even when they exist and look the same                                                 |

gitlab_cp/sync.py

+45 −4
Original line number Diff line number Diff line
@@ -722,6 +722,13 @@ def dirname(path: Optional[str]) -> Optional[str]: 
def to_url(api_url: str) -> str:

    return api_url[0:-7] if api_url.endswith("/api/v4") else api_url



def read_paths_list_from_file(file_path: str) -> list[str]:

    # read a list of paths from a file (one per line), ignoring comments,

    # blank lines and leading/trailing spaces

    with open(file_path, "r") as f:

        lines = (line.partition('#')[0] for line in f)

        lines = [line.strip() for line in lines if line.strip()]

        return lines



def run() -> None:

    # define command parser
@@ -772,12 +779,22 @@ def run() -> None: 
        default=os.getenv("EXCLUDE").split(",") if os.getenv("EXCLUDE") else [],

        help="project/group path to exclude from processing (relative to --src-sync-path)",

    )

    parser.add_argument(

        "--exclude-from",

        default=os.getenv("EXCLUDE_FROM"),

        help="a file which lists paths to exclude (one per line); incompatible with --exclude",

    )

    parser.add_argument(

        "--include",

        action="append",

        default=os.getenv("INCLUDE").split(",") if os.getenv("INCLUDE") else [],

        help="project/group path to include for processing (relative to --src-sync-path); all paths are included by default",

    )

    parser.add_argument(

        "--include-from",

        default=os.getenv("INCLUDE_FROM"),

        help="a file which lists paths to include (one per line); incompatible with --include",

    )

    parser.add_argument(

        "--insecure",

        action="store_true",
@@ -842,6 +859,30 @@ def run() -> None: 
    assert (

        args.dest_token

    ), "Dest token is required (use --dest-token CLI option or DEST_TOKEN variable)"

    assert (

        not args.exclude or not args.exclude_from

    ), "Cannot use both --exclude and --exclude-from"

    assert (

        not args.exclude_from or Path(args.exclude_from).is_file()

    ), "No such file: " + args.exclude_from

    assert (

        not args.include or not args.include_from

    ), "Cannot use both --include and --include-from"

    assert (

        not args.include_from or Path(args.include_from).is_file()

    ), "No such file: " + args.include_from



    exclude_list = []

    if args.exclude:

        exclude_list = args.exclude

    elif args.exclude_from:

        exclude_list = read_paths_list_from_file(args.exclude_from)



    include_list = []

    if args.include:

        include_list = args.include

    elif args.include_from:

        include_list = read_paths_list_from_file(args.include_from)



    print("Synchronizing GitLab group")

    print(
@@ -860,10 +901,10 @@ def run() -> None: 
        f"- max visi.   (--max-visibility) : {AnsiColors.CYAN}{args.max_visibility}{AnsiColors.RESET}"

    )

    print(

        f"- exclude     (--exclude)        : {AnsiColors.CYAN}{', '.join(args.exclude)}{AnsiColors.RESET}"

        f"- exclude     (--exclude(-from)) : {AnsiColors.CYAN}{', '.join(exclude_list)}{AnsiColors.RESET}"

    )

    print(

        f"- include     (--include)        : {AnsiColors.CYAN}{', '.join(args.include)}{AnsiColors.RESET}"

        f"- include     (--include(-from)) : {AnsiColors.CYAN}{', '.join(include_list)}{AnsiColors.RESET}"

    )

    print(

        f"- insecure    (--insecure)       : {AnsiColors.CYAN}{args.insecure}{AnsiColors.RESET}"
@@ -916,8 +957,8 @@ def run() -> None: 
        dest_sync_path,

        work_dir,

        args.max_visibility,

        args.exclude,

        args.include,

        exclude_list,

        include_list,

        args.update_release,

        not args.no_group_description,

        not args.no_project_description,