Commit cf34b410 authored by Pierre Smeyers's avatar Pierre Smeyers
Browse files

fix: sanitize variable substitution pattern 

replace '${foo+repl}' pattern with '${foo:+repl}' (latest supports $foo being defined but empty)
parent d351e8ea

.gitlab-ci.yml

+2 −2
Original line number Diff line number Diff line
@@ -43,7 +43,7 @@ sync-tbc: 
    # we can't use update-ca-certificates --fresh because $CUSTOM_CA_CERTS can contains several certificates

    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"

  script:

    - bash ${CI_DEBUG_TRACE+-x} ./gitlab-sync.sh

    - bash ${CI_DEBUG_TRACE:+-x} ./gitlab-sync.sh

  rules:

    # disable on gitlab.com

    - if: '$CI_SERVER_HOST == "gitlab.com"'
@@ -57,7 +57,7 @@ sync-tbc-dryrun: 
  extends: sync-tbc

  script:

    # unset dest GitLab (dryrun)

    - bash ${CI_DEBUG_TRACE+-x} ./gitlab-sync.sh --dest-api ""

    - bash ${CI_DEBUG_TRACE:+-x} ./gitlab-sync.sh --dest-api ""

  rules:

    # run only on gitlab.com (to test at least the script correctly crawls tbc projects)

    - if: '$CI_SERVER_HOST == "gitlab.com"'

gitlab-sync.sh

+24 −24
Original line number Diff line number Diff line
@@ -71,7 +71,7 @@ function maybe_create_group() { 
    echo "null"

  else

    group_id=${group_path//\//%2f}

    group_status=$(curl ${INSECURE+-k} -s -o /dev/null -I -w "%{http_code}" -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$group_id")

    group_status=$(curl ${INSECURE:+-k} -s -o /dev/null -I -w "%{http_code}" -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$group_id")

    if [[ "$group_status" == 404* ]]

    then

      # group does not exist: create
@@ -81,7 +81,7 @@ function maybe_create_group() { 
      log_info "... group \\e[33;1m$group_path\\e[0m not found: create group \\e[33;1m$group_name\\e[0m with parent \\e[33;1m$parent_path\\e[0m"

      parent_id=$(maybe_create_group "$parent_path")

      # then create group

      group_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X POST "$DEST_GITLAB_API/groups" \

      group_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X POST "$DEST_GITLAB_API/groups" \

        --data "{

          \"name\": \"$group_name\",

          \"path\": \"$group_name\",
@@ -92,7 +92,7 @@ function maybe_create_group() { 
    then

      # group exists: retrieve ID

      log_info "... group \\e[33;1m$group_path\\e[0m found: retrieve ID"

      group_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$group_id")

      group_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$group_id")

    else

      # another error: abort

      fail "... unexpected error while getting group \\e[33;1m$group_path\\e[0m: $group_status"
@@ -122,12 +122,12 @@ function sync_project() { 
  if [[ "$DEST_GITLAB_API" ]]

  then

    dest_visibility=$(adjust_visibility "$(echo "$src_project_json" | jq -r .visibility)")

    dest_project_status=$(curl ${INSECURE+-k} -s -o /dev/null -I -w "%{http_code}" -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/projects/$dest_project_id")

    dest_project_status=$(curl ${INSECURE:+-k} -s -o /dev/null -I -w "%{http_code}" -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/projects/$dest_project_id")

    if [[ "$dest_project_status" == 404* ]]

    then

      # dest project does not exist: create (disable MR and issues as they are cloned projects)

      log_info "... destination project not found: create with visibility \\e[33;1m${dest_visibility}\\e[0m"

      dest_project_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X POST "$DEST_GITLAB_API/projects" \

      dest_project_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X POST "$DEST_GITLAB_API/projects" \

        --data "{

          \"path\": $(echo "$src_project_json" | jq .path),

          \"name\": $(echo "$src_project_json" | jq .name),
@@ -143,13 +143,13 @@ function sync_project() { 
      log_info "... destination project found: synchronize"

      if [[ "${PROJECT_DESCRIPTION_DISABLED}" == "true" ]]

      then

        dest_project_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/projects/$dest_project_id" \

        dest_project_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/projects/$dest_project_id" \

          --data "{

            \"name\": $(echo "$src_project_json" | jq .name),

            \"visibility\": \"$dest_visibility\"

          }")

      else

        dest_project_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/projects/$dest_project_id" \

        dest_project_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/projects/$dest_project_id" \

          --data "{

            \"name\": $(echo "$src_project_json" | jq .name),

            \"visibility\": \"$dest_visibility\",
@@ -157,7 +157,7 @@ function sync_project() { 
          }")

      fi

        # \"visibility\": \"$(echo "$src_project_json" | jq -r .visibility)\",

      dest_latest_commit=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/projects/$dest_project_id/repository/commits?ref_name=$src_default_branch&per_page=1" | jq -r '.[0].id')

      dest_latest_commit=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/projects/$dest_project_id/repository/commits?ref_name=$src_default_branch&per_page=1" | jq -r '.[0].id')

    else

      # another error: abort

      fail "... unexpected error: $dest_project_status"
@@ -171,15 +171,15 @@ function sync_project() { 
    then

      log_info "... update avatar image ($src_avatar_url)"

      avatar_filename=/tmp/$(basename "$src_avatar_url")

      curl ${INSECURE+-k} -sSfL --output "$avatar_filename" "$src_avatar_url"

      dest_project_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" --form "avatar=@$avatar_filename" -X PUT "$DEST_GITLAB_API/projects/$dest_project_id")

      curl ${INSECURE:+-k} -sSfL --output "$avatar_filename" "$src_avatar_url"

      dest_project_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" --form "avatar=@$avatar_filename" -X PUT "$DEST_GITLAB_API/projects/$dest_project_id")

    fi

  fi



  # 2: sync Git repository

  if [[ "$DEST_GITLAB_API" ]]

  then

    src_latest_commit=$(curl ${INSECURE+-k} -sSf -H "${SRC_TOKEN+PRIVATE-TOKEN: $SRC_TOKEN}" "$SRC_GITLAB_API/projects/$src_project_id/repository/commits?ref_name=$src_default_branch&per_page=1" | jq -r '.[0].id')

    src_latest_commit=$(curl ${INSECURE:+-k} -sSf -H "${SRC_TOKEN:+PRIVATE-TOKEN: $SRC_TOKEN}" "$SRC_GITLAB_API/projects/$src_project_id/repository/commits?ref_name=$src_default_branch&per_page=1" | jq -r '.[0].id')

    if [[ "$src_latest_commit" == "$dest_latest_commit" ]]

    then

      # Git sync is not required: skip
@@ -202,7 +202,7 @@ function sync_project() { 
      if [[ "$dest_project_status" == 200* ]]

      then

        log_info "... unprotect $src_default_branch branch (allow failure)"

        curl ${INSECURE+-k} -sS -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -X DELETE "$DEST_GITLAB_API/projects/$dest_project_id/protected_branches/$src_default_branch" > /dev/null

        curl ${INSECURE:+-k} -sS -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -X DELETE "$DEST_GITLAB_API/projects/$dest_project_id/protected_branches/$src_default_branch" > /dev/null

      fi

 

      cd "$repo_name"
@@ -213,12 +213,12 @@ function sync_project() { 
        # shellcheck disable=SC2001

        dest_repo_url=$(echo "$dest_repo_url" | sed -e "s|://|://token:${DEST_TOKEN}@|")

      fi

      git ${INSECURE+-c http.sslVerify=false} push --force "$dest_repo_url" --tags "$src_default_branch"

      git ${INSECURE:+-c http.sslVerify=false} push --force "$dest_repo_url" --tags "$src_default_branch"

      cd ..



      # after sync: protect default branch again

      log_info "... protect $src_default_branch branch"

      curl ${INSECURE+-k} -sS -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -X POST "$DEST_GITLAB_API/projects/$dest_project_id/protected_branches?name=$src_default_branch" > /dev/null

      curl ${INSECURE:+-k} -sS -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -X POST "$DEST_GITLAB_API/projects/$dest_project_id/protected_branches?name=$src_default_branch" > /dev/null

    fi

  fi

}
@@ -236,7 +236,7 @@ function sync_group() { 
  local dest_group_full_path=$4

  local dest_group_name=${dest_group_full_path//\//%2f}

  log_info "Synchronizing group \\e[33;1m${src_group_full_path}\\e[0m (parent group ID \\e[33;1m${dest_parent_id:-none (dry run)}\\e[0m)"

  src_group_json=$(curl ${INSECURE+-k} -sSf -H "${SRC_TOKEN+PRIVATE-TOKEN: $SRC_TOKEN}" "$SRC_GITLAB_API/groups/$src_group_id")

  src_group_json=$(curl ${INSECURE:+-k} -sSf -H "${SRC_TOKEN:+PRIVATE-TOKEN: $SRC_TOKEN}" "$SRC_GITLAB_API/groups/$src_group_id")

  # dump group json (for debug)

  echo "$src_group_json" > "group-$src_group_id.json"
@@ -244,12 +244,12 @@ function sync_group() { 
  if [[ "$DEST_GITLAB_API" ]]

  then

    dest_visibility=$(adjust_visibility "$(echo "$src_group_json" | jq -r .visibility)")

    dest_group_status=$(curl ${INSECURE+-k} -s -o /dev/null -I -w "%{http_code}" -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$dest_group_name")

    dest_group_status=$(curl ${INSECURE:+-k} -s -o /dev/null -I -w "%{http_code}" -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$dest_group_name")

    if [[ "$dest_group_status" == 404* ]]

    then

      # dest group does not exist: create

      log_info "... destination group not found: create with visibility \\e[33;1m${dest_visibility}\\e[0m"

      dest_group_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X POST "$DEST_GITLAB_API/groups" \

      dest_group_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X POST "$DEST_GITLAB_API/groups" \

        --data "{

          \"path\": $(echo "$src_group_json" | jq .path),

          \"name\": $(echo "$src_group_json" | jq .name),
@@ -263,13 +263,13 @@ function sync_group() { 
      log_info "... destination group found: synchronize"

      if [[ "${GROUP_DESCRIPTION_DISABLED}" == "true" ]]

      then

        dest_group_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/groups/$dest_group_name" \

        dest_group_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/groups/$dest_group_name" \

          --data "{

            \"name\": $(echo "$src_group_json" | jq .name),

            \"visibility\": \"$dest_visibility\"

          }")

      else

        dest_group_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/groups/$dest_group_name" \

        dest_group_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" -H "Content-Type: application/json" -X PUT "$DEST_GITLAB_API/groups/$dest_group_name" \

          --data "{

            \"name\": $(echo "$src_group_json" | jq .name),

            \"visibility\": \"$dest_visibility\",
@@ -285,10 +285,10 @@ function sync_group() { 
    local dest_group_id=$(echo "$dest_group_json" | jq -r .id)



    # create TBC_NAMESPACE variable in top group

    if [[ "$dest_group_full_path" == "$DEST_SYNC_PATH" ]] && ! curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$dest_group_id/variables/TBC_NAMESPACE" >/dev/null 2>/dev/null

    if [[ "$dest_group_full_path" == "$DEST_SYNC_PATH" ]] && ! curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" "$DEST_GITLAB_API/groups/$dest_group_id/variables/TBC_NAMESPACE" >/dev/null 2>/dev/null

    then

      log_info "... set TBC_NAMESPACE group variable ($DEST_SYNC_PATH)"

      curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" --form "key=TBC_NAMESPACE" --form "value=$DEST_SYNC_PATH" -X POST "$DEST_GITLAB_API/groups/$dest_group_id/variables"

      curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" --form "key=TBC_NAMESPACE" --form "value=$DEST_SYNC_PATH" -X POST "$DEST_GITLAB_API/groups/$dest_group_id/variables"

    fi



    # set/update avatar url
@@ -298,9 +298,9 @@ function sync_group() { 
    then

      log_info "... update avatar image ($src_avatar_url)"

      avatar_filename=/tmp/$(basename "$src_avatar_url")

      if curl ${INSECURE+-k} -sSfL --output "$avatar_filename" "$src_avatar_url"

      if curl ${INSECURE:+-k} -sSfL --output "$avatar_filename" "$src_avatar_url"

      then

        dest_group_json=$(curl ${INSECURE+-k} -sSf -H "${DEST_TOKEN+PRIVATE-TOKEN: $DEST_TOKEN}" --form "avatar=@$avatar_filename" -X PUT "$DEST_GITLAB_API/groups/$dest_group_id")

        dest_group_json=$(curl ${INSECURE:+-k} -sSf -H "${DEST_TOKEN:+PRIVATE-TOKEN: $DEST_TOKEN}" --form "avatar=@$avatar_filename" -X PUT "$DEST_GITLAB_API/groups/$dest_group_id")

      else

        log_warn "... failed downloading avatar image ($src_avatar_url)"

      fi
@@ -324,7 +324,7 @@ function sync_group() { 
  done



  # 3: sync sub-groups

  src_subgroups_json=$(curl ${INSECURE+-k} -sSf -H "${SRC_TOKEN+PRIVATE-TOKEN: $SRC_TOKEN}" "$SRC_GITLAB_API/groups/$src_group_id/subgroups")

  src_subgroups_json=$(curl ${INSECURE:+-k} -sSf -H "${SRC_TOKEN:+PRIVATE-TOKEN: $SRC_TOKEN}" "$SRC_GITLAB_API/groups/$src_group_id/subgroups")

  # dump subgroups json (for debug)

  echo "$src_subgroups_json" > "subgroups-$src_group_id.json"

  # shellcheck disable=SC2155