Commit 255906e0 authored by Pierre Smeyers's avatar Pierre Smeyers
Browse files

feat: shared bucket support

parent c115bfe6

README.md

+51 −2
Original line number Diff line number Diff line
@@ -68,7 +68,7 @@ The S3 template uses some global configuration used throughout all jobs. 
| :lock: `S3_ACCESS_KEY` | Default S3 service Access Key                 | **has to be defined** |

| :lock: `S3_SECRET_KEY` | Default S3 service Secret Key                 | **has to be defined** |

| `S3_BASE_BUCKET_NAME`  | Base bucket name                              | `$CI_PROJECT_NAME` ([see GitLab doc](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)) |

| `S3_ROOT_PATH` | Default root path where files will be uploaded in the S3 bucket (:warning: don't forget the starting `/`)  | _none_ |

| `S3_ROOT_PATH` | Default root path (_prefix_) where files will be uploaded in the S3 bucket (:warning: don't forget the starting `/`)  | _none_ |



### Secrets management
@@ -176,6 +176,52 @@ Here are variables supported to configure the production environment: 
| `AUTODEPLOY_TO_PROD`     | Set this variable to auto-deploy to production. If not set deployment to production will be `manual` (default behaviour). | _none_ (disabled) |

| `S3_PROD_ROOT_PATH`  |  S3 bucket root path (prefix) for `production` env _(only define if different from default)_ | `S3_ROOT_PATH` |



### Buckets namespacing



In its default configuration, the template manages (create/sync/delete) one S3 bucket per environment.



But you may also configure it to implement other policies. Here are several examples of alternate policies.



#### A single bucket with separate prefixes for each env



Here the `.gitlab-ci.yml` configuration for one shared bucket for all envs, each separated by prefix:



```yaml

variables:

  # use same bucket for all envs

  S3_REVIEW_BUCKET_NAME: "acme-bucket-shared"

  S3_INTEG_BUCKET_NAME: "acme-bucket-shared"

  S3_STAGING_BUCKET_NAME: "acme-bucket-shared"

  S3_PROD_BUCKET_NAME: "acme-bucket-shared"

  # segregate envs with prefixes

  S3_ROOT_PATH: "$CI_ENVIRONMENT_SLUG"

  # envs url

  S3_REVIEW_ENVIRONMENT_URL: "https://acme-bucket-shared.s3-website.public.domain.com/$CI_ENVIRONMENT_SLUG"

  S3_INTEG_ENVIRONMENT_URL: "https://acme-bucket-shared.s3-website.public.domain.com/$CI_ENVIRONMENT_SLUG"

  S3_STAGING_ENVIRONMENT_URL: "https://acme-bucket-shared.s3-website.public.domain.com/$CI_ENVIRONMENT_SLUG"

  S3_PROD_ENVIRONMENT_URL: "https://acme-bucket-shared.s3-website.public.domain.com/$CI_ENVIRONMENT_SLUG"

```



#### Hybrid policy



Here the `.gitlab-ci.yml` configuration for one shared bucket for review envs and separate buckets for others:



```yaml

variables:

  # use same bucket for all review envs

  S3_REVIEW_BUCKET_NAME: "acme-bucket-review"

  S3_INTEG_BUCKET_NAME: "acme-bucket-integ"

  S3_STAGING_BUCKET_NAME: "acme-bucket-staging"

  S3_PROD_BUCKET_NAME: "acme-bucket-prod"

  # segregate review envs with prefixes

  S3_REVIEW_ROOT_PATH: "$CI_ENVIRONMENT_SLUG"

  # envs url

  S3_REVIEW_ENVIRONMENT_URL: "https://acme-bucket-review.s3-website.public.domain.com/$CI_ENVIRONMENT_SLUG"

  S3_INTEG_ENVIRONMENT_URL: "https://acme-bucket-integ.s3-website.public.domain.com"

  S3_STAGING_ENVIRONMENT_URL: "https://acme-bucket-staging.s3-website.public.domain.com"

  S3_PROD_ENVIRONMENT_URL: "https://acme-bucket-prod.s3-website.public.domain.com"

```



### Deployment jobs



Each environment has its own deployment job (associated with the right branch).
@@ -192,9 +238,12 @@ It uses the following variables: 
If need be you could add your own hook script `s3-pre-deploy.sh` that will be triggered right before deploying files to

the S3 bucket.



If the target bucket doesn't appear to exist, the template tries to create it.



### `s3-cleanup-review` job



This job allows destroying each review environment. Simply deletes the associated bucket.

This job allows destroying each review environment. Simply deletes the associated objects in the bucket.

After objects removal, if the bucket appears to be empty, also tries to delete the bucket.



### `s3-cleanup-all-review` job

templates/gitlab-ci-s3.yml

+37 −21
Original line number Diff line number Diff line
@@ -284,11 +284,11 @@ stages: 
    fi

  }



  # deploy application

  # upload/sync files to bucket

  function deploy() {

    export env=$1

    bucket=$2

    root_path=$3

    prefix=$3



    # extract hostname from $CI_ENVIRONMENT_URL

    hostname=$(echo "$CI_ENVIRONMENT_URL" | awk -F[/:] '{print $4}')
@@ -296,7 +296,7 @@ stages: 


    log_info "--- \\e[32mdeploy\\e[0m (env: \\e[33;1m${env}\\e[0m)"

    log_info "--- bucket: \\e[33;1m${bucket}\\e[0m"

    log_info "--- root_path: \\e[33;1m${root_path:-(none)}\\e[0m"

    log_info "--- prefix: \\e[33;1m${prefix:-(none)}\\e[0m"

    log_info "--- env: \\e[33;1m${env}\\e[0m"

    log_info "--- hostname: \\e[33;1m${hostname}\\e[0m"
@@ -324,7 +324,7 @@ stages: 
    # sync files

    log_info "... synchronize files"

    # shellcheck disable=SC2086

    s3cmd ${TRACE+-v} $S3_DEPLOY_ARGS $S3_DEPLOY_FILES "s3://${bucket}${root_path}"

    s3cmd ${TRACE+-v} $S3_DEPLOY_ARGS $S3_DEPLOY_FILES "s3://${bucket}${prefix}"



    # create website

    if [[ "$S3_WEBSITE_DISABLED" != "true" ]]
@@ -339,31 +339,44 @@ stages: 
    echo -e "environment_type=$env\\nenvironment_name=$bucket\\nenvironment_url=$CI_ENVIRONMENT_URL" > s3.env

  }



  # delete application (and dependencies)

  # delete application

  function delete() {

    export env=$1

    bucket=$2

    prefix=$3



    log_info "--- \\e[32mdelete\\e[0m (env: ${env})"

    log_info "--- bucket: \\e[33;1m${bucket}\\e[0m"

    log_info "--- prefix: \\e[33;1m${prefix:-(none)}\\e[0m"

    log_info "--- env: \\e[33;1m${env}\\e[0m"



    # delete bucket if exists

    if s3cmd info "s3://${bucket}" >/dev/null 2>&1

    # check if bucket exists

    if ! s3cmd info "s3://${bucket}" >/dev/null

    then

      log_info "... delete all files from S3 bucket \\e[33;1m${bucket}\\e[0m"

      s3cmd ${TRACE+-v} rm --recursive --force "s3://${bucket}"



      log_info "... delete S3 bucket \\e[33;1m${bucket}\\e[0m"

      s3cmd ${TRACE+-v} rb "s3://${bucket}"

    else

      log_info "... S3 bucket \\e[33;1m${bucket}\\e[0m not found: skip"

      return

    fi



    # delete files

    log_info "... delete files from \\e[33;1m${bucket}${prefix}\\e[0m"

    s3cmd ${TRACE+-v} rm --recursive --force "s3://${bucket}${prefix}"



    # finally delete bucket if empty

    objs_count=$(s3cmd du "s3://${bucket}" | tr -s ' ' | cut -d' ' -f3)

    if [[ "$objs_count" == "0" ]]

    then

      log_info "... bucket is now empty: try to delete"

      if ! s3cmd ${TRACE+-v} rb "s3://${bucket}"

      then

        log_warn "... delete \\e[33;1m${bucket}\\e[0m failed"

      fi

    fi

  }



  function delete_all() {

    export env=$1

    appnameproto=$2

    prefix=$3



    # make appname regex by replacing $CI_COMMIT_REF_SLUG with .*

    appnameregex=$(echo "$appnameproto" | sed -r "s/$CI_COMMIT_REF_SLUG/.*/g")
@@ -371,13 +384,13 @@ stages: 
    matchingbuckets=$(s3cmd ls | awk '{print $3}' | cut -d'/' -f3 | awk "/$appnameregex/")

    matchcount=$(echo "$matchingbuckets" | wc -w)



    log_info "--- \\e[32mdelete all\\e[0m (env: \\e[33;1m${env}\\e[0m, bucket matcher: \\e[33;1m${appnameregex}\\e[0m): \\e[33;1m${matchcount}\\e[0m buckets found"

    log_info "--- \\e[32mdelete all\\e[0m (env: \\e[33;1m${env}\\e[0m, bucket matcher: \\e[33;1m${appnameregex}\\e[0m, prefix: \\e[33;1m${prefix:-(none)}\\e[0m): \\e[33;1m${matchcount}\\e[0m buckets found"



    rc=0

    for bckt in $matchingbuckets

    do

      echo -e "\\e[1;93m-------------------------------------------------------------------------------\\e[0m"

      if ! delete "$env" "$endpoint_host" "$host_bucket" "$access_key" "$secret_key" "$bckt"

      if ! delete "$env" "$bckt" "$prefix"

      then

        log_warn "... failed deleting bucket \\e[33;1m${bckt}\\e[0m (see logs)"

        rc=1
@@ -409,6 +422,7 @@ stages: 
#

# @arg ENV_TYPE         : environment type

# @arg ENV_BUCKET_NAME  : env-specific S3 bucket name

# @arg ENV_ROOT_PATH    : env-specific S3 prefix

# @arg ENV_APP_SUFFIX   : env-specific application suffix

# @arg ENV_ENDPOINT_HOST: env-specific S3 endpoint host

# @arg ENV_ACCESS_KEY   : env-specific S3 access key
@@ -440,6 +454,7 @@ stages: 
#

# @arg ENV_TYPE         : environment type

# @arg ENV_BUCKET_NAME  : env-specific S3 bucket name

# @arg ENV_ROOT_PATH    : env-specific S3 prefix

# @arg ENV_APP_SUFFIX   : env-specific application suffix

# @arg ENV_ENDPOINT_HOST: env-specific S3 endpoint host

# @arg ENV_ACCESS_KEY   : env-specific S3 access key
@@ -462,7 +477,7 @@ stages: 
    - assert_defined "${ENV_SECRET_KEY:-$S3_SECRET_KEY}" 'Missing required S3 secret key'

    - login "${ENV_ENDPOINT_HOST:-$S3_ENDPOINT_HOST}" "${S3_HOST_BUCKET}" "${ENV_ACCESS_KEY:-$S3_ACCESS_KEY}" "${ENV_SECRET_KEY:-$S3_SECRET_KEY}"

  script:

    - delete "$ENV_TYPE" "${ENV_BUCKET_NAME:-${S3_BASE_BUCKET_NAME}${ENV_APP_SUFFIX}}"

    - delete "$ENV_TYPE" "${ENV_BUCKET_NAME:-${S3_BASE_BUCKET_NAME}${ENV_APP_SUFFIX}}" "${ENV_ROOT_PATH:-${S3_ROOT_PATH}}"

  environment:

    action: stop
@@ -472,10 +487,10 @@ s3-review: 
  variables:

    ENV_TYPE: review

    ENV_BUCKET_NAME: "$S3_REVIEW_BUCKET_NAME"

    ENV_ROOT_PATH: "$S3_REVIEW_ROOT_PATH"

    ENV_ENDPOINT_HOST: "$S3_REVIEW_ENDPOINT_HOST"

    ENV_ACCESS_KEY: "$S3_REVIEW_ACCESS_KEY"

    ENV_SECRET_KEY: "$S3_REVIEW_SECRET_KEY"

    ENV_ROOT_PATH: "$S3_REVIEW_ROOT_PATH"

  environment:

    name: review/$CI_COMMIT_REF_NAME

    url: "${S3_REVIEW_ENVIRONMENT_SCHEME}://${CI_PROJECT_NAME}-${CI_ENVIRONMENT_SLUG}.${S3_REVIEW_ENVIRONMENT_DOMAIN}"
@@ -500,6 +515,7 @@ s3-cleanup-review: 
  variables:

    ENV_TYPE: review

    ENV_BUCKET_NAME: "$S3_REVIEW_BUCKET_NAME"

    ENV_ROOT_PATH: "$S3_REVIEW_ROOT_PATH"

    ENV_ENDPOINT_HOST: "$S3_REVIEW_ENDPOINT_HOST"

    ENV_ACCESS_KEY: "$S3_REVIEW_ACCESS_KEY"

    ENV_SECRET_KEY: "$S3_REVIEW_SECRET_KEY"
@@ -540,7 +556,7 @@ s3-cleanup-all-review: 
    - assert_defined "${S3_REVIEW_SECRET_KEY:-$S3_SECRET_KEY}" 'Missing required env $S3_REVIEW_SECRET_KEY or $S3_SECRET_KEY'

    - login "${S3_REVIEW_ENDPOINT_HOST:-$S3_ENDPOINT_HOST}" "${S3_HOST_BUCKET}" "${S3_REVIEW_ACCESS_KEY:-$S3_ACCESS_KEY}" "${S3_REVIEW_SECRET_KEY:-$S3_SECRET_KEY}"

  script:

    - delete_all review "${S3_REVIEW_BUCKET_NAME:-${S3_BASE_BUCKET_NAME}-review-.*}"

    - delete_all review "${S3_REVIEW_BUCKET_NAME:-${S3_BASE_BUCKET_NAME}-review-.*}" "${S3_REVIEW_ROOT_PATH:-${S3_ROOT_PATH}}"

  rules:

    # exclude tags

    - if: $CI_COMMIT_TAG
@@ -563,10 +579,10 @@ s3-integration: 
  variables:

    ENV_TYPE: integration

    ENV_BUCKET_NAME: "$S3_INTEG_BUCKET_NAME"

    ENV_ROOT_PATH: "$S3_INTEG_ROOT_PATH"

    ENV_ENDPOINT_HOST: "$S3_INTEG_ENDPOINT_HOST"

    ENV_ACCESS_KEY: "$S3_INTEG_ACCESS_KEY"

    ENV_SECRET_KEY: "$S3_INTEG_SECRET_KEY"

    ENV_ROOT_PATH: "$S3_INTEG_ROOT_PATH"

  environment:

    name: integration

    url: "${S3_INTEG_ENVIRONMENT_URL}"
@@ -585,10 +601,10 @@ s3-staging: 
  variables:

    ENV_TYPE: staging

    ENV_BUCKET_NAME: "$S3_STAGING_BUCKET_NAME"

    ENV_ROOT_PATH: "$S3_STAGING_ROOT_PATH"

    ENV_ENDPOINT_HOST: "$S3_STAGING_ENDPOINT_HOST"

    ENV_ACCESS_KEY: "$S3_STAGING_ACCESS_KEY"

    ENV_SECRET_KEY: "$S3_STAGING_SECRET_KEY"

    ENV_ROOT_PATH: "$S3_STAGING_ROOT_PATH"

  environment:

    name: staging

    url: "${S3_STAGING_ENVIRONMENT_URL}"
@@ -609,10 +625,10 @@ s3-production: 
    ENV_TYPE: production

    ENV_APP_SUFFIX: "" # no suffix for prod

    ENV_BUCKET_NAME: "$S3_PROD_BUCKET_NAME"

    ENV_ROOT_PATH: "$S3_PROD_ROOT_PATH"

    ENV_ENDPOINT_HOST: "$S3_PROD_ENDPOINT_HOST"

    ENV_ACCESS_KEY: "$S3_PROD_ACCESS_KEY"

    ENV_SECRET_KEY: "$S3_PROD_SECRET_KEY"

    ENV_ROOT_PATH: "$S3_PROD_ROOT_PATH"

  environment:

    name: production

    url: "${S3_PROD_ENVIRONMENT_URL}"