Commit d4c8927e authored by Pierre Smeyers's avatar Pierre Smeyers
Browse files

Merge branch 'master' into 'master' 

feat: add vault variant for validation template

Closes #6

See merge request to-be-continuous/tools/gitlab-ci!4
parents 9a876a5d a16a8879

.gitlab-ci.yml

+1 −1
Original line number Diff line number Diff line
@@ -5,4 +5,4 @@ stages: 
  - build



variables:

  GITLAB_CI_FILES: "templates/*.yml"
 
  GITLAB_CI_FILES: "templates/extract.yml templates/validation.yml"

README.md

+58 −0
Original line number Diff line number Diff line
@@ -24,3 +24,61 @@ stages: 
variables:

  GITLAB_CI_FILES: "templates/*.yml"

```

## Variants



The default validation template is designed to work on untagged runners, without any proxy configuration using Docker images

from the internet and using Default Trusted Certificate Authorities.



Nevertheless there are template variants available to cover specific cases.



### Vault variant



This variant allows delegating your secrets management to a [Vault](https://www.vaultproject.io/) server.



#### Configuration



In order to be able to communicate with the Vault server, the variant requires the additional configuration parameters:



| Input / Variable | Description                            | Default value     |

| ----------------- | -------------------------------------- | ----------------- |

| `TBC_VAULT_IMAGE` | The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use (can be overridden) | `registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest` |

| `vault-base-url` / `VAULT_BASE_URL` | The Vault server base API url          | **must be defined** |

| `vault-oidc-aud` / `VAULT_OIDC_AUD` | The `aud` claim for the JWT | `$CI_SERVER_URL` |

| :lock: `VAULT_ROLE_ID`   | The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID | _none_ |

| :lock: `VAULT_SECRET_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID | _none_ |



By default, the variant will authentifacte using a [JWT ID token](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html). To use [AppRole](https://www.vaultproject.io/docs/auth/approle) instead the `VAULT_ROLE_ID` and `VAULT_SECRET_ID` should be defined as secret project variables.



#### Usage



Then you may retrieve any of your secret(s) from Vault using the following syntax:



```text

@url@http://vault-secrets-provider/api/secrets/{secret_path}?field={field}

```



With:



| Parameter                        | Description                            |

| -------------------------------- | -------------------------------------- |

| `secret_path` (_path parameter_) | this is your secret location in the Vault server |

| `field` (_query parameter_)      | parameter to access a single basic field from the secret JSON payload |



#### Example



```yaml

include:

  # main template

  - component: $CI_SERVER_FQDN/to-be-continuous/tools/gitlab-ci/validation@master

  # Vault variant

  - component: $CI_SERVER_FQDN/to-be-continuous/tools/gitlab-ci/validation-vault@master

    inputs:

      # audience claim for JWT

      vault-oidc-aud: "https://vault.acme.host"

      vault-base-url: "https://vault.acme.host/v1"



variables:

  # Secrets managed by Vault

  GITLAB_TOKEN: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-infra/gitlab?field=token"

```

templates/validation-vault.yml

0 → 100644
+31 −0
Original line number Diff line number Diff line 
 # ====================================================================================================================

 # === Vault template variant

 # ====================================================================================================================

spec:

  inputs:

    vault-base-url:

      description: The Vault server base API url

      default: ''

    vault-oidc-aud:

      description: the `aud` claim for the JWT

      default: $CI_SERVER_URL

---

variables:

  # variabilized vault-secrets-provider image

  TBC_VAULT_IMAGE: registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest

  # variables have to be explicitly declared in the YAML to be exported to the service

  VAULT_ROLE_ID: $VAULT_ROLE_ID

  VAULT_SECRET_ID: $VAULT_SECRET_ID

  VAULT_OIDC_AUD: $[[ inputs.vault-oidc-aud ]]



  VAULT_BASE_URL: $[[ inputs.vault-base-url ]]



.gitlab-ci-base:

  services:

    - name: "$TBC_VAULT_IMAGE"

      alias: "vault-secrets-provider"

  variables:

    VAULT_JWT_TOKEN: "$VAULT_JWT_TOKEN"

  id_tokens:

    VAULT_JWT_TOKEN:

      aud: "$VAULT_OIDC_AUD"

templates/validation.yml

+167 −3
Original line number Diff line number Diff line
@@ -12,6 +12,11 @@ 
    echo -e "[\e[1;91mERROR\e[0m] $*" >&2

  }



  function fail() {

    log_error "$*"

    exit 1

  }



  function install_ca_certs() {

    certs=$1

    if [[ -z "$certs" ]]
@@ -30,6 +35,159 @@ 
    fi

  }



  function unscope_variables() {

    _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)

    if [[ -z "$_scoped_vars" ]]; then return; fi

    log_info "Processing scoped variables..."

    for _scoped_var in $_scoped_vars

    do

      _fields=${_scoped_var//__/:}

      _condition=$(echo "$_fields" | cut -d: -f3)

      case "$_condition" in

      if) _not="";;

      ifnot) _not=1;;

      *)

        log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"

        continue

      ;;

      esac

      _target_var=$(echo "$_fields" | cut -d: -f2)

      _cond_var=$(echo "$_fields" | cut -d: -f4)

      _cond_val=$(eval echo "\$${_cond_var}")

      _test_op=$(echo "$_fields" | cut -d: -f5)

      case "$_test_op" in

      defined)

        if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue; 

        elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue; 

        fi

        ;;

      equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)

        # comparison operator

        # sluggify actual value

        _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')

        # retrieve comparison value

        _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"

        _cmp_val=${_scoped_var#"$_cmp_val_prefix"}

        # manage 'ignore case'

        if [[ "$_test_op" == *_ic ]]

        then

          # lowercase everything

          _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')

          _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')

        fi

        case "$_test_op" in

        equals*)

          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue; 

          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue; 

          fi

          ;;

        startswith*)

          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue; 

          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue; 

          fi

          ;;

        endswith*)

          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue; 

          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue; 

          fi

          ;;

        contains*)

          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue; 

          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue; 

          fi

          ;;

        in*)

          if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue; 

          elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue; 

          fi

          ;;

        esac

        ;;

      *)

        log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"

        continue

        ;;

      esac

      # matches

      _val=$(eval echo "\$${_target_var}")

      log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"

      _val=$(eval echo "\$${_scoped_var}")

      export "${_target_var}"="${_val}"

    done

    log_info "... done"

  }



  # evaluate and export a secret

  # - $1: secret variable name

  function eval_secret() {

    name=$1

    value=$(eval echo "\$${name}")

    case "$value" in

    @b64@*)

      decoded=$(mktemp)

      errors=$(mktemp)

      if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"

      then

        # shellcheck disable=SC2086

        export ${name}="$(cat ${decoded})"

        log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"

      else

        fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"

      fi

      ;;

    @hex@*)

      decoded=$(mktemp)

      errors=$(mktemp)

      if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"

      then

        # shellcheck disable=SC2086

        export ${name}="$(cat ${decoded})"

        log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"

      else

        fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"

      fi

      ;;

    @url@*)

      url=$(echo "$value" | cut -c6-)

      if command -v curl > /dev/null

      then

        decoded=$(mktemp)

        errors=$(mktemp)

        if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"

        then

          # shellcheck disable=SC2086

          export ${name}="$(cat ${decoded})"

          log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"

        else

          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"

        fi

      elif command -v wget > /dev/null

      then

        decoded=$(mktemp)

        errors=$(mktemp)

        if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"

        then

          # shellcheck disable=SC2086

          export ${name}="$(cat ${decoded})"

          log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"

        else

          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"

        fi

      else

        log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"

      fi

      ;;

    esac

  }



  function eval_all_secrets() {

    encoded_vars=$(env | grep -Ev '(^|.*_ENV_)scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')

    for var in $encoded_vars

    do

      eval_secret "$var"

    done

  }



  # validates an input GitLab CI YAML file

  function ci_lint() {

    rc=0
@@ -50,12 +208,18 @@ 
    exit $rc

  }



gitlab-ci-lint:

  image: registry.hub.docker.com/badouralix/curl-jq:latest

  stage: build

  unscope_variables

  eval_all_secrets



.gitlab-ci-base:

  before_script:

    - !reference [.lint-scripts]

    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"



gitlab-ci-lint:

  extends: .gitlab-ci-base

  stage: build

  image: registry.hub.docker.com/badouralix/curl-jq:latest

  script:

    - ci_lint

  rules: