Commit 8ecaac62 authored by Cédric OLIVIER's avatar Cédric OLIVIER
Browse files

feat(gcp): switch to ADC google authentification

parent 9f88f7c2

README.md

+3 −3
Original line number Diff line number Diff line
@@ -254,7 +254,8 @@ With a common default `GCP_OIDC_PROVIDER` and `GCP_OIDC_ACCOUNT` configuration f 
      gcp-prod-oidc-account: "<name>@$<gcp_prod_proj_id>.iam.gserviceaccount.com"

```



To setup dbt to use `GOOGLE_OAUTH_ACCESS_TOKEN`, apply this configuration in dbt `profiles.yml`:

The variant provide [Google Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)

To setup dbt for bigquery, apply this configuration in dbt `profiles.yml` as described in [dbt documentation](https://docs.getdbt.com/docs/core/connect-data-platform/bigquery-setup#oauth-via-gcloud):



```yaml

my-bigquery-db:
@@ -262,11 +263,10 @@ my-bigquery-db: 
  outputs:

    dev:

      type: bigquery

      method: oauth-secrets

      method: oauth

      project: [GCP project id]

      dataset: [the name of your dbt dataset]

      threads: [1 or more]

      token: "{{ env_var('GOOGLE_OAUTH_ACCESS_TOKEN') }}"

      <optional_config>: <value>

```

templates/gitlab-ci-dbt-gcp.yml

+57 −12
Original line number Diff line number Diff line
@@ -38,8 +38,6 @@ spec: 
      default: ''

---

variables:

  # variabilized gcp-auth-provider image

  TBC_GCP_PROVIDER_IMAGE: registry.gitlab.com/to-be-continuous/tools/gcp-auth-provider:latest

  GCP_OIDC_AUD: $[[ inputs.gcp-oidc-aud ]]

  GCP_OIDC_ACCOUNT: $[[ inputs.gcp-oidc-account ]]

  GCP_OIDC_PROVIDER: $[[ inputs.gcp-oidc-provider ]]
@@ -51,16 +49,63 @@ variables: 
  GCP_STAGING_OIDC_PROVIDER: $[[ inputs.gcp-staging-oidc-provider ]]

  GCP_PROD_OIDC_ACCOUNT: $[[ inputs.gcp-prod-oidc-account ]]

  GCP_PROD_OIDC_PROVIDER: $[[ inputs.gcp-prod-oidc-provider ]]

.dbt-base:

  services:

    - name: "$TBC_TRACKING_IMAGE"

      command: ["--service", "dbt", "3.2.3"]

    - name: "$TBC_GCP_PROVIDER_IMAGE"

      alias: "gcp-auth-provider"

  variables:

    #  have to be explicitly declared in the YAML to be exported to the service

    GCP_JWT: $GCP_JWT

    GOOGLE_OAUTH_ACCESS_TOKEN: "@url@http://gcp-auth-provider/token"



.gcp-provider-auth:

  before_script:

    - echo "Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"

    - echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt"

    - |-

      if [[ "$ENV_TYPE" ]]

      then

        case "$ENV_TYPE" in

        review*)

          env_prefix=REVIEW;;

        integ*)

          env_prefix=INTEG;;

        staging*)

          env_prefix=STAGING;;

        prod*)

          env_prefix=PROD;;

        *)

          ;;

        esac

        env_oidc_provider=$(eval echo "\$GCP_${env_prefix}_OIDC_PROVIDER")

        env_oidc_account=$(eval echo "\$GCP_${env_prefix}_OIDC_ACCOUNT")

      fi

      oidc_provider="${env_oidc_provider:-$GCP_OIDC_PROVIDER}"

      oidc_account="${env_oidc_account:-$GCP_OIDC_ACCOUNT}"

    - |-

      cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json"

      {

        "type": "external_account",

        "audience": "//iam.googleapis.com/${oidc_provider}",

        "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",

        "token_url": "https://sts.googleapis.com/v1/token",

        "credential_source": {

          "file": "$CI_BUILDS_DIR/.auth_token.jwt"

        },

        "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${oidc_account}:generateAccessToken"

      }

      EOF

    - export GOOGLE_APPLICATION_CREDENTIALS="$CI_BUILDS_DIR/google_application_credentials.json"





.dbt-build:

  extends: .dbt-base

  id_tokens:

    GCP_JWT:

      aud: "$GCP_OIDC_AUD"

  before_script:

    - !reference [.gcp-provider-auth, before_script]

    - !reference [.dbt-base, before_script]

    



.dbt-deploy:

  extends: .dbt-base    

  id_tokens:

    GCP_JWT:

      aud: "$GCP_OIDC_AUD"

  before_script:

    - !reference [.gcp-provider-auth, before_script]

    - !reference [.dbt-base, before_script]