Merge branch 'switch-to-ADC' into 'master' (35b2e374) · Commits · to be continuous... / dbt

README.md

+3 −3

Original line number	Diff line number	Diff line
		@@ -254,7 +254,8 @@ With a common default `GCP_OIDC_PROVIDER` and `GCP_OIDC_ACCOUNT` configuration f
		gcp-prod-oidc-account: "<name>@$<gcp_prod_proj_id>.iam.gserviceaccount.com"
		```

		To setup dbt to use `GOOGLE_OAUTH_ACCESS_TOKEN`, apply this configuration in dbt `profiles.yml`:
		The variant provide [Google Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc)
		To setup dbt for bigquery, apply this configuration in dbt `profiles.yml` as described in [dbt documentation](https://docs.getdbt.com/docs/core/connect-data-platform/bigquery-setup#oauth-via-gcloud):

		```yaml
		my-bigquery-db:
		@@ -262,11 +263,10 @@ my-bigquery-db:
		outputs:
		dev:
		type: bigquery
		method: oauth-secrets
		method: oauth
		project: [GCP project id]
		dataset: [the name of your dbt dataset]
		threads: [1 or more]
		token: "{{ env_var('GOOGLE_OAUTH_ACCESS_TOKEN') }}"
		<optional_config>: <value>
		```

templates/gitlab-ci-dbt-gcp.yml

+57 −12

Original line number	Diff line number	Diff line
		@@ -38,8 +38,6 @@ spec:
		default: ''
		---
		variables:
		# variabilized gcp-auth-provider image
		TBC_GCP_PROVIDER_IMAGE: registry.gitlab.com/to-be-continuous/tools/gcp-auth-provider:latest
		GCP_OIDC_AUD: $[[ inputs.gcp-oidc-aud ]]
		GCP_OIDC_ACCOUNT: $[[ inputs.gcp-oidc-account ]]
		GCP_OIDC_PROVIDER: $[[ inputs.gcp-oidc-provider ]]
		@@ -51,16 +49,63 @@ variables:
		GCP_STAGING_OIDC_PROVIDER: $[[ inputs.gcp-staging-oidc-provider ]]
		GCP_PROD_OIDC_ACCOUNT: $[[ inputs.gcp-prod-oidc-account ]]
		GCP_PROD_OIDC_PROVIDER: $[[ inputs.gcp-prod-oidc-provider ]]
		.dbt-base:
		services:
		- name: "$TBC_TRACKING_IMAGE"
		command: ["--service", "dbt", "3.2.3"]
		- name: "$TBC_GCP_PROVIDER_IMAGE"
		alias: "gcp-auth-provider"
		variables:
		# have to be explicitly declared in the YAML to be exported to the service
		GCP_JWT: $GCP_JWT
		GOOGLE_OAUTH_ACCESS_TOKEN: "@url@http://gcp-auth-provider/token"

		.gcp-provider-auth:
		before_script:
		- echo "Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"
		- echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt"
		- \|-
		if [[ "$ENV_TYPE" ]]
		then
		case "$ENV_TYPE" in
		review*)
		env_prefix=REVIEW;;
		integ*)
		env_prefix=INTEG;;
		staging*)
		env_prefix=STAGING;;
		prod*)
		env_prefix=PROD;;
		*)
		;;
		esac
		env_oidc_provider=$(eval echo "\$GCP_${env_prefix}_OIDC_PROVIDER")
		env_oidc_account=$(eval echo "\$GCP_${env_prefix}_OIDC_ACCOUNT")
		fi
		oidc_provider="${env_oidc_provider:-$GCP_OIDC_PROVIDER}"
		oidc_account="${env_oidc_account:-$GCP_OIDC_ACCOUNT}"
		- \|-
		cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json"
		{
		"type": "external_account",
		"audience": "//iam.googleapis.com/${oidc_provider}",
		"subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
		"token_url": "https://sts.googleapis.com/v1/token",
		"credential_source": {
		"file": "$CI_BUILDS_DIR/.auth_token.jwt"
		},
		"service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${oidc_account}:generateAccessToken"
		}
		EOF
		- export GOOGLE_APPLICATION_CREDENTIALS="$CI_BUILDS_DIR/google_application_credentials.json"


		.dbt-build:
		extends: .dbt-base
		id_tokens:
		GCP_JWT:
		aud: "$GCP_OIDC_AUD"
		before_script:
		- !reference [.gcp-provider-auth, before_script]
		- !reference [.dbt-base, before_script]


		.dbt-deploy:
		extends: .dbt-base
		id_tokens:
		GCP_JWT:
		aud: "$GCP_OIDC_AUD"
		before_script:
		- !reference [.gcp-provider-auth, before_script]
		- !reference [.dbt-base, before_script]