Merge branch 'feat/gcp-auth-provider' into 'master' (fd28fd0a) · Commits · to be continuous... / dbt

README.md

+67 −1

Original line number	Diff line number	Diff line
		@@ -187,3 +187,69 @@ include:
		ref: '1.1.1'
		file: '/templates/gitlab-ci-dbt-pages.yml'
		```

		### Google Cloud variant

		This variant allows retrieving an [OAuth access token](https://developers.google.com/identity/protocols/oauth2) for the [dbt BigQuery Adapter](https://docs.getdbt.com/reference/resource-configs/bigquery-configs) (using the [GCP Auth Provider](https://gitlab.com/to-be-continuous/tools/gcp-auth-provider) as a _service container_).

		Provided you successfully configured the [federated authentication using OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/), this variant automatically obtains a temporary OAuth token and stores it in the `$GOOGLE_OAUTH_ACCESS_TOKEN` variable (supported by the [dbt BigQuery Config Setup](https://docs.getdbt.com/reference/warehouse-setups/bigquery-setup#oauth-token-based) as an authentication credential).

		#### Configuration

		The variant requires the additional configuration parameters:

		\| Name \| description \| default value \|
		\| ----------------- \| -------------------------------------- \| ----------------- \|
		\| `TBC_GCP_PROVIDER_IMAGE` \| The [GCP Auth Provider](https://gitlab.com/to-be-continuous/tools/gcp-auth-provider) image to use (can be overridden) \| `$CI_REGISTRY/to-be-continuous/tools/gcp-auth-provider:main` \|
		\| `GCP_OIDC_PROVIDER` \| Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) \| _none_ \|
		\| `GCP_OIDC_ACCOUNT` \| Default Service Account to which impersonate with OpenID Connect authentication \| _none_ \|
		\| `GCP_REVIEW_OIDC_PROVIDER` \| Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `review` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_REVIEW_OIDC_ACCOUNT` \| Service Account to which impersonate with OpenID Connect authentication on `review` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_INTEG_OIDC_PROVIDER` \| Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `integration` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_INTEG_OIDC_ACCOUNT` \| Service Account to which impersonate with OpenID Connect authentication on `integration` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_STAGING_OIDC_PROVIDER` \| Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `staging` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_STAGING_OIDC_ACCOUNT` \| Service Account to which impersonate with OpenID Connect authentication on `staging` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_PROD_OIDC_PROVIDER` \| Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `production` environment _(only define if different from default)_ \| _none_ \|
		\| `GCP_PROD_OIDC_ACCOUNT` \| Service Account to which impersonate with OpenID Connect authentication on `production` environment _(only define if different from default)_ \| _none_ \|

		#### Example

		With a common default `GCP_OIDC_PROVIDER` and `GCP_OIDC_ACCOUNT` configuration for non-prod environments, and a specific one for production:

		```yaml
		include:
		# main template
		- project: 'to-be-continuous/dbt'
		ref: '1.1.1'
		file: '/templates/gitlab-ci-dbt.yml'
		# `Google Cloud` variant
		- project: 'to-be-continuous/dbt'
		ref: '1.1.1'
		file: '/templates/gitlab-ci-dbt-gcp.yml'

		variables:
		# common OIDC config for non-prod envs
		GCP_OIDC_PROVIDER: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"
		GCP_OIDC_ACCOUNT: "<name>@$<gcp_nonprod_proj_id>.iam.gserviceaccount.com"
		# specific OIDC config for prod
		GCP_PROD_OIDC_PROVIDER: "projects/<gcp_prod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"
		GCP_PROD_OIDC_ACCOUNT: "<name>@$<gcp_prod_proj_id>.iam.gserviceaccount.com"
		```

		To setup dbt to use `GOOGLE_OAUTH_ACCESS_TOKEN`, apply this configuration in dbt `profiles.yml`:

		```yaml
		my-bigquery-db:
		target: dev
		outputs:
		dev:
		type: bigquery
		method: oauth-secrets
		project: [GCP project id]
		dataset: [the name of your dbt dataset]
		threads: [1 or more]
		token: "{{ env_var('GOOGLE_OAUTH_ACCESS_TOKEN') }}"
		<optional_config>: <value>
		```

kicker.json

+62 −1

Original line number	Diff line number	Diff line
		@@ -115,7 +115,68 @@
		"name": "GitLab Pages",
		"description": "Adds a job to publish the generated site to GitLab pages",
		"template_path": "templates/gitlab-ci-dbt-pages.yml"
		},
		{
		"id": "gcp-auth-provider",
		"name": "Google Cloud",
		"description": "Retrieves an [OAuth access token](https://developers.google.com/identity/protocols/oauth2) for [dbt BigQuery setup](https://docs.getdbt.com/reference/warehouse-setups/bigquery-setup)",
		"template_path": "templates/gitlab-ci-dbt-gcp.yml",
		"variables": [
		{
		"name": "TBC_GCP_PROVIDER_IMAGE",
		"description": "The [GCP Auth Provider](https://gitlab.com/to-be-continuous/tools/gcp-auth-provider) image to use",
		"default": "$CI_REGISTRY/to-be-continuous/tools/vault-secrets-provider:master",
		"advanced": true
		},
		{
		"name": "GCP_OIDC_ACCOUNT",
		"description": "Default Service Account to which impersonate with OpenID Connect authentication"
		},
		{
		"name": "GCP_OIDC_PROVIDER",
		"description": "Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/)"
		},
		{
		"name": "GCP_REVIEW_OIDC_ACCOUNT",
		"description": "Service Account to which impersonate with OpenID Connect authentication on `review` environment",
		"advanced": true
		},
		{
		"name": "GCP_REVIEW_OIDC_PROVIDER",
		"description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `review` environment",
		"advanced": true
		},
		{
		"name": "GCP_INTEG_OIDC_ACCOUNT",
		"description": "Service Account to which impersonate with OpenID Connect authentication on `integration` environment",
		"advanced": true
		},
		{
		"name": "GCP_INTEG_OIDC_PROVIDER",
		"description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `integration` environment",
		"advanced": true
		},
		{
		"name": "GCP_STAGING_OIDC_ACCOUNT",
		"description": "Service Account to which impersonate with OpenID Connect authentication on `staging` environment",
		"advanced": true
		},
		{
		"name": "GCP_STAGING_OIDC_PROVIDER",
		"description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `staging` environment",
		"advanced": true
		},
		{
		"name": "GCP_PROD_OIDC_ACCOUNT",
		"description": "Service Account to which impersonate with OpenID Connect authentication on `production` environment",
		"advanced": true
		},
		{
		"name": "GCP_PROD_OIDC_PROVIDER",
		"description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `production` environment",
		"advanced": true
		}
		]
		}
		]

		}

templates/gitlab-ci-dbt-gcp.yml

0 → 100644

+17 −0

Original line number	Diff line number	Diff line
		# =====================================================================================================================
		# === Google Cloud template variant
		# =====================================================================================================================
		variables:
		# variabilized gcp-auth-provider image
		TBC_GCP_PROVIDER_IMAGE: $CI_REGISTRY/to-be-continuous/tools/gcp-auth-provider:main

		.dbt-base:
		services:
		- name: "$TBC_TRACKING_IMAGE"
		command: ["--service", "dbt", "1.1.1" ]
		- name: "$TBC_GCP_PROVIDER_IMAGE"
		alias: "gcp-auth-provider"
		variables:
		# have to be explicitly declared in the YAML to be exported to the service
		CI_JOB_JWT_V2: $CI_JOB_JWT_V2
		GOOGLE_OAUTH_ACCESS_TOKEN: "@url@http://gcp-auth-provider/token"