feat(vault): add vault variant (eed3f655) · Commits · to be continuous... / dbt

.gitignore

+2 −1

Original line number	Diff line number	Diff line
		@@ -20,3 +20,4 @@
		/dist/
		/nbdist/
		/.nb-gradle/
		.gitlab-ci-local/*

README.md

+52 −0

Original line number	Diff line number	Diff line
		@@ -303,3 +303,55 @@ my-bigquery-db:
		```


		### Vault variant

		This variant allows delegating your secrets management to a [Vault](https://www.vaultproject.io/) server.

		#### Configuration

		In order to be able to communicate with the Vault server, the variant requires the additional configuration parameters:

		\| Input / Variable \| Description \| Default value \|
		\| ----------------- \| -------------------------------------- \| ----------------- \|
		\| `TBC_VAULT_IMAGE` \| The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use (can be overridden) \| `registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest` \|
		\| `vault-base-url` / `VAULT_BASE_URL` \| The Vault server base API url \| must be defined \|
		\| `vault-oidc-aud` / `VAULT_OIDC_AUD` \| The `aud` claim for the JWT \| `$CI_SERVER_URL` \|
		\| :lock: `VAULT_ROLE_ID` \| The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID \| _none_ \|
		\| :lock: `VAULT_SECRET_ID` \| The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID \| _none_ \|

		By default, the variant will authentifacte using a [JWT ID token](https://docs.gitlab.com/ci/secrets/id_token_authentication/). To use [AppRole](https://www.vaultproject.io/docs/auth/approle) instead the `VAULT_ROLE_ID` and `VAULT_SECRET_ID` should be defined as secret project variables.

		#### Usage

		Then you may retrieve any of your secret(s) from Vault using the following syntax:

		```text
		@url@http://vault-secrets-provider/api/secrets/{secret_path}?field={field}
		```

		With:

		\| Parameter \| Description \|
		\| -------------------------------- \| -------------------------------------- \|
		\| `secret_path` (_path parameter_) \| this is your secret location in the Vault server \|
		\| `field` (_query parameter_) \| parameter to access a single basic field from the secret JSON payload \|

		#### Example

		```yaml
		include:
		# main template
		- component: $CI_SERVER_FQDN/to-be-continuous/dbt/gitlab-ci-dbt@4.8.4
		# Vault variant
		- component: $CI_SERVER_FQDN/to-be-continuous/dbt/gitlab-ci-dbt-vault@4.8.4
		inputs:
		# audience claim for JWT
		vault-oidc-aud: "https://vault.acme.host"
		vault-base-url: "https://vault.acme.host/v1"

		variables:
		# Secrets managed by Vault
		PG_USER: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-backend/postgres?field=username"
		PG_PASSWORD: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-backend/postgres?field=password"
		```

kicker.json

+35 −0

Original line number	Diff line number	Diff line
		@@ -225,6 +225,41 @@
		"advanced": true
		}
		]
		},
		{
		"id": "vault",
		"name": "Vault",
		"description": "Retrieve secrets from a [Vault](https://www.vaultproject.io/) server",
		"template_path": "templates/gitlab-ci-dbt-vault.yml",
		"variables": [
		{
		"name": "TBC_VAULT_IMAGE",
		"description": "The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use",
		"default": "registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest",
		"advanced": true
		},
		{
		"name": "VAULT_BASE_URL",
		"description": "The Vault server base API url"
		},
		{
		"name": "VAULT_OIDC_AUD",
		"description": "The `aud` claim for the JWT",
		"default": "$CI_SERVER_URL"
		},
		{
		"name": "VAULT_ROLE_ID",
		"description": "The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID",
		"mandatory": true,
		"secret": true
		},
		{
		"name": "VAULT_SECRET_ID",
		"description": "The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID",
		"mandatory": true,
		"secret": true
		}
		]
		}
		]
		}

templates/gitlab-ci-dbt-vault.yml

0 → 100644

+38 −0

Original line number	Diff line number	Diff line
		# =====================================================================================================================
		# === Vault template variant
		# =====================================================================================================================
		spec:
		inputs:
		vault-base-url:
		description: The Vault server base API url
		default: ''
		vault-oidc-aud:
		description: The `aud` claim for the JWT
		default: $CI_SERVER_URL

		---
		variables:
		# variabilized vault-secrets-provider image
		TBC_VAULT_IMAGE: registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest
		VAULT_BASE_URL: $[[ inputs.vault-base-url ]]
		# variables have to be explicitly declared in the YAML to be exported to the service
		VAULT_ROLE_ID: "$VAULT_ROLE_ID"
		VAULT_SECRET_ID: "$VAULT_SECRET_ID"
		VAULT_OIDC_AUD: $[[ inputs.vault-oidc-aud ]]

		.dbt-base:
		services:
		- name: "$TBC_TRACKING_IMAGE"
		command: ["--service", "dbt", "4.8.4"]
		- name: "$TBC_VAULT_IMAGE"
		alias: "vault-secrets-provider"
		variables:
		SKIP_SSL: "false"
		variables:
		VAULT_JWT_TOKEN: "$VAULT_JWT_TOKEN"
		VAULT_CA_CERTS: \|
		$DEFAULT_CA_CERTS
		$CUSTOM_CA_CERTS
		id_tokens:
		VAULT_JWT_TOKEN:
		aud: "$VAULT_OIDC_AUD"

templates/gitlab-ci-dbt.yml

+12 −0

Original line number	Diff line number	Diff line
		@@ -407,6 +407,18 @@ stages:
		else
		log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${decoded}")\\n... -----\\n$(sed 's/^/... /g' "${errors}")"
		fi
		elif command -v python3 > /dev/null
		then
		decoded=$(mktemp)
		errors=$(mktemp)
		# shellcheck disable=SC2086
		if python3 -c "import urllib.request ; urllib.request.urlretrieve(\"$url\",\"${decoded}\")" > "${errors}" 2>&1
		then
		export ${name}="$(cat ${decoded})"
		log_info "Successfully fetched secret \\e[33;1m${name}\\e[0m"
		else
		log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
		fi
		else
		log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
		fi