Commit 1a26a46f authored by Cédric OLIVIER's avatar Cédric OLIVIER Committed by Pierre Smeyers
Browse files

feat(test): add dbt test support (unit testing)

parent 32be007b

.devcontainer/devcontainer.json

0 → 100644
+32 −0
Original line number Diff line number Diff line 
// For format details, see https://aka.ms/devcontainer.json. For config options, see the

// README at: https://github.com/devcontainers/templates/tree/main/src/debian

{

	"name": "dbt-devcontainer",

	"image": "docker.io/ubuntu:latest",



	// Features to add to the dev container. More info: https://containers.dev/features.

    "features": {

        "ghcr.io/devcontainers/features/common-utils:2": {},		

		"ghcr.io/devcontainers/features/docker-in-docker:2": {},

		"ghcr.io/msclock/features/gitlab-ci-local:0": {}

    },



	// Use 'forwardPorts' to make a list of ports inside the container available locally.

	// "forwardPorts": [],



	// Use 'postCreateCommand' to run commands after the container is created.

	"postCreateCommand": "git config core.editor 'code --wait'",

	

	// Configure tool-specific properties.

	"customizations": {

		"vscode": {

			"extensions": [

				"mhutchie.git-graph"

			]

		}

	}

	



	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.

	// "remoteUser": "root"

}

README.md

+15 −0
Original line number Diff line number Diff line
@@ -160,6 +160,21 @@ This job performs **build, doc generation** and documentation **coverage**. 
`dbt-build` generates executable SQL from source model, test, and analysis files





### `dbt-test` job



This job performs **test**. It is activated by setting $DBT_TEST_ENABLED to true



`dbt-test` execute [dbt test command](https://docs.getdbt.com/reference/commands/test) to performs [dbt unit tests](https://docs.getdbt.com/docs/build/unit-tests):



It is bound to the test stage, and uses the following variables:



| Input / Variable | Description                            | Default value     |

| ------------------------ | -------------------------------------- | ----------------- |

| `test-enabled` / `DBT_TEST_ENABLED` | set to `true` to enable dbt test  | _none_ (disabled) |

| `test-args` / `DBT_TEST_ARGS` | dbt test [options and arguments](https://docs.getdbt.com/reference/commands/test) | `--select resource_type:unit_test` |



:warning: By default this job is focused on [unit testing](https://docs.getdbt.com/docs/build/unit-tests), this job can also run [data testing](https://docs.getdbt.com/docs/build/data-tests#related-reference-docs) when overidding `test-args` input.



### `dbt-sqlfluff-lint` job



This job performs **SQL Lint**.

kicker.json

+14 −0
Original line number Diff line number Diff line
@@ -56,6 +56,20 @@ 
        }    

      ]

    },

    {

      "id": "dbt-test",

      "name": "dbt test",

      "description": "Unit testing on dbt project",

      "enable_with": "DBT_TEST_ENABLED",

      "variables":[

        {

          "name": "DBT_TEST_ARGS",

          "description": "dbt test [options and arguments](https://docs.getdbt.com/reference/commands/test)",

          "default": "--select resource_type:unit_test",

          "advanced": true

        }

      ]

    },

    {

      "id": "dbt-deploy",

      "name": "dbt deploy",

templates/gitlab-ci-dbt-gcp.yml

+8 −0
Original line number Diff line number Diff line
@@ -99,6 +99,14 @@ variables: 
    - !reference [.gcp-provider-auth, before_script]

    - !reference [.dbt-base, before_script]

    

.dbt-test:

  extends: .dbt-base

  id_tokens:

    GCP_JWT:

      aud: "$GCP_OIDC_AUD"

  before_script:

    - !reference [.gcp-provider-auth, before_script]      

    - !reference [.dbt-base, before_script]



.dbt-deploy:

  extends: .dbt-base

templates/gitlab-ci-dbt.yml

+114 −0
Original line number Diff line number Diff line
@@ -34,6 +34,13 @@ spec: 
    build-args:

      description: 'Arguments used by [dbt cli](https://docs.getdbt.com/reference/global-configs#command-line-flags) '

      default: ''

    test-args:

      description: 'Arguments used by [dbt test](https://docs.getdbt.com/reference/commands/test)'

      default: '--select resource_type:unit_test'

    test-enabled:

      description: Enable dbt test

      type: boolean

      default: false

    sqlfluff-enabled:

      description: Enable SQLFluff lint

      type: boolean
@@ -128,6 +135,8 @@ variables: 
  DBT_ADAPTER: $[[ inputs.adapter ]]

  DBT_TARGET: $[[ inputs.target ]]

  DBT_BUILD_ARGS: $[[ inputs.build-args ]]

  DBT_TEST_ARGS: $[[ inputs.test-args ]]

  DBT_TEST_ENABLED: $[[ inputs.test-enabled ]]

  DBT_SQLFLUFF_ENABLED: $[[ inputs.sqlfluff-enabled ]]

  DBT_SQLFLUFF_LINT_ARGS: $[[ inputs.sqlfluff-lint-args ]]

  SQLFLUFF_WORKING_DIR: $[[ inputs.sqlfluff-working-dir ]]
@@ -387,10 +396,20 @@ stages: 
      pip install "dbt-${DBT_ADAPTER}" dbt-coverage

    fi



    if ! command -v dbt-coverage > /dev/null

    then

      pip install dbt-coverage

    fi



    if ! command -v sqlfluff > /dev/null

    then

      pip install sqlfluff-templater-dbt

    fi



    if ! command -v dbt_junit_report > /dev/null

    then

      pip install dbt-junit-report

    fi

  }



  unscope_variables
@@ -407,6 +426,10 @@ stages: 
  services:

    - name: "$TBC_TRACKING_IMAGE"

      command: ["--service", "dbt", "4.2.0"]

  variables:

    # set local cache dir; most Python tools honour XDG specs

    XDG_CACHE_HOME: "$CI_PROJECT_DIR/.cache"

    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"

  before_script:

    - !reference [.dbt-scripts]

    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
@@ -419,6 +442,7 @@ stages: 
    key: "$CI_COMMIT_REF_SLUG-dbt"

    paths:

      - .cache/

      - ${DBT_PROJECT_DIR}/target



dbt-sqlfluff-lint:

  extends: .dbt-base
@@ -491,6 +515,96 @@ dbt-build-production: 
    # only on production branch(es)

    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF'



# compile job

.dbt-test:

  extends: .dbt-base

  stage: test

  needs: []

  script:

    - mkdir -p ${DBT_PROJECT_DIR}/reports

    - dbt test --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}" ${DBT_TEST_ARGS}

    - dbt_junit_report ${DBT_PROJECT_DIR}/target/run_results.json ${DBT_PROJECT_DIR}/reports/dbt-test.xunit.xml

  # keep build artifacts and test reports (see: https://docs.gitlab.com/ee/ci/yaml/#artifactsreportsjunit)

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"

    expire_in: 1 day

    reports:

      junit:

        - ${DBT_PROJECT_DIR}/reports/dbt-test.xunit.xml

    paths:

        - ${DBT_PROJECT_DIR}/target/run_results.json

        - ${DBT_PROJECT_DIR}/reports/dbt-test.xunit.xml

  rules:

    - if: '$DBT_TEST_ENABLED != "true"'

      when: never

    - !reference [.test-policy, rules]



dbt-test-review:

  extends: .dbt-test

  variables:

    ENV_TYPE: review

    ENV_TARGET: "$DBT_REVIEW_TARGET"

  rules:

    - if: '$DBT_TEST_ENABLED != "true"'

      when: never

    # exclude if $DBT_REVIEW_TARGET not set

    - if: '$DBT_REVIEW_TARGET == null || $DBT_REVIEW_TARGET == ""'

      when: never

    # exclude review tests on integration or prod branch

    - if: '$CI_COMMIT_REF_NAME =~ $INTEG_REF || $CI_COMMIT_REF_NAME =~ $PROD_REF'

      when: never

    - !reference [.test-policy, rules]



dbt-test-integration:

  extends: .dbt-test

  variables:

    ENV_TYPE: integration

    ENV_TARGET: "$DBT_INTEG_TARGET"

  rules:

    - if: '$DBT_TEST_ENABLED != "true"'

      when: never

    # exclude if $DBT_INTEG_TARGET not set

    - if: '$DBT_INTEG_TARGET == null || $DBT_INTEG_TARGET == ""'

      when: never

    # exclude integration tests on prod branch

    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF'

      when: never

    - !reference [.test-policy, rules]



dbt-test-staging:

  extends: .dbt-test

  variables:

    ENV_TYPE: staging

    ENV_TARGET: "$DBT_STAGING_TARGET"

  rules:

    - if: '$DBT_TEST_ENABLED != "true"'

      when: never

    # exclude if $DBT_INTEG_TARGET not set

    - if: '$DBT_STAGING_TARGET == null || $DBT_STAGING_TARGET == ""'

      when: never

    # exclude non-production branches

    - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'

      when: never    

    # exclude if $DBT_PROD_TARGET not set

    - if: '$DBT_PROD_TARGET == null || $DBT_PROD_TARGET == ""'

      when: never

    - !reference [.test-policy, rules]        



dbt-test-production:

  extends: .dbt-test

  variables:

    ENV_TYPE: production

    ENV_TARGET: "$DBT_PROD_TARGET"

  rules:

    - if: '$DBT_TEST_ENABLED != "true"'

      when: never

    # exclude non-production branches

    - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'

      when: never    

    # exclude if $DBT_PROD_TARGET not set

    - if: '$DBT_PROD_TARGET == null || $DBT_PROD_TARGET == ""'

      when: never

    - !reference [.test-policy, rules]        



.dbt-deploy:

  extends: .dbt-base