Commit f4009d09 authored by Cédric OLIVIER's avatar Cédric OLIVIER
Browse files

Merge branch '7-enhancement-add-dbt_project_evaluator-job' into 'master' 

feat: add dbt-project-evaluator job

Closes #7

See merge request to-be-continuous/dbt!40
parents d36dea1a e0e1afc7

README.md

+14 −0
Original line number Diff line number Diff line
@@ -175,6 +175,20 @@ It is bound to the test stage, and uses the following variables: 


:warning: By default this job is focused on [unit testing](https://docs.getdbt.com/docs/build/unit-tests), this job can also run [data testing](https://docs.getdbt.com/docs/build/data-tests#related-reference-docs) when overidding `test-args` input.



### `dbt-project-evaluator-xxx` jobs



Those jobs perform a project evaluation to highlight misalignement with dbt Labs' best practices, based on [dbt_project_evaluator](https://dbt-labs.github.io/dbt-project-evaluator).

They can be enabled by setting `evaluator-enabled` / `DBT_EVALUATOR_ENABLED` to `true`



It is bound to the test stage, and uses the following variables:



| Input / Variable | Description                            | Default value     |

| ------------------------ | -------------------------------------- | ----------------- |

| `evaluator-enabled` / `DBT_EVALUATOR_ENABLED` | set to `true` to enable dbt project evaluator  | _none_ (disabled) |

| `evaluator-args` / `DBT_EVALUATOR_ARGS` | Arguments used by [dbt cli](https://docs.getdbt.com/reference/global-configs#command-line-flags) for dbt_project_evaluator | `--warn-error --select package:dbt_project_evaluator dbt_project_evaluator_exceptions` |



:warning: To enable this job, you also need to include `dbt-labs/dbt_project_evaluator` package in your dbt `packages.yml` file.



### `dbt-sqlfluff-lint` job



This job performs **SQL Lint**.

kicker.json

+14 −0
Original line number Diff line number Diff line
@@ -70,6 +70,20 @@ 
        }

      ]

    },

    {

      "id": "dbt-project-evaluator",

      "name": "dbt project evaluator",

      "description": "Highlights areas of a dbt project that are misaligned with dbt Labs' best practices",

      "enable_with": "DBT_EVALUATOR_ENABLED",

      "variables":[

        {

          "name": "DBT_EVALUATOR_ARGS",

          "description": "Arguments used by [dbt cli](https://docs.getdbt.com/reference/global-configs#command-line-flags) for [dbt_project_evaluator](https://dbt-labs.github.io/dbt-project-evaluator)",

          "default": "--warn-error --select package:dbt_project_evaluator dbt_project_evaluator_exceptions",

          "advanced": true

        }

      ]

    },

    {

      "id": "dbt-deploy",

      "name": "dbt deploy",

templates/gitlab-ci-dbt-gcp.yml

+8 −0
Original line number Diff line number Diff line
@@ -119,3 +119,11 @@ variables: 
    - !reference [.dbt-base, before_script]

    - !reference [.dbt-gcp-adc]

    

.dbt-project-evaluator:

  extends: .dbt-base    

  id_tokens:

    GCP_JWT:

      aud: "$GCP_OIDC_AUD"

  before_script:

    - !reference [.dbt-base, before_script]

    - !reference [.dbt-gcp-adc]
 
 No newline at end of file

templates/gitlab-ci-dbt.yml

+94 −1
Original line number Diff line number Diff line
@@ -73,6 +73,13 @@ spec: 
      - manual

      - auto

      default: manual

    evaluator-args:

      description: Arguments used by [dbt cli](https://docs.getdbt.com/reference/global-configs#command-line-flags) for [dbt_project_evaluator](https://dbt-labs.github.io/dbt-project-evaluator)

      default: '--warn-error --select package:dbt_project_evaluator dbt_project_evaluator_exceptions'

    evaluator-enabled:

      description: Enable [dbt_project_evaluator](https://dbt-labs.github.io/dbt-project-evaluator)

      type: boolean

      default: false    

---

workflow:

  rules:
@@ -145,6 +152,9 @@ variables: 
  DBT_INTEG_TARGET: $[[ inputs.integ-target ]]

  DBT_STAGING_TARGET: $[[ inputs.staging-target ]]

  DBT_PROD_TARGET: $[[ inputs.prod-target ]]

  DBT_EVALUATOR_ARGS: $[[ inputs.evaluator-args ]]

  DBT_EVALUATOR_ENABLED: $[[ inputs.evaluator-enabled ]]



stages:

  - build

  - test
@@ -393,7 +403,7 @@ stages: 
  function install_requirements(){

    if ! command -v dbt > /dev/null

    then

      pip install "dbt-${DBT_ADAPTER}" dbt-coverage

      pip install "dbt-${DBT_ADAPTER}"

    fi



    if ! command -v dbt-coverage > /dev/null
@@ -540,6 +550,89 @@ dbt-build-production: 
      when: never

    - !reference [.test-policy, rules]



.dbt-project-evaluator:

  extends: .dbt-base

  stage: test

  needs: []

  script:

    - mkdir -p ${DBT_PROJECT_DIR}/reports

    - dbt deps --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}"

    - dbt build --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}" --target "${ENV_TARGET:-${DBT_TARGET}}" ${DBT_EVALUATOR_ARGS}



  rules:

    - if: '$DBT_TEST_ENABLED != "true"'

      when: never

    - !reference [.test-policy, rules]





dbt-project-evaluator-review:

  extends: .dbt-project-evaluator

  variables:

    ENV_TYPE: review

    ENV_TARGET: "$DBT_REVIEW_TARGET"

  rules:

    - if: '$DBT_EVALUATOR_ENABLED != "true"'

      when: never

    # exclude if $DBT_REVIEW_TARGET not set

    - if: '$DBT_REVIEW_TARGET == null || $DBT_REVIEW_TARGET == ""'

      when: never

    # exclude review tests on integration or prod branch

    - if: '$CI_COMMIT_REF_NAME =~ $INTEG_REF || $CI_COMMIT_REF_NAME =~ $PROD_REF'

      when: never

    - !reference [.test-policy, rules]



dbt-project-evaluator-integration:

  extends: .dbt-project-evaluator

  variables:

    ENV_TYPE: integration

    ENV_TARGET: "$DBT_INTEG_TARGET"

  rules:

    - if: '$DBT_EVALUATOR_ENABLED != "true"'

      when: never

    # exclude if $DBT_INTEG_TARGET not set

    - if: '$DBT_INTEG_TARGET == null || $DBT_INTEG_TARGET == ""'

      when: never

    # exclude integration tests on prod branch

    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF'

      when: never

    - !reference [.test-policy, rules]



dbt-project-evaluator-staging:

  extends: .dbt-project-evaluator

  variables:

    ENV_TYPE: staging

    ENV_TARGET: "$DBT_STAGING_TARGET"

  rules:

    - if: '$DBT_EVALUATOR_ENABLED != "true"'

      when: never

    # exclude if $DBT_INTEG_TARGET not set

    - if: '$DBT_STAGING_TARGET == null || $DBT_STAGING_TARGET == ""'

      when: never

    # exclude non-production branches

    - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'

      when: never    

    # exclude if $DBT_PROD_TARGET not set

    - if: '$DBT_PROD_TARGET == null || $DBT_PROD_TARGET == ""'

      when: never

    - !reference [.test-policy, rules]        



dbt-project-evaluator-production:

  extends: .dbt-project-evaluator

  variables:

    ENV_TYPE: production

    ENV_TARGET: "$DBT_PROD_TARGET"

  rules:

    - if: '$DBT_EVALUATOR_ENABLED != "true"'

      when: never

    # exclude non-production branches

    - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'

      when: never    

    # exclude if $DBT_PROD_TARGET not set

    - if: '$DBT_PROD_TARGET == null || $DBT_PROD_TARGET == ""'

      when: never

    - !reference [.test-policy, rules]  





dbt-test-review:

  extends: .dbt-test

  variables: