Commit ae779716 authored by Cédric OLIVIER's avatar Cédric OLIVIER
Browse files

Merge branch '8-dep-management' into 'master' 

feat: allow dependency management (versioned) from the user

Closes #8

See merge request to-be-continuous/dbt!50
parents 6f69c2c1 cea4fd7b

README.md

+1 −0
Original line number Diff line number Diff line
@@ -83,6 +83,7 @@ The dbt template uses some global configuration used throughout all jobs. 
| `profiles-dir` / `DBT_PROFILES_DIR` | The dbt [profile](https://docs.getdbt.com/dbt-cli/configure-your-profile) location       | `.`   |

| `adapter` / `DBT_ADAPTER` | The dbt [adapter](https://docs.getdbt.com/docs/available-adapters) to use       | __none__ (required)  |

| `target` / `DBT_TARGET` | The dbt [target](https://docs.getdbt.com/reference/dbt-jinja-functions/target) to use  |  __none__ (required)  |

| `reqs-file` / `DBT_REQS_FILE` | [Requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) to install | `requirements.txt` |

| `build-args` / `DBT_BUILD_ARGS` | Arguments used by [`dbt cli`](https://docs.getdbt.com/reference/global-configs#command-line-flags)      | __none__          |



`dbt` official images depending your adapter can be found on [dbt documentation](https://docs.getdbt.com/docs/core/docker-install#install-a-dbt-docker-image-from-github-packages) and on [dbt github packages](https://github.com/orgs/dbt-labs/packages?visibility=public)

kicker.json

+5 −0
Original line number Diff line number Diff line
@@ -30,6 +30,11 @@ 
      "name": "DBT_TARGET",

      "description": "The dbt [target](https://docs.getdbt.com/reference/dbt-jinja-functions/target) to use "

    },

    {

      "name": "DBT_REQS_FILE",

      "description": "[Requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) to install",

      "default": "requirements.txt"

    },

    {

      "name": "DBT_BUILD_ARGS",

      "description": "Arguments used by [dbt cli](https://docs.getdbt.com/reference/global-configs#command-line-flags) ",

templates/gitlab-ci-dbt.yml

+21 −11
Original line number Diff line number Diff line
@@ -31,6 +31,9 @@ spec: 
    target:

      description: 'The dbt [target](https://docs.getdbt.com/reference/dbt-jinja-functions/target) to use '

      default: ''

    reqs-file:

      description: '[Requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) to install'

      default: requirements.txt

    build-args:

      description: 'Arguments used by [dbt cli](https://docs.getdbt.com/reference/global-configs#command-line-flags) '

      default: ''
@@ -160,6 +163,7 @@ variables: 


  DBT_ADAPTER: $[[ inputs.adapter ]]

  DBT_TARGET: $[[ inputs.target ]]

  DBT_REQS_FILE: $[[ inputs.reqs-file ]]

  DBT_BUILD_ARGS: $[[ inputs.build-args ]]

  DBT_TEST_ARGS: $[[ inputs.test-args ]]

  DBT_TEST_ENABLED: $[[ inputs.test-enabled ]]
@@ -420,20 +424,23 @@ stages: 
    fi

  }



  function install_requirements(){

    if ! command -v dbt > /dev/null

    then

      pip install "dbt-${DBT_ADAPTER}"

    fi

  function maybe_install() {

    cmd=$1

    package=$2



    if ! command -v sqlfluff > /dev/null

    if ! command -v "$cmd" > /dev/null 

    then

      pip install sqlfluff-templater-dbt

      log_info "--- command $cmd not found installing $package, provide \\e[33;1m${DBT_REQS_FILE}\\e[0m to fix version"

      # shellcheck disable=SC2086

      pip install "$package"

    fi

  }



    if ! command -v dbt_junit_report > /dev/null

    then

      pip install dbt-junit-report

  function install_requirements(){

    if [[ -f "${DBT_REQS_FILE}" ]]; then

      log_info "--- installing main requirements from \\e[33;1m${DBT_REQS_FILE}\\e[0m"

      # shellcheck disable=SC2086

      pip install -r "${DBT_REQS_FILE}"

    fi     

  }
@@ -472,6 +479,7 @@ dbt-sqlfluff-lint: 
  extends: .dbt-base

  stage: build

  script:

    - maybe_install sqlfluff sqlfluff-templater-dbt

    - cd "${DBT_PROJECT_DIR}"

    - sqlfluff lint ${TRACE+--verbose} --format human $DBT_SQLFLUFF_LINT_ARGS

  rules:
@@ -485,6 +493,7 @@ dbt-sqlfluff-lint: 
  stage: build

  script:

    - assert_defined "${ENV_TARGET:-${DBT_TARGET}}" 'Missing required dbt target'

    - maybe_install dbt dbt-${DBT_ADAPTER}

    - mkdir -p ${DBT_PROJECT_DIR}/reports

    - dbt deps --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}"

    - dbt compile --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}" --target "${ENV_TARGET:-${DBT_TARGET}}" ${DBT_BUILD_ARGS}
@@ -546,6 +555,7 @@ dbt-build-production: 
  needs: []

  script:

    - mkdir -p ${DBT_PROJECT_DIR}/reports

    - maybe_install dbt_junit_report dbt-junit-report

    - dbt deps --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}"    

    - dbt test --project-dir "${DBT_PROJECT_DIR}" --profiles-dir "${DBT_PROFILES_DIR}" --target "${ENV_TARGET:-${DBT_TARGET}}" ${DBT_TEST_ARGS}

    - dbt_junit_report ${DBT_PROJECT_DIR}/target/run_results.json ${DBT_PROJECT_DIR}/reports/dbt-test.xunit.xml