Commit f0406deb authored by Cédric OLIVIER's avatar Cédric OLIVIER Committed by Olivier Cedric
Browse files

feat: fully integration of poetry 

goal: prepare to use other package manager like pipenv
switch from requirements.txt file generation from poetry pyproject to poetry install
use of poetry for package, publish and release
parent ef724bb2

README.md

+2 −3
Original line number Diff line number Diff line
@@ -226,18 +226,17 @@ It is bound to the `test` stage, and uses the following variables: 


This job outputs a **textual report** in the console, and in case of failure also exports a JSON report in the `reports/`

directory _(relative to project root dir)_.



### Package jobs



#### `py-package` job



This job is performs a packaging of your Python code.

This job is **disabled by default** and performs a packaging of your Python code.



It is bound to the `package-build` stage, applies only on git tags and uses the following variables:



| Name            | description                                          | default value |

| --------------- | ---------------------------------------------------- | ------------- |

| `PYTHON_FORCE_PACKAGE` | Force the packaging even if not on tag related event | _none_        |

| `PYTHON_FORCE_PACKAGE` | Set to `true` to force the packaging even if not on tag related event | _none_ (disabled) |



### Publish jobs

templates/gitlab-ci-python.yml

+146 −129
Original line number Diff line number Diff line
@@ -69,90 +69,9 @@ variables: 
    fi

  }



  function install_test_requirements() {

    if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      if  [[ ! -f "poetry.lock" ]]; then 

        log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files"

        exit 1 

      fi

      log_info "--- Poetry detected: generating \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m from poetry.lock"      

      pip install poetry

      poetry export --without-hashes ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} --dev -f requirements.txt --output "${TEST_REQUIREMENTS_FILE}"

    fi



    if [[ -f "${TEST_REQUIREMENTS_FILE}" ]]; then

      log_info "--- installing from ${TEST_REQUIREMENTS_FILE} file"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} -r "${TEST_REQUIREMENTS_FILE}"

    else

      log_info "--- no test requirements file found from env or file ${TEST_REQUIREMENTS_FILE} does not exist"

    fi

  }



  function install_requirements() {

    if  [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      if  [[ ! -f "poetry.lock" ]]; then 

        log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files"

        exit 1 

      fi

      log_info "--- Poetry detected: generating \\e[33;1m${REQUIREMENTS_FILE}\\e[0m from poetry.lock"      

      pip install poetry

      poetry export --without-hashes ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} -f requirements.txt --output "${REQUIREMENTS_FILE}"

    fi



    if [[ -f "${REQUIREMENTS_FILE}" ]]; then

      log_info "--- installing from ${REQUIREMENTS_FILE} file"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} -r "${REQUIREMENTS_FILE}"

    elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then

      log_info "--- installing from ${SETUP_PY_DIR}/setup.py file"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} "${SETUP_PY_DIR}/"

    else

      log_info "--- no requirements or setup.py file found from env or file ${REQUIREMENTS_FILE} - ${SETUP_PY_DIR}/setup.py does not exist"

    fi

  }



  function install_doc_requirements() {

    if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      if  [[ ! -f "poetry.lock" ]]; then 

        log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files"

        exit 1 

      fi

      log_info "--- Poetry detected: generating \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m from poetry.lock"     

      pip install poetry

      poetry export --without-hashes ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} -f requirements.txt --output "${DOCS_REQUIREMENTS_FILE}"

    fi



    if [[ -f "${DOCS_REQUIREMENTS_FILE}" ]]; then

      log_info "--- installing from ${DOCS_REQUIREMENTS_FILE} file"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} -r "${DOCS_REQUIREMENTS_FILE}"

    elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then

      log_info "--- installing from ${SETUP_PY_DIR}/setup.py file"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} "${SETUP_PY_DIR}/"

    else

      log_info "--- no doc requirements file found from env or file ${DOCS_REQUIREMENTS_FILE} - ${SETUP_PY_DIR}/setup.py does not exist"

    fi

  }



  function release_args() {

    if [[ -f ".bumpversion.cfg" ]]; then

      log_info "--- .bumpversion.cfg file found "

      export bumpversion_args="${RELEASE_VERSION_PART} --verbose"

    else

      log_info "---  No .bumpversion.cfg file found "

      if [[ -f "setup.py" ]]; then

        log_info "---  Getting current version of setup.py file "

        current_version=$(python setup.py --version)

        export bumpversion_args=" --verbose --current-version ${current_version} --tag --tag-name {new_version} --commit ${RELEASE_VERSION_PART} setup.py"

      else

        log_warn "---  No setup.py file found. Cannot perform release."

      fi

    fi

    log_info "--- Release args: ${bumpversion_args}"

  }



  function install_ca_certs() {

    certs=$1
@@ -289,6 +208,113 @@ variables: 
    log_info "... done"

  }



  # install requirements

  # arg1: 'build' (build only) or 'test' (build + test)

  function install_requirements() {

    target=$1

    if  [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      if  [[ ! -f "poetry.lock" ]]; then 

        log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files"

        exit 1

      fi

      pip install -U poetry

      if [[ "$target" == "build" ]]; then

        log_info "--- Poetry detected: install build only requirements"

        poetry install --no-dev ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"}

      else

        log_info "--- Poetry detected: install build and dev requirements"      

        poetry install ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"}

      fi

    elif [[ -f "${REQUIREMENTS_FILE}" ]]; then

      log_info "--- installing build requirements from \\e[33;1m${REQUIREMENTS_FILE}\\e[0m"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} -r "${REQUIREMENTS_FILE}"

      if [[ "$target" == "test" ]] && [[ -f "${TEST_REQUIREMENTS_FILE}" ]]; then

        log_info "--- installing test requirements from \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m"

        # shellcheck disable=SC2086

        pip install ${PIP_OPTS} -r "${TEST_REQUIREMENTS_FILE}"

      fi

    elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then

      log_info "--- installing requirements from \\e[33;1m${SETUP_PY_DIR}/setup.py\\e[0m"

      # shellcheck disable=SC2086

      pip install ${PIP_OPTS} "${SETUP_PY_DIR}/"

    else

      log_info "--- no dependency management tool, nor requirements file nor setup.py file found: skip install dependencies"

    fi

  }



  function _run() {

    if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      if ! command -v poetry > /dev/null

      then

        pip install -U poetry

      fi

      poetry run "$@"

    else

      "$@"

    fi

  }



  function _python() {

    _run python "$@"

  }



  function _pip() {

    _run pip "$@"

  }



  function _package(){

     if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      pip install -U poetry

      poetry build

    else

      python setup.py sdist bdist_wheel

    fi

  }

  function _publish() {

    if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      pip install -U poetry

      poetry config repositories.user_defined  "$TWINE_REPOSITORY_URL"

      poetry publish --username "$TWINE_USERNAME" --password "$TWINE_PASSWORD" --repository user_defined

    else

      pip install -U twine setuptools

      pip list



      twine upload --verbose dist/*.tar.gz

      twine upload --verbose dist/*.whl

    fi

  }



  function _release() {

    if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then

      pip install -U poetry

      poetry version "${RELEASE_VERSION_PART}"

    else

      pip install -U bumpversion

      release_args

      bumpversion "${bumpversion_args}"

    fi

  }

  function release_args() {

    if [[ -f ".bumpversion.cfg" ]]; then

      log_info "--- .bumpversion.cfg file found "

      export bumpversion_args="${RELEASE_VERSION_PART} --verbose"

    else

      log_info "---  No .bumpversion.cfg file found "

      if [[ -f "setup.py" ]]; then

        log_info "---  Getting current version of setup.py file "

        current_version=$(python setup.py --version)

        export bumpversion_args=" --verbose --current-version ${current_version} --tag --tag-name {new_version} --commit ${RELEASE_VERSION_PART} setup.py"

      else

        log_warn "---  No setup.py file found. Cannot perform release."

      fi

    fi

    log_info "--- Release args: ${bumpversion_args}"

  }









  function get_latest_template_version() {

    tag_json=$(wget -T 5 -q -O - "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" || echo "")

    echo "$tag_json" | sed -rn 's/^.*"name":"([^"]*)".*$/\1/p'
@@ -347,20 +373,20 @@ py-lint: 
  extends: .python-base

  stage: build

  script:

    - install_requirements

    - pip install pylint_gitlab

    - mkdir -p reports

    - chmod o+rwx reports

    - install_requirements build

    - _pip install -U pylint_gitlab

    - |

      if ! pylint --ignore=.cache --output-format=text  ${PYLINT_ARGS}  ${PYLINT_FILES:-$(find -type f -name "*.py")}

      if ! _run pylint --ignore=.cache --output-format=text  ${PYLINT_ARGS}  ${PYLINT_FILES:-$(find -type f -name "*.py")}

      then

        # failed: also generate codeclimate report

        mkdir -p reports

        chmod o+rwx reports

        pylint --ignore=.cache --output-format=pylint_gitlab.GitlabCodeClimateReporter  ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} > reports/pylint-codeclimate.json



        _run pylint --ignore=.cache --output-format=pylint_gitlab.GitlabCodeClimateReporter  ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} > reports/pylint-codeclimate.json

        exit 1

      else

        # success: generate empty codeclimate report (required by GitLab :( )

        mkdir -p reports

        chmod o+rwx reports



        echo "[]" > reports/pylint-codeclimate.json

      fi

  artifacts:
@@ -387,8 +413,8 @@ py-compile: 
  extends: .python-base

  stage: build

  script:

    - install_requirements

    - python -m compileall $PYTHON_COMPILE_ARGS

    - install_requirements build

    - _python -m compileall $PYTHON_COMPILE_ARGS

  rules:

    # exclude merge requests

    - if: $CI_MERGE_REQUEST_ID
@@ -405,15 +431,14 @@ py-unittest: 
  script:

    - mkdir -p reports

    - chmod o+rwx reports

    - install_requirements

    - install_test_requirements

    - install_requirements test

    # code coverage

    - pip install -U coverage

    - _pip install -U coverage

    # JUnit XML report

    - pip install -U unittest-xml-reporting

    - coverage run -m xmlrunner discover -o "reports/" $UNITTEST_ARGS

    - coverage report -m

    - coverage xml -o "reports/coverage.xml"

    - _pip install -U unittest-xml-reporting

    - _run coverage run -m xmlrunner discover -o "reports/" $UNITTEST_ARGS

    - _run coverage report -m

    - _run coverage xml -o "reports/coverage.xml"

  coverage: /^TOTAL.+?(\d+\%)$/

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
@@ -436,12 +461,11 @@ py-pytest: 
  extends: .python-base

  stage: build

  script:

    - install_requirements

    - install_test_requirements

    - mkdir -p reports

    - chmod o+rwx reports

    - pip install -U pytest pytest-cov coverage

    - python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term  --cov-report xml:reports/coverage.xml ${PYTEST_ARGS}

    - install_requirements test

    - _pip install -U pytest pytest-cov coverage

    - _python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term  --cov-report xml:reports/coverage.xml ${PYTEST_ARGS}

  coverage: /^TOTAL.+?(\d+\%)$/

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
@@ -464,11 +488,10 @@ py-nosetests: 
  extends: .python-base

  stage: build

  script:

    - install_requirements

    - install_test_requirements

    - mkdir -p reports

    - chmod o+rwx reports

    - nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/coverage.xml --cover-html --cover-html-dir=reports/coverage ${NOSETESTS_ARGS}

    - install_requirements test

    - _run nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/coverage.xml --cover-html --cover-html-dir=reports/coverage ${NOSETESTS_ARGS}

  coverage: /^TOTAL.+?(\d+\%)$/

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
@@ -494,14 +517,14 @@ py-bandit: 
  # force no dependencies

  dependencies: []

  script:

    - pip install -U bandit

    - mkdir -p reports

    - chmod o+rwx reports

    - _pip install -U bandit

    - |

      if ! bandit ${TRACE+--verbose} ${BANDIT_ARGS}

      if ! _run bandit ${TRACE+--verbose} ${BANDIT_ARGS}

      then

        # failed: also generate JSON report

        mkdir -p reports

        chmod o+rwx reports

        bandit ${TRACE+--verbose} --format json --output reports/bandit.json ${BANDIT_ARGS}

        _run bandit ${TRACE+--verbose} --format json --output reports/bandit.json ${BANDIT_ARGS}

        exit 1

      fi

  artifacts:
@@ -531,14 +554,15 @@ py-safety: 
  # force no dependencies

  dependencies: []

  script:

    - install_requirements

    - mkdir -p reports

    - chmod o+rwx reports

    - install_requirements build

    - |

      if ! pip freeze | safety check --stdin ${SAFETY_ARGS}

      if ! _pip freeze | safety check --stdin ${SAFETY_ARGS}

      then

        # failed: also generate JSON report

        mkdir -p reports

        chmod o+rwx reports

        pip freeze | safety check --stdin --json --output reports/safety.json ${SAFETY_ARGS}



        _pip freeze | safety check --stdin --json --output reports/safety.json ${SAFETY_ARGS}

        exit 1

      fi

  artifacts:
@@ -559,10 +583,8 @@ py-safety: 
    - if: '$SAFETY_ENABLED == "true"'

      when: manual

      allow_failure: true





###############################################################################################

#                                      pakage stage                                           #

#                                      package stage                                           #

###############################################################################################



# (on tag creation): create packages as artifacts
@@ -570,7 +592,7 @@ py-package: 
  extends: .python-base

  stage: package-build

  script:

    - python setup.py sdist bdist_wheel

    - _package

  artifacts:

    paths:

      - $PYTHON_PROJECT_DIR/dist/*.tar.gz
@@ -580,6 +602,7 @@ py-package: 
    - if: '$CI_COMMIT_TAG'

    - if: '$PYTHON_FORCE_PACKAGE == "true"'





###############################################################################################

#                                      publish stage                                           #

###############################################################################################
@@ -591,27 +614,23 @@ py-publish: 
  script:

    - assert_defined "$TWINE_USERNAME" 'Missing required env $TWINE_USERNAME'

    - assert_defined "$TWINE_PASSWORD" 'Missing required env $TWINE_PASSWORD'

    - pip install -U twine setuptools

    - pip list

    - twine upload --verbose dist/*.tar.gz

    - twine upload --verbose dist/*.whl

    - _publish

  rules:

    # on tags with $TWINE_USERNAME set

    - if: '$TWINE_USERNAME && $CI_COMMIT_TAG'



# (on tag creation): generates the documentation

py-docs:

  extends: .python-base

  stage: publish

  script:

    - install_doc_requirements

    - pip install -U sphinx

    - run_python -m pip install -U sphinx

    - cd ${DOCS_DIRECTORY}

    - make ${DOCS_MAKE_ARGS}

  artifacts:

    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"

    paths:

      - $DOCS_BUILD_DIR

      - ${DOCS_DIRECTORY}/$DOCS_BUILD_DIR

  rules:

    # on tags with $DOCS_ENABLED set

    - if: '$DOCS_ENABLED == "true" && $CI_COMMIT_TAG'
@@ -624,9 +643,7 @@ py-release: 
    - git config --global user.email '$GITLAB_USER_EMAIL'

    - git config --global user.name '$GITLAB_USER_LOGIN'

    - git checkout -B $CI_BUILD_REF_NAME

    - pip install --upgrade bumpversion

    - release_args

    - bumpversion ${bumpversion_args}

    - _release

    - git_url_base=`echo ${CI_REPOSITORY_URL} | cut -d\@ -f2`

    - git push https://${RELEASE_USERNAME}:${RELEASE_ACCESS_TOKEN}@${git_url_base} --tags

    - git push https://${RELEASE_USERNAME}:${RELEASE_ACCESS_TOKEN}@${git_url_base} $CI_BUILD_REF_NAME