Commit b8fbe0f0 authored by Ruben ten Hove's avatar Ruben ten Hove
Browse files

Merge branch '64-refactor-extract-extras-discovery-from-python-pip-licenses' into 'main' 

feat: generalized python package extras discovery

Closes #64

See merge request just-ci/templates!125
parents ff489f56 2d9c9ec0

python/extras-discovery.yml

0 → 100644
+27 −0
Original line number Diff line number Diff line 
---

include:

  - local: python/generic.yml

  - local: python/package-discovery.yml



python:extras-discovery:

  extends: .python:pre

  variables:

    PIP_PACKAGE_INSTALL_EXTRA_ARGS: --no-deps

  stage: build

  script:

    - !reference [".python:pre-install", script]

    - |

      printf "PYTHON_PACKAGE_EXTRAS=" > ${CI_PROJECT_DIR}/package-extras.env

      printf "[*] Auto discovered package extras: "

      python -c "from importlib.metadata import metadata; print(metadata('${PYTHON_PACKAGE}').get_all('Provides-Extra', []))" | tr -d "[]' " | tee -a ${CI_PROJECT_DIR}/package-extras.env

      echo "[*] If this is not correct, provide the package extras variable PYTHON_PACKAGE_EXTRAS=<package_extra_1>,<package_extra_2> in your .gitlab-ci.yml"

  needs:

    - job: python:package-discovery

      optional: true

  artifacts:

    reports:

      dotenv: package-extras.env

  rules:

    - if: $PYTHON_PACKAGE_EXTRAS == "*"

      when: always

    - when: never

python/install-extras.yml

0 → 100644
+24 −0
Original line number Diff line number Diff line 
---

include:

  - local: python/extras-discovery.yml

  - local: python/generic.yml



variables:

  PYTHON_PACKAGE_EXTRAS: "" # Comma-separated list of extras that are to be installed. If "*", then this variable is populated by python:extras-discovery.



.python:install-extras:

  extends:

    - .python:pre

  script:

    - |

      if [ -z "${PYTHON_PACKAGE_EXTRAS}" ]; then

        echo "[*] No extras selected."

      else

        echo "[*] The following extras are selected for installation with package ${PYTHON_PACKAGE}: ${PYTHON_PACKAGE_EXTRAS}"

        PIP_PACKAGE_INSTALL_EXTRA_ARGS="${PIP_PACKAGE_INSTALL_EXTRA_ARGS} .[${PYTHON_PACKAGE_EXTRAS}]"

        echo "[*] Extended PIP_PACKAGE_INSTALL_EXTRA_ARGS with .[${PYTHON_PACKAGE_EXTRAS}]"

      fi

    - !reference [".python:pre-install", script]

  needs:

    - job: python:extras-discovery

      optional: true

python/pip-licenses.yml

+6 −25
Original line number Diff line number Diff line
@@ -3,38 +3,19 @@ 
# https://pypi.org/project/pip-licenses/

include:

  - local: python/generic.yml

  - local: python/install-extras.yml



variables:

  PIP_LICENSES_ALLOWED: "" # Empty means all. Semicolon separated list. See https://pypi.org/project/pip-licenses/#option-allow-only

  PIP_LICENSES_SKIP_EXTRAS: "" # Comma-separated list of extras that are to be skipped



python:pip-licenses:

  extends:

    - .python:pre

    - .python:pre-install

  extends: .python:install-extras

  variables:

    JOB_PACKAGE: pip-licenses importlib_metadata

    PIP_PACKAGE_INSTALL_EXTRA_ARGS: --no-deps

    JOB_PACKAGE: pip-licenses

  script:

    - !reference [".python:pre", script]

    - !reference [".python:pre-install", script]

    # Extract package extras from metadata

    - EXTRAS=$(python -c "from importlib_metadata import metadata;

      print(','.join((extra for extra in

      metadata('${PYTHON_PACKAGE}').get_all('Provides-Extra', []) if not extra

      in '${PIP_LICENSES_SKIP_EXTRAS}'.split(','))), end='')")

    # work in venv

    # isolate installation to ignore licenses of pre-installed packages

    - python3 -m venv .venv

    - source .venv/bin/activate

    - pip3 install pip-licenses

    # if any extra dependencies are found, install them too

    - |

      if [ -z "${EXTRAS}" ]; then

        echo "[*] Installing ${PYTHON_PACKAGE} without extras."

        pip3 install .

      else

        echo "[*] Installing ${PYTHON_PACKAGE} with extras: ${EXTRAS}"

        pip3 install .[${EXTRAS}]

      fi

    # perform test

    - !reference [".python:pre", script]

    - !reference [".python:install-extras", script]

    - pip-licenses --from=mixed --allow-only="${PIP_LICENSES_ALLOWED}"

python/pylic.yml

+6 −32
Original line number Diff line number Diff line 
---

# Check the package repo on how to configure this job

# Check the package page on how to configure this job

# https://github.com/sandrochuber/pylic

include:

  - local: python/generic.yml



variables:

  PYLIC_SKIP_EXTRAS: "" # Comma-separated list of extras that are to be skipped

  - local: python/install-extras.yml



python:pylic:

  extends:

    - .python:pre

    - .python:pre-install

  extends: .python:install-extras

  variables:

    INSTALL_USER_PACKAGE: "true"

    JOB_PACKAGE: importlib_metadata

    PIP_PACKAGE_INSTALL_EXTRA_ARGS: --no-deps

    JOB_PACKAGE: pip-licenses

  script:

    # work in venv

    # isolate installation to ignore licenses of pre-installed packages

    - python3 -m venv .venv

    - source .venv/bin/activate

    - !reference [".python:pre", script]

    - !reference [".python:pre-install", script]

    # Extract package extras from metadata

    - EXTRAS=$(python -c "from importlib_metadata import metadata;

      print(','.join((extra for extra in

      metadata('${PYTHON_PACKAGE}').get_all('Provides-Extra', []) if not extra

      in '${PYLIC_SKIP_EXTRAS}'.split(','))), end='')")

    # work in venv

    - python3 -m venv .venv

    - source .venv/bin/activate

    - pip3 install pylic

    # if any extra dependencies are found, install them too

    - |

      if [ -z "${EXTRAS}" ]; then

        echo "[*] Installing package without extras."

        pip3 install .

      else

        echo "[*] Installing package with extras: ${EXTRAS}"

        pip3 install .[${EXTRAS}]

      fi

    - echo "[*] The following packages, with their respective version, will be

      checked:"

    - !reference [".python:install-extras", script]

    - pylic list

    - pylic check