Commit a50195c5 authored by Thomas Boni's avatar Thomas Boni
Browse files

Merge branch '204-add-a-list-of-linked-issues-to-the-jobs-in-their-documentation' into latest

parents 0c307fa6 0e1e3bde

.gitlab-ci.yml

+39 −16
Original line number Diff line number Diff line 
stages:

  - static_tests

  - project_setup

  - build

  - deployment

  - deploy



include:

  - remote: 'https://jobs.r2devops.io/1.2.0/mkdocs.yml?scope=0.yml'

  - remote: 'https://jobs.r2devops.io/0.2.0/pages.yml?scope=0.yml'



mkdocs:

  variables:

    PIPENV_PIPFILE: tools/builder/Pipfile

  before_script:

    - pip install pipenv && pipenv install

    - pipenv run python tools/builder/builder.py

    - pip3 install mkdocs-awesome-pages-plugin==2.4.0 # temporary, it will be removed after #202



pages:

  rules:

    - if: '$CI_COMMIT_BRANCH == "latest"'

  - remote: 'https://jobs.r2devops.io/1.4.0/mkdocs.yml?scope=0.yml'

  - remote: 'https://jobs.r2devops.io/0.3.0/pages.yml?scope=0.yml'



ci_linter:

  image: golang:1.15.2-alpine3.12
@@ -135,13 +124,43 @@ scan_images_dep: 
    expire_in: 30 days

    when: always





# See https://docs.gitlab.com/ee/api/labels.html

job_labels:

  image: python:3.9.1-alpine

  stage: project_setup

  variables:

    PIPENV_PIPFILE: tools/job_labels/Pipfile

    JOB_LOGFILE: "job_labels.log"

  before_script:

    - pip install --ignore-installed distlib pipenv

    - pipenv install

  script:

    - pipenv run python3 tools/job_labels/job_labels.py

  artifacts:

    paths:

      - ${JOB_LOGFILE}

    expire_in: 30 days

    when: always

  rules:

    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH_SLUG == "r2devops/hub"'





mkdocs:

  variables:

    PIPENV_PIPFILE: tools/builder/Pipfile

  before_script:

    - pip install pipenv && pipenv install

    - pipenv run python tools/builder/builder.py





# See https://docs.gitlab.com/ee/api/releases/

# We can only control the link to the hub, the release is still storing the source code

release:

  stage: deploy

  image:

    name: alpine/httpie:2.2.0

    entrypoint: [""]

  stage: deployment

  variables:

    PIPENV_PIPFILE: tools/notify/Pipfile

  before_script:
@@ -180,3 +199,7 @@ release: 
    done

  rules:

    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'



pages:

  rules:

    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'

Pipfile

+1 −0
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ mkdocs-git-revision-date-localized-plugin = "==0.7.2" 
mkdocs-material = "==6.1.5"

mkdocs-material-extensions = "==1.0.1"

mkdocs-minify-plugin = "==0.3.0"

requests = "=2.25.0"



[requires]

python_version = "3"

Pipfile.lock

+70 −16
Original line number Diff line number Diff line 
{

    "_meta": {

        "hash": {

            "sha256": "0d87b85e161b299977203ec926d3886315e0919718d7172acf16588a4693c853"

            "sha256": "cb5b1b484f0e405938fa20105ab8dadcedb7244d9b1aaafe34ea1c591ed42ae0"

        },

        "pipfile-spec": 6,

        "requires": {
@@ -21,19 +21,37 @@ 
                "sha256:9d35c22fcc79893c3ecc85ac4a56cde1ecf3f19c540bba0922308a6c06ca6fa5",

                "sha256:da031ab54472314f210b0adcff1588ee5d1d1d0ba4dbd07b94dba82bde791e05"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==2.9.0"

        },

        "certifi": {

            "hashes": [

                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",

                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"

            ],

            "version": "==2020.12.5"

        },

        "chardet": {

            "hashes": [

                "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",

                "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",

            "version": "==4.0.0"

        },

        "click": {

            "hashes": [

                "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",

                "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",

            "version": "==7.1.2"

        },

        "future": {

            "hashes": [

                "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"

            ],

            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==0.18.2"

        },

        "gitdb": {
@@ -41,14 +59,16 @@ 
                "sha256:91f36bfb1ab7949b3b40e23736db18231bf7593edada2ba5c3a174a7b23657ac",

                "sha256:c9e1f2d0db7ddb9a704c2a0217be31214e91a4fe1dea1efad19ae42ba0c285c9"

            ],

            "markers": "python_version >= '3.4'",

            "version": "==4.0.5"

        },

        "gitpython": {

            "hashes": [

                "sha256:6eea89b655917b500437e9668e4a12eabdcf00229a0df1762aabd692ef9b746b",

                "sha256:befa4d101f91bad1b632df4308ec64555db684c360bd7d2130b4807d49ce86b8"

                "sha256:42dbefd8d9e2576c496ed0059f3103dcef7125b9ce16f9d5f9c834aed44a1dac",

                "sha256:867ec3dfb126aac0f8296b19fb63b8c4a399f32b4b6fafe84c4b10af5fa9f7b5"

            ],

            "version": "==3.1.11"

            "markers": "python_version >= '3.4'",

            "version": "==3.1.12"

        },

        "htmlmin": {

            "hashes": [
@@ -56,19 +76,29 @@ 
            ],

            "version": "==0.1.12"

        },

        "idna": {

            "hashes": [

                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",

                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==2.10"

        },

        "jinja2": {

            "hashes": [

                "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0",

                "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",

            "version": "==2.11.2"

        },

        "joblib": {

            "hashes": [

                "sha256:698c311779f347cf6b7e6b8a39bb682277b8ee4aba8cf9507bc0cf4cd4737b72",

                "sha256:9e284edd6be6b71883a63c9b7f124738a3c16195513ad940eae7e3438de885d5"

                "sha256:75ead23f13484a2a414874779d69ade40d4fa1abe62b222a23cd50d4bc822f6f",

                "sha256:7ad866067ac1fdec27d51c8678ea760601b70e32ff1881d4dc8e1171f2b64b24"

            ],

            "version": "==0.17.0"

            "markers": "python_version >= '3.6'",

            "version": "==1.0.0"

        },

        "jsmin": {

            "hashes": [
@@ -97,6 +127,7 @@ 
                "sha256:5d9f2b5ca24bc4c7a390d22323ca4bad200368612b5aaa7796babf971d2b2f18",

                "sha256:c109c15b7dc20a9ac454c9e6025927d44460b85bd039da028d85e2b6d0bcc328"

            ],

            "markers": "python_version >= '3.6'",

            "version": "==3.3.3"

        },

        "markupsafe": {
@@ -135,6 +166,7 @@ 
                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",

                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==1.1.1"

        },

        "mkdocs": {
@@ -142,6 +174,7 @@ 
                "sha256:096f52ff52c02c7e90332d2e53da862fde5c062086e1b5356a6e392d5d60f5e9",

                "sha256:f0b61e5402b99d7789efa032c7a74c90a20220a9c81749da06dbfbcbd52ffb39"

            ],

            "markers": "python_version >= '3.5'",

            "version": "==1.1.2"

        },

        "mkdocs-awesome-pages-plugin": {
@@ -192,10 +225,11 @@ 
        },

        "pygments": {

            "hashes": [

                "sha256:ccf3acacf3782cbed4a989426012f1c535c9a90d3a7fc3f16d231b9372d2b716",

                "sha256:f275b6c0909e5dafd2d6269a656aa90fa58ebf4a74f8fcf9053195d226b24a08"

                "sha256:bc9591213a8f0e0ca1a5e68a479b4887fdc3e75d0774e5c71c31920c427de435",

                "sha256:df49d09b498e83c1a73128295860250b0b7edd4c723a32e9bc0d295c7c2ec337"

            ],

            "version": "==2.7.3"

            "markers": "python_version >= '3.5'",

            "version": "==2.7.4"

        },

        "pymdown-extensions": {

            "hashes": [
@@ -207,10 +241,10 @@ 
        },

        "pytz": {

            "hashes": [

                "sha256:3e6b7dd2d1e0a59084bcee14a17af60c5c562cdc16d828e8eba2e683d3a7e268",

                "sha256:5c55e189b682d420be27c6995ba6edce0c0a77dd67bfbe2ae6607134d5851ffd"

                "sha256:16962c5fb8db4a8f63a26646d8886e9d769b6c511543557bc84e9569fb9a9cb4",

                "sha256:180befebb1927b16f6b57101720075a984c019ac16b1b7575673bea42c6c3da5"

            ],

            "version": "==2020.4"

            "version": "==2020.5"

        },

        "pyyaml": {

            "hashes": [
@@ -276,11 +310,20 @@ 
            ],

            "version": "==2020.11.13"

        },

        "requests": {

            "hashes": [

                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",

                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"

            ],

            "index": "pypi",

            "version": "==2.25.1"

        },

        "six": {

            "hashes": [

                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",

                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==1.15.0"

        },

        "smmap": {
@@ -288,6 +331,7 @@ 
                "sha256:54c44c197c819d5ef1991799a7e30b662d1e520f2ac75c9efbeb54a742214cf4",

                "sha256:9c98bbd1f9786d22f14b3d4126894d56befb835ec90cef151af566c7e19b5d24"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==3.0.4"

        },

        "tornado": {
@@ -334,14 +378,24 @@ 
                "sha256:fa2ba70284fa42c2a5ecb35e322e68823288a4251f9ba9cc77be04ae15eada68",

                "sha256:fba85b6cd9c39be262fcd23865652920832b61583de2a2ca907dbd8e8a8c81e5"

            ],

            "markers": "python_version >= '3.5'",

            "version": "==6.1"

        },

        "tqdm": {

            "hashes": [

                "sha256:38b658a3e4ecf9b4f6f8ff75ca16221ae3378b2e175d846b6b33ea3a20852cf5",

                "sha256:d4f413aecb61c9779888c64ddf0c62910ad56dcbe857d8922bb505d4dbff0df1"

                "sha256:4621f6823bab46a9cc33d48105753ccbea671b68bab2c50a9f0be23d4065cb5a",

                "sha256:fe3d08dd00a526850568d542ff9de9bbc2a09a791da3c334f3213d8d0bbbca65"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==4.56.0"

        },

        "urllib3": {

            "hashes": [

                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",

                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"

            ],

            "version": "==4.54.1"

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",

            "version": "==1.26.2"

        }

    },

    "develop": {}

docs/css/extra.css

+2 −2
Original line number Diff line number Diff line
@@ -237,11 +237,11 @@ 
  padding-left: 5px;

}



button.md-button-center {

a.md-button-center {

    display:block !important;

    margin:0 auto !important;

    padding-bottom:1em !important;

    padding-top:0.8em !important

    padding-top:0.8em !important;

}



button.md-button-center:hover {

tools/builder/builder.py

+53 −2
Original line number Diff line number Diff line
@@ -21,6 +21,7 @@ from datetime import datetime 
from distutils.version import LooseVersion

from os import listdir, makedirs

from shutil import copyfile

from urllib.parse import quote, urlencode

import requests

from yaml import full_load, YAMLError

from jinja2 import Environment, FileSystemLoader, TemplateNotFound
@@ -41,9 +42,15 @@ MARKDOWN_EXTENSION = ".md" 
MKDOCS_DIR_JOBS_IMAGES = "images/jobs"

# Directory name to use for the jobs screenshot

SCREENSHOTS_DIR = "screenshots"

ISSUES_LIMIT = 5



# Requests variable

GITLAB_BASE_URL = "https://gitlab.com/"

GITLAB_API_URL = "https://gitlab.com/api/v4/"

R2DEVOPS_URL = "https://jobs.r2devops.io/"

JOB_TOKEN = getenv("API_TOKEN")

PROJECT_NAME = "r2devops/hub"

JOBS_SCOPE_LABEL = "Jobs::"



# Templates variables

BUILDER_DIR = "tools/builder"
@@ -250,6 +257,47 @@ def get_job_raw_content(job_name): 
                                                    JOBS_EXTENSION)

        sys.exit(1)



# https://docs.gitlab.com/ee/api/issues.html#list-project-issues (for the structure of the response)

def get_linked_issues(job_name, opened=True):

    """Get a list of linked issues for a job



    Parameters:

    -----------

    job_name : str

        The name of the job

    opened : boolean

        If we get only opened issues or all of them (default: True)



    Returns:

    --------

    list

        A list of issues linked to the job with their name and url

    str

        Url to list the issues related to the job

    str

        Url to create a new issue for the job

    """

    linked_issues = []

    base_url = f"{GITLAB_API_URL}/projects/{quote(PROJECT_NAME, safe='')}/issues"

    url = f"{base_url}?labels={JOBS_SCOPE_LABEL}{job_name}"

    if opened:

        url += "&state=opened"

    r = requests.get(url)



    for issue in r.json():

        linked_issues.append({

            "name": issue['title'],

            "url": issue['web_url'],

            "iid": issue['iid']

        })

    issues_base_url = f"{GITLAB_BASE_URL}/{PROJECT_NAME}"

    linked_issues_payload = {

        "label_name": f"{JOBS_SCOPE_LABEL}{job_name}"

    }

    linked_issues_url = f"{issues_base_url}/issues?{urlencode(linked_issues_payload)}"

    create_issue_payload = f"issue[title]=[job][{job_name}]"

    create_issue_url = f"{issues_base_url}/issues/new?{quote(create_issue_payload, safe='=')}%20-%20"

    return (linked_issues, linked_issues_url, create_issue_url)



def create_job_doc(job):

    job_path = JOBS_DIR + "/" + job
@@ -310,7 +358,11 @@ def create_job_doc(job): 
                code_owner_url = user["web_url"],

                screenshot_path = screenshot_path,

                screenshots_files = screenshots_files,

                job_raw_content = ''.join(job_raw_content)

                job_raw_content = ''.join(job_raw_content),

                linked_issues = linked_issues,

                linked_issues_limit = ISSUES_LIMIT,

                linked_issues_url = linked_issues_url,

                create_issue_url = create_issue_url

        ))

    except Exception as error:

        logging.error("Failed to create final file for job %s", job)
@@ -338,7 +390,6 @@ def main(): 


    # Iterate over every directories in jobs directory to create their job.md for the documentation

    jobs = listdir(JOBS_DIR)



    for job in jobs:

        create_job_doc(job)