Commit 2b4a86eb authored by Aurelien's avatar Aurelien
Browse files

Merge branch '326-add-an-arg-to-the-builder-to-limit-the-number-of-job-to-build' into 'latest' 

Resolve "Add an arg to the builder to limit the number of job to build"

Closes #326

See merge request r2devops/hub!205
parents b001fe99 2624a176

tools/builder/Pipfile

+1 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ verify_ssl = true 
pyyaml = "==5.3.1"

jinja2 = "==2.11.2"

requests = "==2.25.0"

watchdog = "==2.0.2"



[requires]

python_version = "3"

tools/builder/Pipfile.lock

+50 −5
Original line number Diff line number Diff line 
{

    "_meta": {

        "hash": {

            "sha256": "a93aa223717a2f5950d587d63518f9caaa0645d8f22d0a3ffaa0371d03331010"

            "sha256": "627e9be33a8ed690f1bceddf67b4bd637411013cd943043370deffade2201aae"

        },

        "pipfile-spec": 6,

        "requires": {
@@ -35,6 +35,7 @@ 
                "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",

                "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==2.10"

        },

        "jinja2": {
@@ -52,8 +53,12 @@ 
                "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235",

                "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5",

                "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42",

                "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f",

                "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39",

                "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff",

                "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b",

                "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014",

                "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f",

                "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1",

                "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e",

                "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183",
@@ -62,25 +67,41 @@ 
                "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1",

                "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15",

                "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1",

                "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85",

                "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1",

                "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e",

                "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b",

                "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905",

                "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850",

                "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0",

                "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735",

                "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d",

                "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb",

                "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e",

                "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d",

                "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c",

                "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1",

                "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2",

                "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21",

                "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2",

                "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5",

                "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7",

                "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b",

                "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8",

                "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6",

                "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193",

                "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f",

                "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b",

                "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f",

                "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2",

                "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5",

                "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c",

                "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032",

                "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7",

                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be"

                "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be",

                "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621"

            ],

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",

            "version": "==1.1.1"

        },

        "pyyaml": {
@@ -112,10 +133,34 @@ 
        },

        "urllib3": {

            "hashes": [

                "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08",

                "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473"

                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",

                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"

            ],

            "version": "==1.26.2"

            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",

            "version": "==1.26.4"

        },

        "watchdog": {

            "hashes": [

                "sha256:035f4816daf3c62e03503c267620f3aa8fc7472df85ff3ef1e0c100ea1ed2744",

                "sha256:0f7e9de9ba84af15e9e9fc29c3b13c972daa4d2b11de29aa86b26a26bc877c06",

                "sha256:13c9ff58508dce55ba416eb0ef7af5aa5858558f2ec51112f099fd03503b670b",

                "sha256:19675b8d1f00dabe74a0e66d87980623250d9360a21612e8c27b70a4b214ceeb",

                "sha256:1cd715c4fb803581ded8943f39a51f21c17375d009ca9e3398d6b20638863a70",

                "sha256:1f518a6940cde8720b8826a705c164e6b9bd6cf8c00f14269ffac51e017e06ec",

                "sha256:3e933f3567c4521dd1a5d59fd54a522cae90bebcbeb8b74b84a2f33c90f08388",

                "sha256:41b1a773f364f232b5bc184688e8d60451745d9e0971ac60c648bd47be8f4733",

                "sha256:532fedd993e75554671faa36cd04c580ced3fae084254a779afbbd8aaf00566b",

                "sha256:74528772516228f6a015a647027057939ff0b695a0b864cb3037e8e1aabc7ca0",

                "sha256:89102465764e453609463cf620e744da1b0aa1f9f321b05961e2e7e15b3c9d8b",

                "sha256:a412b1914e27f67b0a10e1ee19b5d035a9f7c115a062bbbd640653d9820ba4c8",

                "sha256:ac6adbdf32e1d180574f9d0819e80259ae48e68727e80c3d950ed5a023714c3e",

                "sha256:adda34bfe6db05485c1dfcd98232bdec385f991fe16358750c2163473eefb985",

                "sha256:d2fcbc15772a82cd139c803a513c45b0fbc72a10a8a34dc2a8b429110b6f1236",

                "sha256:d54e187b76053982180532cb7fd31152201c438b348c456f699929f8a89e786d",

                "sha256:e0114e48ee981b38e328eaa0d5a625c7b4fc144b8dc7f7637749d6b5f7fefb0e"

            ],

            "index": "pypi",

            "version": "==2.0.2"

        }

    },

    "develop": {}

tools/builder/builder.py

+139 −56
Original line number Diff line number Diff line
@@ -19,21 +19,25 @@ 
import logging

import re

import sys

import time

from datetime import datetime

from distutils.version import LooseVersion

from os import listdir, makedirs

from shutil import copyfile

from urllib.parse import quote, urlencode

import argparse

import logging

import requests

from yaml import full_load, YAMLError

from jinja2 import Environment, FileSystemLoader, TemplateNotFound

import argparse



from watchdog.observers import Observer

from watchdog.events import RegexMatchingEventHandler



# Import the Config module and set the path to run the script from root project

# /!\ This instruction is only working if you run this script from the root of the project

sys.path.insert(0, "./")

from tools.utils.utils import Config



utils = Config()
@@ -61,6 +65,7 @@ def get_conf(job_path): 
        logging.error(error)

        sys.exit(1)





def get_description(job_path):

    """Fetch the README file from job
@@ -79,6 +84,7 @@ def get_description(job_path): 
        logging.error(error)

        sys.exit(1)





def get_changelogs(job_path, job_name):

    """Fetch the changelogs file from job
@@ -101,7 +107,8 @@ def get_changelogs(job_path, job_name): 
    logging.info("Parsing changelogs for job %s", job_name)

    try:

        for version in versions:

            with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION, encoding="utf-8") as changelog_file:

            with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION,

                      encoding="utf-8") as changelog_file:

                changelogs.append({

                    "version": version,

                    "url": utils.R2DEVOPS_URL + version + "/" + job_name + utils.JOBS_EXTENSION,
@@ -114,6 +121,7 @@ def get_changelogs(job_path, job_name): 


    return (latest, changelogs)





def get_license(license_name, copyright_holder):

    """Return the license file
@@ -126,7 +134,8 @@ def get_license(license_name, copyright_holder): 
    """

    logging.info("Getting licence %s", license_name)

    try:

        env = Environment(loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR))

        env = Environment(

            loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR))

        template = env.get_template(license_name + utils.MARKDOWN_EXTENSION + ".j2")

        license_content = template.render(

            year=datetime.now().year,
@@ -134,12 +143,14 @@ def get_license(license_name, copyright_holder): 
        ).split('\n')

        license_content = [line + '\n' for line in license_content]

    except TemplateNotFound as error:

        logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder)

        logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name,

                      copyright_holder)

        logging.error(error)

        sys.exit(1)



    return license_content





def get_screenshots(job_path, job_name):

    """Create the job directory for the job documentation

       Gets the jobs screenshots and copy them to the documentation directory
@@ -161,7 +172,8 @@ def get_screenshots(job_path, job_name): 


    logging.info("Getting screenshots for job %s", job_name)

    # Create screenshots folder in docs for the job

    makedirs(utils.MKDOCS_DIR+"/"+utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR,0o777,True)

    makedirs(utils.MKDOCS_DIR + "/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR,

             0o777, True)



    # Get all screenshots of the job

    regex = re.compile('(.png|.jpg|.jpeg)$')
@@ -170,10 +182,12 @@ def get_screenshots(job_path, job_name): 


    # Copy all screenshot of the job into screenshots folder for the doc

    for screenshot in screenshot_list:

        copyfile(job_path + "/" + utils.SCREENSHOTS_DIR+"/"+screenshot, utils.MKDOCS_DIR+ "/"+ utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR+"/"+screenshot)

        copyfile(job_path + "/" + utils.SCREENSHOTS_DIR + "/" + screenshot,

                 utils.MKDOCS_DIR + "/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR + "/" + screenshot)



    return ("/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR, screenshot_list)





def get_user(code_owner):

    """Fetch the job maintainer Gitlab user
@@ -204,6 +218,7 @@ def get_user(code_owner): 


    return None





def get_job_raw_content(job_name):

    """Return the raw content of a job
@@ -229,6 +244,7 @@ def get_job_raw_content(job_name): 
                      utils.JOBS_EXTENSION)

        sys.exit(1)





# https://docs.gitlab.com/ee/api/issues.html#list-project-issues (for the structure of the response)

def get_linked_issues(job_name, opened=True):

    """Get a list of linked issues for a job
@@ -272,6 +288,7 @@ def get_linked_issues(job_name, opened=True): 
    create_issue_url = f"{issues_base_url}/issues/new?{quote(create_issue_payload, safe='=')}%20-%20"

    return (linked_issues, linked_issues_url, create_issue_url)





def create_job_doc(job):

    """Create the Markdown documentation file for a job
@@ -293,7 +310,8 @@ def create_job_doc(job): 
    stage = conf.get("default_stage")



    if not code_owner or not license_name or not stage:

        logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, utils.JOB_METADATA_FILE)

        logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job,

                      utils.JOB_METADATA_FILE)

        sys.exit(1)



    utils.INDEX[stage]["content"].append(conf)
@@ -401,6 +419,7 @@ def add_placeholder(): 
                template = env.get_template(utils.TEMPLATE_PLACEHOLDER)

                file_handle.write(template.render())





def create_arrange_pages():

    """ Create arrange .pages for mkdocs to sort the list of stage in job page
@@ -424,6 +443,7 @@ def create_arrange_pages(): 
        logging.error(error)

        sys.exit(1)





def argparse_setup():

    """Setup argparse
@@ -432,9 +452,69 @@ def argparse_setup(): 
    obj

        Python object with arguments parsed

    """

    parser = argparse.ArgumentParser()

    parser = argparse.ArgumentParser(description="Build R2Devops jobs' documentation")



    parser.add_argument("--job", '-j',

                        type=str, help="Build a specific R2DevOps job's name")



    parser.add_argument("--watch", help="Put in watch mode",

                        action="store_true")



    return parser.parse_args()





def create_jobs_doc(arg_job=None):

    """Build all the jobs (or a single) doc

    """

    # Iterate over every directories in jobs directory to create their job.md for the documentation

    jobs = listdir(utils.JOBS_DIR)



    if arg_job and arg_job not in jobs:

        logging.error("Job %s not found", arg_job)

        sys.exit(1)



    if arg_job:

        create_job_doc(arg_job)

    else:

        for job in jobs:

            create_job_doc(job)





def run_watcher(watch_path):

    watched_jobs = {}



    def on_modified(event):

        path = event.src_path.split("/")

        job = path[len(path) - 2]



        reloaded = watched_jobs.get(job)



        if reloaded is None:

            watched_jobs[job] = 0



        watched_jobs[job] = watched_jobs[job] + 1

        if watched_jobs[job] == 2:

            watched_jobs[job] = 0

        else:

            return



        logging.info(f"New modification detected on {event.src_path}")

        create_jobs_doc(path[len(path) - 2])



    event_handler = RegexMatchingEventHandler(["^.*\.md$"], ignore_directories=False, case_sensitive=True)

    event_handler.on_modified = on_modified



    observer = Observer()

    observer.schedule(event_handler, watch_path, recursive=True)

    observer.start()

    try:

        while True:

            time.sleep(1)

    except KeyboardInterrupt:

        observer.stop()

    observer.join()





def main():

    """

    Main function, multiple-purpose:
@@ -456,10 +536,15 @@ def main(): 
        ]

    )



    # Iterate over every directories in jobs directory to create their job.md for the documentation

    jobs = listdir(utils.JOBS_DIR)

    for job in jobs:

        create_job_doc(job)

    watch_path = utils.JOBS_DIR

    if args.job:

        create_jobs_doc(args.job)

        watch_path = f"{watch_path}/{args.job}"

    else:

        create_jobs_doc()



    if args.watch:

        run_watcher(watch_path)



    # Verify that there is a .md file for every stage, or mkdocs will break

    add_placeholder()
@@ -472,11 +557,9 @@ def main(): 
    template = env.get_template(utils.TEMPLATE_INDEX)

    index_content = template.render(index=utils.INDEX)









    with open(utils.MKDOCS_DIR + "/" + utils.JOBS_DIR + "/" + utils.INDEX_FILE, "w+", encoding="utf-8") as index_file:

        index_file.write(index_content)





if __name__ == "__main__":

    main()