Loading tools/builder/Pipfile +1 −0 Original line number Diff line number Diff line Loading @@ -9,6 +9,7 @@ verify_ssl = true pyyaml = "==5.3.1" jinja2 = "==2.11.2" requests = "==2.25.0" watchdog = "==2.0.2" [requires] python_version = "3" tools/builder/Pipfile.lock +50 −5 Original line number Diff line number Diff line { "_meta": { "hash": { "sha256": "a93aa223717a2f5950d587d63518f9caaa0645d8f22d0a3ffaa0371d03331010" "sha256": "627e9be33a8ed690f1bceddf67b4bd637411013cd943043370deffade2201aae" }, "pipfile-spec": 6, "requires": { Loading Loading @@ -35,6 +35,7 @@ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, "jinja2": { Loading @@ -52,8 +53,12 @@ "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", Loading @@ -62,25 +67,41 @@ "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" }, "pyyaml": { Loading Loading @@ -112,10 +133,34 @@ }, "urllib3": { "hashes": [ "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], "version": "==1.26.2" "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.4" }, "watchdog": { "hashes": [ "sha256:035f4816daf3c62e03503c267620f3aa8fc7472df85ff3ef1e0c100ea1ed2744", "sha256:0f7e9de9ba84af15e9e9fc29c3b13c972daa4d2b11de29aa86b26a26bc877c06", "sha256:13c9ff58508dce55ba416eb0ef7af5aa5858558f2ec51112f099fd03503b670b", "sha256:19675b8d1f00dabe74a0e66d87980623250d9360a21612e8c27b70a4b214ceeb", "sha256:1cd715c4fb803581ded8943f39a51f21c17375d009ca9e3398d6b20638863a70", "sha256:1f518a6940cde8720b8826a705c164e6b9bd6cf8c00f14269ffac51e017e06ec", "sha256:3e933f3567c4521dd1a5d59fd54a522cae90bebcbeb8b74b84a2f33c90f08388", "sha256:41b1a773f364f232b5bc184688e8d60451745d9e0971ac60c648bd47be8f4733", "sha256:532fedd993e75554671faa36cd04c580ced3fae084254a779afbbd8aaf00566b", "sha256:74528772516228f6a015a647027057939ff0b695a0b864cb3037e8e1aabc7ca0", "sha256:89102465764e453609463cf620e744da1b0aa1f9f321b05961e2e7e15b3c9d8b", "sha256:a412b1914e27f67b0a10e1ee19b5d035a9f7c115a062bbbd640653d9820ba4c8", "sha256:ac6adbdf32e1d180574f9d0819e80259ae48e68727e80c3d950ed5a023714c3e", "sha256:adda34bfe6db05485c1dfcd98232bdec385f991fe16358750c2163473eefb985", "sha256:d2fcbc15772a82cd139c803a513c45b0fbc72a10a8a34dc2a8b429110b6f1236", "sha256:d54e187b76053982180532cb7fd31152201c438b348c456f699929f8a89e786d", "sha256:e0114e48ee981b38e328eaa0d5a625c7b4fc144b8dc7f7637749d6b5f7fefb0e" ], "index": "pypi", "version": "==2.0.2" } }, "develop": {} Loading tools/builder/builder.py +116 −63 Original line number Diff line number Diff line Loading @@ -19,21 +19,25 @@ import logging import re import sys import time from datetime import datetime from distutils.version import LooseVersion from os import listdir, makedirs from shutil import copyfile from urllib.parse import quote, urlencode import argparse import logging import requests from yaml import full_load, YAMLError from jinja2 import Environment, FileSystemLoader, TemplateNotFound import argparse from watchdog.observers import Observer from watchdog.events import RegexMatchingEventHandler # Import the Config module and set the path to run the script from root project # /!\ This instruction is only working if you run this script from the root of the project sys.path.insert(0, "./") from tools.utils.utils import Config utils = Config() Loading Loading @@ -61,6 +65,7 @@ def get_conf(job_path): logging.error(error) sys.exit(1) def get_description(job_path): """Fetch the README file from job Loading @@ -79,6 +84,7 @@ def get_description(job_path): logging.error(error) sys.exit(1) def get_changelogs(job_path, job_name): """Fetch the changelogs file from job Loading @@ -101,7 +107,8 @@ def get_changelogs(job_path, job_name): logging.info("Parsing changelogs for job %s", job_name) try: for version in versions: with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION, encoding="utf-8") as changelog_file: with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION, encoding="utf-8") as changelog_file: changelogs.append({ "version": version, "url": utils.R2DEVOPS_URL + version + "/" + job_name + utils.JOBS_EXTENSION, Loading @@ -114,6 +121,7 @@ def get_changelogs(job_path, job_name): return (latest, changelogs) def get_license(license_name, copyright_holder): """Return the license file Loading @@ -126,7 +134,8 @@ def get_license(license_name, copyright_holder): """ logging.info("Getting licence %s", license_name) try: env = Environment(loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR)) env = Environment( loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR)) template = env.get_template(license_name + utils.MARKDOWN_EXTENSION + ".j2") license_content = template.render( year=datetime.now().year, Loading @@ -134,12 +143,14 @@ def get_license(license_name, copyright_holder): ).split('\n') license_content = [line + '\n' for line in license_content] except TemplateNotFound as error: logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder) logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder) logging.error(error) sys.exit(1) return license_content def get_screenshots(job_path, job_name): """Create the job directory for the job documentation Gets the jobs screenshots and copy them to the documentation directory Loading @@ -161,7 +172,8 @@ def get_screenshots(job_path, job_name): logging.info("Getting screenshots for job %s", job_name) # Create screenshots folder in docs for the job makedirs(utils.MKDOCS_DIR+"/"+utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR,0o777,True) makedirs(utils.MKDOCS_DIR + "/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR, 0o777, True) # Get all screenshots of the job regex = re.compile('(.png|.jpg|.jpeg)$') Loading @@ -170,10 +182,12 @@ def get_screenshots(job_path, job_name): # Copy all screenshot of the job into screenshots folder for the doc for screenshot in screenshot_list: copyfile(job_path + "/" + utils.SCREENSHOTS_DIR+"/"+screenshot, utils.MKDOCS_DIR+ "/"+ utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR+"/"+screenshot) copyfile(job_path + "/" + utils.SCREENSHOTS_DIR + "/" + screenshot, utils.MKDOCS_DIR + "/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR + "/" + screenshot) return ("/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR, screenshot_list) def get_user(code_owner): """Fetch the job maintainer Gitlab user Loading Loading @@ -204,6 +218,7 @@ def get_user(code_owner): return None def get_job_raw_content(job_name): """Return the raw content of a job Loading @@ -229,6 +244,7 @@ def get_job_raw_content(job_name): utils.JOBS_EXTENSION) sys.exit(1) # https://docs.gitlab.com/ee/api/issues.html#list-project-issues (for the structure of the response) def get_linked_issues(job_name, opened=True): """Get a list of linked issues for a job Loading Loading @@ -272,6 +288,7 @@ def get_linked_issues(job_name, opened=True): create_issue_url = f"{issues_base_url}/issues/new?{quote(create_issue_payload, safe='=')}%20-%20" return (linked_issues, linked_issues_url, create_issue_url) def create_job_doc(job): """Create the Markdown documentation file for a job Loading @@ -293,7 +310,8 @@ def create_job_doc(job): stage = conf.get("default_stage") if not code_owner or not license_name or not stage: logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, utils.JOB_METADATA_FILE) logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, utils.JOB_METADATA_FILE) sys.exit(1) utils.INDEX[stage]["content"].append(conf) Loading Loading @@ -401,6 +419,7 @@ def add_placeholder(): template = env.get_template(utils.TEMPLATE_PLACEHOLDER) file_handle.write(template.render()) def create_arrange_pages(): """ Create arrange .pages for mkdocs to sort the list of stage in job page Loading @@ -424,6 +443,7 @@ def create_arrange_pages(): logging.error(error) sys.exit(1) def argparse_setup(): """Setup argparse Loading @@ -437,8 +457,46 @@ def argparse_setup(): parser.add_argument("--job", '-j', type=str, help="Build a specific R2DevOps job's name") parser.add_argument("--watch", help="Put in watch mode", action="store_true") return parser.parse_args() def create_jobs_doc(arg_job=None): """Build all the jobs (or a single) doc """ # Iterate over every directories in jobs directory to create their job.md for the documentation jobs = listdir(utils.JOBS_DIR) if arg_job and arg_job not in jobs: logging.error("Job %s not found", arg_job) sys.exit(1) if arg_job: create_job_doc(arg_job) else: for job in jobs: create_job_doc(job) def run_watcher(watch_path): def on_modified(event): logging.info(f"New modification detected on {event.src_path}") create_jobs_doc(event.src_path.split("/")[1]) event_handler = RegexMatchingEventHandler(["^.*\.md$"], ignore_directories=False, case_sensitive=True) event_handler.on_modified = on_modified observer = Observer() observer.schedule(event_handler, watch_path, recursive=True) observer.start() try: while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join() def main(): """ Main function, multiple-purpose: Loading @@ -460,18 +518,15 @@ def main(): ] ) # Iterate over every directories in jobs directory to create their job.md for the documentation jobs = listdir(utils.JOBS_DIR) if args.job and args.job not in jobs: logging.error("Job %s not found", args.job) sys.exit(1) elif args.job: create_job_doc(args.job) watch_path = utils.JOBS_DIR if args.job: create_jobs_doc(args.job) watch_path = f"{watch_path}/{args.job}" else: for job in jobs: create_job_doc(job) create_jobs_doc() if args.watch: run_watcher(watch_path) # Verify that there is a .md file for every stage, or mkdocs will break add_placeholder() Loading @@ -484,11 +539,9 @@ def main(): template = env.get_template(utils.TEMPLATE_INDEX) index_content = template.render(index=utils.INDEX) with open(utils.MKDOCS_DIR + "/" + utils.JOBS_DIR + "/" + utils.INDEX_FILE, "w+", encoding="utf-8") as index_file: index_file.write(index_content) if __name__ == "__main__": main() Loading
tools/builder/Pipfile +1 −0 Original line number Diff line number Diff line Loading @@ -9,6 +9,7 @@ verify_ssl = true pyyaml = "==5.3.1" jinja2 = "==2.11.2" requests = "==2.25.0" watchdog = "==2.0.2" [requires] python_version = "3"
tools/builder/Pipfile.lock +50 −5 Original line number Diff line number Diff line { "_meta": { "hash": { "sha256": "a93aa223717a2f5950d587d63518f9caaa0645d8f22d0a3ffaa0371d03331010" "sha256": "627e9be33a8ed690f1bceddf67b4bd637411013cd943043370deffade2201aae" }, "pipfile-spec": 6, "requires": { Loading Loading @@ -35,6 +35,7 @@ "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.10" }, "jinja2": { Loading @@ -52,8 +53,12 @@ "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", "sha256:195d7d2c4fbb0ee8139a6cf67194f3973a6b3042d742ebe0a9ed36d8b6f0c07f", "sha256:22c178a091fc6630d0d045bdb5992d2dfe14e3259760e713c490da5323866c39", "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", "sha256:2beec1e0de6924ea551859edb9e7679da6e4870d32cb766240ce17e0a0ba2014", "sha256:3b8a6499709d29c2e2399569d96719a1b21dcd94410a586a18526b143ec8470f", "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", Loading @@ -62,25 +67,41 @@ "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", "sha256:6f1e273a344928347c1290119b493a1f0303c52f5a5eae5f16d74f48c15d4a85", "sha256:6fffc775d90dcc9aed1b89219549b329a9250d918fd0b8fa8d93d154918422e1", "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", "sha256:7fed13866cf14bba33e7176717346713881f56d9d2bcebab207f7a036f41b850", "sha256:84dee80c15f1b560d55bcfe6d47b27d070b4681c699c572af2e3c7cc90a3b8e0", "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", "sha256:98bae9582248d6cf62321dcb52aaf5d9adf0bad3b40582925ef7c7f0ed85fceb", "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", "sha256:a6a744282b7718a2a62d2ed9d993cad6f5f585605ad352c11de459f4108df0a1", "sha256:acf08ac40292838b3cbbb06cfe9b2cb9ec78fce8baca31ddb87aaac2e2dc3bc2", "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", "sha256:b1dba4527182c95a0db8b6060cc98ac49b9e2f5e64320e2b56e47cb2831978c7", "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", "sha256:b7d644ddb4dbd407d31ffb699f1d140bc35478da613b441c582aeb7c43838dd8", "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", "sha256:bf5aa3cbcfdf57fa2ee9cd1822c862ef23037f5c832ad09cfea57fa846dec193", "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", "sha256:caabedc8323f1e93231b52fc32bdcde6db817623d33e100708d9a68e1f53b26b", "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", "sha256:d53bc011414228441014aa71dbec320c66468c1030aae3a6e29778a3382d96e5", "sha256:d73a845f227b0bfe8a7455ee623525ee656a9e2e749e4742706d80a6065d5e2c", "sha256:d9be0ba6c527163cbed5e0857c451fcd092ce83947944d6c14bc95441203f032", "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be", "sha256:feb7b34d6325451ef96bc0e36e1a6c0c1c64bc1fbec4b854f4529e51887b1621" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.1" }, "pyyaml": { Loading Loading @@ -112,10 +133,34 @@ }, "urllib3": { "hashes": [ "sha256:19188f96923873c92ccb987120ec4acaa12f0461fa9ce5d3d0772bc965a39e08", "sha256:d8ff90d979214d7b4f8ce956e80f4028fc6860e4431f731ea4a8c08f23f99473" "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], "version": "==1.26.2" "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", "version": "==1.26.4" }, "watchdog": { "hashes": [ "sha256:035f4816daf3c62e03503c267620f3aa8fc7472df85ff3ef1e0c100ea1ed2744", "sha256:0f7e9de9ba84af15e9e9fc29c3b13c972daa4d2b11de29aa86b26a26bc877c06", "sha256:13c9ff58508dce55ba416eb0ef7af5aa5858558f2ec51112f099fd03503b670b", "sha256:19675b8d1f00dabe74a0e66d87980623250d9360a21612e8c27b70a4b214ceeb", "sha256:1cd715c4fb803581ded8943f39a51f21c17375d009ca9e3398d6b20638863a70", "sha256:1f518a6940cde8720b8826a705c164e6b9bd6cf8c00f14269ffac51e017e06ec", "sha256:3e933f3567c4521dd1a5d59fd54a522cae90bebcbeb8b74b84a2f33c90f08388", "sha256:41b1a773f364f232b5bc184688e8d60451745d9e0971ac60c648bd47be8f4733", "sha256:532fedd993e75554671faa36cd04c580ced3fae084254a779afbbd8aaf00566b", "sha256:74528772516228f6a015a647027057939ff0b695a0b864cb3037e8e1aabc7ca0", "sha256:89102465764e453609463cf620e744da1b0aa1f9f321b05961e2e7e15b3c9d8b", "sha256:a412b1914e27f67b0a10e1ee19b5d035a9f7c115a062bbbd640653d9820ba4c8", "sha256:ac6adbdf32e1d180574f9d0819e80259ae48e68727e80c3d950ed5a023714c3e", "sha256:adda34bfe6db05485c1dfcd98232bdec385f991fe16358750c2163473eefb985", "sha256:d2fcbc15772a82cd139c803a513c45b0fbc72a10a8a34dc2a8b429110b6f1236", "sha256:d54e187b76053982180532cb7fd31152201c438b348c456f699929f8a89e786d", "sha256:e0114e48ee981b38e328eaa0d5a625c7b4fc144b8dc7f7637749d6b5f7fefb0e" ], "index": "pypi", "version": "==2.0.2" } }, "develop": {} Loading
tools/builder/builder.py +116 −63 Original line number Diff line number Diff line Loading @@ -19,21 +19,25 @@ import logging import re import sys import time from datetime import datetime from distutils.version import LooseVersion from os import listdir, makedirs from shutil import copyfile from urllib.parse import quote, urlencode import argparse import logging import requests from yaml import full_load, YAMLError from jinja2 import Environment, FileSystemLoader, TemplateNotFound import argparse from watchdog.observers import Observer from watchdog.events import RegexMatchingEventHandler # Import the Config module and set the path to run the script from root project # /!\ This instruction is only working if you run this script from the root of the project sys.path.insert(0, "./") from tools.utils.utils import Config utils = Config() Loading Loading @@ -61,6 +65,7 @@ def get_conf(job_path): logging.error(error) sys.exit(1) def get_description(job_path): """Fetch the README file from job Loading @@ -79,6 +84,7 @@ def get_description(job_path): logging.error(error) sys.exit(1) def get_changelogs(job_path, job_name): """Fetch the changelogs file from job Loading @@ -101,7 +107,8 @@ def get_changelogs(job_path, job_name): logging.info("Parsing changelogs for job %s", job_name) try: for version in versions: with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION, encoding="utf-8") as changelog_file: with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION, encoding="utf-8") as changelog_file: changelogs.append({ "version": version, "url": utils.R2DEVOPS_URL + version + "/" + job_name + utils.JOBS_EXTENSION, Loading @@ -114,6 +121,7 @@ def get_changelogs(job_path, job_name): return (latest, changelogs) def get_license(license_name, copyright_holder): """Return the license file Loading @@ -126,7 +134,8 @@ def get_license(license_name, copyright_holder): """ logging.info("Getting licence %s", license_name) try: env = Environment(loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR)) env = Environment( loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR)) template = env.get_template(license_name + utils.MARKDOWN_EXTENSION + ".j2") license_content = template.render( year=datetime.now().year, Loading @@ -134,12 +143,14 @@ def get_license(license_name, copyright_holder): ).split('\n') license_content = [line + '\n' for line in license_content] except TemplateNotFound as error: logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder) logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder) logging.error(error) sys.exit(1) return license_content def get_screenshots(job_path, job_name): """Create the job directory for the job documentation Gets the jobs screenshots and copy them to the documentation directory Loading @@ -161,7 +172,8 @@ def get_screenshots(job_path, job_name): logging.info("Getting screenshots for job %s", job_name) # Create screenshots folder in docs for the job makedirs(utils.MKDOCS_DIR+"/"+utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR,0o777,True) makedirs(utils.MKDOCS_DIR + "/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR, 0o777, True) # Get all screenshots of the job regex = re.compile('(.png|.jpg|.jpeg)$') Loading @@ -170,10 +182,12 @@ def get_screenshots(job_path, job_name): # Copy all screenshot of the job into screenshots folder for the doc for screenshot in screenshot_list: copyfile(job_path + "/" + utils.SCREENSHOTS_DIR+"/"+screenshot, utils.MKDOCS_DIR+ "/"+ utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR+"/"+screenshot) copyfile(job_path + "/" + utils.SCREENSHOTS_DIR + "/" + screenshot, utils.MKDOCS_DIR + "/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR + "/" + screenshot) return ("/" + utils.MKDOCS_DIR_JOBS_IMAGES + "/" + job_name + "/" + utils.SCREENSHOTS_DIR, screenshot_list) def get_user(code_owner): """Fetch the job maintainer Gitlab user Loading Loading @@ -204,6 +218,7 @@ def get_user(code_owner): return None def get_job_raw_content(job_name): """Return the raw content of a job Loading @@ -229,6 +244,7 @@ def get_job_raw_content(job_name): utils.JOBS_EXTENSION) sys.exit(1) # https://docs.gitlab.com/ee/api/issues.html#list-project-issues (for the structure of the response) def get_linked_issues(job_name, opened=True): """Get a list of linked issues for a job Loading Loading @@ -272,6 +288,7 @@ def get_linked_issues(job_name, opened=True): create_issue_url = f"{issues_base_url}/issues/new?{quote(create_issue_payload, safe='=')}%20-%20" return (linked_issues, linked_issues_url, create_issue_url) def create_job_doc(job): """Create the Markdown documentation file for a job Loading @@ -293,7 +310,8 @@ def create_job_doc(job): stage = conf.get("default_stage") if not code_owner or not license_name or not stage: logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, utils.JOB_METADATA_FILE) logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, utils.JOB_METADATA_FILE) sys.exit(1) utils.INDEX[stage]["content"].append(conf) Loading Loading @@ -401,6 +419,7 @@ def add_placeholder(): template = env.get_template(utils.TEMPLATE_PLACEHOLDER) file_handle.write(template.render()) def create_arrange_pages(): """ Create arrange .pages for mkdocs to sort the list of stage in job page Loading @@ -424,6 +443,7 @@ def create_arrange_pages(): logging.error(error) sys.exit(1) def argparse_setup(): """Setup argparse Loading @@ -437,8 +457,46 @@ def argparse_setup(): parser.add_argument("--job", '-j', type=str, help="Build a specific R2DevOps job's name") parser.add_argument("--watch", help="Put in watch mode", action="store_true") return parser.parse_args() def create_jobs_doc(arg_job=None): """Build all the jobs (or a single) doc """ # Iterate over every directories in jobs directory to create their job.md for the documentation jobs = listdir(utils.JOBS_DIR) if arg_job and arg_job not in jobs: logging.error("Job %s not found", arg_job) sys.exit(1) if arg_job: create_job_doc(arg_job) else: for job in jobs: create_job_doc(job) def run_watcher(watch_path): def on_modified(event): logging.info(f"New modification detected on {event.src_path}") create_jobs_doc(event.src_path.split("/")[1]) event_handler = RegexMatchingEventHandler(["^.*\.md$"], ignore_directories=False, case_sensitive=True) event_handler.on_modified = on_modified observer = Observer() observer.schedule(event_handler, watch_path, recursive=True) observer.start() try: while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join() def main(): """ Main function, multiple-purpose: Loading @@ -460,18 +518,15 @@ def main(): ] ) # Iterate over every directories in jobs directory to create their job.md for the documentation jobs = listdir(utils.JOBS_DIR) if args.job and args.job not in jobs: logging.error("Job %s not found", args.job) sys.exit(1) elif args.job: create_job_doc(args.job) watch_path = utils.JOBS_DIR if args.job: create_jobs_doc(args.job) watch_path = f"{watch_path}/{args.job}" else: for job in jobs: create_job_doc(job) create_jobs_doc() if args.watch: run_watcher(watch_path) # Verify that there is a .md file for every stage, or mkdocs will break add_placeholder() Loading @@ -484,11 +539,9 @@ def main(): template = env.get_template(utils.TEMPLATE_INDEX) index_content = template.render(index=utils.INDEX) with open(utils.MKDOCS_DIR + "/" + utils.JOBS_DIR + "/" + utils.INDEX_FILE, "w+", encoding="utf-8") as index_file: index_file.write(index_content) if __name__ == "__main__": main()
