Commit 021e803f authored by Thomas Boni's avatar Thomas Boni
Browse files

Merge branch '51-give-explicit-errors-when-builder-fail' into 'latest' 

Resolve "Give explicit errors when builder fail"

Closes #51

See merge request r2devops/hub!131
parents 98fe1319 61689f48

tools/builder/builder.py

+138 −45
Original line number Diff line number Diff line
@@ -22,8 +22,8 @@ from distutils.version import LooseVersion 
from os import listdir, makedirs

from shutil import copyfile

import requests

from yaml import full_load

from jinja2 import Environment, FileSystemLoader

from yaml import full_load, YAMLError

from jinja2 import Environment, FileSystemLoader, TemplateNotFound



# Job variables

JOBS_DIR = "jobs"
@@ -35,6 +35,7 @@ JOB_CHANGELOG_DIR = "versions" 
JOB_DESCRIPTION_FILE = "README.md"

JOB_METADATA_FILE = "job.yml"

JOBS_EXTENSION = ".yml"

MARKDOWN_EXTENSION = ".md"



# Path to images used for the built job documentation

MKDOCS_DIR_JOBS_IMAGES = "images/jobs"
@@ -65,16 +66,56 @@ index = { 
}



def get_conf(job_path):

    # Load yaml file

    """Parse the YAML config of the job



    Args:

        job_path (string): Path to job.yml



    Returns:

        (dict): Object of parsed YAML

    """

    try:

        with open(job_path + "/" + JOB_METADATA_FILE) as conf_file:

            return full_load(conf_file)

    except YAMLError as error:

        logging.error("Failed to parse job config '%s/%s", job_path,

                      JOB_METADATA_FILE )

        logging.error(error)

        sys.exit(1)

    except OSError as error:

        logging.error("Failed to open and read job config '%s/%s",

                      job_path, JOB_METADATA_FILE )

        logging.error(error)

        sys.exit(1)



def get_description(job_path):

    # Concatenate description to final file

    """Fetch the README file from job



    Args:

        job_path (string): Path the job folder



    Returns:

        (string): Full README file

    """

    try:

        with open(job_path + "/" + JOB_DESCRIPTION_FILE) as readme_file:

            return readme_file.read()

    except OSError as error:

        logging.error("Failed to open and read file README %s", job_path)

        logging.error(error)

        sys.exit(1)



def get_changelogs(job_path, job_name):

    """Fetch the changelogs file from job



    Args:

        job_path (string): Path the job folder

        job_name (string): Name of the job



    Returns:

        latest (dict): data about latest version

        changelogs (list): list of data about each versions

    """

    versions = listdir(job_path + "/" + JOB_CHANGELOG_DIR)

    versions = [version[:-3] for version in versions]

    versions = sorted(versions, key=LooseVersion, reverse=True)
@@ -83,23 +124,44 @@ def get_changelogs(job_path, job_name): 
      "url": R2DEVOPS_URL + job_name + JOBS_EXTENSION

    }

    changelogs = []

    try:

        for version in versions:

        with open(job_path + "/" + JOB_CHANGELOG_DIR + "/" + version + ".md") as changelog_file:

            with open(job_path + "/" + JOB_CHANGELOG_DIR + "/" + version + MARKDOWN_EXTENSION) as changelog_file:

                changelogs.append({

                    "version": version,

                    "url": R2DEVOPS_URL + version + "/" + job_name + JOBS_EXTENSION,

                    "changelog": changelog_file.readlines()

                })

    except OSError as error:

        logging.error("Failed to open and read versions files of %s", job_name)

        logging.error(error)

        sys.exit(1)



    return (latest, changelogs)



def get_license(license_name, copyright_holder):

    """Return the license file



    Args:

        licence_name (string): name of the licence

        copyright_holder (string): gitlab name of the job maitainer



    Returns:

        license_content (string): content of the license

    """

    try:

        env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR + "/" + TEMPLATE_LICENSE_DIR))

    template = env.get_template(license_name + ".md.j2")

        template = env.get_template(license_name + MARKDOWN_EXTENSION + ".j2")

        license_content = template.render(

            year = datetime.now().year,

            copyright_holder = copyright_holder

        ).split('\n')

        license_content = [ line + '\n' for line in license_content]

    except TemplateNotFound as error:

        logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder)

        logging.error(error)

        sys.exit(1)



    return license_content



def get_screenshots(job_path, job_name):
@@ -136,12 +198,32 @@ def get_screenshots(job_path, job_name): 
    return ("/" + MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+SCREENSHOTS_DIR, screenshot_list)



def get_user(code_owner):

    """Fetch the job maintainer Gitlab user



    Args:

        code_owner (string): gitlab name of the job maitainer



    Returns:

        (dict): user data

    """

    url = GITLAB_API_URL + "users?username=" + code_owner



    response = requests.request("GET", url)



    try:

        if response.status_code == 200:

            return response.json()[0]



        raise Exception('Expected 200 status code, but got {}'.format(response.status_code))

    except IndexError as error:

        logging.error("Unexpected error when retrieving user %s", code_owner)

        logging.error(error)

        sys.exit(1)

    except Exception as error:

        logging.error("User %s was not found", code_owner)

        logging.error(error)

        sys.exit(1)



    return None



def get_job_raw_content(job_name):
@@ -176,9 +258,15 @@ def create_job_doc(job): 
    conf = get_conf(job_path)

    code_owner = conf.get("maintainer")

    license_name = conf.get("license")

    index[conf["default_stage"]].append(conf)

    stage = conf.get("default_stage")



    mkdocs_file_path = MKDOCS_DIR + "/" + JOBS_DIR + "/" + conf["default_stage"] + "/" + job + ".md"

    if not code_owner or not license_name or not stage:

        logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, JOB_METADATA_FILE)

        sys.exit(1)



    index[stage].append(conf)



    mkdocs_file_path = MKDOCS_DIR + "/" + JOBS_DIR + "/" + stage + "/" + job + MARKDOWN_EXTENSION



    # Get variables for jinja

    description = get_description(job_path)
@@ -190,6 +278,7 @@ def create_job_doc(job): 
    job_icon = conf.get("icon")



    # Write final file

    try:

        with open(mkdocs_file_path, 'w+') as doc_file:

            env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR))

            template = env.get_template(TEMPLATE_DOC)
@@ -209,6 +298,10 @@ def create_job_doc(job): 
                screenshots_files = screenshots_files,

                job_raw_content = ''.join(job_raw_content)

        ))

    except Exception as error:

        logging.error("Failed to create final file for job %s", job)

        logging.error(error)

        sys.exit(1)



def add_placeholder():

    # Verify that there is a .md file for every stage, or mkdocs will break