Resolve "Give explicit errors when builder fail"

from os import listdir, makedirs

from shutil import copyfile

import requests

from yaml import full_load

from jinja2 import Environment, FileSystemLoader

from yaml import full_load, YAMLError

from jinja2 import Environment, FileSystemLoader, TemplateNotFound

# Job variables

JOBS_DIR = "jobs"

JOB_DESCRIPTION_FILE = "README.md"

JOB_METADATA_FILE = "job.yml"

JOBS_EXTENSION = ".yml"

MARKDOWN_EXTENSION = ".md"

# Path to images used for the built job documentation

MKDOCS_DIR_JOBS_IMAGES = "images/jobs"

}

def get_conf(job_path):

    # Load yaml file

    """Parse the YAML config of the job

    Args:

        job_path (string): Path to job.yml

    Returns:

        (dict): Object of parsed YAML

    """

    try:

        with open(job_path + "/" + JOB_METADATA_FILE) as conf_file:

            return full_load(conf_file)

    except YAMLError as error:

        logging.error("Failed to parse job config '%s/%s", job_path,

                      JOB_METADATA_FILE )

        logging.error(error)

        sys.exit(1)

    except OSError as error:

        logging.error("Failed to open and read job config '%s/%s",

                      job_path, JOB_METADATA_FILE )

        logging.error(error)

        sys.exit(1)

def get_description(job_path):

    # Concatenate description to final file

    """Fetch the README file from job

    Args:

        job_path (string): Path the job folder

    Returns:

        (string): Full README file

    """

    try:

        with open(job_path + "/" + JOB_DESCRIPTION_FILE) as readme_file:

            return readme_file.read()

    except OSError as error:

        logging.error("Failed to open and read file README %s", job_path)

        logging.error(error)

        sys.exit(1)

def get_changelogs(job_path, job_name):

    """Fetch the changelogs file from job

    Args:

        job_path (string): Path the job folder

        job_name (string): Name of the job

    Returns:

        latest (dict): data about latest version

        changelogs (list): list of data about each versions

    """

    versions = listdir(job_path + "/" + JOB_CHANGELOG_DIR)

    versions = [version[:-3] for version in versions]

    versions = sorted(versions, key=LooseVersion, reverse=True)

      "url": R2DEVOPS_URL + job_name + JOBS_EXTENSION

    }

    changelogs = []

    try:

        for version in versions:

        with open(job_path + "/" + JOB_CHANGELOG_DIR + "/" + version + ".md") as changelog_file:

            with open(job_path + "/" + JOB_CHANGELOG_DIR + "/" + version + MARKDOWN_EXTENSION) as changelog_file:

                changelogs.append({

                    "version": version,

                    "url": R2DEVOPS_URL + version + "/" + job_name + JOBS_EXTENSION,

                    "changelog": changelog_file.readlines()

                })

    except OSError as error:

        logging.error("Failed to open and read versions files of %s", job_name)

        logging.error(error)

        sys.exit(1)

    return (latest, changelogs)

def get_license(license_name, copyright_holder):

    """Return the license file

    Args:

        licence_name (string): name of the licence

        copyright_holder (string): gitlab name of the job maitainer

    Returns:

        license_content (string): content of the license

    """

    try:

        env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR + "/" + TEMPLATE_LICENSE_DIR))

    template = env.get_template(license_name + ".md.j2")

        template = env.get_template(license_name + MARKDOWN_EXTENSION + ".j2")

        license_content = template.render(

            year = datetime.now().year,

            copyright_holder = copyright_holder

        ).split('\n')

        license_content = [ line + '\n' for line in license_content]

    except TemplateNotFound as error:

        logging.error("Failed to fetch the template for license %s and copyright holder: %s", license_name, copyright_holder)

        logging.error(error)

        sys.exit(1)

    return license_content

def get_screenshots(job_path, job_name):

    return ("/" + MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+SCREENSHOTS_DIR, screenshot_list)

def get_user(code_owner):

    """Fetch the job maintainer Gitlab user

    Args:

        code_owner (string): gitlab name of the job maitainer

    Returns:

        (dict): user data

    """

    url = GITLAB_API_URL + "users?username=" + code_owner

    response = requests.request("GET", url)

    try:

        if response.status_code == 200:

            return response.json()[0]

        raise Exception('Expected 200 status code, but got {}'.format(response.status_code))

    except IndexError as error:

        logging.error("Unexpected error when retrieving user %s", code_owner)

        logging.error(error)

        sys.exit(1)

    except Exception as error:

        logging.error("User %s was not found", code_owner)

        logging.error(error)

        sys.exit(1)

    return None

def get_job_raw_content(job_name):

    conf = get_conf(job_path)

    code_owner = conf.get("maintainer")

    license_name = conf.get("license")

    index[conf["default_stage"]].append(conf)

    stage = conf.get("default_stage")

    mkdocs_file_path = MKDOCS_DIR + "/" + JOBS_DIR + "/" + conf["default_stage"] + "/" + job + ".md"

    if not code_owner or not license_name or not stage:

        logging.error("Job %s is missing fields (code_owner, license_name, stage) in '%s'", job, JOB_METADATA_FILE)

        sys.exit(1)

    index[stage].append(conf)

    mkdocs_file_path = MKDOCS_DIR + "/" + JOBS_DIR + "/" + stage + "/" + job + MARKDOWN_EXTENSION

    # Get variables for jinja

    description = get_description(job_path)

    job_icon = conf.get("icon")

    # Write final file

    try:

        with open(mkdocs_file_path, 'w+') as doc_file:

            env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR))

            template = env.get_template(TEMPLATE_DOC)

                screenshots_files = screenshots_files,

                job_raw_content = ''.join(job_raw_content)

        ))

    except Exception as error:

        logging.error("Failed to create final file for job %s", job)

        logging.error(error)

        sys.exit(1)

def add_placeholder():

    # Verify that there is a .md file for every stage, or mkdocs will break