Commit 70c8cf88 authored by Fulcrand Guilhem's avatar Fulcrand Guilhem
Browse files

Merge branch '55-change-builder-wrappers-to-use-jinja2-templates' into 'latest' 

Resolve "Change builder wrappers to use jinja2 templates"

Closes #55

See merge request go2scale/hub!22
parents dba09bde eba5043b

builder/builder.py

+117 −116
Original line number Diff line number Diff line 
#!/usr/bin/env python3



# This script is used to build the documentation that hub.go2scale.com will be using for every job added to the hub

# This script is used to build the documentation that

# hub.go2scale.com will be using for every job added to the hub



# Directory skeleton for a job:

# ── jobs
@@ -13,122 +14,122 @@ 
#             ├── 0.1.0.md

#             └──...



from pathlib import Path

from os import listdir

from yaml import full_load

from jinja2 import Environment, FileSystemLoader

import requests



# Job variables

jobs_dir = "jobs"

mkdocs_dir = "docs"

mkdocs_placeholder_file = "placeholder.md"

job_changelog_dir = "versions"

job_description_file = "README.md"

job_license_file = "LICENSE"

job_metadata_file = "job.yml"



mk_changelog_wrapper = "\n## Changelog\n\n* **[latest]**(current -> `<LATEST_RELEASE>`) : `<TAG_URL>`\n"

mk_license_wrapper = "??? License\n"

mk_placeholder_wrapper = "# 🚧 *Work in progress*\n\nThere is no job for this stage for now"

mk_code_owner_wrapper = "\n\n-- 🔨 Maintainer: <img src='<GITLAB_IMAGE>' alt='avatar' style='width: 20px; height: 20px; border-radius: 50%'> [<CODE_OWNER_NAME>](<CODE_OWNER_URL>) @<CODE_OWNER>\n"



gitlab_api_url = "https://gitlab.com/api/v4/"



# Index variables

builder_dir = "builder"

template_dir = "templates"

template_name = "index.md.j2"

index_file = "index.md"

index = {"static_tests": [], "build": [], "dynamic_tests": [], "review": [], "deployment": []}



def get_conf(job_path, job_name):

JOBS_DIR = "jobs"

MKDOCS_DIR = "docs"

MKDOCS_PLACEHOLDER_FILE = "placeholder.md"

JOB_CHANGELOG_DIR = "versions"

JOB_DESCRIPTION_FILE = "README.md"

JOB_LICENSE_FILE = "LICENSE"

JOB_METADATA_FILE = "job.yml"



GITLAB_API_URL = "https://gitlab.com/api/v4/"

GO2SCALE_URL = "https://hub.go2scale.io/"



# Templates variables

BUILDER_DIR = "builder"

TEMPLATE_DIR = "templates"

TMEPLATE_INDEX = "index.md.j2"

TEMPLATE_DOC = "job_documentation.md.j2"

TEMPLATE_PLACEHOLDER = "placeholder.md.j2"

INDEX_FILE = "index.md"

index = {

    "static_tests": [],

    "build": [],

    "dynamic_tests": [],

    "review": [],

    "deployment": []

}



def get_conf(job_path):

    # Load yaml file

  with open(job_path + "/" + job_metadata_file) as file:

    return full_load(file)

    with open(job_path + "/" + JOB_METADATA_FILE) as conf_file:

        return full_load(conf_file)



def add_description(job_path, job_name, mkdocs_job_content):

def get_description(job_path):

    # Concatenate description to final file

  with open(job_path + "/" + job_description_file) as file:

    mkdocs_job_content += file.read()

  return mkdocs_job_content





def add_changelog(job_path, job_name, mkdocs_job_content):

  # Concatenate changelog to final file

  mkdocs_job_content += mk_changelog_wrapper



  # For now, we are just getting the latest release, but we will be adding a full link to the release later

  latest_release = listdir(job_path + "/" + job_changelog_dir)[-1][0:-3]

  mkdocs_job_content = mkdocs_job_content.replace("<LATEST_RELEASE>", latest_release)



  for release in listdir(job_path + "/" + job_changelog_dir):

    with open(job_path + "/" + job_changelog_dir + "/" + release) as file:

      mkdocs_job_content += file.read() + "\n"

      # TODO replace <TAG_URL> by the link to the release

  

  # Adding a new line for consistency

  mkdocs_job_content += "\n"

  return mkdocs_job_content



def add_license(job_path, job_name, mkdocs_job_content):

  # Concatenate license to final file

  mkdocs_job_content += mk_license_wrapper



  with open(job_path + "/" + job_license_file) as file:

    for line in file.readlines():

      mkdocs_job_content += "    " + line

  return mkdocs_job_content



def add_code_owner(job_path, job, mkdocs_job_content, code_owner):

  url = gitlab_api_url + "users?username=" + code_owner

    with open(job_path + "/" + JOB_DESCRIPTION_FILE) as readme_file:

        return readme_file.read()



def get_changelogs(job_path, job_name):

    latest = {

      "version": listdir(job_path + "/" + JOB_CHANGELOG_DIR)[0][0:-3],

      "url": GO2SCALE_URL + job_name + ".yml"

    }



    changelogs = []

    for version in listdir(job_path + "/" + JOB_CHANGELOG_DIR):

        with open(job_path + "/" + JOB_CHANGELOG_DIR + "/" + version) as changelog_file:

            changelogs.append({

                "version": version[0:-3],

                "url": GO2SCALE_URL + version[0:-3] + "/" + job_name + ".yml",

                "changelog": changelog_file.readlines()

            })

    return (latest, changelogs)



def get_license(job_path):

    with open(job_path + "/" + JOB_LICENSE_FILE) as license_file:

        return license_file.readlines()



def get_user(code_owner):

    url = GITLAB_API_URL + "users?username=" + code_owner



    response = requests.request("GET", url)



    if response.status_code == 200:

    user = response.json()[0]

    mkdocs_job_content += mk_code_owner_wrapper

    mkdocs_job_content = mkdocs_job_content.replace("<GITLAB_IMAGE>", user["avatar_url"])

    mkdocs_job_content = mkdocs_job_content.replace("<CODE_OWNER_NAME>", user["name"])

    mkdocs_job_content = mkdocs_job_content.replace("<CODE_OWNER>", code_owner)

    mkdocs_job_content = mkdocs_job_content.replace("<CODE_OWNER_URL>", user["web_url"])

  return mkdocs_job_content

        return response.json()[0]

    return None



def create_job_doc(job):

  job_path = jobs_dir + "/" + job

  mkdocs_job_content = ""

    job_path = JOBS_DIR + "/" + job



    # Getting conf for indexing

  conf = get_conf(job_path, job)

    conf = get_conf(job_path)

    code_owner = conf.get("maintainer")

    index[conf["default_stage"]].append(conf)



  mkdocs_file_path = mkdocs_dir + "/" + jobs_dir + "/" + conf["default_stage"] + "/" + job + ".md"

    mkdocs_file_path = MKDOCS_DIR + "/" + JOBS_DIR + "/" + conf["default_stage"] + "/" + job + ".md"



  mkdocs_job_content = add_description(job_path, job, mkdocs_job_content)

  mkdocs_job_content = add_changelog(job_path, job, mkdocs_job_content)

  mkdocs_job_content = add_license(job_path, job, mkdocs_job_content)

  code_owner = conf.get("code-owner")

  if code_owner:

    mkdocs_job_content = add_code_owner(job_path, job, mkdocs_job_content, conf["code-owner"])

    # Get variables for jinja

    description = get_description(job_path)

    latest, changelogs = get_changelogs(job_path, job)

    license_content = get_license(job_path)

    user = get_user(code_owner)



    # Write final file

  with open(mkdocs_file_path, 'w+') as file:

    file.write(mkdocs_job_content)

    with open(mkdocs_file_path, 'w+') as doc_file:

        env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR))

        template = env.get_template(TEMPLATE_DOC)

        doc_file.write(template.render(

            readme = description,

            license = license_content,

            latest = latest,

            changelogs = changelogs,

            gitlab_image = user["avatar_url"],

            code_owner_name = user["name"],

            code_owner = code_owner,

            code_owner_url = user["web_url"]

      ))



def add_placeholder():

    # Verify that there is a .md file for every stage, or mkdocs will break

  stages = listdir(mkdocs_dir + "/" + jobs_dir)

  stages.remove(".pages")



  for stage in stages:

    if len(listdir(mkdocs_dir + "/" + jobs_dir + "/" + stage)) == 1:

    for stage_key, _ in index.items():

        placeholder_path = MKDOCS_DIR + "/" + JOBS_DIR + "/" + stage_key

        if len(listdir(placeholder_path)) == 1:

            # There is only the .pages file, so mkdocs will break

      with open(mkdocs_dir + "/" + jobs_dir + "/" + stage + "/" + mkdocs_placeholder_file, "w+") as file:

        file.write(mk_placeholder_wrapper)

            with open(placeholder_path + "/" + MKDOCS_PLACEHOLDER_FILE, "w+") as file_handle:

                env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR))

                template = env.get_template(TEMPLATE_PLACEHOLDER)

                file_handle.write(template.render())



if __name__ == "__main__":

    # Iterate over every directories in jobs directory to create their job.md for the documentation

  jobs = listdir(jobs_dir)

    jobs = listdir(JOBS_DIR)



    for job in jobs:

        create_job_doc(job)
@@ -137,9 +138,9 @@ if __name__ == "__main__": 
    add_placeholder()



    # Using jinja2 with a template to create the index

  env = Environment(loader=FileSystemLoader(builder_dir + "/" + 'templates'))

  template = env.get_template(template_name)

    env = Environment(loader=FileSystemLoader(BUILDER_DIR + "/" + TEMPLATE_DIR))

    template = env.get_template(TMEPLATE_INDEX)

    index_content = template.render(index=index)



  with open(mkdocs_dir + "/" + jobs_dir + "/" + index_file, "w+") as file:

    file.write(index_content)
 
    with open(MKDOCS_DIR + "/" + JOBS_DIR + "/" + INDEX_FILE, "w+") as index_file:

        index_file.write(index_content)

builder/templates/job_documentation.md.j2

0 → 100644
+24 −0
Original line number Diff line number Diff line 
{{ readme }}



??? License

{% for line in license %}

  {% if line %}

    {{ line }}

  {% endif %}

{% endfor %}



# Changelog

* **[latest]({{ latest.url }})** (current -> `{{ latest.version }}`): `{{ latest.url }}`

{% for version in changelogs %}

??? {{ version.version }}

    `{{ version.url }}`

  {% for line in version.changelog %}

    {% if line %}

    {{ line }}

    {% endif %}

  {% endfor %}

{% endfor %}



{% if code_owner %}

-- 🔨 Maintainer: <img src='{{ gitlab_image }}' alt='avatar' style='width: 20px; height: 20px; border-radius: 50%'> [{{ code_owner_name }}]({{ code_owner_url }}) @{{ code_owner }}

{% endif %}
 
 No newline at end of file

builder/templates/placeholder.md.j2

0 → 100644
+3 −0
Original line number Diff line number Diff line 
# 🚧 *Work in progress*



There is no job for this stage for now
 
 No newline at end of file

jobs/apidoc/job.yml

+1 −1
Original line number Diff line number Diff line
@@ -2,4 +2,4 @@ name: apidoc 
description: A ready-to-use apidoc job to build a web documentation of your API

default_stage: build

icon: 📒

code-owner: thomasboni
 
 No newline at end of file
 
maintainer: thomasboni
 
 No newline at end of file

jobs/apidoc/versions/2020-08-07_1.md

+1 −2
Original line number Diff line number Diff line 
* **Tag `2020-08-07_1`** : `https://jobs.go2scale.io/2020-08-07_1/apidoc.yml`

    * Initial version
 
 No newline at end of file
 
Initial version
 
 No newline at end of file