Commit ec63d2a6 authored by Thomas Boni's avatar Thomas Boni
Browse files

Merge branch '227-verify-job-structure' into 'latest' 

Resolve "Verify job structure"

Closes #227

See merge request r2devops/hub!115
parents a1b0de9f 33b7cc82

.gitlab-ci.yml

+17 −0
Original line number Diff line number Diff line
@@ -30,6 +30,23 @@ ci_linter: 
    -   gitlab-ci-linter ${JOB_PATH}/${JOB}.yml

    - done



structure:

  image: python:3.9.1-alpine

  stage: static_tests

  variables:

    PIPENV_PIPFILE: tools/job_structure/Pipfile

    JOB_LOGFILE: "job_structure.log"

  before_script:

    - pip install --ignore-installed distlib pipenv

    - pipenv install

  script:

    - pipenv run python3 tools/job_structure/job_structure.py

  artifacts:

    paths:

      - ${JOB_LOGFILE}

    expire_in: 30 days

    when: always



# See https://docs.gitlab.com/ee/api/releases/

# We can only control the link to the hub, the release is still storing the source code

release:

tools/builder/builder.py

+0 −1
Original line number Diff line number Diff line
@@ -25,7 +25,6 @@ import requests 
from yaml import full_load

from jinja2 import Environment, FileSystemLoader





# Job variables

JOBS_DIR = "jobs"

MKDOCS_DIR = "docs"

tools/job_structure/Pipfile

0 → 100644
+10 −0
Original line number Diff line number Diff line 
[[source]]

name = "pypi"

url = "https://pypi.org/simple"

verify_ssl = true



[requires]

python_version = "3"



[packages]

pyyaml = "==5.3.1"

tools/job_structure/Pipfile.lock

0 → 100644
+40 −0
Original line number Diff line number Diff line 
{

    "_meta": {

        "hash": {

            "sha256": "393c026ec5df70f675b7d8e3a09d37d15b99d5312608065a0443886c8a752316"

        },

        "pipfile-spec": 6,

        "requires": {

            "python_version": "3"

        },

        "sources": [

            {

                "name": "pypi",

                "url": "https://pypi.org/simple",

                "verify_ssl": true

            }

        ]

    },

    "default": {

        "pyyaml": {

            "hashes": [

                "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",

                "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",

                "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",

                "sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",

                "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",

                "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",

                "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",

                "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",

                "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",

                "sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",

                "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",

                "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",

                "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"

            ],

            "index": "pypi",

            "version": "==5.3.1"

        }

    },

    "develop": {}

}

tools/job_structure/job_structure.py

0 → 100644
+133 −0
Original line number Diff line number Diff line 
#!/usr/bin/env python3



import os

import logging

import sys

import yaml



# Job variables

JOBS_DIR = "jobs"

TOOLS_DIR = "tools"

JOB_TEMPLATE_DIR = "job_template"

JOB_DIR = "job_name"

JOB_YAML = "job.yml"

LOGFILE_NAME = os.getenv("JOB_LOGFILE")

EXIT_SUCCESS = 0

EXIT_FAILURE = 1



def check_job_yaml(job):

    """Verify the content of job.yaml for every job



    Parameters

    ----------

    job

        Name of the job



    Returns

    -------

    0

        On success

    1

        If at least one key is missing

    """

    ret = EXIT_SUCCESS



    with open(f"{TOOLS_DIR}/{JOB_TEMPLATE_DIR}/{JOB_DIR}/{JOB_YAML}", "r") as template_yml, open(f"{JOBS_DIR}/{job}/{JOB_YAML}", "r") as job_yml:

        template_content = yaml.load(template_yml, Loader=yaml.FullLoader)

        job_content = yaml.load(job_yml, Loader=yaml.FullLoader)



        logging.info(f"Checking the content of {JOB_YAML} in job {job}")

        diff = set(template_content.keys()) - set(job_content.keys())

        if len(diff) > 0:

            for item in diff:

                logging.error(f"Key {item} in {JOB_YAML} of job {job} is missing")

            ret = EXIT_FAILURE

        else:

            logging.info(f"{JOB_YAML} for job {job} is complete")

    return ret



def check_directory_structure(template_structure, job):

    """Verify every file and directories in template to be sure they are present in the job



    Parameters

    ----------

    template_structure

        Structure of the template

    job

        Name of the job



    Returns

    -------

    0

        On success

    1

        If at least a file/directory is missing

    """



    # Change "job_name" in template for the actual job name for comparison

    template_structure_tmp = [obj.replace("job_name", f"{job}") for obj in template_structure]



    job_structure = [os.path.join(parent, name) for (parent, subdirs, files) in os.walk(f"{JOBS_DIR}/{job}") for name in files + subdirs]

    # Adding the job directory

    job_structure.append(f"{job}")



    # Clear the first directory

    job_structure = [obj[obj.find('/') + 1:] for obj in job_structure]



    # Check if file/directory is empty

    logging.info(f"Checking empty file/directory in job structure of job {job}")

    for item in job_structure:

        if os.path.isfile(item):

            if os.path.getsize(item) == 0:

                logging.error(f"File {item} for job {job} is empty")

        elif os.path.isdir(item):

            if len(os.listdir(item)) == 0:

                logging.error(f"Directory {item} for job {job} is empty")



    ret = EXIT_SUCCESS

    if len(set(template_structure_tmp).intersection(job_structure)) != len(template_structure_tmp):

        # Not every file and directories in template_structure_tmp matched the job structure

        logging.error(f"Job structure of {job} does not match the template:")

        for item in set(template_structure_tmp) - set(template_structure_tmp).intersection(job_structure):

            logging.error(f"\tFile/directory missing: {item}")

        ret = EXIT_FAILURE

    else:

        logging.info(f"Job structure of {job} matches the template")

    return ret





if __name__ == "__main__":

    """Main function, iterating over job structures to compare to the tempale one



    Returns

    -------

    0

        In case nothing was mismatched

    Number

        Number of jobs that doesn't match the template

    """

    # Setup logging

    logging.basicConfig(

        encoding="utf-8",

        level=logging.INFO,

        format="%(asctime)s [%(levelname)s] %(message)s",

        handlers=[

            logging.FileHandler(LOGFILE_NAME),

            logging.StreamHandler()

        ]

    )



    # Iterate over every directories in jobs directory to create their job.md for the documentation

    jobs = os.listdir(JOBS_DIR)

    template_structure = [os.path.join(parent, name) for (parent, subdirs, files) in os.walk(f"{TOOLS_DIR}/{JOB_TEMPLATE_DIR}") for name in files + subdirs]

    # Clear the first 2 directories

    template_structure = [obj[obj.find('/') + 1:] for obj in template_structure]

    template_structure = [obj[obj.find('/') + 1:] for obj in template_structure]



    ret = EXIT_SUCCESS

    for job in jobs:

        if check_directory_structure(template_structure, job) != EXIT_SUCCESS:

            ret = EXIT_FAILURE

        elif check_job_yaml(job) != EXIT_SUCCESS:

            ret = EXIT_FAILURE

    sys.exit(ret)