Commit 632d605f authored by coconux's avatar coconux
Browse files

add config in all .py file

parent 51cbafd5

tools/builder/builder.py

+1 −2
Original line number Diff line number Diff line
@@ -26,11 +26,10 @@ import requests 
from yaml import full_load, YAMLError

from jinja2 import Environment, FileSystemLoader, TemplateNotFound



# Import the config module

from tools.utils.utils import Config



utils = Config()





def get_conf(job_path):

    """Parse the YAML config of the job

tools/job_customs/job_customs.py

+10 −12
Original line number Diff line number Diff line
@@ -7,12 +7,10 @@ import yaml 
import argparse

import re



# Job variables

JOBS_DIR = "jobs"

JOBS_EXTENSION = "yml"

LOGFILE_NAME = os.getenv("JOB_LOGFILE")

EXIT_SUCCESS = 0

EXIT_FAILURE = 1

# Import the config module

from tools.utils.utils import Config

utils = Config()





patterns = [

    "git commit",
@@ -54,18 +52,18 @@ if __name__ == "__main__": 
        level=logging.INFO,

        format="%(asctime)s [%(levelname)s] %(message)s",

        handlers=[

            logging.FileHandler(LOGFILE_NAME),

            logging.FileHandler(utils.LOGFILE_NAME),

            logging.StreamHandler()

        ]

    )



    return_code = EXIT_SUCCESS

    for job in os.listdir(JOBS_DIR):

    return_code = utils.EXIT_SUCCESS

    for job in os.listdir(utils.JOBS_DIR):



        logging.info(f"Getting the script for job {job}")



        data = {}

        with open(f"{JOBS_DIR}/{job}/{job}.{JOBS_EXTENSION}", 'r') as file:

        with open(f"{utils.JOBS_DIR}/{job}/{job}.{utils.JOBS_EXTENSION}", 'r') as file:

            data = yaml.load(file, Loader=yaml.FullLoader)



        for script in scripts:
@@ -75,13 +73,13 @@ if __name__ == "__main__": 
                    for line in data[job][script]:

                        if any(re.match(pattern, line) for pattern in patterns):

                            logging.error(f"Code modification discovered in script of job {job}")

                            return_code = EXIT_FAILURE

                            return_code = utils.EXIT_FAILURE

                elif "extends" in data[job].keys():

                    if script in data[data[job]['extends']].keys():

                        for line in data[data[job]['extends']][script]:

                            if any(re.match(pattern, line) for pattern in patterns):

                                logging.error(f"Code modification discovered in script of job {job}")

                                return_code = EXIT_FAILURE

                                return_code = utils.EXIT_FAILURE

            # If the extended job isn't in the file, it produce a KeyError

            except KeyError :

                logging.warning('The job %s extends a job not declared in the file, we aren\'t able to check what %s does',

tools/job_image/job_image.py

+6 −9
Original line number Diff line number Diff line
@@ -6,12 +6,9 @@ import logging 
import yaml

import argparse



# Job variables

JOBS_DIR = "jobs"

JOBS_EXTENSION = "yml"

LOGFILE_NAME = os.getenv("JOB_LOGFILE")

EXIT_SUCCESS = 0

EXIT_FAILURE = 1

# Import the config module

from tools.utils.utils import Config

utils = Config()



def argparse_setup():

    """Setup argparse
@@ -48,19 +45,19 @@ if __name__ == "__main__": 
        level=logging.INFO,

        format="%(asctime)s [%(levelname)s] %(message)s",

        handlers=[

            logging.FileHandler(LOGFILE_NAME),

            logging.FileHandler(utils.LOGFILE_NAME),

            logging.StreamHandler(sys.stderr)

        ]

    )



    if args.job is None:

        logging.error(f"No argument provided")

        sys.exit(EXIT_FAILURE)

        sys.exit(utils.EXIT_FAILURE)



    logging.info(f"Getting the image for job {args.job}")



    data = {}

    with open(f"{JOBS_DIR}/{args.job}/{args.job}.{JOBS_EXTENSION}", 'r') as file:

    with open(f"{utils.JOBS_DIR}/{args.job}/{args.job}.{utils.JOBS_EXTENSION}", 'r') as file:

        data = yaml.load(file, Loader=yaml.FullLoader)



    # If image option is directly specified in the job

tools/job_labels/job_labels.py

+16 −21
Original line number Diff line number Diff line
@@ -9,16 +9,11 @@ from urllib.parse import quote, urlencode 
from os import getenv, listdir

import requests



# API Variables

PROJECT_NAME = "r2devops/hub"

BASE_API_URL = "https://gitlab.com/api/v4"

JOB_TOKEN = getenv("API_TOKEN")

# Import the config module

from tools.utils.utils import Config

utils = Config()





# Job variables

JOBS_DIR = "jobs"

JOBS_SCOPE_LABEL = "Jobs::"

LABEL_COLOR = "fuchsia"

LOGFILE_NAME = getenv("JOB_LOGFILE")



def get_labels(project_name, with_counts=False, include_ancestor_groups=True, search=""):

    """Get labels of the project, can also serach for a specific label with search filter
@@ -40,19 +35,19 @@ def get_labels(project_name, with_counts=False, include_ancestor_groups=True, se 
        Response to the request

    """

    headers = {

        'PRIVATE-TOKEN': JOB_TOKEN

        'PRIVATE-TOKEN': utils.JOB_TOKEN

    }

    payload = {

        'with_counts': with_counts,

        'include_ancestors_groups': include_ancestor_groups,

        'search': search

    }

    base_label_url = f"{BASE_API_URL}/projects/{quote(project_name, safe='')}" + "/labels"

    base_label_url = f"{utils.GITLAB_API_URL}/projects/{quote(project_name, safe='')}" + "/labels"

    url = f"{base_label_url}?{urlencode(payload)}"

    logging.info(f"Getting the list of issues from the project {project_name} filtered by {search}")

    return (requests.get(url, headers=headers))



def create_label(project_name, label_name, label_color=LABEL_COLOR):

def create_label(project_name, label_name, label_color=utils.LABEL_COLOR):

    """Create a new label for a job



    Parameters:
@@ -71,14 +66,14 @@ def create_label(project_name, label_name, label_color=LABEL_COLOR): 
        Response to the request

    """

    headers = {

        'PRIVATE-TOKEN': JOB_TOKEN

        'PRIVATE-TOKEN': utils.JOB_TOKEN

    }

    payload = {

        "name": label_name,

        "color": label_color,

        "description": f"Issues related to {label_name}"

    }

    url = f"{BASE_API_URL}/projects/{quote(project_name, safe='')}/labels"

    url = f"{utils.GITLAB_API_URL}/projects/{quote(project_name, safe='')}/labels"

    logging.info(f"Creating a label {label_name} for the project {project_name}")

    return (requests.post(url, headers=headers, data=payload))
@@ -98,9 +93,9 @@ def delete_label(project_name, label_name): 
        Response to the request

    """

    headers = {

        'PRIVATE-TOKEN': JOB_TOKEN

        'PRIVATE-TOKEN': utils.JOB_TOKEN

    }

    url = f"{BASE_API_URL}/projects/{quote(project_name, safe='')}/labels/{label_name}"

    url = f"{utils.GITLAB_API_URL}/projects/{quote(project_name, safe='')}/labels/{label_name}"

    logging.info(f"Deleting a label {label_name} for the project {project_name}")

    return (requests.delete(url, headers=headers))
@@ -120,21 +115,21 @@ if __name__ == "__main__": 
        level=logging.INFO,

        format="%(asctime)s [%(levelname)s] %(message)s",

        handlers=[

            logging.FileHandler(LOGFILE_NAME),

            logging.FileHandler(utils.LOGFILE_NAME),

            logging.StreamHandler()

        ]

    )



    jobs = listdir(JOBS_DIR)

    jobs = listdir(utils.JOBS_DIR)

    for job in jobs:

        job_label = JOBS_SCOPE_LABEL + job

        label = get_labels(PROJECT_NAME, search=job_label)

        job_label = utils.JOBS_SCOPE_LABEL + job

        label = get_labels(utils.PROJECT_NAME, search=job_label)

        if "Unauthorized" not in label.text:

            if label.json():

                logging.info(f"Label {job} exist")

            else:

                logging.info(f"Label {job} does not exist, creating one now")

                create_label(PROJECT_NAME, job_label)

                create_label(utils.PROJECT_NAME, job_label)

        else:

            logging.error("Not Authorized, verify the API_TOKEN environment variable in the gitlab project")

            sys.exit(1)

tools/job_structure/job_structure.py

+1 −1
Original line number Diff line number Diff line
@@ -5,8 +5,8 @@ import logging 
import sys

import yaml



# Import the config module

from tools.utils.utils import Config



utils = Config()



def check_job_yaml(job):