Resolve "Verify job structure" (33b7cc82) · Commits · mirror-bot / Hub

.gitlab-ci.yml

+17 −0

Original line number	Diff line number	Diff line
		@@ -30,6 +30,23 @@ ci_linter:
		- gitlab-ci-linter ${JOB_PATH}/${JOB}.yml
		- done

		structure:
		image: python:3.9.1-alpine
		stage: static_tests
		variables:
		PIPENV_PIPFILE: tools/job_structure/Pipfile
		JOB_LOGFILE: "job_structure.log"
		before_script:
		- pip install --ignore-installed distlib pipenv
		- pipenv install
		script:
		- pipenv run python3 tools/job_structure/job_structure.py
		artifacts:
		paths:
		- ${JOB_LOGFILE}
		expire_in: 30 days
		when: always

		# See https://docs.gitlab.com/ee/api/releases/
		# We can only control the link to the hub, the release is still storing the source code
		release:

tools/builder/builder.py

+0 −1

Original line number	Diff line number	Diff line
		@@ -25,7 +25,6 @@ import requests
		from yaml import full_load
		from jinja2 import Environment, FileSystemLoader


		# Job variables
		JOBS_DIR = "jobs"
		MKDOCS_DIR = "docs"

tools/job_structure/Pipfile

0 → 100644

+10 −0

Original line number	Diff line number	Diff line
		[[source]]
		name = "pypi"
		url = "https://pypi.org/simple"
		verify_ssl = true

		[requires]
		python_version = "3"

		[packages]
		pyyaml = "==5.3.1"

tools/job_structure/Pipfile.lock

0 → 100644

+40 −0

Original line number	Diff line number	Diff line
		{
		"_meta": {
		"hash": {
		"sha256": "393c026ec5df70f675b7d8e3a09d37d15b99d5312608065a0443886c8a752316"
		},
		"pipfile-spec": 6,
		"requires": {
		"python_version": "3"
		},
		"sources": [
		{
		"name": "pypi",
		"url": "https://pypi.org/simple",
		"verify_ssl": true
		}
		]
		},
		"default": {
		"pyyaml": {
		"hashes": [
		"sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97",
		"sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76",
		"sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2",
		"sha256:6034f55dab5fea9e53f436aa68fa3ace2634918e8b5994d82f3621c04ff5ed2e",
		"sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648",
		"sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf",
		"sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f",
		"sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2",
		"sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee",
		"sha256:ad9c67312c84def58f3c04504727ca879cb0013b2517c85a9a253f0cb6380c0a",
		"sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d",
		"sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c",
		"sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a"
		],
		"index": "pypi",
		"version": "==5.3.1"
		}
		},
		"develop": {}
		}

tools/job_structure/job_structure.py

0 → 100644

+133 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python3

		import os
		import logging
		import sys
		import yaml

		# Job variables
		JOBS_DIR = "jobs"
		TOOLS_DIR = "tools"
		JOB_TEMPLATE_DIR = "job_template"
		JOB_DIR = "job_name"
		JOB_YAML = "job.yml"
		LOGFILE_NAME = os.getenv("JOB_LOGFILE")
		EXIT_SUCCESS = 0
		EXIT_FAILURE = 1

		def check_job_yaml(job):
		"""Verify the content of job.yaml for every job

		Parameters
		----------
		job
		Name of the job

		Returns
		-------
		0
		On success
		1
		If at least one key is missing
		"""
		ret = EXIT_SUCCESS

		with open(f"{TOOLS_DIR}/{JOB_TEMPLATE_DIR}/{JOB_DIR}/{JOB_YAML}", "r") as template_yml, open(f"{JOBS_DIR}/{job}/{JOB_YAML}", "r") as job_yml:
		template_content = yaml.load(template_yml, Loader=yaml.FullLoader)
		job_content = yaml.load(job_yml, Loader=yaml.FullLoader)

		logging.info(f"Checking the content of {JOB_YAML} in job {job}")
		diff = set(template_content.keys()) - set(job_content.keys())
		if len(diff) > 0:
		for item in diff:
		logging.error(f"Key {item} in {JOB_YAML} of job {job} is missing")
		ret = EXIT_FAILURE
		else:
		logging.info(f"{JOB_YAML} for job {job} is complete")
		return ret

		def check_directory_structure(template_structure, job):
		"""Verify every file and directories in template to be sure they are present in the job

		Parameters
		----------
		template_structure
		Structure of the template
		job
		Name of the job

		Returns
		-------
		0
		On success
		1
		If at least a file/directory is missing
		"""

		# Change "job_name" in template for the actual job name for comparison
		template_structure_tmp = [obj.replace("job_name", f"{job}") for obj in template_structure]

		job_structure = [os.path.join(parent, name) for (parent, subdirs, files) in os.walk(f"{JOBS_DIR}/{job}") for name in files + subdirs]
		# Adding the job directory
		job_structure.append(f"{job}")

		# Clear the first directory
		job_structure = [obj[obj.find('/') + 1:] for obj in job_structure]

		# Check if file/directory is empty
		logging.info(f"Checking empty file/directory in job structure of job {job}")
		for item in job_structure:
		if os.path.isfile(item):
		if os.path.getsize(item) == 0:
		logging.error(f"File {item} for job {job} is empty")
		elif os.path.isdir(item):
		if len(os.listdir(item)) == 0:
		logging.error(f"Directory {item} for job {job} is empty")

		ret = EXIT_SUCCESS
		if len(set(template_structure_tmp).intersection(job_structure)) != len(template_structure_tmp):
		# Not every file and directories in template_structure_tmp matched the job structure
		logging.error(f"Job structure of {job} does not match the template:")
		for item in set(template_structure_tmp) - set(template_structure_tmp).intersection(job_structure):
		logging.error(f"\tFile/directory missing: {item}")
		ret = EXIT_FAILURE
		else:
		logging.info(f"Job structure of {job} matches the template")
		return ret


		if __name__ == "__main__":
		"""Main function, iterating over job structures to compare to the tempale one

		Returns
		-------
		0
		In case nothing was mismatched
		Number
		Number of jobs that doesn't match the template
		"""
		# Setup logging
		logging.basicConfig(
		encoding="utf-8",
		level=logging.INFO,
		format="%(asctime)s [%(levelname)s] %(message)s",
		handlers=[
		logging.FileHandler(LOGFILE_NAME),
		logging.StreamHandler()
		]
		)

		# Iterate over every directories in jobs directory to create their job.md for the documentation
		jobs = os.listdir(JOBS_DIR)
		template_structure = [os.path.join(parent, name) for (parent, subdirs, files) in os.walk(f"{TOOLS_DIR}/{JOB_TEMPLATE_DIR}") for name in files + subdirs]
		# Clear the first 2 directories
		template_structure = [obj[obj.find('/') + 1:] for obj in template_structure]
		template_structure = [obj[obj.find('/') + 1:] for obj in template_structure]

		ret = EXIT_SUCCESS
		for job in jobs:
		if check_directory_structure(template_structure, job) != EXIT_SUCCESS:
		ret = EXIT_FAILURE
		elif check_job_yaml(job) != EXIT_SUCCESS:
		ret = EXIT_FAILURE
		sys.exit(ret)