Merge conflicts (d1966fd4) · Commits · mirror-bot / Hub

.gitlab-ci.yml

+4 −0

Original line number	Diff line number	Diff line
		@@ -183,9 +183,13 @@ mkdocs:
		variables:
		PIPENV_PIPFILE: tools/builder/Pipfile
		PYTHONPATH: "./:$PYTHONPATH"
		JOB_LOGFILE: "builder.log"
		before_script:
		- pip install pipenv && pipenv install
		- pipenv run python tools/builder/builder.py
		artifacts:
		paths:
		- "${JOB_LOGIFILE}"


		# See https://docs.gitlab.com/ee/api/releases/

tools/builder/builder.py

+23 −3

Original line number	Diff line number	Diff line
		@@ -43,6 +43,7 @@ def get_conf(job_path):
		Returns:
		(dict): Object of parsed YAML
		"""
		logging.info("Parsing conf file")
		try:
		with open(job_path + "/" + utils.JOB_METADATA_FILE) as conf_file:
		return full_load(conf_file)
		@@ -66,6 +67,7 @@ def get_description(job_path):
		Returns:
		(string): Full README file
		"""
		logging.info("Parsing readme for job %s", job_path)
		try:
		with open(job_path + "/" + utils.JOB_DESCRIPTION_FILE) as readme_file:
		return readme_file.read()
		@@ -93,6 +95,7 @@ def get_changelogs(job_path, job_name):
		"url": utils.R2DEVOPS_URL + job_name + utils.JOBS_EXTENSION
		}
		changelogs = []
		logging.info("Parsing changelogs for job %s", job_name)
		try:
		for version in versions:
		with open(job_path + "/" + utils.JOB_CHANGELOG_DIR + "/" + version + utils.MARKDOWN_EXTENSION) as changelog_file:
		@@ -118,6 +121,7 @@ def get_license(license_name, copyright_holder):
		Returns:
		license_content (string): content of the license
		"""
		logging.info("Getting licence %s", license_name)
		try:
		env = Environment(loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR + "/" + utils.TEMPLATE_LICENSE_DIR))
		template = env.get_template(license_name + utils.MARKDOWN_EXTENSION + ".j2")
		@@ -152,6 +156,7 @@ def get_screenshots(job_path, job_name):
		List of all screenshots name for the job
		"""

		logging.info("Getting screenshots for job %s", job_name)
		# Create screenshots folder in docs for the job
		makedirs(utils.MKDOCS_DIR+"/"+utils.MKDOCS_DIR_JOBS_IMAGES+"/"+job_name+"/"+utils.SCREENSHOTS_DIR,0o777,True)

		@@ -179,6 +184,7 @@ def get_user(code_owner):

		response = requests.request("GET", url)

		logging.info("Getting user link for %s", code_owner)
		try:
		if response.status_code == 200:
		return response.json()[0]
		@@ -208,6 +214,7 @@ def get_job_raw_content(job_name):
		yaml
		Raw content of the job
		"""
		logging.info("Parsing content of the job %s", job_name)
		try:
		with open("{}/{job}/{job}{}".format(utils.JOBS_DIR, utils.JOBS_EXTENSION,
		job=job_name), 'r') as job:
		@@ -239,6 +246,7 @@ def get_linked_issues(job_name, opened=True):
		str
		Url to create a new issue for the job
		"""
		logging.info(f"Getting list of linked issues for job %s", job_name)
		linked_issues = []
		base_url = f"{utils.GITLAB_API_URL}/projects/{quote(utils.PROJECT_NAME, safe='')}/issues"
		url = f"{base_url}?labels={utils.JOBS_SCOPE_LABEL}{job_name}"
		@@ -382,6 +390,7 @@ def add_placeholder():
		placeholder_path = utils.MKDOCS_DIR + "/" + utils.JOBS_DIR + "/" + stage_key
		if len(listdir(placeholder_path)) == 1:
		# There is only the .pages file, so mkdocs will break
		logging.info("Creating a placeholder file for the stage %s", stage_key)
		with open(placeholder_path + "/" + utils.MKDOCS_PLACEHOLDER_FILE, "w+") as file_handle:
		env = Environment(loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR))
		template = env.get_template(utils.TEMPLATE_PLACEHOLDER)
		@@ -433,8 +442,15 @@ def main():
		# Setup argparse
		args = argparse_setup()

		# logging
		logging.basicConfig(level=logging.INFO)
		# Setup logging
		logging.basicConfig(
		level=logging.INFO,
		format="%(asctime)s [%(levelname)s] %(message)s",
		handlers=[
		logging.FileHandler(LOGFILE_NAME),
		logging.StreamHandler()
		]
		)

		# Verify that there is a .md file for every stage, or mkdocs will break
		add_placeholder()
		@@ -448,10 +464,14 @@ def main():


		# Using jinja2 with a template to create the index
		logging.info("Creating index of jobs")
		env = Environment(loader=FileSystemLoader(utils.BUILDER_DIR + "/" + utils.TEMPLATE_DIR))
		template = env.get_template(utils.TEMPLATE_INDEX)
		index_content = template.render(index=utils.INDEX)




		with open(utils.MKDOCS_DIR + "/" + utils.JOBS_DIR + "/" + utils.INDEX_FILE, "w+") as index_file:
		index_file.write(index_content)

tools/job_customs/job_customs.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -60,7 +60,7 @@ if __name__ == "__main__":
		return_code = utils.EXIT_SUCCESS
		for job in os.listdir(utils.JOBS_DIR):

		logging.info(f"Getting the script for job {job}")
		logging.info("Getting the script for job %s", job)

		data = {}
		with open(f"{utils.JOBS_DIR}/{job}/{job}{utils.JOBS_EXTENSION}", 'r') as file:
		@@ -72,13 +72,13 @@ if __name__ == "__main__":
		if script in data[job].keys():
		for line in data[job][script]:
		if any(re.match(pattern, line) for pattern in patterns):
		logging.error(f"Code modification discovered in script of job {job}")
		logging.error("Code modification discovered in script of job %s", job)
		return_code = utils.EXIT_FAILURE
		elif "extends" in data[job].keys():
		if script in data[data[job]['extends']].keys():
		for line in data[data[job]['extends']][script]:
		if any(re.match(pattern, line) for pattern in patterns):
		logging.error(f"Code modification discovered in script of job {job}")
		logging.error("Code modification discovered in script of job %s", job)
		return_code = utils.EXIT_FAILURE
		# If the extended job isn't in the file, it produce a KeyError
		except KeyError :

tools/job_gitlab_labels/job_gitlab_labels.py

+5 −5

Original line number	Diff line number	Diff line
		@@ -43,7 +43,7 @@ def get_labels(project_name, with_counts=False, include_ancestor_groups=True, se
		}
		base_label_url = f"{utils.GITLAB_API_URL}/projects/{quote(project_name, safe='')}" + "/labels"
		url = f"{base_label_url}?{urlencode(payload)}"
		logging.info(f"Getting the list of issues from the project {project_name} filtered by {search}")
		logging.info("Getting the list of issues from the project %s filtered by %s", project_name, search)
		return (requests.get(url, headers=headers))

		def create_label(project_name, label_name, label_color=utils.LABEL_COLOR):
		@@ -73,7 +73,7 @@ def create_label(project_name, label_name, label_color=utils.LABEL_COLOR):
		"description": f"Issues related to {label_name}"
		}
		url = f"{utils.GITLAB_API_URL}/projects/{quote(project_name, safe='')}/labels"
		logging.info(f"Creating a label {label_name} for the project {project_name}")
		logging.info("Creating a label %s for the project %s", label_name, project_name)
		return (requests.post(url, headers=headers, data=payload))

		def delete_label(project_name, label_name):
		@@ -95,7 +95,7 @@ def delete_label(project_name, label_name):
		'PRIVATE-TOKEN': utils.JOB_TOKEN
		}
		url = f"{utils.GITLAB_API_URL}/projects/{quote(project_name, safe='')}/labels/{label_name}"
		logging.info(f"Deleting a label {label_name} for the project {project_name}")
		logging.info("Deleting a label %s for the project %s", label_name, project_name)
		return (requests.delete(url, headers=headers))

		def argparse_setup():
		@@ -139,9 +139,9 @@ if __name__ == "__main__":
		label = get_labels(utils.PROJECT_NAME, search=job_label)
		if "Unauthorized" not in label.text:
		if label.json():
		logging.info(f"Label {job} exist")
		logging.info("Label %s exist", job)
		else:
		logging.info(f"Label {job} does not exist, creating one now")
		logging.info("Label %s does not exist, creating one now", job)
		create_label(utils.PROJECT_NAME, job_label)
		else:
		logging.error("Not Authorized, verify the API_TOKEN environment variable in the gitlab project")

tools/job_image/job_image.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -51,10 +51,10 @@ if __name__ == "__main__":
		)

		if args.job is None:
		logging.error(f"No argument provided")
		logging.error("No argument provided")
		sys.exit(utils.EXIT_FAILURE)

		logging.info(f"Getting the image for job {args.job}")
		logging.info("Getting the image for job %s", args.job)

		data = {}
		with open(f"{utils.JOBS_DIR}/{args.job}/{args.job}{utils.JOBS_EXTENSION}", 'r') as file: