No OneTemporary
Actions

Size

70 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/Containerfile.worker b/Containerfile.worker
	index 4899af6..a96f87c 100644
	--- a/Containerfile.worker
	+++ b/Containerfile.worker
	@@ -1,25 +1,26 @@

	FROM alpine

	RUN apk add --no-cache podman podman-compose python3 \
	py3-virtualenv dumb-init bash shadow git openssh rsync php php-curl \
	&& virtualenv --python=python3 /buildbot_venv \
	&& /buildbot_venv/bin/pip3 install 'twisted[tls]' pycobertura \
	&& mkdir /buildbot \
	&& useradd -ms /bin/bash buildbot \
	&& mkdir -pv /tools \
	&& cd /tools \
	&& git clone https://github.com/phorgeit/arcanist.git \
	&& cd arcanist \
	&& git checkout stable

	COPY . /usr/src/buildbot-worker
	COPY docker/buildbot.tac /buildbot/buildbot.tac

	RUN /buildbot_venv/bin/pip3 install /usr/src/buildbot-worker && \
	chown -R buildbot /buildbot

	USER buildbot
	WORKDIR /buildbot
	+ENV PATH="/buildbot_venv/bin:/tools/arcanist/bin:$PATH"

	CMD ["/usr/bin/dumb-init", "/buildbot_venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]
	diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
	index aa0340b..4008e3d 100644
	--- a/lilybuild/lilybuild/ci_steps.py
	+++ b/lilybuild/lilybuild/ci_steps.py
	@@ -1,396 +1,446 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from twisted.internet import defer
	from .ci_syntax import ci_file
	from .helpers import rsync_rules_from_artifacts, get_job_script, normalize_image
	+from .phorge import SendCoverageToPhorge
	import re
	import sys
	import json

	SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'
	+COVERAGE_EXEC = '/lilybuild/lilybuild/coverage.py'

	def on_success(step):
	return step.build.results == util.SUCCESS

	+def on_always(_step):
	+ return True
	+
	def fill_list(*args):
	return list(args)

	class RunCIJobStep(steps.BuildStep):
	# 200 MiB
	artifact_max_size = 200 * 1024 * 1024
	default_image = 'alpine'
	master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
	artifact_file_name = 'artifacts.tar'
	master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
	+ reports_file_name = 'reports.tar'
	+ master_reports_file_name_pattern = master_job_artifact_dir_pattern + '/' + reports_file_name
	master_pages_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/pages'
	+ phorge_coverage_file_name = 'coverage-phorge.json'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	job_prop=None,
	**kwargs):
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	super().__init__(name='Run step', **kwargs)

	@defer.inlineCallbacks
	def run(self):
	job_prop = self.getProperty('lilybuild_job_prop')
	job = ci_file.CIJob.from_prop(job_prop)
	job_index = self.getProperty('lilybuild_job_index')

	next_steps = yield self.job_to_steps(job, job_index)

	self.build.addStepsAfterCurrentStep(next_steps)
	return util.SUCCESS

	@defer.inlineCallbacks
	def get_ci_variables(self, job):
	res = {}
	res.update(job.get_predefined_ci_variables())
	res.update(self.getProperty('lilybuild_pipeline_vars'))
	res['CI_JOB_IMAGE'] = job.image or self.default_image
	res['CI_JOB_URL'] = yield self.build.getUrl()
	res['CI_JOB_ID'] = self.build.buildid
	res['CI_PROJECT_DIR'] = '/build'
	try:
	repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
	variables = yield repo['variables_getter'](self.build)
	res_vars = {}
	for var in variables:
	value = variables[var]
	if IRenderable.providedBy(value):
	value = yield self.build.render(value)
	res_vars[var] = value
	res.update(res_vars)
	except Exception as e:
	self.addCompleteLog('exception', f'{e}')
	return res

	+ def get_upload_artifacts_jobs(self, short_name, artifact_name, base_dir, paths, exclude, master_pattern, job_index, doStepIf=on_success):
	+ archive_artifact_step = steps.ShellCommand(
	+ name=f'Archive artifacts: {short_name}',
	+ command=[
	+ SAFETAR_EXEC,
	+ ],
	+ initialStdin=json.dumps({
	+ 'op': 'create',
	+ 'archive_file': artifact_name,
	+ 'base_dir': base_dir,
	+ 'content': paths,
	+ 'items_to_exclude': exclude,
	+ }),
	+ workdir=self.work_root_dir,
	+ doStepIf=doStepIf,
	+ )
	+ masterdest = util.Interpolate(
	+ master_pattern,
	+ st=self.storage_dir,
	+ job=job_index,
	+ doStepIf=doStepIf,
	+ )
	+ upload_artifact_step = steps.FileUpload(
	+ workersrc=artifact_name,
	+ maxsize=self.artifact_max_size,
	+ name=f'Upload artifacts: {short_name}',
	+ masterdest=masterdest,
	+ workdir=self.work_root_dir,
	+ doStepIf=doStepIf,
	+ )
	+ return [archive_artifact_step, upload_artifact_step]
	+
	@defer.inlineCallbacks
	def job_to_steps(self, job, job_index):
	script_name = self.script_dir + '/run.sh'
	variables = yield self.get_ci_variables(job)
	script_step = steps.StringDownload(
	get_job_script(variables, job),
	name='Set up script',
	workerdest=script_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	chmod_step = steps.ShellCommand(
	name='Make script executable',
	command=['chmod', '+x', script_name],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps = []
	dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
	if dep_job_indices:
	for i in dep_job_indices:
	download_job = steps.FileDownload(
	mastersrc=util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=i,
	),
	maxsize=self.artifact_max_size,
	name=f'Download artifacts from job #{i}',
	workerdest=self.artifact_file_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	unarchive_job = steps.ShellCommand(
	name=f'Unarchive artifacts from job #{i}',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'extract',
	'archive_file': self.artifact_file_name,
	'target_dir': self.src_relative,
	}),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps += [download_job, unarchive_job]

	run_step = steps.ShellCommand(
	name='Run script in container',
	command=[
	'/lilybuild/podman-helper',
	normalize_image(job.image or self.default_image),
	self.src_relative,
	self.script_dir,
	self.result_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	clean_script_step = steps.ShellCommand(
	name='Clean script dir',
	command=[
	'rm',
	'-rf',
	self.script_dir,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)

	steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
	if 'paths' in job.artifacts:
	- archive_artifact_step = steps.ShellCommand(
	- name='Archive artifacts',
	+ steps_to_run += self.get_upload_artifacts_jobs(
	+ 'files',
	+ self.artifact_file_name,
	+ self.result_relative,
	+ job.artifacts.get('paths', []),
	+ job.artifacts.get('exclude', []),
	+ self.master_job_artifact_file_name_pattern,
	+ job_index
	+ )
	+ if job.has_supported_coverage_report():
	+ steps_to_run += [steps.ShellCommand(
	+ name='Process reports',
	command=[
	- SAFETAR_EXEC,
	+ COVERAGE_EXEC,
	],
	initialStdin=json.dumps({
	- 'op': 'create',
	- 'archive_file': self.artifact_file_name,
	- 'base_dir': self.result_relative,
	- 'content': job.artifacts.get('paths', []),
	- 'items_to_exclude': job.artifacts.get('exclude', []),
	+ 'source_dir': self.src_relative,
	+ 'result_dir': self.result_relative,
	+ 'untrusted_coverage_file': job.artifacts['reports']['coverage_report']['path'],
	+ 'output_dir': self.artifact_stage_relative,
	}),
	workdir=self.work_root_dir,
	- doStepIf=on_success,
	- )
	- masterdest = util.Interpolate(
	- self.master_job_artifact_file_name_pattern,
	- st=self.storage_dir,
	- job=job_index,
	- doStepIf=on_success,
	- )
	- upload_artifact_step = steps.FileUpload(
	- workersrc=self.artifact_file_name,
	- maxsize=self.artifact_max_size,
	- name='Upload artifacts',
	- masterdest=masterdest,
	+ doStepIf=on_always,
	+ )] + self.get_upload_artifacts_jobs(
	+ 'reports',
	+ self.reports_file_name,
	+ self.artifact_stage_relative,
	+ ['*'],
	+ [],
	+ self.master_reports_file_name_pattern,
	+ job_index,
	+ doStepIf=on_always
	+ ) + [SendCoverageToPhorge(
	+ self.lbc,
	+ self.artifact_stage_relative + '/' + self.phorge_coverage_file_name,
	workdir=self.work_root_dir,
	- doStepIf=on_success,
	- )
	- steps_to_run += [archive_artifact_step, upload_artifact_step]
	+ )]
	+
	clean_stage_dir_again_step = steps.ShellCommand(
	name='Clean stage, result and artifact',
	command=[
	'rm',
	'-rf',
	self.result_relative,
	self.artifact_file_name,
	+ self.artifact_stage_relative,
	+ self.reports_file_name,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)
	steps_to_run.append(clean_stage_dir_again_step)

	if job.is_pages() and 'paths' in job.artifacts:
	deploy_pages_step = steps.MasterShellCommand(
	command=util.Transform(fill_list,
	sys.executable,
	'-m', 'lilybuild.pages',
	util.Interpolate(
	self.master_pages_dir_pattern,
	st=self.storage_dir,
	),
	masterdest,
	),
	name='Deploy pages',
	logEnviron=False,
	doStepIf=on_success,
	)
	steps_to_run.append(deploy_pages_step)

	return steps_to_run

	class TriggerMultipleJobsStep(steps.Trigger):
	properties_to_keep = [
	'branch',
	'revision',
	'repository',
	'harbormaster_build_target_phid',
	'harbormaster_variable_buildable.diff',
	'harbormaster_variable_repository.staging.ref',
	'harbormaster_variable_repository.staging.uri',
	'harbormaster_variable_repository.uri',
	'lilybuild_repo',
	'lilybuild_repo_id',
	'lilybuild_pipeline_vars',
	]

	def __init__(self, lbc, jobs_with_data, **kwargs):
	self.lbc = lbc
	self.jobs = jobs_with_data
	super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)

	def getSchedulersAndProperties(self):
	ret = []
	common_properties = {
	'lilybuild_root_build_id': self.build.buildid,
	}
	for prop in self.properties_to_keep:
	if self.hasProperty(prop):
	common_properties[prop] = self.getProperty(prop)

	for (job, i, dep_job_indices) in self.jobs:
	properties = common_properties.copy()
	properties['lilybuild_job_prop'] = job.to_prop()
	properties['lilybuild_job_index'] = i
	properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
	properties['lilybuild_dependency_job_indices'] = dep_job_indices
	ret.append({
	'sched_name': self.lbc.triggerable_scheduler_name,
	'props_to_set': properties,
	'unimportant': False,
	})

	return ret

	class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
	ci_def_file = '.gitlab-ci.yml'
	build_target_prop_name = 'harbormaster_build_target_phid'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	**kwargs):
	kwargs['name'] = 'Analyze CI file'
	kwargs['command'] = ['cat', self.ci_def_file]
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	kwargs['workdir'] = self.src_dir
	kwargs = self.setupShellMixin(kwargs)
	super().__init__(**kwargs)
	self.observer = logobserver.BufferLogObserver()
	self.addLogObserver('stdio', self.observer)

	def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
	jobs_with_data = []
	for job in stage_jobs:
	dep_job_names = [
	jn
	for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
	if ci_file.jobs[jn].has_artifacts_archive()
	]
	dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
	jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))

	trigger = TriggerMultipleJobsStep(
	name=stage_name,
	lbc=self.lbc,
	jobs_with_data=jobs_with_data,
	waitForFinish=True,
	doStepIf=on_success,
	)

	return trigger

	def get_steps(self, stdout):
	f = ci_file.CIFile(stdout)

	stages = f.get_grouped_jobs()
	jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
	job_names = [job.name for job in jobs]
	job_name_to_index_map = {}
	for (i, j) in enumerate(jobs):
	job_name_to_index_map[j.name] = i

	steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
	print('steps:', steps)
	return steps

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_ref_and_type(self):
	ref_type = 'branch'
	ref = self.getProperty('branch')

	if self.getProperty('category') == 'tag':
	ref_type = 'tag'

	if ref is not None:
	m = re.match(r'^refs/(heads\|tags)/(.+)$', ref)
	if m:
	ref = m.group(2)

	return (ref, ref_type)

	@defer.inlineCallbacks
	def get_pipeline_ci_vars(self):
	url = yield self.build.getUrl()
	res = {
	'CI_PIPELINE_ID': self.build.buildid,
	'CI_PIPELINE_IID': self.build.buildid,
	'CI_PIPELINE_URL': url,
	'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
	'CI_CONFIG_PATH': self.ci_def_file,
	}
	if not self.get_is_phorge():
	res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
	res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
	(ref, ref_type) = self.get_ref_and_type()
	res['CI_COMMIT_REF_NAME'] = ref
	res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
	res['CI_COMMIT_REF_PROTECTED'] = 'false'
	if ref_type == 'tag':
	res['CI_COMMIT_TAG'] = ref
	elif ref_type == 'branch':
	res['CI_COMMIT_BRANCH'] = ref
	return res

	@defer.inlineCallbacks
	def run(self):
	# run './build.sh --list-stages' to generate the list of stages
	cmd = yield self.makeRemoteShellCommand()
	yield self.runCommand(cmd)

	# if the command passes extract the list of stages
	result = cmd.results()
	if result == util.SUCCESS:
	pipeline_vars = yield self.get_pipeline_ci_vars()
	self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
	# create a ShellCommand for each stage and add them to the build
	self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))

	return result
	diff --git a/lilybuild/lilybuild/ci_syntax/ci_file.py b/lilybuild/lilybuild/ci_syntax/ci_file.py
	index fc78869..7d8e8fe 100644
	--- a/lilybuild/lilybuild/ci_syntax/ci_file.py
	+++ b/lilybuild/lilybuild/ci_syntax/ci_file.py
	@@ -1,229 +1,237 @@

	import yaml
	import jsonschema
	import json
	import os
	import re

	# https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json
	schema_file = os.path.join(os.path.dirname(__file__), 'ci.json')
	schema = None
	extend_limit = 11

	class CIValidationError(Exception):
	pass

	def get_schema():
	global schema
	if schema is not None:
	return schema
	else:
	with open(schema_file, 'r') as f:
	schema = json.loads(f.read())
	return schema

	def toplevel_entries():
	return get_schema()['properties']

	def normalize_script(script):
	if script is None:
	return []
	elif isinstance(script, str):
	return [script]
	else:
	res = []
	for l in script:
	if isinstance(l, str):
	res.append(l)
	else:
	res += l
	return res

	slug_re = re.compile('[^0-9a-z]')
	def ci_slugify(s):
	return re.sub(slug_re, '-', s.lower()[:63]).strip('-')

	def get_job_extend_seq(job_name, all_jobs, depth=0):
	# DFS traversal; child-first, self-last order
	res = []
	if job_name not in all_jobs:
	raise CIValidationError(f'Job "{job_name}" does not exist, but occurs in `extends`.')
	job_struct = all_jobs.get(job_name) or {}
	parents = job_struct.get('extends', [])
	if depth > extend_limit:
	raise CIValidationError(f'`extends` depth is over the limit of {extend_limit}.')
	if isinstance(parents, str):
	parents = [parents]
	for p in parents:
	res += get_job_extend_seq(p, all_jobs, depth + 1)
	res.append(job_name)

	return res

	depth_limit = 20
	def merge_job_deep(res, parent, depth=0):
	res_keys = set(res.keys())
	if depth > depth_limit:
	raise CIValidationError(f'depth is over the limit when merging job.')
	for k in parent:
	if k not in res_keys:
	res[k] = parent[k]
	elif isinstance(parent[k], dict) and isinstance(res[k], dict):
	child = res[k]
	# Necessary to avoid changing anything inside res[k] that was shallow-copied
	res[k] = {}
	merge_job_deep(res[k], child, depth + 1)
	merge_job_deep(res[k], parent[k], depth + 1)
	else:
	# Presenting in both parent and child, and it's not a dict,
	# so the child should take preference.
	pass

	def expand_job(job_name, all_jobs, defaults):
	seq = get_job_extend_seq(job_name, all_jobs)
	res = {}
	for ancestor_name in reversed(seq):
	merge_job_deep(res, all_jobs[ancestor_name])
	merge_job_deep(res, defaults)
	if 'extends' in res:
	del res['extends']
	return res

	class CIJob:
	def __init__(self, job_name, job_stage, job_struct):
	self.name = job_name
	self.stage = job_stage
	self.struct_raw = job_struct
	self.image = job_struct.get('image')
	self.before_script = normalize_script(job_struct.get('before_script'))
	self.script = normalize_script(job_struct.get('script'))
	self.after_script = normalize_script(job_struct.get('after_script'))
	self.artifacts = job_struct.get('artifacts') or {}
	self.dependencies = job_struct.get('dependencies')

	def get_predefined_ci_variables(self):
	return {
	'CI': 'true',
	'CI_JOB_NAME': self.name,
	'CI_JOB_NAME_SLUG': ci_slugify(self.name),
	'CI_JOB_STAGE': self.stage,
	}

	def has_artifacts_archive(self):
	return self.artifacts and 'paths' in self.artifacts

	def to_prop(self):
	return {
	'name': self.name,
	'stage': self.stage,
	'struct_raw': self.struct_raw,
	}

	@classmethod
	def from_prop(cls, prop):
	return cls(
	prop['name'],
	prop['stage'],
	prop['struct_raw']
	)

	def is_pages(self):
	return self.name == 'pages' or self.struct_raw.get('pages', False)

	+ def has_supported_coverage_report(self):
	+ return (
	+ 'reports' in self.artifacts
	+ and self.artifacts['reports'].get('coverage_report')
	+ and self.artifacts['reports']['coverage_report'].get('coverage_format') == 'cobertura'
	+ and self.artifacts['reports']['coverage_report'].get('path')
	+ )
	+
	OLD_TOPLEVEL_DEFAULTS = ['image', 'services', 'cache', 'before_script', 'after_script']
	DEFAULT_STAGES = ['.pre', 'build', 'test', 'deploy', '.post']
	DEFAULT_JOB_STAGE = 'test'

	class CIFile:
	'''
	Class for parsing CI file.
	'''

	def __init__(self, file_content):
	'''
	Construct a CI File from its text content.
	'''
	f = yaml.safe_load(file_content)
	if f is None:
	self.stages = []
	self.jobs = {}
	return

	jsonschema.validate(instance=f, schema=get_schema())

	self.stages = f.get('stages', DEFAULT_STAGES)
	self.jobs = {}

	defaults = f.get('default', {})
	if f.get('variables'):
	defaults['variables'] = f['variables']
	for kw in OLD_TOPLEVEL_DEFAULTS:
	if kw not in defaults and kw in f:
	defaults[kw] = f[kw]

	all_jobs = {}
	for job_name, job_struct in f.items():
	# 'pages' is a special job
	if job_name != 'pages' and job_name in toplevel_entries():
	continue
	all_jobs[job_name] = job_struct

	for job_name in all_jobs:
	# jobs starting with . will only be used for base jobs of other jobs,
	# they themselves are not run
	if job_name.startswith('.'):
	continue
	job_struct = expand_job(job_name, all_jobs, defaults)
	job_stage = job_struct.get('stage', DEFAULT_JOB_STAGE)
	if job_stage not in self.stages:
	raise CIValidationError(f'Job "{job_name}": Stage "{job_stage}" is not specified in CI file')
	self.jobs[job_name] = CIJob(job_name, job_stage, job_struct)

	self.validate_logic()

	def validate_logic(self):
	for job_name in self.jobs:
	self.validate_job(job_name)

	def validate_job(self, job_name):
	j = self.jobs[job_name]
	my_stage_order = self.stage_order(job_name)
	if j.dependencies is not None:
	for d in j.dependencies:
	try:
	stage_order = self.stage_order(d)
	except KeyError:
	raise CIValidationError(f'Dependency "{d}" of job "{job_name}" does not exist')
	if stage_order >= my_stage_order:
	raise CIValidationError(f'Dependency "{d}" of job "{job_name}" is not before the job in stage')

	def stage_order(self, jn):
	return self.stages.index(self.jobs[jn].stage)

	def get_grouped_jobs(self):
	groups = {}
	for job_name in self.jobs:
	job = self.jobs[job_name]
	if job.stage not in groups:
	groups[job.stage] = []
	groups[job.stage].append(job)
	res = []
	for stage in self.stages:
	if stage in groups:
	res.append((stage, groups[stage]))
	return res

	def get_jobs_to_pull_artifacts_from(self, job_name):
	'''
	Get a list of names of jobs of which the artifacts will be pulled
	from for the job named `job_name`.
	'''
	dependencies = self.jobs[job_name].dependencies
	if dependencies is None:
	dependencies = []
	cur_job_stage_order = self.stage_order(job_name)
	for jn in self.jobs:
	so = self.stage_order(jn)
	if so < cur_job_stage_order:
	dependencies.append(jn)
	return dependencies
	diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
	index 451fcf0..096ac73 100644
	--- a/lilybuild/lilybuild/config.py
	+++ b/lilybuild/lilybuild/config.py
	@@ -1,253 +1,256 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from twisted.internet import defer

	from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand, on_success
	from .phorge import CheckoutSourceFromPhorge, SendArtifactLinkToPhorge, SendBuildStatusToPhorge, MaybeRequireApprovalForPhorge
	from .helpers import normalize_base_url, phorge_token_to_arcrc

	import yaml

	def to_params(d):
	res = []
	for n in d:
	res.append(util.FixedParameter(name=n, default=d[n]))
	return res

	work_root = util.Interpolate('repos/%(prop:lilybuild_repo_id)s')
	src_relative = 'sources'
	src_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/sources')
	result_relative = 'result'
	result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
	artifact_stage_relative = 'stage'
	artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
	storage_dir = '/buildbot/storage'

	report_phorge = 'phorge'
	report_forgejo = 'forgejo'

	class LilyBuildConfig:
	builder_name = 'lilybuild'
	builder_name_run_job = 'lilybuild-job'
	builder_name_force = 'lilybuild-force'
	main_builder_tag = 'lilybuild-pipeline'
	triggerable_scheduler_name = 'lilybuild-triggerable'
	force_triggerable_scheduler_name = 'lilybuild-force-triggerable'

	def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None):
	self.c = c
	self.poll_interval = 300
	self.workernames = workernames
	if reports is None:
	reports = [report_phorge, report_forgejo]
	self.reports = reports
	self.repos = {}
	self.repo_id_by_url = {}
	self.branch_skip_re = '^phabricator/'
	self.phorge_base_url = normalize_base_url(phorge_base_url)
	self.phorge_token = phorge_token
	self.ssh_priv_key = ssh_priv_key
	self.ssh_known_hosts = ssh_known_hosts

	def configure_factory_and_builder(self):
	self.add_lilybuild_builder()
	self.add_lilybuild_job_builder()

	def create_source_step(self):
	return CheckoutSourceFromPhorge(
	lbc=self,
	repourl=util.Property('lilybuild_repo'),
	mode='full',
	workdir=src_dir,
	submodules=True
	)

	def get_arcrc_for_repo(self, repo_id):
	return util.Transform(
	phorge_token_to_arcrc,
	self.repos[repo_id]['phorge_base_url'],
	self.repos[repo_id]['phorge_token'],
	)

	+ def get_phorge_base_url_for_repo(self, repo_id):
	+ return self.repos[repo_id]['phorge_base_url']
	+
	def add_lilybuild_builder(self):
	factory = util.BuildFactory()
	factory.addStep(self.create_source_step())
	if report_phorge in self.reports:
	factory.addStep(MaybeRequireApprovalForPhorge(lbc=self, workdir=src_dir))
	factory.addStep(SendArtifactLinkToPhorge(lbc=self, workdir=src_dir))
	factory.addStep(AnalyzeCIFileCommand(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	storage_dir=storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	doStepIf=on_success,
	))
	if report_phorge in self.reports:
	factory.addStep(SendBuildStatusToPhorge(lbc=self, workdir=src_dir))

	builder = util.BuilderConfig(
	name=self.builder_name,
	workernames=self.workernames,
	factory=factory,
	tags=[self.main_builder_tag],
	)

	self.c['builders'].append(builder)

	def add_lilybuild_job_builder(self):
	factory_job = util.BuildFactory()
	factory_job.addStep(self.create_source_step())
	factory_job.addStep(RunCIJobStep(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	storage_dir=storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	))

	builder_job = util.BuilderConfig(
	name=self.builder_name_run_job,
	workernames=self.workernames,
	factory=factory_job,
	# Tell buildbot to never collapse build requests for this one
	# because otherwise triggering multiple jobs will cause it to only run one
	# https://docs.buildbot.net/current/manual/configuration/builders.html#collapsing-build-requests
	collapseRequests=False,
	)
	self.c['builders'].append(builder_job)

	s = schedulers.Triggerable(self.triggerable_scheduler_name, builderNames=[self.builder_name_run_job])
	self.c['schedulers'].append(s)

	def record_url(self, repo_id, repo_url):
	if repo_url in self.repo_id_by_url:
	raise Exception(f'repo url "{repo_url}" already recorded')
	self.repo_id_by_url[repo_url] = repo_id

	def add_repo(
	self,
	repo_id,
	repo_url,
	do_poll=False,
	alternative_urls=None,
	phorge_base_url=None,
	phorge_token=None,
	# Function(build -> dict(str -> str \| renderable<str>))
	variables_getter=None,
	):
	if not alternative_urls:
	alternative_urls = []

	self.repos[repo_id] = {
	'repo_id': repo_id,
	'repo_url': repo_url,
	'do_poll': do_poll,
	'alternative_urls': alternative_urls,
	'phorge_base_url': normalize_base_url(phorge_base_url) or self.phorge_base_url,
	'phorge_token': phorge_token or self.phorge_token,
	'variables_getter': variables_getter or (lambda _build: {}),
	}
	self.record_url(repo_id, repo_url)
	for u in alternative_urls:
	self.record_url(repo_id, u)

	def get_builder_name_for_repo(self, repo_def):
	main_repo_url = repo_def['repo_url']
	return f'lilybuild - {main_repo_url}'

	def add_one_repo_def(self, repo_def):
	print('Adding repo', repo_def)
	repo_urls = [repo_def['repo_url']] + repo_def['alternative_urls']
	repo_id = repo_def['repo_id']
	do_poll = repo_def['do_poll']
	main_repo_url = repo_def['repo_url']
	properties = {
	'lilybuild_repo': main_repo_url,
	'lilybuild_repo_id': repo_id,
	'virtual_builder_name': self.get_builder_name_for_repo(repo_def),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	self.c['schedulers'].append(schedulers.AnyBranchScheduler(
	name=f'lilybuild-anybranch - {main_repo_url}',
	change_filter=util.ChangeFilter(
	repository=repo_urls,
	),
	treeStableTimer=None,
	builderNames=[self.builder_name],
	properties=properties))

	if do_poll:
	for repo in repo_urls:
	print('Adding poller')
	self.c['change_source'].append(changes.GitPoller(
	repo,
	workdir=f'gitpoller/{repo_id}', branches=True,
	pollInterval=self.poll_interval))

	def configure_pipeline_defs(self):
	for repo_id in self.repos:
	self.add_one_repo_def(self.repos[repo_id])

	self.add_force_builder()
	self.add_force_scheduler()

	def add_force_builder(self):
	def get_repo_id_by_url(url):
	return self.repo_id_by_url[url]

	factory_force = util.BuildFactory()
	factory_force.addStep(steps.Trigger(
	name='trigger lilybuild',
	schedulerNames=[self.force_triggerable_scheduler_name],
	waitForFinish=True,
	set_properties={
	'lilybuild_repo': util.Property('lilybuild_repo'),
	'lilybuild_repo_id': util.Transform(get_repo_id_by_url, util.Property('lilybuild_repo')),
	'virtual_builder_name': util.Interpolate('lilybuild - %(prop:lilybuild_repo)s'),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	))

	builder_force = util.BuilderConfig(
	name=self.builder_name_force,
	workernames=self.workernames,
	factory=factory_force,
	)

	self.c['builders'].append(builder_force)
	# this is triggered by lilybuild-force, and it triggers lilybuild
	# via its virtual builder
	s = schedulers.Triggerable(
	self.force_triggerable_scheduler_name,
	builderNames=[self.builder_name],
	)
	self.c['schedulers'].append(s)


	def add_force_scheduler(self):
	repo_urls = []
	for r in self.repos.values():
	repo_urls.append(r['repo_url'])

	repo_param = util.ChoiceStringParameter(
	name='lilybuild_repo',
	required=True,
	strict=True,
	choices=repo_urls,
	)

	self.c['schedulers'].append(schedulers.ForceScheduler(
	name="force",
	builderNames=[self.builder_name_force],
	properties=[repo_param],
	))
	diff --git a/lilybuild/lilybuild/phorge.py b/lilybuild/lilybuild/phorge.py
	index 0fe5631..d4f4a03 100644
	--- a/lilybuild/lilybuild/phorge.py
	+++ b/lilybuild/lilybuild/phorge.py
	@@ -1,294 +1,346 @@

	import stat
	import os
	import json
	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from buildbot.steps.download_secret_to_worker import DownloadSecretsToWorker, RemoveWorkerFileSecret
	from buildbot.steps.worker import CompositeStepMixin
	from buildbot.util import bytes2unicode
	from twisted.internet import defer
	from twisted.python import log

	+SEND_COVERAGE_EXEC = '/lilybuild/lilybuild/send_coverage.py'
	+
	class PhorgeMixin(CompositeStepMixin, buildstep.ShellMixin):
	'''
	Should be inherited by a BuildStep.

	Required properties:
	self.lbc: LilyBuildConfig
	self.secret_dir: str
	'''
	staging_uri_prop_name = 'harbormaster_variable_repository.staging.uri'
	staging_ref_prop_name = 'harbormaster_variable_repository.staging.ref'
	diff_id_prop_name = 'harbormaster_variable_buildable.diff'
	build_target_prop_name = 'harbormaster_build_target_phid'
	arc_command = '/tools/arcanist/bin/arc'

	def setup_phorge_mixin(self, kwargs):
	shell_mixin_kwargs = {}
	shell_mixin_inherit_args = ['workdir']
	for a in shell_mixin_inherit_args:
	if a in kwargs:
	shell_mixin_kwargs[a] = kwargs[a]

	self.setupShellMixin(shell_mixin_kwargs)

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_secret_dir(self):
	# Taken from buildbot.util.git
	workerbuilddir = bytes2unicode(self.build.builder.config.workerbuilddir)
	workdir = self.workdir.rstrip('/\\')

	if os.path.isabs(workdir):
	parent_path = os.path.dirname(workdir)
	else:
	assert self.worker is not None
	parent_path = os.path.join(
	self.worker.worker_basedir, os.path.dirname(workdir)
	)

	basename = f'.{workerbuilddir}.{os.path.basename(workdir)}.lilybuild-phorge'
	secret_dir = os.path.join(parent_path, basename)
	log.msg('Arc secret dir is', secret_dir)
	return secret_dir

	@defer.inlineCallbacks
	- def make_arc_command(self, args, **kwargs):
	+ def make_command(self, **kwargs):
	cmd = yield self.makeRemoteShellCommand(
	- command=[self.arc_command] + args,
	env={'HOME': self.get_secret_dir()},
	**kwargs
	)
	return cmd

	@defer.inlineCallbacks
	- def run_arc_with_secret(self, args, **kwargs):
	+ def make_arc_command(self, args, **kwargs):
	+ cmd = yield self.make_command(
	+ command=[self.arc_command] + args,
	+ **kwargs
	+ )
	+ return cmd
	+
	+ @defer.inlineCallbacks
	+ def run_command_with_secret(self, **kwargs):
	try:
	res = yield self.download_arc_secret()
	if res != util.SUCCESS:
	return res
	- cmd = yield self.make_arc_command(args, **kwargs)
	+ cmd = yield self.make_command(**kwargs)
	yield self.runCommand(cmd)
	return cmd.results()
	except Exception as e:
	raise e
	finally:
	yield self.remove_arc_secret()


	+ @defer.inlineCallbacks
	+ def run_arc_with_secret(self, args, **kwargs):
	+ res = yield self.run_command_with_secret(
	+ command=[self.arc_command] + args,
	+ **kwargs
	+ )
	+ return res
	+
	@defer.inlineCallbacks
	def download_arc_secret(self):
	arcrc = self.lbc.get_arcrc_for_repo(self.getProperty('lilybuild_repo_id'))
	if not arcrc:
	self.addCompleteLog('error', 'arcrc secret is not specified for the repository')
	return util.FAILURE

	arcrc_content = yield self.build.render(arcrc)

	path = os.path.join(self.get_secret_dir(), '.arcrc')

	yield self.downloadFileContentToWorker(
	path, arcrc_content, mode=stat.S_IRUSR \| stat.S_IWUSR, workdir=self.workdir
	)
	# If failed, it will raise
	return util.SUCCESS

	@defer.inlineCallbacks
	def remove_arc_secret(self):
	path = self.get_secret_dir()

	yield self.runRmdir(path, abandonOnFailure=False)
	return util.SUCCESS

	class CheckoutSourceFromPhorge(PhorgeMixin, steps.Git):
	'''
	Set a property `lilybuild_source`:
	`non-phorge` if the source is not a Differential Diff,
	`staging` if the source is checked out from staging area,
	`arc-patch` if the source is checked out by `arc patch`, either because
	there is no staging area defined, or because the staging area does not
	have the relevant diffs.
	'''
	def __init__(self, lbc, **kwargs):
	self.lbc = lbc
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	sshPrivateKey=lbc.ssh_priv_key,
	sshKnownHosts=lbc.ssh_known_hosts,
	**kwargs)

	@defer.inlineCallbacks
	def run_vc(self, branch, revision, patch):
	def set_prop(k, v):
	self.setProperty(k, v, self.__class__.__name__)

	is_phorge = self.get_is_phorge()
	if not is_phorge:
	set_prop('lilybuild_source', 'non-phorge')
	res = yield super().run_vc(branch, revision, patch)
	return res

	staging_uri = self.getProperty(self.staging_uri_prop_name)
	staging_ref = self.getProperty(self.staging_ref_prop_name)

	if staging_uri:
	old_repourl = self.repourl
	self.repourl = staging_uri
	# Assume branch is the same as staging ref
	try:
	res = yield super().run_vc(branch, revision, patch)
	if res == util.SUCCESS:
	# If we can get the source from the staging area, then we are done
	set_prop('lilybuild_source', 'staging')
	return res
	except:
	pass
	self.addCompleteLog('log', 'Cannot fetch source from staging area, trying to `arc patch`')
	self.repourl = old_repourl
	else:
	self.addCompleteLog('log', 'No staging area defined, trying to `arc patch`')

	res = yield super().run_vc(branch='HEAD', revision=None, patch=None)
	if res != util.SUCCESS:
	self.addCompleteLog('error', 'Cannot checkout repository head, unable to proceed')
	return res

	diff_id = self.getProperty(self.diff_id_prop_name)
	if not diff_id:
	self.addCompleteLog('error', 'Diff id is not present')
	return util.FAILURE

	res = yield self.run_arc_with_secret([
	'patch',
	'--diff', diff_id,
	'--nobranch', '--nocommit',
	])
	set_prop('lilybuild_source', 'arc-patch')
	return res

	class SendArtifactLinkToPhorge(PhorgeMixin, steps.BuildStep):
	def __init__(self, lbc, **kwargs):
	self.lbc = lbc
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	name='Send artifact link to Phorge',
	doStepIf=lambda step: step.get_is_phorge(),
	alwaysRun=True,
	**kwargs
	)

	@defer.inlineCallbacks
	def run(self):
	if self.getProperty('lilybuild_require_approval'):
	# Do not send another artifact if this requires approval
	return util.SKIPPED

	build_url = yield self.build.getUrl()

	data = {
	'buildTargetPHID': self.getProperty(self.build_target_prop_name),
	'artifactKey': f'lilybuild-{self.build.buildid}',
	'artifactType': 'uri',
	'artifactData': {
	'uri': build_url,
	'name': f'Buildbot build #{self.build.buildid}',
	'ui.external': True,
	},
	}
	encoded_data = json.dumps(data)
	res = yield self.run_arc_with_secret([
	'call-conduit',
	'--',
	'harbormaster.createartifact',
	], initialStdin=encoded_data)
	return res

	class MaybeRequireApprovalForPhorge(PhorgeMixin, steps.BuildStep):
	def __init__(self, lbc, sources_need_approval=None, **kwargs):
	self.lbc = lbc
	if sources_need_approval is None:
	sources_need_approval = ['arc-patch']
	self.sources_need_approval = sources_need_approval
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	name='Maybe require approval for phorge sources',
	doStepIf=lambda step: step.get_is_phorge(),
	alwaysRun=True,
	**kwargs
	)

	@defer.inlineCallbacks
	def run(self):
	def set_prop(k, v):
	self.setProperty(k, v, self.__class__.__name__)

	if self.getProperty('lilybuild_source') not in self.sources_need_approval:
	set_prop('lilybuild_require_approval', False)
	return util.SUCCESS

	build_data = yield self.master.data.get(('builds', self.build.buildid))
	build_request_data = yield self.master.data.get(('buildrequests', build_data['buildrequestid']))
	buildset_data = yield self.master.data.get(('buildsets', build_request_data['buildsetid']))
	rebuilt_buildid = buildset_data['rebuilt_buildid']
	if rebuilt_buildid is not None:
	# This is rebuilt, because someone approved it earlier
	set_prop('lilybuild_require_approval', False)
	return util.SUCCESS

	self.addCompleteLog('info', 'This build requires approval. To approve, rebuild this build.')
	build_url = yield self.build.getUrl()

	data = {
	'buildTargetPHID': self.getProperty(self.build_target_prop_name),
	'artifactKey': f'lilybuild-{self.build.buildid}-pending',
	'artifactType': 'uri',
	'artifactData': {
	'uri': build_url,
	'name': f'Requires approval',
	'ui.external': True,
	},
	}
	encoded_data = json.dumps(data)
	res = yield self.run_arc_with_secret([
	'call-conduit',
	'--',
	'harbormaster.createartifact',
	], initialStdin=encoded_data)
	set_prop('lilybuild_require_approval', True)
	return util.FAILURE


	class SendBuildStatusToPhorge(PhorgeMixin, steps.BuildStep):
	def __init__(self, lbc, **kwargs):
	self.lbc = lbc
	self.setup_phorge_mixin(kwargs)
	super().__init__(
	name='Send build status to Phorge',
	doStepIf=lambda step: step.get_is_phorge(),
	alwaysRun=True,
	**kwargs
	)

	@defer.inlineCallbacks
	def run(self):
	if self.getProperty('lilybuild_require_approval'):
	# Do not send final build result if this is skipped, because
	# otherwise we cannot update it later.
	return util.SKIPPED

	build_url = yield self.build.getUrl()

	data = {
	'receiver': self.getProperty(self.build_target_prop_name),
	'type': 'pass' if self.build.results == util.SUCCESS else 'fail',
	}
	encoded_data = json.dumps(data)
	res = yield self.run_arc_with_secret([
	'call-conduit',
	'--',
	'harbormaster.sendmessage',
	], initialStdin=encoded_data)
	return res
	+
	+class SendCoverageToPhorge(PhorgeMixin, steps.BuildStep):
	+ def __init__(self, lbc, coverage_file, **kwargs):
	+ self.lbc = lbc
	+ self.coverage_file = coverage_file
	+ self.setup_phorge_mixin(kwargs)
	+ super().__init__(
	+ name='Send coverage to Phorge',
	+ doStepIf=lambda step: step.get_is_phorge(),
	+ alwaysRun=True,
	+ **kwargs
	+ )
	+
	+ @defer.inlineCallbacks
	+ def run(self):
	+ phorge_url = yield self.build.render(self.lbc.get_phorge_base_url_for_repo(self.getProperty('lilybuild_repo_id')))
	+ build_url = yield self.build.getUrl()
	+ filename = yield self.build.render(self.coverage_file)
	+ receiver = self.getProperty(self.build_target_prop_name)
	+ job_name = self.getProperty('lilybuild_job_prop').get('name')
	+ is_success = self.build.results == util.SUCCESS
	+
	+ data = {
	+ 'filename': filename,
	+ 'build_url': build_url,
	+ 'receiver': receiver,
	+ 'is_success': is_success,
	+ 'phorge_url': phorge_url,
	+ 'job_name': job_name,
	+ }
	+ encoded_data = json.dumps(data)
	+ res = yield self.run_command_with_secret(command=[
	+ SEND_COVERAGE_EXEC,
	+ ], initialStdin=encoded_data)
	+ return res
	diff --git a/lilybuild/lilybuild/send_coverage.py b/lilybuild/lilybuild/send_coverage.py
	new file mode 100755
	index 0000000..7746ca2
	--- /dev/null
	+++ b/lilybuild/lilybuild/send_coverage.py
	@@ -0,0 +1,43 @@
	+#!/usr/bin/env python3
	+
	+import sys
	+import json
	+import subprocess
	+
	+def main(args):
	+ filename = args['filename']
	+ build_url = args['build_url']
	+ is_success = args['is_success']
	+ receiver = args['receiver']
	+ phorge_url = args['phorge_url']
	+ job_name = args['job_name']
	+
	+ with open(filename) as f:
	+ coverage = json.loads(f.read())
	+
	+ data = {
	+ 'receiver': receiver,
	+ 'type': 'work',
	+ 'unit': [{
	+ 'name': f'Coverage ({job_name})',
	+ 'result': 'pass' if is_success else 'fail',
	+ 'details': build_url,
	+ 'format': 'remarkup',
	+ 'coverage': coverage,
	+ }],
	+ }
	+
	+ subprocess.run([
	+ 'arc',
	+ '--config',
	+ 'phabricator.uri=' + phorge_url,
	+ 'call-conduit',
	+ '--',
	+ 'harbormaster.sendmessage',
	+ ], check=True, input=json.dumps(data), encoding='utf-8')
	+
	+if __name__ == '__main__':
	+ if len(sys.argv) > 1:
	+ main(json.loads(sys.argv[1]))
	+ else:
	+ main(json.loads(sys.stdin.read()))
	diff --git a/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py b/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
	index c612adf..79ef83a 100644
	--- a/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
	+++ b/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
	@@ -1,304 +1,311 @@

	import unittest
	import yaml
	from lilybuild.ci_syntax.ci_file import CIFile, CIValidationError, get_job_extend_seq, merge_job_deep
	from lilybuild.tests.resources import get_res

	class CIFileTest(unittest.TestCase):
	def test_empty(self):
	r = CIFile('')
	self.assertEqual(r.stages, [])
	self.assertEqual(r.jobs, {})

	def test_invalid(self):
	with self.assertRaises(yaml.composer.ComposerError) as m:
	r = CIFile('*xxx')

	def test_simple(self):
	r = CIFile(get_res('simple'))
	self.assertEqual(r.stages, ['build'])
	self.assertEqual(list(r.jobs), ['build-a'])
	self.assertEqual(r.jobs['build-a'].image, 'test')
	self.assertEqual(r.jobs['build-a'].before_script, ['ls'])
	self.assertEqual(r.jobs['build-a'].script, ['make', 'make install'])
	self.assertEqual(r.jobs['build-a'].after_script, ['find', 'echo ok'])

	def test_noscript(self):
	r = CIFile(get_res('noscript'))

	self.assertEqual(r.stages, ['build'])
	self.assertEqual(list(r.jobs), ['build-a'])
	self.assertEqual(r.jobs['build-a'].image, 'test')
	self.assertEqual(r.jobs['build-a'].script, [])

	def test_defaults(self):
	r = CIFile(get_res('defaults'))
	self.assertEqual(r.stages, ['build'])
	self.assertEqual(list(r.jobs), ['build-a'])
	self.assertEqual(r.jobs['build-a'].image, 'test')
	self.assertEqual(r.jobs['build-a'].before_script, ['ls'])
	self.assertEqual(r.jobs['build-a'].script, ['make', 'make install'])
	self.assertEqual(r.jobs['build-a'].after_script, ['find', 'echo ok'])

	def test_pages(self):
	r = CIFile(get_res('pages'))
	self.assertEqual(r.stages, ['test'])
	self.assertEqual(list(r.jobs), ['pages'])
	self.assertEqual(r.jobs['pages'].script, ['make docs'])
	self.assertTrue(r.jobs['pages'].is_pages())

	def test_pages_attr(self):
	r = CIFile(get_res('pages_attr'))
	self.assertFalse(r.jobs['not-pages'].is_pages())
	self.assertTrue(r.jobs['is-pages'].is_pages())

	def test_dotjobs(self):
	r = CIFile(get_res('dotjobs'))
	self.assertEqual(list(r.jobs), [])

	def test_no_such_stage(self):
	with self.assertRaises(CIValidationError) as m:
	r = CIFile(get_res('no_such_stage'))

	def test_default_stages(self):
	r = CIFile(get_res('default_stages'))
	self.assertEqual(r.jobs['a'].stage, 'build')
	self.assertEqual(r.jobs['b'].stage, 'test')
	self.assertEqual(r.jobs['c'].stage, 'deploy')
	self.assertEqual(r.jobs['d'].stage, '.pre')
	self.assertEqual(r.jobs['e'].stage, '.post')
	self.assertEqual(r.jobs['f'].stage, 'test')

	def test_group_jobs(self):
	r = CIFile(get_res('group_jobs'))
	groups = r.get_grouped_jobs()
	self.assertEqual(len(groups), 2)
	self.assertEqual(groups[0][0], 'build')
	self.assertEqual(len(groups[0][1]), 2)
	self.assertTrue(next(j for j in groups[0][1] if j.name == 'a'))
	self.assertTrue(next(j for j in groups[0][1] if j.name == 'b'))
	self.assertEqual(groups[1][0], 'test')
	self.assertEqual(len(groups[1][1]), 2)
	self.assertTrue(next(j for j in groups[1][1] if j.name == 'c'))
	self.assertTrue(next(j for j in groups[1][1] if j.name == 'd'))

	def test_dependencies(self):
	r = CIFile(get_res('dependencies'))
	self.assertEqual(r.get_jobs_to_pull_artifacts_from('a'), [])
	self.assertEqual(r.get_jobs_to_pull_artifacts_from('a2'), [])
	self.assertEqual(r.get_jobs_to_pull_artifacts_from('b'), ['a'])
	self.assertEqual(set(r.get_jobs_to_pull_artifacts_from('b2')), set(['a', 'a2']))

	def test_nonexistent_dep(self):
	with self.assertRaises(CIValidationError) as m:
	r = CIFile(get_res('nonexistent_dep'))
	self.assertEqual(str(m.exception), 'Dependency "a3" of job "b" does not exist')

	def test_late_dep(self):
	with self.assertRaises(CIValidationError) as m:
	r = CIFile(get_res('late_dep'))
	self.assertEqual(str(m.exception), 'Dependency "a2" of job "a" is not before the job in stage')

	def test_artifacts(self):
	r = CIFile(get_res('has_artifacts_archive'))
	self.assertEqual(r.jobs['has_archive'].artifacts, { 'paths': ['a'] })
	self.assertEqual(r.jobs['no_archive'].artifacts, {})
	self.assertEqual(r.jobs['no_artifacts'].artifacts, {})

	def test_has_artifacts_archive(self):
	r = CIFile(get_res('has_artifacts_archive'))
	self.assertTrue(r.jobs['has_archive'].has_artifacts_archive())
	self.assertFalse(r.jobs['no_archive'].has_artifacts_archive())
	self.assertFalse(r.jobs['no_artifacts'].has_artifacts_archive())

	+ def test_artifacts_reports(self):
	+ r = CIFile(get_res('artifacts_reports'))
	+ self.assertFalse(r.jobs['has_archive'].has_supported_coverage_report())
	+ self.assertFalse(r.jobs['empty_reports'].has_supported_coverage_report())
	+ self.assertTrue(r.jobs['yes_reports'].has_supported_coverage_report())
	+ self.assertFalse(r.jobs['no_supported_reports'].has_supported_coverage_report())
	+
	def test_extends(self):
	r = CIFile(get_res('extends'))
	self.assertEqual(r.jobs['build-a'].struct_raw, {
	'image': 'alpine',
	'variables': { 'SECRET': 'abcdef', 'CC': 'gcc' },
	'after_script': ['rm -r cache'],
	'stage': 'build',
	'before_script': ['apk add gcc'],
	'script': ['make a', 'make install'],
	})
	self.assertEqual(r.jobs['build-b'].struct_raw, {
	'image': 'alpine',
	'variables': { 'SECRET': 'abcdef', 'CC': 'cc' },
	'after_script': ['rm -r cache'],
	'stage': 'build',
	'before_script': ['apk add gcc'],
	'script': ['make b', 'make install'],
	})
	self.assertEqual(r.jobs['test'].struct_raw, {
	'image': 'tester',
	'variables': { 'SECRET': 'abcdef' },
	'after_script': ['rm -r cache'],
	'stage': 'test',
	'script': ['make test'],
	})
	self.assertEqual(r.jobs['container-amd64'].struct_raw, {
	'image': 'podman',
	'variables': { 'SECRET': 'abcdef', 'DOCKER': 'podman', 'PLATFORM': 'amd64', 'LOGIN': 'defghi' },
	'after_script': ['rm -r cache'],
	'stage': 'deploy',
	'script': ['podman login', 'make docker'],
	'tags': ['podman']
	})
	self.assertEqual(r.jobs['container-aarch64'].struct_raw, {
	'image': 'podman',
	'variables': { 'SECRET': 'abcdef', 'DOCKER': 'podman', 'PLATFORM': 'aarch64', 'LOGIN': 'defghi' },
	'after_script': ['rm -r cache'],
	'stage': 'deploy',
	'script': ['podman login', 'make docker'],
	'tags': ['podman']
	})
	self.assertEqual(r.jobs['appimage-amd64'].struct_raw, {
	'image': 'alpine',
	'variables': { 'SECRET': 'abcdef', 'PLATFORM': 'amd64', 'LOGIN': 'defghi' },
	'after_script': ['rm -r cache'],
	'stage': 'deploy',
	'script': ['make appimage'],
	'tags': ['amd64']
	})
	self.assertEqual(r.jobs['appimage-aarch64'].struct_raw, {
	'image': 'alpine',
	'variables': { 'SECRET': 'abcdef', 'PLATFORM': 'aarch64', 'LOGIN': 'defghi' },
	'after_script': ['rm -r cache'],
	'stage': 'deploy',
	'script': ['make appimage'],
	'tags': ['aarch64']
	})

	class GetJobExtendSeqTest(unittest.TestCase):
	def test_no_extends(self):
	all_jobs = {
	'nothing': {}
	}
	self.assertEqual(get_job_extend_seq('nothing', all_jobs), ['nothing'])

	def test_multi_extends(self):
	all_jobs = {
	'a': { 'extends': ['b', 'c'] },
	'b': { 'extends': ['d', 'e'] },
	'c': { 'extends': ['f', 'g'] },
	'd': {},
	'e': {},
	'f': {},
	'g': {},
	}
	self.assertEqual(get_job_extend_seq('a', all_jobs), ['d', 'e', 'b', 'f', 'g', 'c', 'a'])

	def test_common_ancestor(self):
	all_jobs = {
	'a': { 'extends': ['b', 'c'] },
	'b': { 'extends': ['d', 'e'] },
	'c': { 'extends': ['e', 'd'] },
	'd': {},
	'e': {},
	'f': {},
	'g': {},
	}
	self.assertEqual(get_job_extend_seq('a', all_jobs), ['d', 'e', 'b', 'e', 'd', 'c', 'a'])

	def test_self_cycle(self):
	all_jobs = {
	'a': { 'extends': ['a'] },
	}
	with self.assertRaises(CIValidationError) as m:
	r = get_job_extend_seq('a', all_jobs)
	self.assertTrue(str(m.exception).startswith('`extends` depth is over the limit of'))

	def test_complex_cycle(self):
	all_jobs = {
	'a': { 'extends': ['b', 'c'] },
	'b': { 'extends': ['d', 'c'] },
	'c': { 'extends': ['d'] },
	'd': { 'extends': ['a'] },
	}
	with self.assertRaises(CIValidationError) as m:
	r = get_job_extend_seq('a', all_jobs)
	self.assertTrue(str(m.exception).startswith('`extends` depth is over the limit of'))

	def test_nonexistent_parent(self):
	all_jobs = {
	'a': { 'extends': ['b', 'c'] },
	'b': { 'extends': ['d', 'c'] },
	'c': { 'extends': ['d'] },
	}
	with self.assertRaises(CIValidationError) as m:
	r = get_job_extend_seq('a', all_jobs)
	self.assertEqual(str(m.exception), 'Job "d" does not exist, but occurs in `extends`.')

	def test_null_parent(self):
	all_jobs = {
	'a': { 'extends': ['b', 'c'] },
	'b': { 'extends': ['d', 'c'] },
	'c': { 'extends': ['d'] },
	'd': None,
	}
	self.assertEqual(get_job_extend_seq('a', all_jobs), ['d', 'd', 'c', 'b', 'd', 'c', 'a'])

	class MergeJobTest(unittest.TestCase):
	def test_simple(self):
	parent = {
	'a': {
	'a/1': {
	'a/1/1': '3',
	'a/1/2': 4,
	},
	'a/2': {},
	},
	'b': 1,
	}
	child = {
	'a': {
	'a/1': {
	'a/1/2': '6',
	'a/1/3': 5,
	},
	'a/2': 'mew',
	},
	'c': 'test',
	}
	merge_job_deep(child, parent)
	self.assertEqual(parent, {
	'a': {
	'a/1': {
	'a/1/1': '3',
	'a/1/2': 4,
	},
	'a/2': {},
	},
	'b': 1,
	})
	self.assertEqual(child, {
	'a': {
	'a/1': {
	'a/1/1': '3',
	'a/1/2': '6',
	'a/1/3': 5,
	},
	'a/2': 'mew',
	},
	'b': 1,
	'c': 'test',
	})

	def test_no_overflow(self):
	a = {
	'1': '2',
	}
	b = {
	'2': '4',
	'a': a,
	}
	a['a'] = b

	res = {}
	merge_job_deep(res, b)
	with self.assertRaises(CIValidationError) as m:
	merge_job_deep(res, a)
	self.assertEqual(str(m.exception), 'depth is over the limit when merging job.')

	if __name__ == '__main__':
	unittest.main()
	diff --git a/lilybuild/lilybuild/tests/ci_syntax/res/artifacts_reports.yaml b/lilybuild/lilybuild/tests/ci_syntax/res/artifacts_reports.yaml
	new file mode 100644
	index 0000000..536e627
	--- /dev/null
	+++ b/lilybuild/lilybuild/tests/ci_syntax/res/artifacts_reports.yaml
	@@ -0,0 +1,30 @@
	+
	+has_archive:
	+ script: echo
	+ artifacts:
	+ paths:
	+ - a
	+
	+empty_reports:
	+ script: echo
	+ artifacts:
	+ paths:
	+ - a
	+ reports:
	+ coverage_report:
	+
	+yes_reports:
	+ script: echo
	+ artifacts:
	+ reports:
	+ coverage_report:
	+ coverage_format: 'cobertura'
	+ path: 'x.xml'
	+
	+no_supported_reports:
	+ script: echo
	+ artifacts:
	+ reports:
	+ coverage_report:
	+ coverage_format: 'jacoco'
	+ path: 'x.xml'
	diff --git a/lilybuild/podman-helper b/lilybuild/podman-helper
	index a09ea9f..795d54a 100755
	--- a/lilybuild/podman-helper
	+++ b/lilybuild/podman-helper
	@@ -1,211 +1,212 @@
	#!/usr/bin/env python3

	import subprocess
	import sys
	import os
	import json
	import random
	import traceback
	import string
	import time

	work_vol_mount_dir = '/build'
	script_vol_mount_dir = '/script'
	script_name = script_vol_mount_dir + '/run.sh'
	volume_helper_image = os.environ.get('LILYBUILD_VOLUME_HELPER_IMAGE', 'r.lily-is.land/infra/lilybuild/volume-helper:servant')
	key_file_pub = '/secrets/lilybuild-volume-helper-key.pub'
	key_file_sub = '/secrets/lilybuild-volume-helper-key'
	ssh_port = '2222'
	ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null'
	worker_container_name = os.environ['HOSTNAME']
	volumes_to_remove = []
	helper_container_id = None
	ssh_max_wait = 10
	ssh_wait_interval_sec = 1

	col_info = '\x1b[1;34m'
	col_success = '\x1b[1;32m'
	col_error = '\x1b[1;31m'
	col_reset = '\x1b[0m'

	def pinfo(args, *kwargs):
	print(col_info, args, col_reset, *kwargs)
	sys.stdout.flush()

	def perror(args, *kwargs):
	print(col_error, args, col_reset, *kwargs)
	sys.stdout.flush()

	def psuccess(args, *kwargs):
	print(col_success, args, col_reset, *kwargs)
	sys.stdout.flush()

	def gen_random_id():
	# https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits
	return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(10))

	def verbose_run(args, *kwargs):
	print('run:', args, kwargs)
	sys.stdout.flush()
	return subprocess.run(args, *kwargs)

	def create_volume(t):
	res = verbose_run([
	'podman', 'volume', 'create',
	'--label', 'lilybuild=' + t,
	], check=True, capture_output=True, encoding='utf-8')
	volname = res.stdout.strip()
	volumes_to_remove.append(volname)
	return volname

	def clean_volumes():
	verbose_run([
	'podman', 'volume', 'rm', '--',
	] + volumes_to_remove, capture_output=True)

	def clean_helper_container():
	verbose_run([
	'podman', 'container', 'rm', '-f', helper_container_id,
	], capture_output=True)

	def start_helper_service(work_volname, script_volname):
	res = verbose_run([
	'podman', 'container', 'inspect', worker_container_name,
	], check=True, capture_output=True, encoding='utf-8')
	container_stat = json.loads(res.stdout)[0]
	pod = container_stat.get('Pod')
	networks = list(container_stat.get('NetworkSettings').get('Networks').keys())
	alias = gen_random_id()
	container_name = 'lilybuild-helper-' + alias

	with open(key_file_pub) as f:
	pub_key = f.readline().strip()

	res = verbose_run([
	'podman', 'run', '--rm', '-d', '--name', container_name,
	f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
	f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
	f'--pod={pod}',
	f'--net={networks[0]}',
	f'--network-alias={alias}',
	'--label', 'lilybuild=helper',
	'-e', 'PUID=0',
	'-e', 'PGID=0',
	'-e', f'PUBLIC_KEY={pub_key}',
	'-e', 'USER_NAME=helper',
	'-e', 'SUDO_ACCESS=true',
	volume_helper_image,
	], check=True, capture_output=True, encoding='utf-8')
	pinfo('Waiting for ssh service to be up...')
	service_up = False
	for i in range(ssh_max_wait):
	chk = verbose_run(['nc', alias, ssh_port], input=b'', capture_output=True)
	if chk.returncode == 0 and chk.stdout is not None and chk.stdout.startswith(b'SSH'):
	service_up = True
	break
	else:
	time.sleep(ssh_wait_interval_sec)
	if not service_up:
	raise RuntimeError('Service is still not up!')
	psuccess('Service is up.')
	return (container_name, alias)

	def import_volume(alias, local_dir, vol_mount_dir):
	# I'll just use the shell instead of pipe2+fork+exec+wait, much easier
	verbose_run([
	'rsync', '-a', '--delete',
	'--rsh', ssh_command,
	f'{local_dir}/',
	f'helper@{alias}:{vol_mount_dir}',
	], check=True)

	def export_volume(alias, local_dir, vol_mount_dir):
	verbose_run([
	'rsync', '-a', '--delete',
	'--rsh', ssh_command,
	f'helper@{alias}:{vol_mount_dir}/',
	local_dir,
	], check=True)

	def image_to_podman_args(image):
	name = image['name']
	args = []
	if 'entrypoint' in image:
	# ci.json requires that the entrypoint is an array of strings
	ep = json.dumps(image['entrypoint'])
	args += ['--entrypoint', ep]
	args += [name]
	return args

	def run_in_container(image, work_volname, script_volname):
	timeout = 60 * 60 * 2 # 2 hours by default
	p = verbose_run([
	'podman', 'run', '--rm',
	f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
	f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
	] + image_to_podman_args(image) + [
	script_name,
	], timeout=timeout)
	return p

	def main():
	image = json.loads(sys.argv[1])
	work_dir = sys.argv[2]
	script_dir = sys.argv[3]
	result_dir = sys.argv[4]

	pinfo('Creating volumes...')
	work_vol = create_volume('work')
	script_vol = create_volume('script')
	psuccess('Created.')

	pinfo('Starting helper service...')
	global helper_container_id
	(helper_container_id, alias) = start_helper_service(work_vol, script_vol)
	psuccess('Started...')

	pinfo('Importing volumes...')
	import_volume(alias, work_dir, work_vol_mount_dir)
	import_volume(alias, script_dir, script_vol_mount_dir)
	psuccess('Imported.')

	pinfo('Running container...')
	try:
	res = run_in_container(image, work_vol, script_vol)
	retcode = res.returncode
	except subprocess.TimeoutExpired:
	perror('Command timed out.')
	retcode = 1

	pinfo(f'Returned {retcode}.')

	if retcode != 0:
	perror('Job failed.')
	else:
	psuccess('Job succeeded.')

	- pinfo('Collecting build changes...')
	- export_volume(alias, result_dir, work_vol_mount_dir)
	- psuccess('Collected.')
	+ # We should collect the result regardless whether it succeeded
	+ pinfo('Collecting build changes...')
	+ export_volume(alias, result_dir, work_vol_mount_dir)
	+ psuccess('Collected.')

	return retcode

	retcode = 1

	try:
	retcode = main()
	except Exception as e:
	perror('Error!', e)
	print(traceback.format_exc())
	raise
	finally:
	if helper_container_id:
	pinfo('Cleaning helper container')
	clean_helper_container()
	psuccess('Cleaned.')

	pinfo('Cleaning volumes...')
	clean_volumes()
	psuccess('Cleaned.')

	sys.exit(retcode)

File Metadata

Mime Type: text/x-diff
Expires: Fri, Oct 9, 3:21 AM (22 h, 38 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1784415
Default Alt Text: (70 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions