No OneTemporary
Actions

Size

39 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/.gitignore b/.gitignore
	index 880c314..c658fdf 100644
	--- a/.gitignore
	+++ b/.gitignore
	@@ -1,8 +1,10 @@
	\#*
	.#*
	*~
	__pycache__
	*.pyc
	/venv
	.coverage
	coverage.xml
	+dist
	+lilybuild.egg-info
	diff --git a/Containerfile.master b/Containerfile.master
	index 8d83e5a..81b96cc 100644
	--- a/Containerfile.master
	+++ b/Containerfile.master
	@@ -1,4 +1,8 @@

	FROM docker.io/buildbot/buildbot-master:v4.2.1

	RUN /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
	+
	+COPY --from=lilybuild . /usr/src/lilybuild
	+
	+RUN /buildbot_venv/bin/pip3 install /usr/src/lilybuild
	diff --git a/build-master.sh b/build-master.sh
	index bb12fb6..6e6f5bf 100755
	--- a/build-master.sh
	+++ b/build-master.sh
	@@ -1,3 +1,3 @@
	#!/bin/sh

	-podman build -f Containerfile.master master "$@"
	+podman build -f Containerfile.master --build-context=lilybuild=lilybuild master "$@"
	diff --git a/lilybuild/lilybuild/artifacts/__init__.py b/lilybuild/lilybuild/artifacts/__init__.py
	new file mode 100644
	index 0000000..ace1953
	--- /dev/null
	+++ b/lilybuild/lilybuild/artifacts/__init__.py
	@@ -0,0 +1,4 @@
	+
	+from .application import ArtifactsApplication
	+
	+ep = ArtifactsApplication()
	diff --git a/lilybuild/lilybuild/artifacts/application.py b/lilybuild/lilybuild/artifacts/application.py
	new file mode 100644
	index 0000000..bb25f9e
	--- /dev/null
	+++ b/lilybuild/lilybuild/artifacts/application.py
	@@ -0,0 +1,12 @@
	+
	+from buildbot.www.plugin import Application
	+from .resource import Api
	+
	+class ArtifactsApplication(Application):
	+ def __init__(self):
	+ self.description = 'LilyBuild Artifacts support'
	+ self.version = '0.0.0'
	+ self.static_dir = ''
	+ self.ui = True
	+ self.api = Api(self)
	+ self.resource = self.api.app.resource()
	diff --git a/lilybuild/lilybuild/artifacts/authz_utils.py b/lilybuild/lilybuild/artifacts/authz_utils.py
	new file mode 100644
	index 0000000..21f3878
	--- /dev/null
	+++ b/lilybuild/lilybuild/artifacts/authz_utils.py
	@@ -0,0 +1,24 @@
	+
	+from twisted.internet import defer
	+from buildbot.www.authz.endpointmatchers import EndpointMatcherBase
	+
	+class AnyArtifactEndpointMatcher(EndpointMatcherBase):
	+ def __init__(self, role, defaultDeny=True):
	+ super().__init__(role, defaultDeny)
	+
	+ def match(self, ep, action='get', options=None):
	+ if isinstance(ep, tuple) and ep[0] == 'lilybuild_artifacts':
	+ return defer.secceed(Match(self.master))
	+ return defer.succeed(None)
	+
	+class BuildArtifactEndpointMatcher(EndpointMatcherBase):
	+ def __init__(self, role, defaultDeny=True):
	+ super().__init__(role, defaultDeny)
	+
	+ @defer.inlineCallbacks
	+ def match(self, ep, action='get', options=None):
	+ if isinstance(ep, tuple) and ep[0] == 'lilybuild_artifacts' and ep[1] == 'builds':
	+ build_id = ep[2]
	+ build = yield self.master.data.get(('builds', build_id))
	+ return Match(self.master, build=build)
	+ return None
	diff --git a/lilybuild/lilybuild/artifacts/resource.py b/lilybuild/lilybuild/artifacts/resource.py
	new file mode 100644
	index 0000000..a8f3fc4
	--- /dev/null
	+++ b/lilybuild/lilybuild/artifacts/resource.py
	@@ -0,0 +1,85 @@
	+
	+import json
	+from klein import Klein
	+from twisted.internet import defer
	+from twisted.web.static import File
	+from buildbot.data.resultspec import ResultSpec, Property
	+import os
	+
	+POSSIBLE_ARTIFACT_TYPES = {
	+ 'archive': 'artifacts.tar',
	+ 'reports': 'reports.tar',
	+}
	+BAD_REQUEST = 'BAD_REQUEST'
	+NOT_FOUND = 'NOT_FOUND'
	+
	+def validate_artifact_type(at):
	+ return at in POSSIBLE_ARTIFACT_TYPES
	+
	+class Api:
	+ app = Klein()
	+
	+ def __init__(self, ep):
	+ self.ep = ep
	+
	+ def lbc(self):
	+ return self.ep.config['lbc']
	+
	+ def www(self):
	+ return self.ep.master.www
	+
	+ @app.route('/builds/<int:build_id>/jobs/<int:job_index>/artifacts/<artifact_type>', methods=['GET'])
	+ @defer.inlineCallbacks
	+ def getArtifact(self, request, build_id, job_index, artifact_type):
	+ if not validate_artifact_type(artifact_type):
	+ request.setResponseCode(400)
	+ request.setHeader('Content-Type', 'application/json')
	+ return json.dumps({'error': BAD_REQUEST})
	+ storage_dir = self.lbc().storage_dir
	+ try:
	+ artifact_file = yield self.get_artifact_location(request, build_id, job_index, artifact_type)
	+ if os.path.exists(artifact_file):
	+ return File(artifact_file)
	+ else:
	+ raise FileNotFoundError('Not Found')
	+ except:
	+ request.setResponseCode(404)
	+ request.setHeader('Content-Type', 'application/json')
	+ return json.dumps({'error': NOT_FOUND})
	+
	+ @defer.inlineCallbacks
	+ def get_artifact_location(self, request, build_id, job_index, artifact_type):
	+ result_spec = ResultSpec(
	+ # see popProperties in ResultSpec
	+ properties=[Property(b'property', 'eq', [
	+ 'lilybuild_repo_id',
	+ 'lilybuild_source',
	+ ])]
	+ )
	+
	+ build = yield self.ep.master.data.get_with_resultspec(
	+ ('builds', build_id), result_spec
	+ )
	+ if not build:
	+ raise FileNotFoundError('Not Found')
	+ if 'lilybuild_repo_id' not in build['properties']:
	+ raise FileNotFoundError('Not Found')
	+ if 'lilybuild_source' not in build['properties']:
	+ source = 'none'
	+ else:
	+ source = build['properties']['lilybuild_source'][0]
	+ # Things not pushed to staging area is considered private
	+ # and can only be seen by project members
	+ if source == 'arc-patch':
	+ # see ForgejoAuthz
	+ yield self.www().assertUserAllowed(request, ('lilybuild_artifacts', 'builds', build_id, 'jobs', job_index, 'artifacts', artifact_type), 'get', {})
	+ repo_id = build['properties']['lilybuild_repo_id'][0]
	+ fn = POSSIBLE_ARTIFACT_TYPES[artifact_type]
	+ res = os.path.join(
	+ self.lbc().storage_dir,
	+ 'repos', str(repo_id),
	+ 'builds', str(build_id),
	+ 'jobs', str(job_index),
	+ 'artifacts', fn
	+ )
	+ return res
	diff --git a/lilybuild/lilybuild/auth.py b/lilybuild/lilybuild/auth.py
	index 3fa2951..7cd8e1e 100644
	--- a/lilybuild/lilybuild/auth.py
	+++ b/lilybuild/lilybuild/auth.py
	@@ -1,114 +1,130 @@

	from buildbot.plugins import *
	from buildbot.www.oauth2 import OAuth2Auth
	from buildbot.www.authz.roles import RolesFromBase, RolesFromOwner
	from buildbot.www.authz.authz import Authz
	from buildbot.process.properties import Properties
	from buildbot.data.builds import BuildEndpoint
	from twisted.internet import defer
	from twisted.python import log
	import requests
	import txrequests

	class ForgejoAuth(OAuth2Auth):
	name = 'Forgejo'
	faIcon = 'fa-git'

	def __init__(self, instanceUri, clientId, clientSecret, **kwargs):
	uri = instanceUri.rstrip('/')
	self.authUri = f'{uri}/login/oauth/authorize'
	self.tokenUri = f'{uri}/login/oauth/access_token'
	self.resourceEndpoint = f'{uri}/api/v1'
	super().__init__(clientId, clientSecret, **kwargs)

	def getUserInfoFromOAuthClient(self, c):
	user = self.get(c, '/user')
	groups = self.get(c, '/user/orgs')
	return {
	'full_name': user['full_name'],
	'username': user['login'],
	'email': user['email'],
	'avatar_url': user['avatar_url'],
	'groups': [g['name'] for g in groups],
	'is_admin': user['is_admin'],
	}

	class RolesFromGroups(RolesFromBase):
	def __init__(self, prefix=''):
	super().__init__()
	self.prefix = prefix

	def getRolesFromUser(self, userDetails):
	roles = []
	if 'groups' in userDetails:
	for group in userDetails['groups']:
	roles.append(self.prefix + group)
	return roles

	class RolesFromAttr(RolesFromBase):
	def __init__(self, attr_name, role_name):
	super().__init__()
	self.attr_name = attr_name
	self.role_name = role_name

	def getRolesFromUser(self, userDetails):
	if userDetails.get(self.attr_name):
	return [self.role_name]
	else:
	return []

	class ForgejoAuthz(Authz):
	def __init__(self, base_url, access_token, **kwargs):
	self.forgejo_base_url = base_url.rstrip('/') + '/'
	self.forgejo_api_url = self.forgejo_base_url + 'api/v1'
	self.forgejo_access_token = access_token
	self.session = txrequests.Session()
	super().__init__(**kwargs)

	@defer.inlineCallbacks
	def assertUserAllowed(self, ep, action, options, userDetails):
	- try:
	- (epobject, epdict) = self.master.data.getEndpoint(ep)
	- if isinstance(epobject, BuildEndpoint):
	- build_props = yield self.master.data.get(('builds', epdict['buildid'], 'properties'))
	- ok = yield self.user_can_access_build(userDetails, build_props)
	+ if isinstance(ep, tuple) and ep[0] == 'lilybuild_artifacts' and ep[1] == 'builds':
	+ try:
	+ build_id = ep[2]
	+ ok = yield self.user_can_access_build_by_id(userDetails, build_id)
	if ok:
	return defer.succeed(None)
	- except Exception as e:
	- print('exception:', e)
	- finally:
	- pass
	+ except Exception as e:
	+ print('exception when checking artifact access', e)
	+ else:
	+ try:
	+ (epobject, epdict) = self.master.data.getEndpoint(ep)
	+ if isinstance(epobject, BuildEndpoint):
	+ ok = yield self.user_can_access_build_by_id(userDetails, epdict['buildid'])
	+ if ok:
	+ return defer.succeed(None)
	+ except Exception as e:
	+ print('exception:', e)
	+ finally:
	+ pass
	res = yield super().assertUserAllowed(ep, action, options, userDetails)
	return res

	+ @defer.inlineCallbacks
	+ def user_can_access_build_by_id(self, userDetails, build_id):
	+ build_props = yield self.master.data.get(('builds', build_id, 'properties'))
	+ ok = yield self.user_can_access_build(userDetails, build_props)
	+ return ok
	+
	@defer.inlineCallbacks
	def user_can_access_build(self, userDetails, build_props):
	if 'lilybuild_repo' not in build_props:
	return False
	repo = build_props['lilybuild_repo'][0]
	if not repo.startswith(self.forgejo_base_url):
	return False
	repo_name = repo[len(self.forgejo_base_url):]
	if repo_name.count('/') != 1:
	return False

	+ if 'username' not in userDetails:
	+ return False
	username = userDetails['username']
	props = Properties()
	props.master = self.master
	access_token = yield props.render(self.forgejo_access_token)

	# https://codeberg.org/forgejo/forgejo/issues/6837
	# If I am not the repo owner, this works
	resp = yield self.session.get(
	f'{self.forgejo_api_url}/repos/{repo_name}/collaborators/{username}',
	headers={'Authorization': f'token {access_token}'}
	)
	if resp.status_code < 300:
	return True

	# If I am the repo owner, this works
	resp = yield self.session.get(
	f'{self.forgejo_api_url}/repos/{repo_name}/collaborators/{username}/permission',
	headers={'Authorization': f'token {access_token}'}
	)
	return resp.status_code < 300
	diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
	index 4008e3d..59b633e 100644
	--- a/lilybuild/lilybuild/ci_steps.py
	+++ b/lilybuild/lilybuild/ci_steps.py
	@@ -1,446 +1,453 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from twisted.internet import defer
	from .ci_syntax import ci_file
	from .helpers import rsync_rules_from_artifacts, get_job_script, normalize_image
	from .phorge import SendCoverageToPhorge
	import re
	import sys
	import json

	SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'
	COVERAGE_EXEC = '/lilybuild/lilybuild/coverage.py'

	def on_success(step):
	return step.build.results == util.SUCCESS

	def on_always(_step):
	return True

	def fill_list(*args):
	return list(args)

	class RunCIJobStep(steps.BuildStep):
	# 200 MiB
	artifact_max_size = 200 * 1024 * 1024
	default_image = 'alpine'
	master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
	artifact_file_name = 'artifacts.tar'
	master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
	reports_file_name = 'reports.tar'
	master_reports_file_name_pattern = master_job_artifact_dir_pattern + '/' + reports_file_name
	master_pages_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/pages'
	phorge_coverage_file_name = 'coverage-phorge.json'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	job_prop=None,
	+ artifact_link_base=None,
	**kwargs):
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	+ self.artifact_link_base = artifact_link_base
	super().__init__(name='Run step', **kwargs)

	@defer.inlineCallbacks
	def run(self):
	job_prop = self.getProperty('lilybuild_job_prop')
	job = ci_file.CIJob.from_prop(job_prop)
	job_index = self.getProperty('lilybuild_job_index')

	next_steps = yield self.job_to_steps(job, job_index)

	self.build.addStepsAfterCurrentStep(next_steps)
	return util.SUCCESS

	@defer.inlineCallbacks
	def get_ci_variables(self, job):
	res = {}
	res.update(job.get_predefined_ci_variables())
	res.update(self.getProperty('lilybuild_pipeline_vars'))
	res['CI_JOB_IMAGE'] = job.image or self.default_image
	res['CI_JOB_URL'] = yield self.build.getUrl()
	res['CI_JOB_ID'] = self.build.buildid
	res['CI_PROJECT_DIR'] = '/build'
	try:
	repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
	variables = yield repo['variables_getter'](self.build)
	res_vars = {}
	for var in variables:
	value = variables[var]
	if IRenderable.providedBy(value):
	value = yield self.build.render(value)
	res_vars[var] = value
	res.update(res_vars)
	except Exception as e:
	self.addCompleteLog('exception', f'{e}')
	return res

	- def get_upload_artifacts_jobs(self, short_name, artifact_name, base_dir, paths, exclude, master_pattern, job_index, doStepIf=on_success):
	+ def get_upload_artifacts_jobs(self, short_name, artifact_type, artifact_name, base_dir, paths, exclude, master_pattern, job_index, doStepIf=on_success):
	archive_artifact_step = steps.ShellCommand(
	name=f'Archive artifacts: {short_name}',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'create',
	'archive_file': artifact_name,
	'base_dir': base_dir,
	'content': paths,
	'items_to_exclude': exclude,
	}),
	workdir=self.work_root_dir,
	doStepIf=doStepIf,
	)
	masterdest = util.Interpolate(
	master_pattern,
	st=self.storage_dir,
	job=job_index,
	doStepIf=doStepIf,
	)
	+ parent_build_id = self.getProperty('lilybuild_pipeline_vars')['CI_PIPELINE_ID']
	+ artifact_url = f'{self.artifact_link_base}/plugins/lilybuild_artifacts/builds/{parent_build_id}/jobs/{job_index}/artifacts/{artifact_type}' if self.artifact_link_base else None
	upload_artifact_step = steps.FileUpload(
	workersrc=artifact_name,
	maxsize=self.artifact_max_size,
	name=f'Upload artifacts: {short_name}',
	masterdest=masterdest,
	workdir=self.work_root_dir,
	+ url=artifact_url,
	doStepIf=doStepIf,
	)
	return [archive_artifact_step, upload_artifact_step]

	@defer.inlineCallbacks
	def job_to_steps(self, job, job_index):
	script_name = self.script_dir + '/run.sh'
	variables = yield self.get_ci_variables(job)
	script_step = steps.StringDownload(
	get_job_script(variables, job),
	name='Set up script',
	workerdest=script_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	chmod_step = steps.ShellCommand(
	name='Make script executable',
	command=['chmod', '+x', script_name],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps = []
	dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
	if dep_job_indices:
	for i in dep_job_indices:
	download_job = steps.FileDownload(
	mastersrc=util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=i,
	),
	maxsize=self.artifact_max_size,
	name=f'Download artifacts from job #{i}',
	workerdest=self.artifact_file_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	unarchive_job = steps.ShellCommand(
	name=f'Unarchive artifacts from job #{i}',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'extract',
	'archive_file': self.artifact_file_name,
	'target_dir': self.src_relative,
	}),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps += [download_job, unarchive_job]

	run_step = steps.ShellCommand(
	name='Run script in container',
	command=[
	'/lilybuild/podman-helper',
	normalize_image(job.image or self.default_image),
	self.src_relative,
	self.script_dir,
	self.result_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	clean_script_step = steps.ShellCommand(
	name='Clean script dir',
	command=[
	'rm',
	'-rf',
	self.script_dir,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)

	steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
	if 'paths' in job.artifacts:
	steps_to_run += self.get_upload_artifacts_jobs(
	'files',
	+ 'archive',
	self.artifact_file_name,
	self.result_relative,
	job.artifacts.get('paths', []),
	job.artifacts.get('exclude', []),
	self.master_job_artifact_file_name_pattern,
	job_index
	)
	if job.has_supported_coverage_report():
	steps_to_run += [steps.ShellCommand(
	name='Process reports',
	command=[
	COVERAGE_EXEC,
	],
	initialStdin=json.dumps({
	'source_dir': self.src_relative,
	'result_dir': self.result_relative,
	'untrusted_coverage_file': job.artifacts['reports']['coverage_report']['path'],
	'output_dir': self.artifact_stage_relative,
	}),
	workdir=self.work_root_dir,
	doStepIf=on_always,
	)] + self.get_upload_artifacts_jobs(
	+ 'reports',
	'reports',
	self.reports_file_name,
	self.artifact_stage_relative,
	['*'],
	[],
	self.master_reports_file_name_pattern,
	job_index,
	doStepIf=on_always
	) + [SendCoverageToPhorge(
	self.lbc,
	self.artifact_stage_relative + '/' + self.phorge_coverage_file_name,
	workdir=self.work_root_dir,
	)]

	clean_stage_dir_again_step = steps.ShellCommand(
	name='Clean stage, result and artifact',
	command=[
	'rm',
	'-rf',
	self.result_relative,
	self.artifact_file_name,
	self.artifact_stage_relative,
	self.reports_file_name,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)
	steps_to_run.append(clean_stage_dir_again_step)

	if job.is_pages() and 'paths' in job.artifacts:
	deploy_pages_step = steps.MasterShellCommand(
	command=util.Transform(fill_list,
	sys.executable,
	'-m', 'lilybuild.pages',
	util.Interpolate(
	self.master_pages_dir_pattern,
	st=self.storage_dir,
	),
	masterdest,
	),
	name='Deploy pages',
	logEnviron=False,
	doStepIf=on_success,
	)
	steps_to_run.append(deploy_pages_step)

	return steps_to_run

	class TriggerMultipleJobsStep(steps.Trigger):
	properties_to_keep = [
	'branch',
	'revision',
	'repository',
	'harbormaster_build_target_phid',
	'harbormaster_variable_buildable.diff',
	'harbormaster_variable_repository.staging.ref',
	'harbormaster_variable_repository.staging.uri',
	'harbormaster_variable_repository.uri',
	'lilybuild_repo',
	'lilybuild_repo_id',
	'lilybuild_pipeline_vars',
	]

	def __init__(self, lbc, jobs_with_data, **kwargs):
	self.lbc = lbc
	self.jobs = jobs_with_data
	super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)

	def getSchedulersAndProperties(self):
	ret = []
	common_properties = {
	'lilybuild_root_build_id': self.build.buildid,
	}
	for prop in self.properties_to_keep:
	if self.hasProperty(prop):
	common_properties[prop] = self.getProperty(prop)

	for (job, i, dep_job_indices) in self.jobs:
	properties = common_properties.copy()
	properties['lilybuild_job_prop'] = job.to_prop()
	properties['lilybuild_job_index'] = i
	properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
	properties['lilybuild_dependency_job_indices'] = dep_job_indices
	ret.append({
	'sched_name': self.lbc.triggerable_scheduler_name,
	'props_to_set': properties,
	'unimportant': False,
	})

	return ret

	class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
	ci_def_file = '.gitlab-ci.yml'
	build_target_prop_name = 'harbormaster_build_target_phid'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	**kwargs):
	kwargs['name'] = 'Analyze CI file'
	kwargs['command'] = ['cat', self.ci_def_file]
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	kwargs['workdir'] = self.src_dir
	kwargs = self.setupShellMixin(kwargs)
	super().__init__(**kwargs)
	self.observer = logobserver.BufferLogObserver()
	self.addLogObserver('stdio', self.observer)

	def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
	jobs_with_data = []
	for job in stage_jobs:
	dep_job_names = [
	jn
	for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
	if ci_file.jobs[jn].has_artifacts_archive()
	]
	dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
	jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))

	trigger = TriggerMultipleJobsStep(
	name=stage_name,
	lbc=self.lbc,
	jobs_with_data=jobs_with_data,
	waitForFinish=True,
	doStepIf=on_success,
	)

	return trigger

	def get_steps(self, stdout):
	f = ci_file.CIFile(stdout)

	stages = f.get_grouped_jobs()
	jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
	job_names = [job.name for job in jobs]
	job_name_to_index_map = {}
	for (i, j) in enumerate(jobs):
	job_name_to_index_map[j.name] = i

	steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
	print('steps:', steps)
	return steps

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_ref_and_type(self):
	ref_type = 'branch'
	ref = self.getProperty('branch')

	if self.getProperty('category') == 'tag':
	ref_type = 'tag'

	if ref is not None:
	m = re.match(r'^refs/(heads\|tags)/(.+)$', ref)
	if m:
	ref = m.group(2)

	return (ref, ref_type)

	@defer.inlineCallbacks
	def get_pipeline_ci_vars(self):
	url = yield self.build.getUrl()
	res = {
	'CI_PIPELINE_ID': self.build.buildid,
	'CI_PIPELINE_IID': self.build.buildid,
	'CI_PIPELINE_URL': url,
	'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
	'CI_CONFIG_PATH': self.ci_def_file,
	}
	if not self.get_is_phorge():
	res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
	res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
	(ref, ref_type) = self.get_ref_and_type()
	res['CI_COMMIT_REF_NAME'] = ref
	res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
	res['CI_COMMIT_REF_PROTECTED'] = 'false'
	if ref_type == 'tag':
	res['CI_COMMIT_TAG'] = ref
	elif ref_type == 'branch':
	res['CI_COMMIT_BRANCH'] = ref
	return res

	@defer.inlineCallbacks
	def run(self):
	# run './build.sh --list-stages' to generate the list of stages
	cmd = yield self.makeRemoteShellCommand()
	yield self.runCommand(cmd)

	# if the command passes extract the list of stages
	result = cmd.results()
	if result == util.SUCCESS:
	pipeline_vars = yield self.get_pipeline_ci_vars()
	self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
	# create a ShellCommand for each stage and add them to the build
	self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))

	return result
	diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
	index 096ac73..924b298 100644
	--- a/lilybuild/lilybuild/config.py
	+++ b/lilybuild/lilybuild/config.py
	@@ -1,256 +1,259 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from twisted.internet import defer

	from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand, on_success
	from .phorge import CheckoutSourceFromPhorge, SendArtifactLinkToPhorge, SendBuildStatusToPhorge, MaybeRequireApprovalForPhorge
	from .helpers import normalize_base_url, phorge_token_to_arcrc

	import yaml

	def to_params(d):
	res = []
	for n in d:
	res.append(util.FixedParameter(name=n, default=d[n]))
	return res

	work_root = util.Interpolate('repos/%(prop:lilybuild_repo_id)s')
	src_relative = 'sources'
	src_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/sources')
	result_relative = 'result'
	result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
	artifact_stage_relative = 'stage'
	artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
	-storage_dir = '/buildbot/storage'
	+default_storage_dir = '/buildbot/storage'

	report_phorge = 'phorge'
	report_forgejo = 'forgejo'

	class LilyBuildConfig:
	builder_name = 'lilybuild'
	builder_name_run_job = 'lilybuild-job'
	builder_name_force = 'lilybuild-force'
	main_builder_tag = 'lilybuild-pipeline'
	triggerable_scheduler_name = 'lilybuild-triggerable'
	force_triggerable_scheduler_name = 'lilybuild-force-triggerable'

	- def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None):
	+ def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None, storage_dir=default_storage_dir, artifact_link_base=None):
	self.c = c
	self.poll_interval = 300
	self.workernames = workernames
	if reports is None:
	reports = [report_phorge, report_forgejo]
	self.reports = reports
	self.repos = {}
	self.repo_id_by_url = {}
	self.branch_skip_re = '^phabricator/'
	self.phorge_base_url = normalize_base_url(phorge_base_url)
	self.phorge_token = phorge_token
	self.ssh_priv_key = ssh_priv_key
	self.ssh_known_hosts = ssh_known_hosts
	+ self.storage_dir = storage_dir
	+ self.artifact_link_base = normalize_base_url(artifact_link_base)

	def configure_factory_and_builder(self):
	self.add_lilybuild_builder()
	self.add_lilybuild_job_builder()

	def create_source_step(self):
	return CheckoutSourceFromPhorge(
	lbc=self,
	repourl=util.Property('lilybuild_repo'),
	mode='full',
	workdir=src_dir,
	submodules=True
	)

	def get_arcrc_for_repo(self, repo_id):
	return util.Transform(
	phorge_token_to_arcrc,
	self.repos[repo_id]['phorge_base_url'],
	self.repos[repo_id]['phorge_token'],
	)

	def get_phorge_base_url_for_repo(self, repo_id):
	return self.repos[repo_id]['phorge_base_url']

	def add_lilybuild_builder(self):
	factory = util.BuildFactory()
	factory.addStep(self.create_source_step())
	if report_phorge in self.reports:
	factory.addStep(MaybeRequireApprovalForPhorge(lbc=self, workdir=src_dir))
	factory.addStep(SendArtifactLinkToPhorge(lbc=self, workdir=src_dir))
	factory.addStep(AnalyzeCIFileCommand(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	- storage_dir=storage_dir,
	+ storage_dir=self.storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	doStepIf=on_success,
	))
	if report_phorge in self.reports:
	factory.addStep(SendBuildStatusToPhorge(lbc=self, workdir=src_dir))

	builder = util.BuilderConfig(
	name=self.builder_name,
	workernames=self.workernames,
	factory=factory,
	tags=[self.main_builder_tag],
	)

	self.c['builders'].append(builder)

	def add_lilybuild_job_builder(self):
	factory_job = util.BuildFactory()
	factory_job.addStep(self.create_source_step())
	factory_job.addStep(RunCIJobStep(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	- storage_dir=storage_dir,
	+ storage_dir=self.storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	+ artifact_link_base=self.artifact_link_base
	))

	builder_job = util.BuilderConfig(
	name=self.builder_name_run_job,
	workernames=self.workernames,
	factory=factory_job,
	# Tell buildbot to never collapse build requests for this one
	# because otherwise triggering multiple jobs will cause it to only run one
	# https://docs.buildbot.net/current/manual/configuration/builders.html#collapsing-build-requests
	collapseRequests=False,
	)
	self.c['builders'].append(builder_job)

	s = schedulers.Triggerable(self.triggerable_scheduler_name, builderNames=[self.builder_name_run_job])
	self.c['schedulers'].append(s)

	def record_url(self, repo_id, repo_url):
	if repo_url in self.repo_id_by_url:
	raise Exception(f'repo url "{repo_url}" already recorded')
	self.repo_id_by_url[repo_url] = repo_id

	def add_repo(
	self,
	repo_id,
	repo_url,
	do_poll=False,
	alternative_urls=None,
	phorge_base_url=None,
	phorge_token=None,
	# Function(build -> dict(str -> str \| renderable<str>))
	variables_getter=None,
	):
	if not alternative_urls:
	alternative_urls = []

	self.repos[repo_id] = {
	'repo_id': repo_id,
	'repo_url': repo_url,
	'do_poll': do_poll,
	'alternative_urls': alternative_urls,
	'phorge_base_url': normalize_base_url(phorge_base_url) or self.phorge_base_url,
	'phorge_token': phorge_token or self.phorge_token,
	'variables_getter': variables_getter or (lambda _build: {}),
	}
	self.record_url(repo_id, repo_url)
	for u in alternative_urls:
	self.record_url(repo_id, u)

	def get_builder_name_for_repo(self, repo_def):
	main_repo_url = repo_def['repo_url']
	return f'lilybuild - {main_repo_url}'

	def add_one_repo_def(self, repo_def):
	print('Adding repo', repo_def)
	repo_urls = [repo_def['repo_url']] + repo_def['alternative_urls']
	repo_id = repo_def['repo_id']
	do_poll = repo_def['do_poll']
	main_repo_url = repo_def['repo_url']
	properties = {
	'lilybuild_repo': main_repo_url,
	'lilybuild_repo_id': repo_id,
	'virtual_builder_name': self.get_builder_name_for_repo(repo_def),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	self.c['schedulers'].append(schedulers.AnyBranchScheduler(
	name=f'lilybuild-anybranch - {main_repo_url}',
	change_filter=util.ChangeFilter(
	repository=repo_urls,
	),
	treeStableTimer=None,
	builderNames=[self.builder_name],
	properties=properties))

	if do_poll:
	for repo in repo_urls:
	print('Adding poller')
	self.c['change_source'].append(changes.GitPoller(
	repo,
	workdir=f'gitpoller/{repo_id}', branches=True,
	pollInterval=self.poll_interval))

	def configure_pipeline_defs(self):
	for repo_id in self.repos:
	self.add_one_repo_def(self.repos[repo_id])

	self.add_force_builder()
	self.add_force_scheduler()

	def add_force_builder(self):
	def get_repo_id_by_url(url):
	return self.repo_id_by_url[url]

	factory_force = util.BuildFactory()
	factory_force.addStep(steps.Trigger(
	name='trigger lilybuild',
	schedulerNames=[self.force_triggerable_scheduler_name],
	waitForFinish=True,
	set_properties={
	'lilybuild_repo': util.Property('lilybuild_repo'),
	'lilybuild_repo_id': util.Transform(get_repo_id_by_url, util.Property('lilybuild_repo')),
	'virtual_builder_name': util.Interpolate('lilybuild - %(prop:lilybuild_repo)s'),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	))

	builder_force = util.BuilderConfig(
	name=self.builder_name_force,
	workernames=self.workernames,
	factory=factory_force,
	)

	self.c['builders'].append(builder_force)
	# this is triggered by lilybuild-force, and it triggers lilybuild
	# via its virtual builder
	s = schedulers.Triggerable(
	self.force_triggerable_scheduler_name,
	builderNames=[self.builder_name],
	)
	self.c['schedulers'].append(s)


	def add_force_scheduler(self):
	repo_urls = []
	for r in self.repos.values():
	repo_urls.append(r['repo_url'])

	repo_param = util.ChoiceStringParameter(
	name='lilybuild_repo',
	required=True,
	strict=True,
	choices=repo_urls,
	)

	self.c['schedulers'].append(schedulers.ForceScheduler(
	name="force",
	builderNames=[self.builder_name_force],
	properties=[repo_param],
	))
	diff --git a/lilybuild/pyproject.toml b/lilybuild/pyproject.toml
	new file mode 100644
	index 0000000..fc9ed1d
	--- /dev/null
	+++ b/lilybuild/pyproject.toml
	@@ -0,0 +1,23 @@
	+[build-system]
	+requires = ["setuptools >= 75.8.0"]
	+build-backend = "setuptools.build_meta"
	+
	+[project]
	+name = "lilybuild"
	+version = "0.0.0"
	+authors = [
	+ { name="tusooa", email="tusooa@kazv.moe" },
	+]
	+description = "A buildbot extension to build GitLab-style CI files"
	+readme = "README.md"
	+requires-python = ">=3.9"
	+
	+[project.urls]
	+Homepage = "https://iron.lily-is.land/diffusion/B/"
	+Issues = "https://iron.lily-is.land/project/view/13/"
	+
	+[project.entry-points."buildbot.util"]
	+LilyBuildConfig = "lilybuild.config:LilyBuildConfig"
	+
	+[project.entry-points."buildbot.www"]
	+lilybuild_artifacts = "lilybuild.artifacts:ep"

File Metadata

Mime Type: text/x-diff
Expires: Fri, Oct 9, 3:03 AM (1 d, 1 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1784409
Default Alt Text: (39 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions