No OneTemporary
Actions

Size

44 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
	index 2ca7b9c..eadabd7 100644
	--- a/lilybuild/lilybuild/ci_steps.py
	+++ b/lilybuild/lilybuild/ci_steps.py
	@@ -1,611 +1,612 @@
	# This file is part of lilybuild.
	# SPDX-FileCopyrightText: 2025-2026 tusooa <tusooa@kazv.moe>
	# SPDX-License-Identifier: GPL-2.0-only

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from twisted.internet import defer
	from .ci_syntax import ci_file
	from .ci_syntax import rules as ci_rules
	from .helpers import (
	rsync_rules_from_artifacts,
	get_job_script,
	normalize_image,
	normalize_services,
	ci_vars_to_env_file,
	generate_metadata_from_job,
	)
	from .phorge import SendCoverageToPhorge
	import re
	import sys
	import json

	SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'
	COVERAGE_EXEC = '/lilybuild/lilybuild/coverage.py'

	def on_success(step):
	return step.build.results == util.SUCCESS

	def on_always(_step):
	return True

	def fill_list(*args):
	return list(args)

	class RunCIJobStep(steps.BuildStep):
	# 200 MiB
	artifact_max_size = 200 * 1024 * 1024
	default_image = 'alpine'
	master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
	artifact_file_name = 'artifacts.tar'
	master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
	reports_file_name = 'reports.tar'
	master_reports_file_name_pattern = master_job_artifact_dir_pattern + '/' + reports_file_name
	master_pages_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/pages'
	phorge_coverage_file_name = 'coverage-phorge.json'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	job_prop=None,
	artifact_link_base=None,
	**kwargs):
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	self.artifact_link_base = artifact_link_base
	super().__init__(name='Run step', **kwargs)

	def get_cur_repo_config(self):
	return self.lbc.repos[self.getProperty('lilybuild_repo_id')]

	@defer.inlineCallbacks
	def run(self):
	job_prop = self.getProperty('lilybuild_job_prop')
	job = ci_file.CIJob.from_prop(job_prop)
	job_index = self.getProperty('lilybuild_job_index')
	variables = yield self.get_ci_variables(job)

	should_run = True
	if len(job.rules):
	should_run = False
	default_when = job.struct_raw.get('when', 'on_success')
	for r in job.rules:
	try:
	when = r.get('when', default_when)
	if when == 'never' or when == 'manual':
	should_run_to_set = False
	else:
	should_run_to_set = True

	rule_str = r.get('if')
	if not rule_str:
	# No condition == always true
	# TODO: `changes` rule
	should_run = should_run_to_set
	break

	res = ci_rules.evaluate_rule(ci_rules.parse_rule(rule_str), variables)
	if not res:
	continue
	should_run = should_run_to_set
	break
	except SyntaxError:
	self.addCompleteLog('error', f'Rule "{rule_str}" has syntax errors')
	except:
	pass

	if should_run:
	next_steps = self.job_to_steps(job, job_index, variables)

	self.build.addStepsAfterCurrentStep(next_steps)
	return util.SUCCESS
	else:
	self.addCompleteLog('info', 'Job skipped by a rule')
	return util.SKIPPED

	@defer.inlineCallbacks
	def get_ci_variables(self, job):
	res = {}
	res.update(job.get_predefined_ci_variables())
	res.update(self.getProperty('lilybuild_pipeline_vars'))
	res['CI_JOB_IMAGE'] = job.image or self.default_image
	res['CI_JOB_URL'] = yield self.build.getUrl()
	res['CI_JOB_ID'] = self.build.buildid
	res['CI_PROJECT_DIR'] = '/build'
	try:
	repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
	variables = yield repo['variables_getter'](self.build)
	res_vars = {}
	for var in variables:
	value = variables[var]
	if IRenderable.providedBy(value):
	value = yield self.build.render(value)
	res_vars[var] = value
	res.update(res_vars)
	except Exception as e:
	self.addCompleteLog('exception', f'{e}')
	return res

	def get_upload_artifacts_jobs(self, short_name, artifact_type, artifact_name, base_dir, paths, exclude, master_pattern, job_index, doStepIf=on_success, has_pages=False):
	archive_artifact_step = steps.ShellCommand(
	name=f'Archive artifacts: {short_name}',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'create',
	'archive_file': artifact_name,
	'base_dir': base_dir,
	'content': paths,
	'items_to_exclude': exclude,
	'compression': 'gz',
	'limit_bytes': self.get_cur_repo_config()['artifact_uncompressed_limit'],
	}),
	workdir=self.work_root_dir,
	doStepIf=doStepIf,
	)
	masterdest = util.Interpolate(
	master_pattern,
	st=self.storage_dir,
	job=job_index,
	doStepIf=doStepIf,
	)
	parent_build_id = self.getProperty('lilybuild_pipeline_vars')['CI_PIPELINE_ID']
	artifact_url = f'{self.artifact_link_base}/plugins/lilybuild_artifacts/builds/{parent_build_id}/jobs/{job_index}/artifacts/{artifact_type}' if self.artifact_link_base else None
	upload_artifact_step = steps.FileUpload(
	workersrc=artifact_name,
	maxsize=self.get_cur_repo_config()['artifact_compressed_limit'],
	name=f'Upload artifacts: {short_name}',
	masterdest=masterdest,
	workdir=self.work_root_dir,
	url=artifact_url,
	doStepIf=doStepIf,
	)
	r = [archive_artifact_step, upload_artifact_step]
	if has_pages:
	r.append(steps.MasterShellCommand(
	command=util.Transform(fill_list,
	sys.executable,
	'-m', 'lilybuild.pages',
	util.Interpolate(
	self.master_pages_dir_pattern,
	st=self.storage_dir,
	),
	masterdest,
	+ str(self.get_cur_repo_config()['pages_uncompressed_limit']),
	),
	name='Deploy pages',
	logEnviron=False,
	doStepIf=doStepIf,
	))
	return r

	def job_to_steps(self, job, job_index, variables):
	script_name = self.script_dir + '/run.sh'
	env_filename = self.script_dir + '/env'
	metadata_filename = self.script_dir + '/metadata.json'

	source_step = self.lbc.create_source_step()

	script_step = steps.StringDownload(
	get_job_script(job),
	name='Set up script',
	workerdest=script_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	env_step = steps.StringDownload(
	ci_vars_to_env_file(variables),
	name='Set up env file',
	workerdest=env_filename,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	metadata_step = steps.StringDownload(
	generate_metadata_from_job(
	self.getProperty('lilybuild_repo_id'),
	job,
	variables,
	),
	name='Set up metadata file',
	workerdest=metadata_filename,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	chmod_step = steps.ShellCommand(
	name='Make script executable',
	command=['chmod', '+x', script_name],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps = []
	dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
	if dep_job_indices:
	for i in dep_job_indices:
	# The steps may not run or may not have an artifact even if it runs
	download_job = steps.FileDownload(
	mastersrc=util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=i,
	),
	maxsize=self.get_cur_repo_config()['artifact_compressed_limit'],
	name=f'Download artifacts from job #{i}',
	workerdest=self.artifact_file_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	haltOnFailure=False,
	flunkOnFailure=False,
	# https://github.com/buildbot/buildbot/issues/3709
	blocksize=256 * 1024,
	)

	unarchive_job = steps.ShellCommand(
	name=f'Unarchive artifacts from job #{i}',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'extract',
	'archive_file': self.artifact_file_name,
	'target_dir': self.src_relative,
	}),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	haltOnFailure=False,
	flunkOnFailure=False,
	)

	artifact_steps += [download_job, unarchive_job]

	run_step = steps.ShellCommand(
	name='Run script in container',
	command=[
	'/lilybuild/podman-helper',
	normalize_image(job.image or self.default_image),
	self.src_relative,
	self.script_dir,
	self.result_relative,
	normalize_services(job.services),
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	# 2h timeout by default
	# TODO support timeout by each job
	timeout=60 * 60 * 2,
	)

	clean_script_step = steps.ShellCommand(
	name='Clean script dir',
	command=[
	'rm',
	'-rf',
	self.script_dir,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)

	steps_to_run = [source_step, script_step, chmod_step, env_step, metadata_step] + artifact_steps + [run_step, clean_script_step]
	if 'paths' in job.artifacts:
	steps_to_run += self.get_upload_artifacts_jobs(
	'files',
	'archive',
	self.artifact_file_name,
	self.result_relative,
	job.artifacts.get('paths', []),
	job.artifacts.get('exclude', []),
	self.master_job_artifact_file_name_pattern,
	job_index,
	has_pages=job.is_pages()
	)
	if job.has_supported_coverage_report():
	steps_to_run += [steps.ShellCommand(
	name='Process reports',
	command=[
	COVERAGE_EXEC,
	],
	initialStdin=json.dumps({
	'source_dir': self.src_relative,
	'result_dir': self.result_relative,
	'untrusted_coverage_file': job.artifacts['reports']['coverage_report']['path'],
	'output_dir': self.artifact_stage_relative,
	}),
	workdir=self.work_root_dir,
	flunkOnFailure=False,
	doStepIf=on_always,
	)] + self.get_upload_artifacts_jobs(
	'reports',
	'reports',
	self.reports_file_name,
	self.artifact_stage_relative,
	['*'],
	[],
	self.master_reports_file_name_pattern,
	job_index,
	doStepIf=on_always
	) + [SendCoverageToPhorge(
	self.lbc,
	self.artifact_stage_relative + '/' + self.phorge_coverage_file_name,
	workdir=self.work_root_dir,
	)]

	clean_stage_dir_again_step = steps.ShellCommand(
	name='Clean stage, result and artifact',
	command=[
	'rm',
	'-rf',
	self.result_relative,
	self.artifact_file_name,
	self.artifact_stage_relative,
	self.reports_file_name,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)
	steps_to_run.append(clean_stage_dir_again_step)

	return steps_to_run

	class TriggerMultipleJobsStep(steps.Trigger):
	properties_to_keep = [
	'branch',
	'revision',
	'repository',
	'harbormaster_build_target_phid',
	'harbormaster_variable_buildable.diff',
	'harbormaster_variable_repository.staging.ref',
	'harbormaster_variable_repository.staging.uri',
	'harbormaster_variable_repository.uri',
	'lilybuild_repo',
	'lilybuild_repo_id',
	'lilybuild_pipeline_vars',
	]

	def __init__(self, lbc, jobs_with_data, **kwargs):
	self.lbc = lbc
	self.jobs = jobs_with_data
	super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)

	def getSchedulersAndProperties(self):
	ret = []
	common_properties = {
	'lilybuild_root_build_id': self.build.buildid,
	}
	for prop in self.properties_to_keep:
	if self.hasProperty(prop):
	common_properties[prop] = self.getProperty(prop)

	for (job, i, dep_job_indices) in self.jobs:
	properties = common_properties.copy()
	properties['lilybuild_job_prop'] = job.to_prop()
	properties['lilybuild_job_index'] = i
	properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
	properties['lilybuild_dependency_job_indices'] = dep_job_indices
	ret.append({
	'sched_name': self.lbc.triggerable_scheduler_name,
	'props_to_set': properties,
	'unimportant': False,
	})

	return ret

	class LatestMixin:
	latest_build_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/latest'
	latest_good_build_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/latest-good'
	latest_name_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/latest/%(kw:name)s'
	latest_good_name_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/latest-good/%(kw:name)s'

	class EnsureLatestDirs(steps.MasterShellCommand, LatestMixin):
	def __init__(self, lbc, storage_dir, **kwargs):
	self.lbc = lbc
	self.storage_dir = storage_dir
	command = util.Transform(
	fill_list,
	'mkdir',
	'-pv',
	util.Interpolate(self.latest_build_dir_pattern, st=self.storage_dir),
	util.Interpolate(self.latest_good_build_dir_pattern, st=self.storage_dir))
	super().__init__(
	name='Ensure latest dirs',
	command=command,
	logEnviron=False,
	doStepIf=on_always,
	**kwargs
	)

	class MarkLatest(steps.MasterShellCommand, LatestMixin):
	def __init__(self, lbc, storage_dir, buildid, ref_name, is_good, **kwargs):
	self.lbc = lbc
	self.storage_dir = storage_dir

	name_pattern = self.latest_good_name_pattern if is_good else self.latest_name_pattern

	command = util.Transform(
	fill_list,
	'ln',
	'-sfvn',
	util.Interpolate('../builds/%(kw:buildid)s', buildid=buildid),
	util.Interpolate(name_pattern, st=self.storage_dir, name=ref_name),
	)
	super().__init__(
	name='Mark build as latest-good' if is_good else 'Mark build as latest',
	command=command,
	logEnviron=False,
	doStepIf=on_success if is_good else on_always,
	**kwargs
	)

	class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
	ci_def_file = '.gitlab-ci.yml'
	build_target_prop_name = 'harbormaster_build_target_phid'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	**kwargs):
	kwargs['name'] = 'Analyze CI file'
	kwargs['command'] = ['cat', self.ci_def_file]
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	kwargs['workdir'] = self.src_dir
	kwargs = self.setupShellMixin(kwargs)
	super().__init__(**kwargs)
	self.observer = logobserver.BufferLogObserver()
	self.addLogObserver('stdio', self.observer)

	def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
	jobs_with_data = []
	for job in stage_jobs:
	dep_job_names = [
	jn
	for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
	if ci_file.jobs[jn].has_artifacts_archive()
	]
	dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
	jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))

	trigger = TriggerMultipleJobsStep(
	name=stage_name,
	lbc=self.lbc,
	jobs_with_data=jobs_with_data,
	waitForFinish=True,
	doStepIf=on_success,
	)

	return trigger

	def get_steps_and_job_map(self, stdout):
	f = ci_file.CIFile(stdout)

	stages = f.get_grouped_jobs()
	jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
	job_names = [job.name for job in jobs]
	job_name_to_index_map = {}
	for (i, j) in enumerate(jobs):
	job_name_to_index_map[j.name] = i

	steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
	print('steps:', steps)
	return (steps, job_name_to_index_map)

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_ref_and_type(self):
	ref_type = 'branch'
	ref = self.getProperty('branch')

	if self.getProperty('category') == 'tag':
	ref_type = 'tag'

	if ref is not None:
	m = re.match(r'^refs/(heads\|tags)/(.+)$', ref)
	if m:
	ref = m.group(2)

	return (ref, ref_type)

	def get_cur_repo_config(self):
	return self.lbc.repos[self.getProperty('lilybuild_repo_id')]

	@defer.inlineCallbacks
	def get_pipeline_ci_vars(self):
	url = yield self.build.getUrl()
	res = {
	'CI_PIPELINE_ID': self.build.buildid,
	'CI_PIPELINE_IID': self.build.buildid,
	'CI_PIPELINE_URL': url,
	'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
	'CI_CONFIG_PATH': self.ci_def_file,
	}
	if not self.get_is_phorge():
	res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
	res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
	(ref, ref_type) = self.get_ref_and_type()
	res['CI_COMMIT_REF_NAME'] = ref
	res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
	res['CI_COMMIT_REF_PROTECTED'] = 'false'
	if ref_type == 'tag':
	res['CI_COMMIT_TAG'] = ref
	elif ref_type == 'branch':
	res['CI_COMMIT_BRANCH'] = ref
	return res

	@defer.inlineCallbacks
	def run(self):
	# run './build.sh --list-stages' to generate the list of stages
	cmd = yield self.makeRemoteShellCommand()
	yield self.runCommand(cmd)

	# if the command passes extract the list of stages
	result = cmd.results()
	if result == util.SUCCESS:
	pipeline_vars = yield self.get_pipeline_ci_vars()
	self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
	# create a ShellCommand for each stage and add them to the build
	(steps, job_map) = self.get_steps_and_job_map(self.observer.getStdout())
	self.setProperty('lilybuild_job_map', job_map, self.__class__.__name__)
	self.build.addStepsAfterCurrentStep(steps)

	latest_branch_map = self.get_cur_repo_config()['artifact_latest_branch_map']
	ref, _ref_type = self.get_ref_and_type()
	if ref in latest_branch_map:
	latest_name = latest_branch_map[ref]
	self.build.addStepsAfterLastStep([
	EnsureLatestDirs(lbc=self.lbc, storage_dir=self.storage_dir),
	MarkLatest(
	lbc=self.lbc,
	storage_dir=self.storage_dir,
	buildid=self.build.buildid,
	ref_name=latest_name,
	is_good=True,
	),
	MarkLatest(
	lbc=self.lbc,
	storage_dir=self.storage_dir,
	buildid=self.build.buildid,
	ref_name=latest_name,
	is_good=False,
	),
	])

	return result
	diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
	index c195979..1c9fed1 100644
	--- a/lilybuild/lilybuild/config.py
	+++ b/lilybuild/lilybuild/config.py
	@@ -1,272 +1,275 @@
	# This file is part of lilybuild.
	# SPDX-FileCopyrightText: 2025-2026 tusooa <tusooa@kazv.moe>
	# SPDX-License-Identifier: GPL-2.0-only

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from twisted.internet import defer

	from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand, on_success
	from .phorge import CheckoutSourceFromPhorge, MaybeRequireApprovalForPhorge
	from .helpers import normalize_base_url, phorge_token_to_arcrc

	import yaml

	def to_params(d):
	res = []
	for n in d:
	res.append(util.FixedParameter(name=n, default=d[n]))
	return res

	work_root = util.Interpolate('repos/%(prop:lilybuild_repo_id)s')
	src_relative = 'sources'
	src_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/sources')
	result_relative = 'result'
	result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
	artifact_stage_relative = 'stage'
	artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
	default_storage_dir = '/buildbot/storage'

	report_phorge = 'phorge'
	report_forgejo = 'forgejo'
	default_artifact_uncompressed_limit = 200 * 1024 * 1024
	default_artifact_compressed_limit = 200 * 1024 * 1024

	class LilyBuildConfig:
	builder_name = 'lilybuild'
	builder_name_run_job = 'lilybuild-job'
	builder_name_force = 'lilybuild-force'
	main_builder_tag = 'lilybuild-pipeline'
	triggerable_scheduler_name = 'lilybuild-triggerable'
	force_triggerable_scheduler_name = 'lilybuild-force-triggerable'

	- def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None, storage_dir=default_storage_dir, artifact_link_base=None, artifact_uncompressed_limit=None, artifact_compressed_limit=None):
	+ def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None, storage_dir=default_storage_dir, artifact_link_base=None, artifact_uncompressed_limit=None, artifact_compressed_limit=None, pages_uncompressed_limit=None):
	self.c = c
	self.poll_interval = 300
	self.workernames = workernames
	if reports is None:
	reports = [report_phorge, report_forgejo]
	self.reports = reports
	self.repos = {}
	self.repo_id_by_url = {}
	self.branch_skip_re = '^phabricator/'
	self.phorge_base_url = normalize_base_url(phorge_base_url)
	self.phorge_token = phorge_token
	self.ssh_priv_key = ssh_priv_key
	self.ssh_known_hosts = ssh_known_hosts
	self.storage_dir = storage_dir
	self.artifact_link_base = normalize_base_url(artifact_link_base)
	self.artifact_uncompressed_limit = artifact_uncompressed_limit or default_artifact_uncompressed_limit
	self.artifact_compressed_limit = artifact_compressed_limit or default_artifact_compressed_limit
	+ self.pages_uncompressed_limit = pages_uncompressed_limit or self.artifact_uncompressed_limit

	def configure_factory_and_builder(self):
	self.add_lilybuild_builder()
	self.add_lilybuild_job_builder()

	def create_source_step(self):
	return CheckoutSourceFromPhorge(
	lbc=self,
	repourl=util.Property('lilybuild_repo'),
	mode='full',
	workdir=src_dir,
	submodules=True
	)

	def get_arcrc_for_repo(self, repo_id):
	return util.Transform(
	phorge_token_to_arcrc,
	self.repos[repo_id]['phorge_base_url'],
	self.repos[repo_id]['phorge_token'],
	)

	def get_phorge_base_url_for_repo(self, repo_id):
	return self.repos[repo_id]['phorge_base_url']

	def get_phorge_token_for_repo(self, repo_id):
	return self.repos[repo_id]['phorge_token']

	def add_lilybuild_builder(self):
	factory = util.BuildFactory()
	factory.addStep(self.create_source_step())
	if report_phorge in self.reports:
	factory.addStep(MaybeRequireApprovalForPhorge(lbc=self, workdir=src_dir))

	factory.addStep(AnalyzeCIFileCommand(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	storage_dir=self.storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	doStepIf=on_success,
	))

	builder = util.BuilderConfig(
	name=self.builder_name,
	workernames=self.workernames,
	factory=factory,
	tags=[self.main_builder_tag],
	)

	self.c['builders'].append(builder)

	def add_lilybuild_job_builder(self):
	factory_job = util.BuildFactory()
	factory_job.addStep(RunCIJobStep(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	storage_dir=self.storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	artifact_link_base=self.artifact_link_base
	))

	builder_job = util.BuilderConfig(
	name=self.builder_name_run_job,
	workernames=self.workernames,
	factory=factory_job,
	# Tell buildbot to never collapse build requests for this one
	# because otherwise triggering multiple jobs will cause it to only run one
	# https://docs.buildbot.net/current/manual/configuration/builders.html#collapsing-build-requests
	collapseRequests=False,
	)
	self.c['builders'].append(builder_job)

	s = schedulers.Triggerable(self.triggerable_scheduler_name, builderNames=[self.builder_name_run_job])
	self.c['schedulers'].append(s)

	def record_url(self, repo_id, repo_url):
	if repo_url in self.repo_id_by_url:
	raise Exception(f'repo url "{repo_url}" already recorded')
	self.repo_id_by_url[repo_url] = repo_id

	def add_repo(
	self,
	repo_id,
	repo_url,
	do_poll=False,
	alternative_urls=None,
	phorge_base_url=None,
	phorge_token=None,
	# Function(build -> dict(str -> str \| renderable<str>))
	variables_getter=None,
	artifact_compressed_limit=None,
	artifact_uncompressed_limit=None,
	artifact_latest_branch_map=None,
	+ pages_uncompressed_limit=None,
	):
	if not alternative_urls:
	alternative_urls = []

	self.repos[repo_id] = {
	'repo_id': repo_id,
	'repo_url': repo_url,
	'do_poll': do_poll,
	'alternative_urls': alternative_urls,
	'phorge_base_url': normalize_base_url(phorge_base_url) or self.phorge_base_url,
	'phorge_token': phorge_token or self.phorge_token,
	'variables_getter': variables_getter or (lambda _build: {}),
	'artifact_compressed_limit': artifact_compressed_limit or self.artifact_compressed_limit,
	'artifact_uncompressed_limit': artifact_uncompressed_limit or self.artifact_uncompressed_limit,
	'artifact_latest_branch_map': artifact_latest_branch_map or {},
	+ 'pages_uncompressed_limit': pages_uncompressed_limit or self.pages_uncompressed_limit,
	}
	self.record_url(repo_id, repo_url)
	for u in alternative_urls:
	self.record_url(repo_id, u)

	def get_builder_name_for_repo(self, repo_def):
	main_repo_url = repo_def['repo_url']
	return f'lilybuild - {main_repo_url}'

	def add_one_repo_def(self, repo_def):
	print('Adding repo', repo_def)
	repo_urls = [repo_def['repo_url']] + repo_def['alternative_urls']
	repo_id = repo_def['repo_id']
	do_poll = repo_def['do_poll']
	main_repo_url = repo_def['repo_url']
	properties = {
	'lilybuild_repo': main_repo_url,
	'lilybuild_repo_id': repo_id,
	'virtual_builder_name': self.get_builder_name_for_repo(repo_def),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	self.c['schedulers'].append(schedulers.AnyBranchScheduler(
	name=f'lilybuild-anybranch - {main_repo_url}',
	change_filter=util.ChangeFilter(
	repository=repo_urls,
	),
	treeStableTimer=None,
	builderNames=[self.builder_name],
	properties=properties))

	if do_poll:
	for repo in repo_urls:
	print('Adding poller')
	self.c['change_source'].append(changes.GitPoller(
	repo,
	workdir=f'gitpoller/{repo_id}', branches=True,
	pollInterval=self.poll_interval))

	def configure_pipeline_defs(self):
	for repo_id in self.repos:
	self.add_one_repo_def(self.repos[repo_id])

	self.add_force_builder()
	self.add_force_scheduler()

	def add_force_builder(self):
	def get_repo_id_by_url(url):
	return self.repo_id_by_url[url]

	factory_force = util.BuildFactory()
	factory_force.addStep(steps.Trigger(
	name='trigger lilybuild',
	schedulerNames=[self.force_triggerable_scheduler_name],
	waitForFinish=True,
	set_properties={
	'lilybuild_repo': util.Property('lilybuild_repo'),
	'lilybuild_repo_id': util.Transform(get_repo_id_by_url, util.Property('lilybuild_repo')),
	'virtual_builder_name': util.Interpolate('lilybuild - %(prop:lilybuild_repo)s'),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	))

	builder_force = util.BuilderConfig(
	name=self.builder_name_force,
	workernames=self.workernames,
	factory=factory_force,
	)

	self.c['builders'].append(builder_force)
	# this is triggered by lilybuild-force, and it triggers lilybuild
	# via its virtual builder
	s = schedulers.Triggerable(
	self.force_triggerable_scheduler_name,
	builderNames=[self.builder_name],
	)
	self.c['schedulers'].append(s)


	def add_force_scheduler(self):
	repo_urls = []
	for r in self.repos.values():
	repo_urls.append(r['repo_url'])

	repo_param = util.ChoiceStringParameter(
	name='lilybuild_repo',
	required=True,
	strict=True,
	choices=repo_urls,
	)

	self.c['schedulers'].append(schedulers.ForceScheduler(
	name="force",
	builderNames=[self.builder_name_force],
	properties=[repo_param],
	))
	diff --git a/lilybuild/lilybuild/pages.py b/lilybuild/lilybuild/pages.py
	index bd07c98..4a24e9a 100644
	--- a/lilybuild/lilybuild/pages.py
	+++ b/lilybuild/lilybuild/pages.py
	@@ -1,87 +1,92 @@
	# This file is part of lilybuild.
	# SPDX-FileCopyrightText: 2025-2026 tusooa <tusooa@kazv.moe>
	# SPDX-License-Identifier: GPL-2.0-only

	import os
	import subprocess
	import shutil
	import sys
	try:
	import tarfile
	tarfile.FilterError
	except AttributeError:
	import backports.tarfile as tarfile

	'''
	This module implements blue-green deployment for pages.
	'''

	FIRST = '0'
	SECOND = '1'
	CURRENT = 'cur'

	def init_deployment(dir_name):
	'''
	Init the deployment at `dir_name` and return which directory we should
	write into for the next deployment.
	'''
	os.makedirs(os.path.join(dir_name, FIRST), exist_ok=True)
	os.makedirs(os.path.join(dir_name, SECOND), exist_ok=True)
	link_file = os.path.join(dir_name, CURRENT)
	try:
	current = os.readlink(link_file)
	except FileNotFoundError as e:
	os.symlink(SECOND, link_file, target_is_directory=True)
	current = SECOND
	return FIRST if current == SECOND else SECOND

	def switch_symlink(dir_name, target):
	subprocess.run([
	'ln',
	'-sfvn',
	'--',
	target,
	os.path.join(dir_name, CURRENT),
	], check=True)

	def empty_directory(dir_name):
	shutil.rmtree(dir_name)
	os.makedirs(dir_name, exist_ok=True)

	class TarPagesFilter:
	public_dir = 'public'
	def __init__(
	self,
	limit_bytes=10010241024 # 100 MiB
	):
	self.total_bytes_extracted = 0
	self.limit_bytes = limit_bytes

	def __call__(self, member, path):
	filtered_member = tarfile.data_filter(member, path)
	if not filtered_member:
	return None
	if self.total_bytes_extracted + filtered_member.size > self.limit_bytes:
	self.total_bytes_extracted = self.limit_bytes + 1
	raise RuntimeError('Limit exceeded')
	name = os.path.normpath(filtered_member.name)
	if not (name == self.public_dir or name.startswith(self.public_dir + '/')):
	return None

	self.total_bytes_extracted += filtered_member.size
	return filtered_member

	-def extract_archive(dir_name, archive_file):
	+def extract_archive(dir_name, archive_file, limit_bytes=None):
	+ if limit_bytes is None:
	+ limit_bytes = 100 * 1024 * 1024
	with tarfile.open(archive_file) as tf:
	tf.errorlevel = 1
	- tf.extractall(dir_name, filter=TarPagesFilter())
	+ tf.extractall(dir_name, filter=TarPagesFilter(limit_bytes=limit_bytes))

	-def deploy_pages(dir_name, archive_file):
	+def deploy_pages(dir_name, archive_file, max_size=None):
	target = init_deployment(dir_name)
	full_target = os.path.join(dir_name, target)
	empty_directory(full_target)
	- extract_archive(full_target, archive_file)
	+ extract_archive(full_target, archive_file, limit_bytes=max_size)
	switch_symlink(dir_name, target)

	if __name__ == '__main__':
	- deploy_pages(sys.argv[1], sys.argv[2])
	+ max_size = None
	+ if len(sys.argv) >= 4:
	+ max_size = int(sys.argv[3])
	+ deploy_pages(sys.argv[1], sys.argv[2], max_size=max_size)
	diff --git a/lilybuild/lilybuild/tests/pages_test.py b/lilybuild/lilybuild/tests/pages_test.py
	index 17461bb..2dbb099 100644
	--- a/lilybuild/lilybuild/tests/pages_test.py
	+++ b/lilybuild/lilybuild/tests/pages_test.py
	@@ -1,131 +1,131 @@
	# This file is part of lilybuild.
	# SPDX-FileCopyrightText: 2025-2026 tusooa <tusooa@kazv.moe>
	# SPDX-License-Identifier: GPL-2.0-only

	import unittest
	import tempfile
	import os
	import stat
	import subprocess
	from lilybuild.pages import (
	FIRST, SECOND, CURRENT, init_deployment,
	switch_symlink, extract_archive, deploy_pages,
	)

	try:
	import tarfile
	tarfile.FilterError
	except AttributeError:
	import backports.tarfile as tarfile


	def is_dir(filename):
	return stat.S_ISDIR(os.lstat(filename).st_mode)

	def make_artifact_archive(root_dir):
	os.makedirs(os.path.join(root_dir, 'public'))
	with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	print('test', file=f)
	os.makedirs(os.path.join(root_dir, 'other'))
	with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
	print('should not be there', file=f)
	archive = os.path.join(root_dir, 'artifacts.tar')
	with tarfile.open(archive, 'w') as f:
	f.add(os.path.join(root_dir, 'public'), 'public')
	f.add(os.path.join(root_dir, 'other'), 'other')
	return archive

	def make_artifact_archive2(root_dir):
	os.makedirs(os.path.join(root_dir, 'public'))
	with open(os.path.join(root_dir, 'public', 'b'), 'w') as f:
	print('test b', file=f)
	archive = os.path.join(root_dir, 'artifacts.tar')
	with tarfile.open(archive, 'w') as f:
	f.add(os.path.join(root_dir, 'public'), 'public')
	return archive

	def make_bad_artifact_archive(root_dir):
	os.makedirs(os.path.join(root_dir, 'public'))
	with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	print('test', file=f)
	os.makedirs(os.path.join(root_dir, 'other'))
	with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
	print('should not be there', file=f)
	archive = os.path.join(root_dir, 'artifacts.tar')
	with tarfile.open(archive, 'w') as f:
	f.add(os.path.join(root_dir, 'public'), 'public')
	f.add(os.path.join(root_dir, 'other'), '../../../other')
	return archive

	class PagesTest(unittest.TestCase):
	def test_init_deployment(self):
	with tempfile.TemporaryDirectory() as dir_name:
	res = init_deployment(dir_name)
	self.assertEqual(res, FIRST)
	self.assertTrue(is_dir(os.path.join(dir_name, FIRST)))
	self.assertTrue(is_dir(os.path.join(dir_name, SECOND)))
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)

	res = init_deployment(dir_name)
	self.assertEqual(res, FIRST)
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)

	def test_switch_symlink(self):
	with tempfile.TemporaryDirectory() as dir_name:
	res = init_deployment(dir_name)
	switch_symlink(dir_name, res)
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)

	def test_extract_archive(self):
	with tempfile.TemporaryDirectory() as root_dir:
	archive_file = make_artifact_archive(root_dir)
	dir_name = os.path.join(root_dir, 'deployment')
	- extract_archive(dir_name, archive_file)
	+ extract_archive(dir_name, archive_file, limit_bytes=10010241024)
	self.assertTrue(is_dir(os.path.join(dir_name, 'public')))
	self.assertTrue(os.path.exists(os.path.join(dir_name, 'public', 'a')))
	self.assertFalse(os.path.exists(os.path.join(dir_name, 'other')))

	def test_extract_bad_archive(self):
	with tempfile.TemporaryDirectory() as root_dir:
	archive_file = make_bad_artifact_archive(root_dir)
	dir_name = os.path.join(root_dir, 'deployment')
	with self.assertRaises(tarfile.FilterError):
	- extract_archive(dir_name, archive_file)
	+ extract_archive(dir_name, archive_file, limit_bytes=10010241024)

	def test_deploy_pages(self):
	with tempfile.TemporaryDirectory() as root_dir:
	archive_file = make_artifact_archive(os.path.join(root_dir, 'a1'))
	bad_archive = make_bad_artifact_archive(os.path.join(root_dir, 'bad'))
	archive_file2 = make_artifact_archive2(os.path.join(root_dir, 'a2'))
	dir_name = os.path.join(root_dir, 'deployment')
	- deploy_pages(dir_name, archive_file)
	+ deploy_pages(dir_name, archive_file, max_size=10010241024)
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
	self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
	with open(os.path.join(dir_name, FIRST, 'public', 'a')) as f:
	self.assertEqual(f.read(), 'test\n')

	# deploy the same thing again, it should go to SECOND
	- deploy_pages(dir_name, archive_file)
	+ deploy_pages(dir_name, archive_file, max_size=10010241024)
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)
	self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
	with open(os.path.join(dir_name, SECOND, 'public', 'a')) as f:
	self.assertEqual(f.read(), 'test\n')

	# try to deploy the bad archive, it should not update the link
	with self.assertRaises(tarfile.FilterError):
	- deploy_pages(dir_name, bad_archive)
	+ deploy_pages(dir_name, bad_archive, max_size=10010241024)
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), SECOND)

	# deploy something else, verify it cleans up all old files
	- deploy_pages(dir_name, archive_file2)
	+ deploy_pages(dir_name, archive_file2, max_size=10010241024)
	self.assertEqual(os.readlink(os.path.join(dir_name, CURRENT)), FIRST)
	self.assertTrue(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'b')))
	self.assertFalse(os.path.exists(os.path.join(dir_name, FIRST, 'public', 'a')))
	self.assertTrue(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'a')))
	self.assertFalse(os.path.exists(os.path.join(dir_name, SECOND, 'public', 'b')))
	with open(os.path.join(dir_name, FIRST, 'public', 'b')) as f:
	self.assertEqual(f.read(), 'test b\n')


	if __name__ == '__main__':
	unittest.main()

File Metadata

Mime Type: text/x-diff
Expires: Sun, Aug 9, 2:14 PM (1 d, 6 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1724656
Default Alt Text: (44 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions