config.py
No OneTemporary
Actions

Size

8 KB

Referenced Files

None

Subscribers

None

config.py
View Options


	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from twisted.internet import defer

	from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand, on_success
	from .phorge import CheckoutSourceFromPhorge, SendArtifactLinkToPhorge, SendBuildStatusToPhorge, MaybeRequireApprovalForPhorge
	from .helpers import normalize_base_url, phorge_token_to_arcrc

	import yaml

	def to_params(d):
	res = []
	for n in d:
	res.append(util.FixedParameter(name=n, default=d[n]))
	return res

	work_root = util.Interpolate('repos/%(prop:lilybuild_repo_id)s')
	src_relative = 'sources'
	src_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/sources')
	result_relative = 'result'
	result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
	artifact_stage_relative = 'stage'
	artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
	storage_dir = '/buildbot/storage'

	report_phorge = 'phorge'
	report_forgejo = 'forgejo'

	class LilyBuildConfig:
	builder_name = 'lilybuild'
	builder_name_run_job = 'lilybuild-job'
	builder_name_force = 'lilybuild-force'
	main_builder_tag = 'lilybuild-pipeline'
	triggerable_scheduler_name = 'lilybuild-triggerable'
	force_triggerable_scheduler_name = 'lilybuild-force-triggerable'

	def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None):
	self.c = c
	self.poll_interval = 300
	self.workernames = workernames
	if reports is None:
	reports = [report_phorge, report_forgejo]
	self.reports = reports
	self.repos = {}
	self.repo_id_by_url = {}
	self.branch_skip_re = '^phabricator/'
	self.phorge_base_url = normalize_base_url(phorge_base_url)
	self.phorge_token = phorge_token

	def configure_factory_and_builder(self):
	self.add_lilybuild_builder()
	self.add_lilybuild_job_builder()

	def create_source_step(self):
	return CheckoutSourceFromPhorge(
	lbc=self,
	repourl=util.Property('lilybuild_repo'),
	mode='full',
	workdir=src_dir,
	submodules=True
	)

	def get_arcrc_for_repo(self, repo_id):
	return util.Transform(
	phorge_token_to_arcrc,
	self.repos[repo_id]['phorge_base_url'],
	self.repos[repo_id]['phorge_token'],
	)

	def add_lilybuild_builder(self):
	factory = util.BuildFactory()
	factory.addStep(self.create_source_step())
	if report_phorge in self.reports:
	factory.addStep(MaybeRequireApprovalForPhorge(lbc=self, workdir=src_dir))
	factory.addStep(SendArtifactLinkToPhorge(lbc=self, workdir=src_dir))
	factory.addStep(AnalyzeCIFileCommand(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	storage_dir=storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	doStepIf=on_success,
	))
	if report_phorge in self.reports:
	factory.addStep(SendBuildStatusToPhorge(lbc=self, workdir=src_dir))

	builder = util.BuilderConfig(
	name=self.builder_name,
	workernames=self.workernames,
	factory=factory,
	tags=[self.main_builder_tag],
	)

	self.c['builders'].append(builder)

	def add_lilybuild_job_builder(self):
	factory_job = util.BuildFactory()
	factory_job.addStep(self.create_source_step())
	factory_job.addStep(RunCIJobStep(
	lbc=self,
	workdir=work_root,
	src_relative=src_relative,
	src_dir=src_dir,
	storage_dir=storage_dir,
	artifact_stage_relative=artifact_stage_relative,
	artifact_stage_dir=artifact_stage_dir,
	result_relative=result_relative,
	result_dir=result_dir,
	))

	builder_job = util.BuilderConfig(
	name=self.builder_name_run_job,
	workernames=self.workernames,
	factory=factory_job,
	# Tell buildbot to never collapse build requests for this one
	# because otherwise triggering multiple jobs will cause it to only run one
	# https://docs.buildbot.net/current/manual/configuration/builders.html#collapsing-build-requests
	collapseRequests=False,
	)
	self.c['builders'].append(builder_job)

	s = schedulers.Triggerable(self.triggerable_scheduler_name, builderNames=[self.builder_name_run_job])
	self.c['schedulers'].append(s)

	def record_url(self, repo_id, repo_url):
	if repo_url in self.repo_id_by_url:
	raise Exception(f'repo url "{repo_url}" already recorded')
	self.repo_id_by_url[repo_url] = repo_id

	def add_repo(
	self,
	repo_id,
	repo_url,
	do_poll=False,
	alternative_urls=None,
	phorge_base_url=None,
	phorge_token=None,
	# Function(build -> dict(str -> str \| renderable<str>))
	variables_getter=None,
	):
	if not alternative_urls:
	alternative_urls = []

	self.repos[repo_id] = {
	'repo_id': repo_id,
	'repo_url': repo_url,
	'do_poll': do_poll,
	'alternative_urls': alternative_urls,
	'phorge_base_url': normalize_base_url(phorge_base_url) or self.phorge_base_url,
	'phorge_token': phorge_token or self.phorge_token,
	'variables_getter': variables_getter or (lambda _build: {}),
	}
	self.record_url(repo_id, repo_url)
	for u in alternative_urls:
	self.record_url(repo_id, u)

	def get_builder_name_for_repo(self, repo_def):
	main_repo_url = repo_def['repo_url']
	return f'lilybuild - {main_repo_url}'

	def add_one_repo_def(self, repo_def):
	print('Adding repo', repo_def)
	repo_urls = [repo_def['repo_url']] + repo_def['alternative_urls']
	repo_id = repo_def['repo_id']
	do_poll = repo_def['do_poll']
	main_repo_url = repo_def['repo_url']
	properties = {
	'lilybuild_repo': main_repo_url,
	'lilybuild_repo_id': repo_id,
	'virtual_builder_name': self.get_builder_name_for_repo(repo_def),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	self.c['schedulers'].append(schedulers.AnyBranchScheduler(
	name=f'lilybuild-anybranch - {main_repo_url}',
	change_filter=util.ChangeFilter(
	repository=repo_urls,
	),
	treeStableTimer=None,
	builderNames=[self.builder_name],
	properties=properties))

	if do_poll:
	for repo in repo_urls:
	print('Adding poller')
	self.c['change_source'].append(changes.GitPoller(
	repo,
	workdir=f'gitpoller/{repo_id}', branches=True,
	pollInterval=self.poll_interval))

	def configure_pipeline_defs(self):
	for repo_id in self.repos:
	self.add_one_repo_def(self.repos[repo_id])

	self.add_force_builder()
	self.add_force_scheduler()

	def add_force_builder(self):
	def get_repo_id_by_url(url):
	return self.repo_id_by_url[url]

	factory_force = util.BuildFactory()
	factory_force.addStep(steps.Trigger(
	name='trigger lilybuild',
	schedulerNames=[self.force_triggerable_scheduler_name],
	waitForFinish=True,
	set_properties={
	'lilybuild_repo': util.Property('lilybuild_repo'),
	'lilybuild_repo_id': util.Transform(get_repo_id_by_url, util.Property('lilybuild_repo')),
	'virtual_builder_name': util.Interpolate('lilybuild - %(prop:lilybuild_repo)s'),
	'virtual_builder_tags': [self.main_builder_tag],
	}
	))

	builder_force = util.BuilderConfig(
	name=self.builder_name_force,
	workernames=self.workernames,
	factory=factory_force,
	)

	self.c['builders'].append(builder_force)
	# this is triggered by lilybuild-force, and it triggers lilybuild
	# via its virtual builder
	s = schedulers.Triggerable(
	self.force_triggerable_scheduler_name,
	builderNames=[self.builder_name],
	)
	self.c['schedulers'].append(s)


	def add_force_scheduler(self):
	repo_urls = []
	for r in self.repos.values():
	repo_urls.append(r['repo_url'])

	repo_param = util.ChoiceStringParameter(
	name='lilybuild_repo',
	required=True,
	strict=True,
	choices=repo_urls,
	)

	self.c['schedulers'].append(schedulers.ForceScheduler(
	name="force",
	builderNames=[self.builder_name_force],
	properties=[repo_param],
	))

File Metadata

Mime Type: text/x-python
Expires: Tue, Jan 20, 9:27 AM (1 d, 22 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 972854
Default Alt Text: config.py (8 KB)

config.pyNo OneTemporaryActions

config.pyView Options

File Metadata

Event Timeline

config.py
No OneTemporary
Actions

config.py
View Options