Page MenuHomePhorge
Files F85803311

No OneTemporary
Actions

Size
20 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
index 58a0786..451fcf0 100644
--- a/lilybuild/lilybuild/config.py
+++ b/lilybuild/lilybuild/config.py
@@ -1,251 +1,253 @@
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from twisted.internet import defer
from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand, on_success
from .phorge import CheckoutSourceFromPhorge, SendArtifactLinkToPhorge, SendBuildStatusToPhorge, MaybeRequireApprovalForPhorge
from .helpers import normalize_base_url, phorge_token_to_arcrc
import yaml
def to_params(d):
res = []
for n in d:
res.append(util.FixedParameter(name=n, default=d[n]))
return res
work_root = util.Interpolate('repos/%(prop:lilybuild_repo_id)s')
src_relative = 'sources'
src_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/sources')
result_relative = 'result'
result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
artifact_stage_relative = 'stage'
artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
storage_dir = '/buildbot/storage'
report_phorge = 'phorge'
report_forgejo = 'forgejo'
class LilyBuildConfig:
builder_name = 'lilybuild'
builder_name_run_job = 'lilybuild-job'
builder_name_force = 'lilybuild-force'
main_builder_tag = 'lilybuild-pipeline'
triggerable_scheduler_name = 'lilybuild-triggerable'
force_triggerable_scheduler_name = 'lilybuild-force-triggerable'
- def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None):
+ def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None):
self.c = c
self.poll_interval = 300
self.workernames = workernames
if reports is None:
reports = [report_phorge, report_forgejo]
self.reports = reports
self.repos = {}
self.repo_id_by_url = {}
self.branch_skip_re = '^phabricator/'
self.phorge_base_url = normalize_base_url(phorge_base_url)
self.phorge_token = phorge_token
+ self.ssh_priv_key = ssh_priv_key
+ self.ssh_known_hosts = ssh_known_hosts
def configure_factory_and_builder(self):
self.add_lilybuild_builder()
self.add_lilybuild_job_builder()
def create_source_step(self):
return CheckoutSourceFromPhorge(
lbc=self,
repourl=util.Property('lilybuild_repo'),
mode='full',
workdir=src_dir,
submodules=True
)
def get_arcrc_for_repo(self, repo_id):
return util.Transform(
phorge_token_to_arcrc,
self.repos[repo_id]['phorge_base_url'],
self.repos[repo_id]['phorge_token'],
)
def add_lilybuild_builder(self):
factory = util.BuildFactory()
factory.addStep(self.create_source_step())
if report_phorge in self.reports:
factory.addStep(MaybeRequireApprovalForPhorge(lbc=self, workdir=src_dir))
factory.addStep(SendArtifactLinkToPhorge(lbc=self, workdir=src_dir))
factory.addStep(AnalyzeCIFileCommand(
lbc=self,
workdir=work_root,
src_relative=src_relative,
src_dir=src_dir,
storage_dir=storage_dir,
artifact_stage_relative=artifact_stage_relative,
artifact_stage_dir=artifact_stage_dir,
result_relative=result_relative,
result_dir=result_dir,
doStepIf=on_success,
))
if report_phorge in self.reports:
factory.addStep(SendBuildStatusToPhorge(lbc=self, workdir=src_dir))
builder = util.BuilderConfig(
name=self.builder_name,
workernames=self.workernames,
factory=factory,
tags=[self.main_builder_tag],
)
self.c['builders'].append(builder)
def add_lilybuild_job_builder(self):
factory_job = util.BuildFactory()
factory_job.addStep(self.create_source_step())
factory_job.addStep(RunCIJobStep(
lbc=self,
workdir=work_root,
src_relative=src_relative,
src_dir=src_dir,
storage_dir=storage_dir,
artifact_stage_relative=artifact_stage_relative,
artifact_stage_dir=artifact_stage_dir,
result_relative=result_relative,
result_dir=result_dir,
))
builder_job = util.BuilderConfig(
name=self.builder_name_run_job,
workernames=self.workernames,
factory=factory_job,
# Tell buildbot to never collapse build requests for this one
# because otherwise triggering multiple jobs will cause it to only run one
# https://docs.buildbot.net/current/manual/configuration/builders.html#collapsing-build-requests
collapseRequests=False,
)
self.c['builders'].append(builder_job)
s = schedulers.Triggerable(self.triggerable_scheduler_name, builderNames=[self.builder_name_run_job])
self.c['schedulers'].append(s)
def record_url(self, repo_id, repo_url):
if repo_url in self.repo_id_by_url:
raise Exception(f'repo url "{repo_url}" already recorded')
self.repo_id_by_url[repo_url] = repo_id
def add_repo(
self,
repo_id,
repo_url,
do_poll=False,
alternative_urls=None,
phorge_base_url=None,
phorge_token=None,
# Function(build -> dict(str -> str | renderable<str>))
variables_getter=None,
):
if not alternative_urls:
alternative_urls = []
self.repos[repo_id] = {
'repo_id': repo_id,
'repo_url': repo_url,
'do_poll': do_poll,
'alternative_urls': alternative_urls,
'phorge_base_url': normalize_base_url(phorge_base_url) or self.phorge_base_url,
'phorge_token': phorge_token or self.phorge_token,
'variables_getter': variables_getter or (lambda _build: {}),
}
self.record_url(repo_id, repo_url)
for u in alternative_urls:
self.record_url(repo_id, u)
def get_builder_name_for_repo(self, repo_def):
main_repo_url = repo_def['repo_url']
return f'lilybuild - {main_repo_url}'
def add_one_repo_def(self, repo_def):
print('Adding repo', repo_def)
repo_urls = [repo_def['repo_url']] + repo_def['alternative_urls']
repo_id = repo_def['repo_id']
do_poll = repo_def['do_poll']
main_repo_url = repo_def['repo_url']
properties = {
'lilybuild_repo': main_repo_url,
'lilybuild_repo_id': repo_id,
'virtual_builder_name': self.get_builder_name_for_repo(repo_def),
'virtual_builder_tags': [self.main_builder_tag],
}
self.c['schedulers'].append(schedulers.AnyBranchScheduler(
name=f'lilybuild-anybranch - {main_repo_url}',
change_filter=util.ChangeFilter(
repository=repo_urls,
),
treeStableTimer=None,
builderNames=[self.builder_name],
properties=properties))
if do_poll:
for repo in repo_urls:
print('Adding poller')
self.c['change_source'].append(changes.GitPoller(
repo,
workdir=f'gitpoller/{repo_id}', branches=True,
pollInterval=self.poll_interval))
def configure_pipeline_defs(self):
for repo_id in self.repos:
self.add_one_repo_def(self.repos[repo_id])
self.add_force_builder()
self.add_force_scheduler()
def add_force_builder(self):
def get_repo_id_by_url(url):
return self.repo_id_by_url[url]
factory_force = util.BuildFactory()
factory_force.addStep(steps.Trigger(
name='trigger lilybuild',
schedulerNames=[self.force_triggerable_scheduler_name],
waitForFinish=True,
set_properties={
'lilybuild_repo': util.Property('lilybuild_repo'),
'lilybuild_repo_id': util.Transform(get_repo_id_by_url, util.Property('lilybuild_repo')),
'virtual_builder_name': util.Interpolate('lilybuild - %(prop:lilybuild_repo)s'),
'virtual_builder_tags': [self.main_builder_tag],
}
))
builder_force = util.BuilderConfig(
name=self.builder_name_force,
workernames=self.workernames,
factory=factory_force,
)
self.c['builders'].append(builder_force)
# this is triggered by lilybuild-force, and it triggers lilybuild
# via its virtual builder
s = schedulers.Triggerable(
self.force_triggerable_scheduler_name,
builderNames=[self.builder_name],
)
self.c['schedulers'].append(s)
def add_force_scheduler(self):
repo_urls = []
for r in self.repos.values():
repo_urls.append(r['repo_url'])
repo_param = util.ChoiceStringParameter(
name='lilybuild_repo',
required=True,
strict=True,
choices=repo_urls,
)
self.c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=[self.builder_name_force],
properties=[repo_param],
))
diff --git a/lilybuild/lilybuild/phorge.py b/lilybuild/lilybuild/phorge.py
index 3db783a..0fe5631 100644
--- a/lilybuild/lilybuild/phorge.py
+++ b/lilybuild/lilybuild/phorge.py
@@ -1,291 +1,294 @@
import stat
import os
import json
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from buildbot.interfaces import IRenderable
from buildbot.steps.download_secret_to_worker import DownloadSecretsToWorker, RemoveWorkerFileSecret
from buildbot.steps.worker import CompositeStepMixin
from buildbot.util import bytes2unicode
from twisted.internet import defer
from twisted.python import log
class PhorgeMixin(CompositeStepMixin, buildstep.ShellMixin):
'''
Should be inherited by a BuildStep.
Required properties:
self.lbc: LilyBuildConfig
self.secret_dir: str
'''
staging_uri_prop_name = 'harbormaster_variable_repository.staging.uri'
staging_ref_prop_name = 'harbormaster_variable_repository.staging.ref'
diff_id_prop_name = 'harbormaster_variable_buildable.diff'
build_target_prop_name = 'harbormaster_build_target_phid'
arc_command = '/tools/arcanist/bin/arc'
def setup_phorge_mixin(self, kwargs):
shell_mixin_kwargs = {}
shell_mixin_inherit_args = ['workdir']
for a in shell_mixin_inherit_args:
if a in kwargs:
shell_mixin_kwargs[a] = kwargs[a]
self.setupShellMixin(shell_mixin_kwargs)
def get_is_phorge(self):
return not not self.getProperty(self.build_target_prop_name)
def get_secret_dir(self):
# Taken from buildbot.util.git
workerbuilddir = bytes2unicode(self.build.builder.config.workerbuilddir)
workdir = self.workdir.rstrip('/\\')
if os.path.isabs(workdir):
parent_path = os.path.dirname(workdir)
else:
assert self.worker is not None
parent_path = os.path.join(
self.worker.worker_basedir, os.path.dirname(workdir)
)
basename = f'.{workerbuilddir}.{os.path.basename(workdir)}.lilybuild-phorge'
secret_dir = os.path.join(parent_path, basename)
log.msg('Arc secret dir is', secret_dir)
return secret_dir
@defer.inlineCallbacks
def make_arc_command(self, args, **kwargs):
cmd = yield self.makeRemoteShellCommand(
command=[self.arc_command] + args,
env={'HOME': self.get_secret_dir()},
**kwargs
)
return cmd
@defer.inlineCallbacks
def run_arc_with_secret(self, args, **kwargs):
try:
res = yield self.download_arc_secret()
if res != util.SUCCESS:
return res
cmd = yield self.make_arc_command(args, **kwargs)
yield self.runCommand(cmd)
return cmd.results()
except Exception as e:
raise e
finally:
yield self.remove_arc_secret()
@defer.inlineCallbacks
def download_arc_secret(self):
arcrc = self.lbc.get_arcrc_for_repo(self.getProperty('lilybuild_repo_id'))
if not arcrc:
self.addCompleteLog('error', 'arcrc secret is not specified for the repository')
return util.FAILURE
arcrc_content = yield self.build.render(arcrc)
path = os.path.join(self.get_secret_dir(), '.arcrc')
yield self.downloadFileContentToWorker(
path, arcrc_content, mode=stat.S_IRUSR | stat.S_IWUSR, workdir=self.workdir
)
# If failed, it will raise
return util.SUCCESS
@defer.inlineCallbacks
def remove_arc_secret(self):
path = self.get_secret_dir()
yield self.runRmdir(path, abandonOnFailure=False)
return util.SUCCESS
class CheckoutSourceFromPhorge(PhorgeMixin, steps.Git):
'''
Set a property `lilybuild_source`:
`non-phorge` if the source is not a Differential Diff,
`staging` if the source is checked out from staging area,
`arc-patch` if the source is checked out by `arc patch`, either because
there is no staging area defined, or because the staging area does not
have the relevant diffs.
'''
def __init__(self, lbc, **kwargs):
self.lbc = lbc
self.setup_phorge_mixin(kwargs)
- super().__init__(**kwargs)
+ super().__init__(
+ sshPrivateKey=lbc.ssh_priv_key,
+ sshKnownHosts=lbc.ssh_known_hosts,
+ **kwargs)
@defer.inlineCallbacks
def run_vc(self, branch, revision, patch):
def set_prop(k, v):
self.setProperty(k, v, self.__class__.__name__)
is_phorge = self.get_is_phorge()
if not is_phorge:
set_prop('lilybuild_source', 'non-phorge')
res = yield super().run_vc(branch, revision, patch)
return res
staging_uri = self.getProperty(self.staging_uri_prop_name)
staging_ref = self.getProperty(self.staging_ref_prop_name)
if staging_uri:
old_repourl = self.repourl
self.repourl = staging_uri
# Assume branch is the same as staging ref
try:
res = yield super().run_vc(branch, revision, patch)
if res == util.SUCCESS:
# If we can get the source from the staging area, then we are done
set_prop('lilybuild_source', 'staging')
return res
except:
pass
self.addCompleteLog('log', 'Cannot fetch source from staging area, trying to `arc patch`')
self.repourl = old_repourl
else:
self.addCompleteLog('log', 'No staging area defined, trying to `arc patch`')
res = yield super().run_vc(branch='HEAD', revision=None, patch=None)
if res != util.SUCCESS:
self.addCompleteLog('error', 'Cannot checkout repository head, unable to proceed')
return res
diff_id = self.getProperty(self.diff_id_prop_name)
if not diff_id:
self.addCompleteLog('error', 'Diff id is not present')
return util.FAILURE
res = yield self.run_arc_with_secret([
'patch',
'--diff', diff_id,
'--nobranch', '--nocommit',
])
set_prop('lilybuild_source', 'arc-patch')
return res
class SendArtifactLinkToPhorge(PhorgeMixin, steps.BuildStep):
def __init__(self, lbc, **kwargs):
self.lbc = lbc
self.setup_phorge_mixin(kwargs)
super().__init__(
name='Send artifact link to Phorge',
doStepIf=lambda step: step.get_is_phorge(),
alwaysRun=True,
**kwargs
)
@defer.inlineCallbacks
def run(self):
if self.getProperty('lilybuild_require_approval'):
# Do not send another artifact if this requires approval
return util.SKIPPED
build_url = yield self.build.getUrl()
data = {
'buildTargetPHID': self.getProperty(self.build_target_prop_name),
'artifactKey': f'lilybuild-{self.build.buildid}',
'artifactType': 'uri',
'artifactData': {
'uri': build_url,
'name': f'Buildbot build #{self.build.buildid}',
'ui.external': True,
},
}
encoded_data = json.dumps(data)
res = yield self.run_arc_with_secret([
'call-conduit',
'--',
'harbormaster.createartifact',
], initialStdin=encoded_data)
return res
class MaybeRequireApprovalForPhorge(PhorgeMixin, steps.BuildStep):
def __init__(self, lbc, sources_need_approval=None, **kwargs):
self.lbc = lbc
if sources_need_approval is None:
sources_need_approval = ['arc-patch']
self.sources_need_approval = sources_need_approval
self.setup_phorge_mixin(kwargs)
super().__init__(
name='Maybe require approval for phorge sources',
doStepIf=lambda step: step.get_is_phorge(),
alwaysRun=True,
**kwargs
)
@defer.inlineCallbacks
def run(self):
def set_prop(k, v):
self.setProperty(k, v, self.__class__.__name__)
if self.getProperty('lilybuild_source') not in self.sources_need_approval:
set_prop('lilybuild_require_approval', False)
return util.SUCCESS
build_data = yield self.master.data.get(('builds', self.build.buildid))
build_request_data = yield self.master.data.get(('buildrequests', build_data['buildrequestid']))
buildset_data = yield self.master.data.get(('buildsets', build_request_data['buildsetid']))
rebuilt_buildid = buildset_data['rebuilt_buildid']
if rebuilt_buildid is not None:
# This is rebuilt, because someone approved it earlier
set_prop('lilybuild_require_approval', False)
return util.SUCCESS
self.addCompleteLog('info', 'This build requires approval. To approve, rebuild this build.')
build_url = yield self.build.getUrl()
data = {
'buildTargetPHID': self.getProperty(self.build_target_prop_name),
'artifactKey': f'lilybuild-{self.build.buildid}-pending',
'artifactType': 'uri',
'artifactData': {
'uri': build_url,
'name': f'Requires approval',
'ui.external': True,
},
}
encoded_data = json.dumps(data)
res = yield self.run_arc_with_secret([
'call-conduit',
'--',
'harbormaster.createartifact',
], initialStdin=encoded_data)
set_prop('lilybuild_require_approval', True)
return util.FAILURE
class SendBuildStatusToPhorge(PhorgeMixin, steps.BuildStep):
def __init__(self, lbc, **kwargs):
self.lbc = lbc
self.setup_phorge_mixin(kwargs)
super().__init__(
name='Send build status to Phorge',
doStepIf=lambda step: step.get_is_phorge(),
alwaysRun=True,
**kwargs
)
@defer.inlineCallbacks
def run(self):
if self.getProperty('lilybuild_require_approval'):
# Do not send final build result if this is skipped, because
# otherwise we cannot update it later.
return util.SKIPPED
build_url = yield self.build.getUrl()
data = {
'receiver': self.getProperty(self.build_target_prop_name),
'type': 'pass' if self.build.results == util.SUCCESS else 'fail',
}
encoded_data = json.dumps(data)
res = yield self.run_arc_with_secret([
'call-conduit',
'--',
'harbormaster.sendmessage',
], initialStdin=encoded_data)
return res

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 9, 4:39 AM (1 d, 7 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1784450
Default Alt Text
(20 KB)

Event Timeline