Page MenuHomePhorge
Files F85831432

No OneTemporary
Actions

Authored By
Unknown
Size
23 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/Containerfile.worker b/Containerfile.worker
index c50d96a..161bac4 100644
--- a/Containerfile.worker
+++ b/Containerfile.worker
@@ -1,20 +1,25 @@
FROM alpine
RUN apk add --no-cache podman podman-compose python3 \
- py3-virtualenv dumb-init bash shadow git openssh rsync \
+ py3-virtualenv dumb-init bash shadow git openssh rsync php php-curl \
&& virtualenv --python=python3 /buildbot_venv \
&& /buildbot_venv/bin/pip3 install 'twisted[tls]' \
&& mkdir /buildbot \
- && useradd -ms /bin/bash buildbot
+ && useradd -ms /bin/bash buildbot \
+ && mkdir -pv /tools \
+ && cd /tools \
+ && git clone https://we.phorge.it/source/arcanist.git \
+ && cd arcanist \
+ && git checkout stable
COPY . /usr/src/buildbot-worker
COPY docker/buildbot.tac /buildbot/buildbot.tac
RUN /buildbot_venv/bin/pip3 install /usr/src/buildbot-worker && \
chown -R buildbot /buildbot
USER buildbot
WORKDIR /buildbot
CMD ["/usr/bin/dumb-init", "/buildbot_venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]
diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
index 8eb71fd..01547bf 100644
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -1,306 +1,307 @@
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from twisted.internet import defer
from .ci_syntax import ci_file
from .helpers import rsync_rules_from_artifacts
def on_success(step):
return step.build.results == util.SUCCESS
DEFAULT_SCRIPT_HEADER = '''\
#!/bin/sh
set -e -x
cd /build
'''
class RunCIJobStep(steps.BuildStep):
# 200 MiB
artifact_max_size = 200 * 1024 * 1024
default_image = 'alpine'
script_header = DEFAULT_SCRIPT_HEADER
master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
artifact_file_name = 'artifacts.tar'
master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
def __init__(
self,
lbc,
src_relative=None,
src_dir=None,
storage_dir=None,
repo_id=None,
result_relative=None,
result_dir=None,
artifact_stage_relative=None,
artifact_stage_dir=None,
job_prop=None,
**kwargs):
self.lbc = lbc
self.src_relative = src_relative
self.src_dir = src_dir
self.work_root_dir = kwargs['workdir']
self.script_dir = 'script'
self.storage_dir = storage_dir
self.repo_id = repo_id
self.artifact_stage_relative = artifact_stage_relative
self.artifact_stage_dir = artifact_stage_dir
self.result_relative = result_relative
self.result_dir = result_dir
super().__init__(name='Run step', **kwargs)
def run(self):
job_prop = self.getProperty('lilybuild_job_prop')
job = ci_file.CIJob.from_prop(job_prop)
job_index = self.getProperty('lilybuild_job_index')
next_steps = self.job_to_steps(job, job_index)
self.build.addStepsAfterCurrentStep(next_steps)
return util.SUCCESS
def job_to_steps(self, job, job_index):
script_name = self.script_dir + '/run.sh'
script_step = steps.StringDownload(
self.script_header +
'\n\n'.join(job.script),
name=f'{job.name}: Set up script',
workerdest=script_name,
workdir=self.work_root_dir,
doStepIf=on_success,
)
chmod_step = steps.ShellCommand(
name=f'{job.name}: Make script executable',
command=['chmod', '+x', script_name],
workdir=self.work_root_dir,
doStepIf=on_success,
)
artifact_steps = []
dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
if dep_job_indices:
for i in dep_job_indices:
download_job = steps.FileDownload(
mastersrc=util.Interpolate(
self.master_job_artifact_file_name_pattern,
st=self.storage_dir,
job=i,
),
maxsize=self.artifact_max_size,
name=f'Download artifacts from job #{i}',
workerdest=self.artifact_file_name,
workdir=self.work_root_dir,
doStepIf=on_success,
)
unarchive_job = steps.ShellCommand(
name=f'Unarchive artifacts from job #{i}',
command=[
'tar',
'-xf',
self.artifact_file_name,
'-C',
self.src_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
artifact_steps += [download_job, unarchive_job]
run_step = steps.ShellCommand(
name=f'{job.name}: Run script in container',
command=[
'/lilybuild/podman-helper',
job.image or self.default_image,
self.src_relative,
self.script_dir,
self.result_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step]
if 'paths' in job.artifacts:
clean_stage_dir_step = steps.ShellCommand(
name=f'{job.name}: Clean stage dir',
command=[
'rm',
'-rf',
self.artifact_stage_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
collect_artifact_step = steps.ShellCommand(
name=f'{job.name}: Collect artifacts',
command=[
'rsync',
'-av',
self.result_relative + '/',
'--delete',
'--prune-empty-dirs',
] + rsync_rules_from_artifacts(job.artifacts) + [
self.artifact_stage_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
archive_artifact_step = steps.ShellCommand(
name=f'{job.name}: Archive artifacts',
command=[
'tar',
'-cf',
self.artifact_file_name,
'-C',
self.artifact_stage_relative,
'.',
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
masterdest = util.Interpolate(
self.master_job_artifact_file_name_pattern,
st=self.storage_dir,
job=job_index,
doStepIf=on_success,
)
upload_artifact_step = steps.FileUpload(
workersrc=self.artifact_file_name,
maxsize=self.artifact_max_size,
name=f'{job.name}: Upload artifacts',
masterdest=masterdest,
workdir=self.work_root_dir,
doStepIf=on_success,
)
steps_to_run += [clean_stage_dir_step, collect_artifact_step, archive_artifact_step, upload_artifact_step]
return steps_to_run
class TriggerMultipleJobsStep(steps.Trigger):
properties_to_keep = [
'branch',
'revision',
'repository',
+ 'harbormaster_build_target_phid',
'harbormaster_variable_buildable.diff',
'harbormaster_variable_repository.staging.ref',
'harbormaster_variable_repository.staging.uri',
'harbormaster_variable_repository.uri',
'lilybuild_repo',
'lilybuild_repo_id',
]
def __init__(self, lbc, jobs_with_data, **kwargs):
self.lbc = lbc
self.jobs = jobs_with_data
super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)
def getSchedulersAndProperties(self):
ret = []
common_properties = {
'lilybuild_root_build_id': self.build.buildid,
}
for prop in self.properties_to_keep:
if self.hasProperty(prop):
common_properties[prop] = self.getProperty(prop)
for (job, i, dep_job_indices) in self.jobs:
properties = common_properties.copy()
properties['lilybuild_job_prop'] = job.to_prop()
properties['lilybuild_job_index'] = i
properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
properties['lilybuild_dependency_job_indices'] = dep_job_indices
ret.append({
'sched_name': self.lbc.triggerable_scheduler_name,
'props_to_set': properties,
'unimportant': False,
})
return ret
class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
ci_def_file = '.gitlab-ci.yml'
def __init__(
self,
lbc,
src_relative=None,
src_dir=None,
storage_dir=None,
repo_id=None,
result_relative=None,
result_dir=None,
artifact_stage_relative=None,
artifact_stage_dir=None,
**kwargs):
kwargs['name'] = 'Analyze CI file'
kwargs['command'] = ['cat', self.ci_def_file]
self.lbc = lbc
self.src_relative = src_relative
self.src_dir = src_dir
self.work_root_dir = kwargs['workdir']
self.script_dir = 'script'
self.storage_dir = storage_dir
self.repo_id = repo_id
self.artifact_stage_relative = artifact_stage_relative
self.artifact_stage_dir = artifact_stage_dir
self.result_relative = result_relative
self.result_dir = result_dir
kwargs['workdir'] = self.src_dir
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver('stdio', self.observer)
def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
jobs_with_data = []
for job in stage_jobs:
dep_job_names = [
jn
for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
if ci_file.jobs[jn].has_artifacts_archive()
]
dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))
trigger = TriggerMultipleJobsStep(
name=stage_name,
lbc=self.lbc,
jobs_with_data=jobs_with_data,
waitForFinish=True,
doStepIf=on_success,
)
return trigger
def get_steps(self, stdout):
f = ci_file.CIFile(stdout)
stages = f.get_grouped_jobs()
jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
job_names = [job.name for job in jobs]
job_name_to_index_map = {}
for (i, j) in enumerate(jobs):
job_name_to_index_map[j.name] = i
steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
print('steps:', steps)
return steps
@defer.inlineCallbacks
def run(self):
# run './build.sh --list-stages' to generate the list of stages
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))
return result
diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
index 49b4175..4699f08 100644
--- a/lilybuild/lilybuild/config.py
+++ b/lilybuild/lilybuild/config.py
@@ -1,197 +1,218 @@
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from twisted.internet import defer
-from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand
+from .ci_steps import RunCIJobStep, AnalyzeCIFileCommand, on_success
+from .phorge import CheckoutSourceFromPhorge
import yaml
def to_params(d):
res = []
for n in d:
res.append(util.FixedParameter(name=n, default=d[n]))
return res
work_root = util.Interpolate('repos/%(prop:lilybuild_repo_id)s')
src_relative = 'sources'
src_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/sources')
result_relative = 'result'
result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
artifact_stage_relative = 'stage'
artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
storage_dir = '/buildbot/storage'
class LilyBuildConfig:
builder_name = 'lilybuild'
builder_name_run_job = 'lilybuild-job'
builder_name_force = 'lilybuild-force'
triggerable_scheduler_name = 'lilybuild-triggerable'
force_triggerable_scheduler_name = 'lilybuild-force-triggerable'
def __init__(self, c, workernames):
self.c = c
self.poll_interval = 300
self.workernames = workernames
self.repos = {}
self.repo_id_by_url = {}
self.branch_skip_re = '^phabricator/'
def configure_factory_and_builder(self):
self.add_lilybuild_builder()
self.add_lilybuild_job_builder()
+ def create_source_step(self):
+ return CheckoutSourceFromPhorge(
+ lbs=self,
+ repourl=util.Property('lilybuild_repo'),
+ mode='full',
+ workdir=src_dir
+ )
+
+ def get_arcrc_for_repo(self, repo_id):
+ return self.repos[repo_id]['arcrc']
+
def add_lilybuild_builder(self):
factory = util.BuildFactory()
- factory.addStep(steps.Git(repourl=util.Property('lilybuild_repo'), mode='full', workdir=src_dir))
+ factory.addStep(self.create_source_step())
factory.addStep(AnalyzeCIFileCommand(
lbc=self,
workdir=work_root,
src_relative=src_relative,
src_dir=src_dir,
storage_dir=storage_dir,
artifact_stage_relative=artifact_stage_relative,
artifact_stage_dir=artifact_stage_dir,
result_relative=result_relative,
result_dir=result_dir,
+ doStepIf=on_success,
))
builder = util.BuilderConfig(
name=self.builder_name,
workernames=self.workernames,
factory=factory)
self.c['builders'].append(builder)
def add_lilybuild_job_builder(self):
factory_job = util.BuildFactory()
- factory_job.addStep(steps.Git(repourl=util.Property('lilybuild_repo'), mode='full', workdir=src_dir))
+ factory_job.addStep(self.create_source_step())
factory_job.addStep(RunCIJobStep(
lbc=self,
workdir=work_root,
src_relative=src_relative,
src_dir=src_dir,
storage_dir=storage_dir,
artifact_stage_relative=artifact_stage_relative,
artifact_stage_dir=artifact_stage_dir,
result_relative=result_relative,
result_dir=result_dir,
))
builder_job = util.BuilderConfig(
name=self.builder_name_run_job,
workernames=self.workernames,
factory=factory_job,
# Tell buildbot to never collapse build requests for this one
# because otherwise triggering multiple jobs will cause it to only run one
# https://docs.buildbot.net/current/manual/configuration/builders.html#collapsing-build-requests
collapseRequests=False,
)
self.c['builders'].append(builder_job)
s = schedulers.Triggerable(self.triggerable_scheduler_name, builderNames=[self.builder_name_run_job])
self.c['schedulers'].append(s)
def record_url(self, repo_id, repo_url):
if repo_url in self.repo_id_by_url:
raise Exception(f'repo url "{repo_url}" already recorded')
self.repo_id_by_url[repo_url] = repo_id
- def add_repo(self, repo_id, repo_url, do_poll=False, alternative_urls=None):
+ def add_repo(
+ self,
+ repo_id,
+ repo_url,
+ do_poll=False,
+ alternative_urls=None,
+ arcrc=None,
+ ):
if not alternative_urls:
alternative_urls = []
self.repos[repo_id] = {
'repo_id': repo_id,
'repo_url': repo_url,
'do_poll': do_poll,
'alternative_urls': alternative_urls,
+ 'arcrc': arcrc,
}
self.record_url(repo_id, repo_url)
for u in alternative_urls:
self.record_url(repo_id, u)
def add_one_repo_def(self, repo_def):
print('Adding repo', repo_def)
repo_urls = [repo_def['repo_url']] + repo_def['alternative_urls']
repo_id = repo_def['repo_id']
do_poll = repo_def['do_poll']
main_repo_url = repo_def['repo_url']
properties = {
'lilybuild_repo': main_repo_url,
'lilybuild_repo_id': repo_id,
'virtual_builder_name': 'lilybuild - ' + main_repo_url,
}
self.c['schedulers'].append(schedulers.AnyBranchScheduler(
name=f'lilybuild-anybranch - {main_repo_url}',
change_filter=util.ChangeFilter(
repository=repo_urls,
),
treeStableTimer=None,
builderNames=[self.builder_name],
properties=properties))
if do_poll:
for repo in repo_urls:
print('Adding poller')
self.c['change_source'].append(changes.GitPoller(
repo,
workdir=f'gitpoller/{repo_id}', branches=True,
pollInterval=self.poll_interval))
def configure_pipeline_defs(self):
for repo_id in self.repos:
self.add_one_repo_def(self.repos[repo_id])
self.add_force_builder()
self.add_force_scheduler()
def add_force_builder(self):
def get_repo_id_by_url(url):
return self.repo_id_by_url[url]
factory_force = util.BuildFactory()
factory_force.addStep(steps.Trigger(
name='trigger lilybuild',
schedulerNames=[self.force_triggerable_scheduler_name],
waitForFinish=True,
set_properties={
'lilybuild_repo': util.Property('lilybuild_repo'),
'lilybuild_repo_id': util.Transform(get_repo_id_by_url, util.Property('lilybuild_repo')),
'virtual_builder_name': util.Interpolate('lilybuild - %(prop:lilybuild_repo)s')
}
))
builder_force = util.BuilderConfig(
name=self.builder_name_force,
workernames=self.workernames,
factory=factory_force,
)
self.c['builders'].append(builder_force)
# this is triggered by lilybuild-force, and it triggers lilybuild
# via its virtual builder
s = schedulers.Triggerable(
self.force_triggerable_scheduler_name,
builderNames=[self.builder_name],
)
self.c['schedulers'].append(s)
def add_force_scheduler(self):
repo_urls = []
for r in self.repos.values():
repo_urls.append(r['repo_url'])
repo_param = util.ChoiceStringParameter(
name='lilybuild_repo',
required=True,
strict=True,
choices=repo_urls,
)
self.c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=[self.builder_name_force],
properties=[repo_param],
))
diff --git a/lilybuild/lilybuild/phorge.py b/lilybuild/lilybuild/phorge.py
new file mode 100644
index 0000000..652b490
--- /dev/null
+++ b/lilybuild/lilybuild/phorge.py
@@ -0,0 +1,83 @@
+
+from buildbot.plugins import *
+from buildbot.process import buildstep, logobserver
+from buildbot.steps.download_secret_to_worker import DownloadSecretsToWorker, RemoveWorkerFileSecret
+from twisted.internet import defer
+
+class CheckoutSourceFromPhorge(steps.Git):
+ staging_uri_prop_name = 'harbormaster_variable_repository.staging.uri'
+ staging_ref_prop_name = 'harbormaster_variable_repository.staging.ref'
+ diff_id_prop_name = 'harbormaster_variable_buildable.diff'
+ build_target_prop_name = 'harbormaster_build_target_phid'
+ arc_command = '/tools/arcanist/bin/arc'
+
+ def __init__(self, lbs, secret_dir='../secrets', **kwargs):
+ self.lbs = lbs
+ self.secret_dir = secret_dir
+ super().__init__(**kwargs)
+
+ @defer.inlineCallbacks
+ def run_vc(self, branch, revision, patch):
+ is_phorge = self.getProperty(self.build_target_prop_name)
+ if not is_phorge:
+ res = yield super().run_vc(branch, revision, patch)
+ return res
+
+ staging_uri = self.getProperty(self.staging_uri_prop_name)
+ staging_ref = self.getProperty(self.staging_ref_prop_name)
+
+ if staging_uri:
+ old_repourl = self.repourl
+ self.repourl = staging_uri
+ # Assume branch is the same as staging ref
+ try:
+ res = yield super().run_vc(branch, revision, patch)
+ if res == util.SUCCESS:
+ # If we can get the source from the staging area, then we are done
+ return res
+ except:
+ pass
+ self.addCompleteLog('log', 'Cannot fetch source from staging area, trying to `arc patch`')
+ self.repourl = old_repourl
+ else:
+ self.addCompleteLog('log', 'No staging area defined, trying to `arc patch`')
+
+ res = yield super().run_vc(branch='HEAD', revision=None, patch=None)
+ if res != util.SUCCESS:
+ self.addCompleteLog('error', 'Cannot checkout repository head, unable to proceed')
+ return res
+
+ arcrc = self.lbs.get_arcrc_for_repo(self.getProperty('lilybuild_repo_id'))
+ if not arcrc:
+ self.addCompleteLog('error', 'arcrc secret is not specified for the repository')
+ return util.FAILURE
+
+ secret_list = [(
+ util.Interpolate(
+ '%(kw:secret_dir)s/.arcrc',
+ secret_dir=self.secret_dir,
+ ),
+ arcrc,
+ )]
+
+ diff_id = self.getProperty(self.diff_id_prop_name)
+ if not diff_id:
+ self.addCompleteLog('error', 'Diff id is not present')
+ return util.FAILURE
+
+ self.build.addStepsAfterCurrentStep([
+ DownloadSecretsToWorker(secret_list, workdir=self.workdir),
+ steps.ShellCommand(
+ name='arc patch',
+ command=[
+ self.arc_command, 'patch',
+ '--diff', diff_id,
+ '--nobranch', '--nocommit',
+ ],
+ env={'HOME': self.secret_dir},
+ workdir=self.workdir,
+ ),
+ RemoveWorkerFileSecret(secret_list, workdir=self.workdir),
+ ])
+
+ return util.SUCCESS

File Metadata

Mime Type
text/x-diff
Expires
Sun, Nov 1, 3:13 PM (12 h, 2 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1803129
Default Alt Text
(23 KB)

Event Timeline