Page MenuHomePhorge
Files F85649490

No OneTemporary
Actions

Size
14 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
index 78ed5c1..c7063bb 100644
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -1,405 +1,405 @@
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from buildbot.interfaces import IRenderable
from twisted.internet import defer
from .ci_syntax import ci_file
from .helpers import rsync_rules_from_artifacts, ci_vars_to_cmds
import re
def on_success(step):
return step.build.results == util.SUCCESS
DEFAULT_SCRIPT_HEADER = '''\
#!/bin/sh
set -e -x
cd /build
'''
class RunCIJobStep(steps.BuildStep):
# 200 MiB
artifact_max_size = 200 * 1024 * 1024
default_image = 'alpine'
script_header = DEFAULT_SCRIPT_HEADER
master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
artifact_file_name = 'artifacts.tar'
master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
def __init__(
self,
lbc,
src_relative=None,
src_dir=None,
storage_dir=None,
repo_id=None,
result_relative=None,
result_dir=None,
artifact_stage_relative=None,
artifact_stage_dir=None,
job_prop=None,
**kwargs):
self.lbc = lbc
self.src_relative = src_relative
self.src_dir = src_dir
self.work_root_dir = kwargs['workdir']
self.script_dir = 'script'
self.storage_dir = storage_dir
self.repo_id = repo_id
self.artifact_stage_relative = artifact_stage_relative
self.artifact_stage_dir = artifact_stage_dir
self.result_relative = result_relative
self.result_dir = result_dir
super().__init__(name='Run step', **kwargs)
@defer.inlineCallbacks
def run(self):
job_prop = self.getProperty('lilybuild_job_prop')
job = ci_file.CIJob.from_prop(job_prop)
job_index = self.getProperty('lilybuild_job_index')
next_steps = yield self.job_to_steps(job, job_index)
self.build.addStepsAfterCurrentStep(next_steps)
return util.SUCCESS
@defer.inlineCallbacks
def get_ci_variables(self, job):
res = {}
res.update(job.get_predefined_ci_variables())
res.update(self.getProperty('lilybuild_pipeline_vars'))
res['CI_JOB_IMAGE'] = job.image or self.default_image
res['CI_JOB_URL'] = yield self.build.getUrl()
res['CI_JOB_ID'] = self.build.buildid
res['CI_PROJECT_DIR'] = '/build'
try:
repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
variables = yield repo['variables_getter'](self.build)
res_vars = {}
for var in variables:
value = variables[var]
- if isinstance(value, IRenderable):
+ if IRenderable.providedBy(value):
value = yield self.build.render(value)
res_vars[var] = value
res.update(res_vars)
except Exception as e:
self.addCompleteLog('exception', f'{e}')
return res
@defer.inlineCallbacks
def job_to_steps(self, job, job_index):
script_name = self.script_dir + '/run.sh'
variables = yield self.get_ci_variables(job)
var_cmds = ci_vars_to_cmds(variables)
script_step = steps.StringDownload(
self.script_header +
'\n' + var_cmds + '\n\n' +
'\n\n'.join(job.script),
name='Set up script',
workerdest=script_name,
workdir=self.work_root_dir,
doStepIf=on_success,
)
chmod_step = steps.ShellCommand(
name='Make script executable',
command=['chmod', '+x', script_name],
workdir=self.work_root_dir,
doStepIf=on_success,
)
artifact_steps = []
dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
if dep_job_indices:
for i in dep_job_indices:
download_job = steps.FileDownload(
mastersrc=util.Interpolate(
self.master_job_artifact_file_name_pattern,
st=self.storage_dir,
job=i,
),
maxsize=self.artifact_max_size,
name=f'Download artifacts from job #{i}',
workerdest=self.artifact_file_name,
workdir=self.work_root_dir,
doStepIf=on_success,
)
unarchive_job = steps.ShellCommand(
name=f'Unarchive artifacts from job #{i}',
command=[
'tar',
'-xf',
self.artifact_file_name,
'-C',
self.src_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
artifact_steps += [download_job, unarchive_job]
run_step = steps.ShellCommand(
name='Run script in container',
command=[
'/lilybuild/podman-helper',
job.image or self.default_image,
self.src_relative,
self.script_dir,
self.result_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
clean_script_step = steps.ShellCommand(
name='Clean script dir',
command=[
'rm',
'-rf',
self.script_dir,
],
workdir=self.work_root_dir,
alwaysRun=True,
)
steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
if 'paths' in job.artifacts:
clean_stage_dir_step = steps.ShellCommand(
name='Clean stage dir',
command=[
'rm',
'-rf',
self.artifact_stage_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
collect_artifact_step = steps.ShellCommand(
name='Collect artifacts',
command=[
'rsync',
'-av',
self.result_relative + '/',
'--delete',
'--prune-empty-dirs',
] + rsync_rules_from_artifacts(job.artifacts) + [
self.artifact_stage_relative,
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
archive_artifact_step = steps.ShellCommand(
name='Archive artifacts',
command=[
'tar',
'-cf',
self.artifact_file_name,
'-C',
self.artifact_stage_relative,
'.',
],
workdir=self.work_root_dir,
doStepIf=on_success,
)
masterdest = util.Interpolate(
self.master_job_artifact_file_name_pattern,
st=self.storage_dir,
job=job_index,
doStepIf=on_success,
)
upload_artifact_step = steps.FileUpload(
workersrc=self.artifact_file_name,
maxsize=self.artifact_max_size,
name='Upload artifacts',
masterdest=masterdest,
workdir=self.work_root_dir,
doStepIf=on_success,
)
steps_to_run += [clean_stage_dir_step, collect_artifact_step, archive_artifact_step, upload_artifact_step]
clean_stage_dir_again_step = steps.ShellCommand(
name='Clean stage, result and artifact',
command=[
'rm',
'-rf',
self.artifact_stage_relative,
self.result_relative,
self.artifact_file_name,
],
workdir=self.work_root_dir,
alwaysRun=True,
)
steps_to_run.append(clean_stage_dir_again_step)
return steps_to_run
class TriggerMultipleJobsStep(steps.Trigger):
properties_to_keep = [
'branch',
'revision',
'repository',
'harbormaster_build_target_phid',
'harbormaster_variable_buildable.diff',
'harbormaster_variable_repository.staging.ref',
'harbormaster_variable_repository.staging.uri',
'harbormaster_variable_repository.uri',
'lilybuild_repo',
'lilybuild_repo_id',
'lilybuild_pipeline_vars',
]
def __init__(self, lbc, jobs_with_data, **kwargs):
self.lbc = lbc
self.jobs = jobs_with_data
super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)
def getSchedulersAndProperties(self):
ret = []
common_properties = {
'lilybuild_root_build_id': self.build.buildid,
}
for prop in self.properties_to_keep:
if self.hasProperty(prop):
common_properties[prop] = self.getProperty(prop)
for (job, i, dep_job_indices) in self.jobs:
properties = common_properties.copy()
properties['lilybuild_job_prop'] = job.to_prop()
properties['lilybuild_job_index'] = i
properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
properties['lilybuild_dependency_job_indices'] = dep_job_indices
ret.append({
'sched_name': self.lbc.triggerable_scheduler_name,
'props_to_set': properties,
'unimportant': False,
})
return ret
class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
ci_def_file = '.gitlab-ci.yml'
build_target_prop_name = 'harbormaster_build_target_phid'
def __init__(
self,
lbc,
src_relative=None,
src_dir=None,
storage_dir=None,
repo_id=None,
result_relative=None,
result_dir=None,
artifact_stage_relative=None,
artifact_stage_dir=None,
**kwargs):
kwargs['name'] = 'Analyze CI file'
kwargs['command'] = ['cat', self.ci_def_file]
self.lbc = lbc
self.src_relative = src_relative
self.src_dir = src_dir
self.work_root_dir = kwargs['workdir']
self.script_dir = 'script'
self.storage_dir = storage_dir
self.repo_id = repo_id
self.artifact_stage_relative = artifact_stage_relative
self.artifact_stage_dir = artifact_stage_dir
self.result_relative = result_relative
self.result_dir = result_dir
kwargs['workdir'] = self.src_dir
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver('stdio', self.observer)
def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
jobs_with_data = []
for job in stage_jobs:
dep_job_names = [
jn
for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
if ci_file.jobs[jn].has_artifacts_archive()
]
dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))
trigger = TriggerMultipleJobsStep(
name=stage_name,
lbc=self.lbc,
jobs_with_data=jobs_with_data,
waitForFinish=True,
doStepIf=on_success,
)
return trigger
def get_steps(self, stdout):
f = ci_file.CIFile(stdout)
stages = f.get_grouped_jobs()
jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
job_names = [job.name for job in jobs]
job_name_to_index_map = {}
for (i, j) in enumerate(jobs):
job_name_to_index_map[j.name] = i
steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
print('steps:', steps)
return steps
def get_is_phorge(self):
return not not self.getProperty(self.build_target_prop_name)
def get_ref_and_type(self):
ref_type = 'branch'
ref = self.getProperty('branch')
if self.getProperty('category') == 'tag':
ref_type = 'tag'
if ref is not None:
m = re.match(r'^refs/(heads|tags)/(.+)$', ref)
if m:
ref = m.group(2)
return (ref, ref_type)
@defer.inlineCallbacks
def get_pipeline_ci_vars(self):
url = yield self.build.getUrl()
res = {
'CI_PIPELINE_ID': self.build.buildid,
'CI_PIPELINE_IID': self.build.buildid,
'CI_PIPELINE_URL': url,
'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
'CI_CONFIG_PATH': self.ci_def_file,
}
if not self.get_is_phorge():
res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
(ref, ref_type) = self.get_ref_and_type()
res['CI_COMMIT_REF_NAME'] = ref
res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
res['CI_COMMIT_REF_PROTECTED'] = 'false'
if ref_type == 'tag':
res['CI_COMMIT_TAG'] = ref
elif ref_type == 'branch':
res['CI_COMMIT_BRANCH'] = ref
return res
@defer.inlineCallbacks
def run(self):
# run './build.sh --list-stages' to generate the list of stages
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
pipeline_vars = yield self.get_pipeline_ci_vars()
self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))
return result

File Metadata

Mime Type
text/x-diff
Expires
Sat, Aug 29, 1:49 PM (1 d, 21 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1737108
Default Alt Text
(14 KB)

Event Timeline