No OneTemporary
Actions

Size

27 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
	index 9e980f7..aa0340b 100644
	--- a/lilybuild/lilybuild/ci_steps.py
	+++ b/lilybuild/lilybuild/ci_steps.py
	@@ -1,396 +1,396 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from twisted.internet import defer
	from .ci_syntax import ci_file
	-from .helpers import rsync_rules_from_artifacts, get_job_script
	+from .helpers import rsync_rules_from_artifacts, get_job_script, normalize_image
	import re
	import sys
	import json

	SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'

	def on_success(step):
	return step.build.results == util.SUCCESS

	def fill_list(*args):
	return list(args)

	class RunCIJobStep(steps.BuildStep):
	# 200 MiB
	artifact_max_size = 200 * 1024 * 1024
	default_image = 'alpine'
	master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
	artifact_file_name = 'artifacts.tar'
	master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
	master_pages_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/pages'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	job_prop=None,
	**kwargs):
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	super().__init__(name='Run step', **kwargs)

	@defer.inlineCallbacks
	def run(self):
	job_prop = self.getProperty('lilybuild_job_prop')
	job = ci_file.CIJob.from_prop(job_prop)
	job_index = self.getProperty('lilybuild_job_index')

	next_steps = yield self.job_to_steps(job, job_index)

	self.build.addStepsAfterCurrentStep(next_steps)
	return util.SUCCESS

	@defer.inlineCallbacks
	def get_ci_variables(self, job):
	res = {}
	res.update(job.get_predefined_ci_variables())
	res.update(self.getProperty('lilybuild_pipeline_vars'))
	res['CI_JOB_IMAGE'] = job.image or self.default_image
	res['CI_JOB_URL'] = yield self.build.getUrl()
	res['CI_JOB_ID'] = self.build.buildid
	res['CI_PROJECT_DIR'] = '/build'
	try:
	repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
	variables = yield repo['variables_getter'](self.build)
	res_vars = {}
	for var in variables:
	value = variables[var]
	if IRenderable.providedBy(value):
	value = yield self.build.render(value)
	res_vars[var] = value
	res.update(res_vars)
	except Exception as e:
	self.addCompleteLog('exception', f'{e}')
	return res

	@defer.inlineCallbacks
	def job_to_steps(self, job, job_index):
	script_name = self.script_dir + '/run.sh'
	variables = yield self.get_ci_variables(job)
	script_step = steps.StringDownload(
	get_job_script(variables, job),
	name='Set up script',
	workerdest=script_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	chmod_step = steps.ShellCommand(
	name='Make script executable',
	command=['chmod', '+x', script_name],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps = []
	dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
	if dep_job_indices:
	for i in dep_job_indices:
	download_job = steps.FileDownload(
	mastersrc=util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=i,
	),
	maxsize=self.artifact_max_size,
	name=f'Download artifacts from job #{i}',
	workerdest=self.artifact_file_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	unarchive_job = steps.ShellCommand(
	name=f'Unarchive artifacts from job #{i}',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'extract',
	'archive_file': self.artifact_file_name,
	'target_dir': self.src_relative,
	}),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps += [download_job, unarchive_job]

	run_step = steps.ShellCommand(
	name='Run script in container',
	command=[
	'/lilybuild/podman-helper',
	- job.image or self.default_image,
	+ normalize_image(job.image or self.default_image),
	self.src_relative,
	self.script_dir,
	self.result_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	clean_script_step = steps.ShellCommand(
	name='Clean script dir',
	command=[
	'rm',
	'-rf',
	self.script_dir,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)

	steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
	if 'paths' in job.artifacts:
	archive_artifact_step = steps.ShellCommand(
	name='Archive artifacts',
	command=[
	SAFETAR_EXEC,
	],
	initialStdin=json.dumps({
	'op': 'create',
	'archive_file': self.artifact_file_name,
	'base_dir': self.result_relative,
	'content': job.artifacts.get('paths', []),
	'items_to_exclude': job.artifacts.get('exclude', []),
	}),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	masterdest = util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=job_index,
	doStepIf=on_success,
	)
	upload_artifact_step = steps.FileUpload(
	workersrc=self.artifact_file_name,
	maxsize=self.artifact_max_size,
	name='Upload artifacts',
	masterdest=masterdest,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	steps_to_run += [archive_artifact_step, upload_artifact_step]
	clean_stage_dir_again_step = steps.ShellCommand(
	name='Clean stage, result and artifact',
	command=[
	'rm',
	'-rf',
	self.result_relative,
	self.artifact_file_name,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)
	steps_to_run.append(clean_stage_dir_again_step)

	if job.is_pages() and 'paths' in job.artifacts:
	deploy_pages_step = steps.MasterShellCommand(
	command=util.Transform(fill_list,
	sys.executable,
	'-m', 'lilybuild.pages',
	util.Interpolate(
	self.master_pages_dir_pattern,
	st=self.storage_dir,
	),
	masterdest,
	),
	name='Deploy pages',
	logEnviron=False,
	doStepIf=on_success,
	)
	steps_to_run.append(deploy_pages_step)

	return steps_to_run

	class TriggerMultipleJobsStep(steps.Trigger):
	properties_to_keep = [
	'branch',
	'revision',
	'repository',
	'harbormaster_build_target_phid',
	'harbormaster_variable_buildable.diff',
	'harbormaster_variable_repository.staging.ref',
	'harbormaster_variable_repository.staging.uri',
	'harbormaster_variable_repository.uri',
	'lilybuild_repo',
	'lilybuild_repo_id',
	'lilybuild_pipeline_vars',
	]

	def __init__(self, lbc, jobs_with_data, **kwargs):
	self.lbc = lbc
	self.jobs = jobs_with_data
	super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)

	def getSchedulersAndProperties(self):
	ret = []
	common_properties = {
	'lilybuild_root_build_id': self.build.buildid,
	}
	for prop in self.properties_to_keep:
	if self.hasProperty(prop):
	common_properties[prop] = self.getProperty(prop)

	for (job, i, dep_job_indices) in self.jobs:
	properties = common_properties.copy()
	properties['lilybuild_job_prop'] = job.to_prop()
	properties['lilybuild_job_index'] = i
	properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
	properties['lilybuild_dependency_job_indices'] = dep_job_indices
	ret.append({
	'sched_name': self.lbc.triggerable_scheduler_name,
	'props_to_set': properties,
	'unimportant': False,
	})

	return ret

	class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
	ci_def_file = '.gitlab-ci.yml'
	build_target_prop_name = 'harbormaster_build_target_phid'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	**kwargs):
	kwargs['name'] = 'Analyze CI file'
	kwargs['command'] = ['cat', self.ci_def_file]
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	kwargs['workdir'] = self.src_dir
	kwargs = self.setupShellMixin(kwargs)
	super().__init__(**kwargs)
	self.observer = logobserver.BufferLogObserver()
	self.addLogObserver('stdio', self.observer)

	def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
	jobs_with_data = []
	for job in stage_jobs:
	dep_job_names = [
	jn
	for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
	if ci_file.jobs[jn].has_artifacts_archive()
	]
	dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
	jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))

	trigger = TriggerMultipleJobsStep(
	name=stage_name,
	lbc=self.lbc,
	jobs_with_data=jobs_with_data,
	waitForFinish=True,
	doStepIf=on_success,
	)

	return trigger

	def get_steps(self, stdout):
	f = ci_file.CIFile(stdout)

	stages = f.get_grouped_jobs()
	jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
	job_names = [job.name for job in jobs]
	job_name_to_index_map = {}
	for (i, j) in enumerate(jobs):
	job_name_to_index_map[j.name] = i

	steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
	print('steps:', steps)
	return steps

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_ref_and_type(self):
	ref_type = 'branch'
	ref = self.getProperty('branch')

	if self.getProperty('category') == 'tag':
	ref_type = 'tag'

	if ref is not None:
	m = re.match(r'^refs/(heads\|tags)/(.+)$', ref)
	if m:
	ref = m.group(2)

	return (ref, ref_type)

	@defer.inlineCallbacks
	def get_pipeline_ci_vars(self):
	url = yield self.build.getUrl()
	res = {
	'CI_PIPELINE_ID': self.build.buildid,
	'CI_PIPELINE_IID': self.build.buildid,
	'CI_PIPELINE_URL': url,
	'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
	'CI_CONFIG_PATH': self.ci_def_file,
	}
	if not self.get_is_phorge():
	res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
	res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
	(ref, ref_type) = self.get_ref_and_type()
	res['CI_COMMIT_REF_NAME'] = ref
	res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
	res['CI_COMMIT_REF_PROTECTED'] = 'false'
	if ref_type == 'tag':
	res['CI_COMMIT_TAG'] = ref
	elif ref_type == 'branch':
	res['CI_COMMIT_BRANCH'] = ref
	return res

	@defer.inlineCallbacks
	def run(self):
	# run './build.sh --list-stages' to generate the list of stages
	cmd = yield self.makeRemoteShellCommand()
	yield self.runCommand(cmd)

	# if the command passes extract the list of stages
	result = cmd.results()
	if result == util.SUCCESS:
	pipeline_vars = yield self.get_pipeline_ci_vars()
	self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
	# create a ShellCommand for each stage and add them to the build
	self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))

	return result
	diff --git a/lilybuild/lilybuild/helpers.py b/lilybuild/lilybuild/helpers.py
	index 52830da..675781f 100644
	--- a/lilybuild/lilybuild/helpers.py
	+++ b/lilybuild/lilybuild/helpers.py
	@@ -1,70 +1,76 @@

	import json
	import shlex

	def normalize_path_for_rsync(path):
	n = path
	if n.startswith('./'):
	n = n[2:]
	if n.endswith('/'):
	n = n[:-1]

	return '/' + n

	def rsync_rules_from_artifacts(artifacts):
	paths = artifacts.get('paths', [])
	# Include all dirs
	rules = ['--include', '*/']
	for p in paths:
	normalized_path = normalize_path_for_rsync(p)
	rules += [
	# If path already has /** at the end, the second will actually do nothing,
	# but it's fine to add it anyway. The directory itself will still
	# be visited because of the --include */ option we add at the beginning.
	'--include', normalized_path,
	'--include', normalized_path + '/**',
	]

	# Exclude everything else
	rules += ['--exclude', '*']
	return rules

	def normalize_base_url(base_url):
	return base_url.rstrip('/') if base_url else None

	def phorge_token_to_arcrc(normalized_base_url, token):
	return json.dumps({
	'hosts': {
	normalized_base_url + '/api/': {
	'token': token,
	},
	},
	})

	def ci_vars_to_cmds(v):
	res = []
	for name in v:
	value = shlex.quote('{}'.format(v[name]))
	res.append(f'export {name}={value}')
	return '\n'.join(res)

	DEFAULT_SCRIPT_HEADER = '''\
	#!/bin/sh

	set -e -x
	cd /build

	'''

	def get_job_script(variables, job):
	var_cmds = ci_vars_to_cmds(variables)

	return (
	DEFAULT_SCRIPT_HEADER +
	'\n' + var_cmds + '\n\n' +
	'\n\n'.join(job.before_script) + '\n\n' +
	'\n\n'.join(job.script) +
	'\n\nset +e\n\n' +
	'\n\n'.join(job.after_script) +
	'\n\nexit 0'
	)
	+
	+def normalize_image(image):
	+ if isinstance(image, str):
	+ return json.dumps({'name': image})
	+ else:
	+ return json.dumps(image)
	diff --git a/lilybuild/lilybuild/tests/helpers_test.py b/lilybuild/lilybuild/tests/helpers_test.py
	index d688019..6c5b510 100644
	--- a/lilybuild/lilybuild/tests/helpers_test.py
	+++ b/lilybuild/lilybuild/tests/helpers_test.py
	@@ -1,132 +1,143 @@

	import unittest
	import json
	from lilybuild.ci_syntax.ci_file import CIFile
	from lilybuild.helpers import (
	rsync_rules_from_artifacts,
	normalize_base_url,
	phorge_token_to_arcrc,
	ci_vars_to_cmds,
	get_job_script,
	DEFAULT_SCRIPT_HEADER,
	+ normalize_image,
	)
	from lilybuild.tests.resources import get_res

	class RsyncRulesTest(unittest.TestCase):
	def test_empty(self):
	self.assertEqual(
	rsync_rules_from_artifacts({}),
	['--include', '/', '--exclude', '']
	)

	def test_simple(self):
	self.assertEqual(
	rsync_rules_from_artifacts({'paths': ['public']}),
	['--include', '*/',
	'--include', '/public',
	'--include', '/public/**',
	'--exclude', '*']
	)

	def test_dotslash(self):
	self.assertEqual(
	rsync_rules_from_artifacts({'paths': ['./public/']}),
	['--include', '*/',
	'--include', '/public',
	'--include', '/public/**',
	'--exclude', '*']
	)

	def test_doublestar(self):
	self.assertEqual(
	rsync_rules_from_artifacts({'paths': ['./public/**']}),
	['--include', '*/',
	'--include', '/public/**',
	'--include', '/public//',
	'--exclude', '*']
	)

	def test_doublestar_middle(self):
	self.assertEqual(
	rsync_rules_from_artifacts({'paths': ['./public/*/.html']}),
	['--include', '*/',
	'--include', '/public/*/.html',
	'--include', '/public/*/.html/**',
	'--exclude', '*']
	)

	def test_dotdotslash(self):
	# No exploit possible because rsync will not visit beyond the source root
	self.assertEqual(
	rsync_rules_from_artifacts({'paths': ['../etc/passwd']}),
	['--include', '*/',
	'--include', '/../etc/passwd',
	'--include', '/../etc/passwd/**',
	'--exclude', '*']
	)

	class PhorgeUtilsTest(unittest.TestCase):
	def test_normalize_base_url(self):
	self.assertEqual(normalize_base_url('https://iron.lily-is.land/'), 'https://iron.lily-is.land')
	self.assertEqual(normalize_base_url('https://iron.lily-is.land'), 'https://iron.lily-is.land')
	self.assertEqual(normalize_base_url(''), None)

	def test_phorge_token_to_arcrc(self):
	self.assertEqual(
	json.loads(phorge_token_to_arcrc('https://iron.lily-is.land', 'some-token')),
	{
	'hosts': {
	'https://iron.lily-is.land/api/': {
	'token': 'some-token',
	},
	},
	},
	)

	class CiVarsTest(unittest.TestCase):
	def test_simple(self):
	self.assertEqual(ci_vars_to_cmds({}), '')
	self.assertEqual(ci_vars_to_cmds({'VAR': 'val'}), 'export VAR=val')
	self.assertEqual(ci_vars_to_cmds({'VAR': 'foo bar'}), "export VAR='foo bar'")

	class GetJobScriptTest(unittest.TestCase):
	def test_only_script(self):
	r = CIFile(get_res('pages_attr'))
	job_script = get_job_script({}, r.jobs['is-pages'])
	self.assertEqual(job_script, f'''\
	{DEFAULT_SCRIPT_HEADER}




	make docs

	set +e



	exit 0''')

	def test_before_and_after(self):
	r = CIFile(get_res('defaults'))
	job_script = get_job_script({}, r.jobs['build-a'])
	self.assertEqual(job_script, f'''\
	{DEFAULT_SCRIPT_HEADER}


	ls

	make

	make install

	set +e

	find

	echo ok

	exit 0''')

	+class NormalizeImageTest(unittest.TestCase):
	+ def test_str(self):
	+ res = normalize_image('alpine')
	+ self.assertEqual(json.loads(res), {'name': 'alpine'})
	+
	+ def test_object(self):
	+ orig = {'name': 'alpine', 'entrypoint': ['/docker-run', '/bin/bb']}
	+ res = normalize_image(orig)
	+ self.assertEqual(json.loads(res), orig)
	+
	if __name__ == '__main__':
	unittest.main()
	diff --git a/lilybuild/podman-helper b/lilybuild/podman-helper
	index 952f75f..a09ea9f 100755
	--- a/lilybuild/podman-helper
	+++ b/lilybuild/podman-helper
	@@ -1,201 +1,211 @@
	#!/usr/bin/env python3

	import subprocess
	import sys
	import os
	import json
	import random
	import traceback
	import string
	import time

	work_vol_mount_dir = '/build'
	script_vol_mount_dir = '/script'
	script_name = script_vol_mount_dir + '/run.sh'
	volume_helper_image = os.environ.get('LILYBUILD_VOLUME_HELPER_IMAGE', 'r.lily-is.land/infra/lilybuild/volume-helper:servant')
	key_file_pub = '/secrets/lilybuild-volume-helper-key.pub'
	key_file_sub = '/secrets/lilybuild-volume-helper-key'
	ssh_port = '2222'
	ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null'
	worker_container_name = os.environ['HOSTNAME']
	volumes_to_remove = []
	helper_container_id = None
	ssh_max_wait = 10
	ssh_wait_interval_sec = 1

	col_info = '\x1b[1;34m'
	col_success = '\x1b[1;32m'
	col_error = '\x1b[1;31m'
	col_reset = '\x1b[0m'

	def pinfo(args, *kwargs):
	print(col_info, args, col_reset, *kwargs)
	sys.stdout.flush()

	def perror(args, *kwargs):
	print(col_error, args, col_reset, *kwargs)
	sys.stdout.flush()

	def psuccess(args, *kwargs):
	print(col_success, args, col_reset, *kwargs)
	sys.stdout.flush()

	def gen_random_id():
	# https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits
	return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(10))

	def verbose_run(args, *kwargs):
	print('run:', args, kwargs)
	sys.stdout.flush()
	return subprocess.run(args, *kwargs)

	def create_volume(t):
	res = verbose_run([
	'podman', 'volume', 'create',
	'--label', 'lilybuild=' + t,
	], check=True, capture_output=True, encoding='utf-8')
	volname = res.stdout.strip()
	volumes_to_remove.append(volname)
	return volname

	def clean_volumes():
	verbose_run([
	'podman', 'volume', 'rm', '--',
	] + volumes_to_remove, capture_output=True)

	def clean_helper_container():
	verbose_run([
	'podman', 'container', 'rm', '-f', helper_container_id,
	], capture_output=True)

	def start_helper_service(work_volname, script_volname):
	res = verbose_run([
	'podman', 'container', 'inspect', worker_container_name,
	], check=True, capture_output=True, encoding='utf-8')
	container_stat = json.loads(res.stdout)[0]
	pod = container_stat.get('Pod')
	networks = list(container_stat.get('NetworkSettings').get('Networks').keys())
	alias = gen_random_id()
	container_name = 'lilybuild-helper-' + alias

	with open(key_file_pub) as f:
	pub_key = f.readline().strip()

	res = verbose_run([
	'podman', 'run', '--rm', '-d', '--name', container_name,
	f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
	f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
	f'--pod={pod}',
	f'--net={networks[0]}',
	f'--network-alias={alias}',
	'--label', 'lilybuild=helper',
	'-e', 'PUID=0',
	'-e', 'PGID=0',
	'-e', f'PUBLIC_KEY={pub_key}',
	'-e', 'USER_NAME=helper',
	'-e', 'SUDO_ACCESS=true',
	volume_helper_image,
	], check=True, capture_output=True, encoding='utf-8')
	pinfo('Waiting for ssh service to be up...')
	service_up = False
	for i in range(ssh_max_wait):
	chk = verbose_run(['nc', alias, ssh_port], input=b'', capture_output=True)
	if chk.returncode == 0 and chk.stdout is not None and chk.stdout.startswith(b'SSH'):
	service_up = True
	break
	else:
	time.sleep(ssh_wait_interval_sec)
	if not service_up:
	raise RuntimeError('Service is still not up!')
	psuccess('Service is up.')
	return (container_name, alias)

	def import_volume(alias, local_dir, vol_mount_dir):
	# I'll just use the shell instead of pipe2+fork+exec+wait, much easier
	verbose_run([
	'rsync', '-a', '--delete',
	'--rsh', ssh_command,
	f'{local_dir}/',
	f'helper@{alias}:{vol_mount_dir}',
	], check=True)

	def export_volume(alias, local_dir, vol_mount_dir):
	verbose_run([
	'rsync', '-a', '--delete',
	'--rsh', ssh_command,
	f'helper@{alias}:{vol_mount_dir}/',
	local_dir,
	], check=True)

	+def image_to_podman_args(image):
	+ name = image['name']
	+ args = []
	+ if 'entrypoint' in image:
	+ # ci.json requires that the entrypoint is an array of strings
	+ ep = json.dumps(image['entrypoint'])
	+ args += ['--entrypoint', ep]
	+ args += [name]
	+ return args
	+
	def run_in_container(image, work_volname, script_volname):
	timeout = 60 * 60 * 2 # 2 hours by default
	p = verbose_run([
	'podman', 'run', '--rm',
	f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
	f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
	- image,
	+ ] + image_to_podman_args(image) + [
	script_name,
	], timeout=timeout)
	return p

	def main():
	- image = sys.argv[1]
	+ image = json.loads(sys.argv[1])
	work_dir = sys.argv[2]
	script_dir = sys.argv[3]
	result_dir = sys.argv[4]

	pinfo('Creating volumes...')
	work_vol = create_volume('work')
	script_vol = create_volume('script')
	psuccess('Created.')

	pinfo('Starting helper service...')
	global helper_container_id
	(helper_container_id, alias) = start_helper_service(work_vol, script_vol)
	psuccess('Started...')

	pinfo('Importing volumes...')
	import_volume(alias, work_dir, work_vol_mount_dir)
	import_volume(alias, script_dir, script_vol_mount_dir)
	psuccess('Imported.')

	pinfo('Running container...')
	try:
	res = run_in_container(image, work_vol, script_vol)
	retcode = res.returncode
	except subprocess.TimeoutExpired:
	perror('Command timed out.')
	retcode = 1

	pinfo(f'Returned {retcode}.')

	if retcode != 0:
	perror('Job failed.')
	else:
	psuccess('Job succeeded.')

	pinfo('Collecting build changes...')
	export_volume(alias, result_dir, work_vol_mount_dir)
	psuccess('Collected.')

	return retcode

	retcode = 1

	try:
	retcode = main()
	except Exception as e:
	perror('Error!', e)
	print(traceback.format_exc())
	raise
	finally:
	if helper_container_id:
	pinfo('Cleaning helper container')
	clean_helper_container()
	psuccess('Cleaned.')

	pinfo('Cleaning volumes...')
	clean_volumes()
	psuccess('Cleaned.')

	sys.exit(retcode)

File Metadata

Mime Type: text/x-diff
Expires: Fri, Oct 9, 3:43 AM (1 d, 8 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1784425
Default Alt Text: (27 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions