No OneTemporary
Actions

Size

38 KB

Referenced Files

None

Subscribers

None

View Options

	diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
	index 64f7a13..f5c2453 100644
	--- a/.gitlab-ci.yml
	+++ b/.gitlab-ci.yml
	@@ -1,45 +1,56 @@

	stages:
	- unit-test
	- build

	default:
	image: quay.io/podman/stable
	before_script:
	- 'if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman login -u "$REGISTRY_USER" --password-stdin "$REGISTRY" <<< "$REGISTRY_PASSWORD"; fi'
	- IMAGE_PREFIX="$REGISTRY/infra/lilybuild"
	- IMAGE_VER="${CI_COMMIT_REF_SLUG-none}"

	.push:
	script: &push
	- if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman push "$IMAGE"; fi

	-unit-test:
	+unit-test-master:
	stage: unit-test
	image: docker.io/buildbot/buildbot-master:v4.2.1
	before_script: []
	script:
	- /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
	- . /buildbot_venv/bin/activate
	- - ./lilybuild/run-tests.sh
	+ - ./lilybuild/run-tests.sh master
	+
	+unit-test-worker:
	+ stage: unit-test
	+ image: alpine
	+ before_script: []
	+ script:
	+ - apk add --no-cache python3 py3-virtualenv
	+ - virtualenv --python=python3 /buildbot_venv
	+ - /buildbot_venv/bin/pip3 install 'twisted[tls]' jsonschema backports.tarfile
	+ - . /buildbot_venv/bin/activate
	+ - ./lilybuild/run-tests.sh worker

	build:master:
	stage: build
	script:
	- IMAGE="$IMAGE_PREFIX/buildbot-master:$IMAGE_VER"
	- sudo -u podman ./build-master.sh -t "$IMAGE"
	- *push

	build:worker:
	stage: build
	script:
	- IMAGE="$IMAGE_PREFIX/buildbot-worker:$IMAGE_VER"
	- sudo -u podman ./build-worker.sh -t "$IMAGE"
	- *push

	build:volume-helper:
	stage: build
	script:
	- IMAGE="$IMAGE_PREFIX/volume-helper:$IMAGE_VER"
	- sudo -u podman ./build-volume-helper.sh -t "$IMAGE"
	- *push
	diff --git a/Containerfile.worker b/Containerfile.worker
	index 161bac4..d277c5c 100644
	--- a/Containerfile.worker
	+++ b/Containerfile.worker
	@@ -1,25 +1,25 @@

	FROM alpine

	RUN apk add --no-cache podman podman-compose python3 \
	py3-virtualenv dumb-init bash shadow git openssh rsync php php-curl \
	&& virtualenv --python=python3 /buildbot_venv \
	&& /buildbot_venv/bin/pip3 install 'twisted[tls]' \
	&& mkdir /buildbot \
	&& useradd -ms /bin/bash buildbot \
	&& mkdir -pv /tools \
	&& cd /tools \
	- && git clone https://we.phorge.it/source/arcanist.git \
	+ && git clone https://github.com/phorgeit/arcanist.git \
	&& cd arcanist \
	&& git checkout stable

	COPY . /usr/src/buildbot-worker
	COPY docker/buildbot.tac /buildbot/buildbot.tac

	RUN /buildbot_venv/bin/pip3 install /usr/src/buildbot-worker && \
	chown -R buildbot /buildbot

	USER buildbot
	WORKDIR /buildbot

	CMD ["/usr/bin/dumb-init", "/buildbot_venv/bin/twistd", "--pidfile=", "-ny", "buildbot.tac"]
	diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
	index 3d839db..9e980f7 100644
	--- a/lilybuild/lilybuild/ci_steps.py
	+++ b/lilybuild/lilybuild/ci_steps.py
	@@ -1,415 +1,396 @@

	from buildbot.plugins import *
	from buildbot.process import buildstep, logobserver
	from buildbot.interfaces import IRenderable
	from twisted.internet import defer
	from .ci_syntax import ci_file
	from .helpers import rsync_rules_from_artifacts, get_job_script
	import re
	import sys
	+import json
	+
	+SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'

	def on_success(step):
	return step.build.results == util.SUCCESS

	def fill_list(*args):
	return list(args)

	class RunCIJobStep(steps.BuildStep):
	# 200 MiB
	artifact_max_size = 200 * 1024 * 1024
	default_image = 'alpine'
	master_job_artifact_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/builds/%(prop:lilybuild_root_build_id)s/jobs/%(kw:job)s/artifacts'
	artifact_file_name = 'artifacts.tar'
	master_job_artifact_file_name_pattern = master_job_artifact_dir_pattern + '/' + artifact_file_name
	master_pages_dir_pattern = '%(kw:st)s/repos/%(prop:lilybuild_repo_id)s/pages'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	job_prop=None,
	**kwargs):
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	super().__init__(name='Run step', **kwargs)

	@defer.inlineCallbacks
	def run(self):
	job_prop = self.getProperty('lilybuild_job_prop')
	job = ci_file.CIJob.from_prop(job_prop)
	job_index = self.getProperty('lilybuild_job_index')

	next_steps = yield self.job_to_steps(job, job_index)

	self.build.addStepsAfterCurrentStep(next_steps)
	return util.SUCCESS

	@defer.inlineCallbacks
	def get_ci_variables(self, job):
	res = {}
	res.update(job.get_predefined_ci_variables())
	res.update(self.getProperty('lilybuild_pipeline_vars'))
	res['CI_JOB_IMAGE'] = job.image or self.default_image
	res['CI_JOB_URL'] = yield self.build.getUrl()
	res['CI_JOB_ID'] = self.build.buildid
	res['CI_PROJECT_DIR'] = '/build'
	try:
	repo = self.lbc.repos[self.getProperty('lilybuild_repo_id')]
	variables = yield repo['variables_getter'](self.build)
	res_vars = {}
	for var in variables:
	value = variables[var]
	if IRenderable.providedBy(value):
	value = yield self.build.render(value)
	res_vars[var] = value
	res.update(res_vars)
	except Exception as e:
	self.addCompleteLog('exception', f'{e}')
	return res

	@defer.inlineCallbacks
	def job_to_steps(self, job, job_index):
	script_name = self.script_dir + '/run.sh'
	variables = yield self.get_ci_variables(job)
	script_step = steps.StringDownload(
	get_job_script(variables, job),
	name='Set up script',
	workerdest=script_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	chmod_step = steps.ShellCommand(
	name='Make script executable',
	command=['chmod', '+x', script_name],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps = []
	dep_job_indices = self.getProperty('lilybuild_dependency_job_indices')
	if dep_job_indices:
	for i in dep_job_indices:
	download_job = steps.FileDownload(
	mastersrc=util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=i,
	),
	maxsize=self.artifact_max_size,
	name=f'Download artifacts from job #{i}',
	workerdest=self.artifact_file_name,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	unarchive_job = steps.ShellCommand(
	name=f'Unarchive artifacts from job #{i}',
	command=[
	- 'tar',
	- '-xf',
	- self.artifact_file_name,
	- '-C',
	- self.src_relative,
	+ SAFETAR_EXEC,
	],
	+ initialStdin=json.dumps({
	+ 'op': 'extract',
	+ 'archive_file': self.artifact_file_name,
	+ 'target_dir': self.src_relative,
	+ }),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	artifact_steps += [download_job, unarchive_job]

	run_step = steps.ShellCommand(
	name='Run script in container',
	command=[
	'/lilybuild/podman-helper',
	job.image or self.default_image,
	self.src_relative,
	self.script_dir,
	self.result_relative,
	],
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)

	clean_script_step = steps.ShellCommand(
	name='Clean script dir',
	command=[
	'rm',
	'-rf',
	self.script_dir,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)

	steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
	if 'paths' in job.artifacts:
	- clean_stage_dir_step = steps.ShellCommand(
	- name='Clean stage dir',
	- command=[
	- 'rm',
	- '-rf',
	- self.artifact_stage_relative,
	- ],
	- workdir=self.work_root_dir,
	- doStepIf=on_success,
	- )
	- collect_artifact_step = steps.ShellCommand(
	- name='Collect artifacts',
	- command=[
	- 'rsync',
	- '-av',
	- self.result_relative + '/',
	- '--delete',
	- '--prune-empty-dirs',
	- ] + rsync_rules_from_artifacts(job.artifacts) + [
	- self.artifact_stage_relative,
	- ],
	- workdir=self.work_root_dir,
	- doStepIf=on_success,
	- )
	archive_artifact_step = steps.ShellCommand(
	name='Archive artifacts',
	command=[
	- 'tar',
	- '-cf',
	- self.artifact_file_name,
	- '-C',
	- self.artifact_stage_relative,
	- '.',
	+ SAFETAR_EXEC,
	],
	+ initialStdin=json.dumps({
	+ 'op': 'create',
	+ 'archive_file': self.artifact_file_name,
	+ 'base_dir': self.result_relative,
	+ 'content': job.artifacts.get('paths', []),
	+ 'items_to_exclude': job.artifacts.get('exclude', []),
	+ }),
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	masterdest = util.Interpolate(
	self.master_job_artifact_file_name_pattern,
	st=self.storage_dir,
	job=job_index,
	doStepIf=on_success,
	)
	upload_artifact_step = steps.FileUpload(
	workersrc=self.artifact_file_name,
	maxsize=self.artifact_max_size,
	name='Upload artifacts',
	masterdest=masterdest,
	workdir=self.work_root_dir,
	doStepIf=on_success,
	)
	- steps_to_run += [clean_stage_dir_step, collect_artifact_step, archive_artifact_step, upload_artifact_step]
	+ steps_to_run += [archive_artifact_step, upload_artifact_step]
	clean_stage_dir_again_step = steps.ShellCommand(
	name='Clean stage, result and artifact',
	command=[
	'rm',
	'-rf',
	- self.artifact_stage_relative,
	self.result_relative,
	self.artifact_file_name,
	],
	workdir=self.work_root_dir,
	alwaysRun=True,
	)
	steps_to_run.append(clean_stage_dir_again_step)

	if job.is_pages() and 'paths' in job.artifacts:
	deploy_pages_step = steps.MasterShellCommand(
	command=util.Transform(fill_list,
	sys.executable,
	'-m', 'lilybuild.pages',
	util.Interpolate(
	self.master_pages_dir_pattern,
	st=self.storage_dir,
	),
	masterdest,
	),
	name='Deploy pages',
	logEnviron=False,
	doStepIf=on_success,
	)
	steps_to_run.append(deploy_pages_step)

	return steps_to_run

	class TriggerMultipleJobsStep(steps.Trigger):
	properties_to_keep = [
	'branch',
	'revision',
	'repository',
	'harbormaster_build_target_phid',
	'harbormaster_variable_buildable.diff',
	'harbormaster_variable_repository.staging.ref',
	'harbormaster_variable_repository.staging.uri',
	'harbormaster_variable_repository.uri',
	'lilybuild_repo',
	'lilybuild_repo_id',
	'lilybuild_pipeline_vars',
	]

	def __init__(self, lbc, jobs_with_data, **kwargs):
	self.lbc = lbc
	self.jobs = jobs_with_data
	super().__init__(schedulerNames=[self.lbc.triggerable_scheduler_name], **kwargs)

	def getSchedulersAndProperties(self):
	ret = []
	common_properties = {
	'lilybuild_root_build_id': self.build.buildid,
	}
	for prop in self.properties_to_keep:
	if self.hasProperty(prop):
	common_properties[prop] = self.getProperty(prop)

	for (job, i, dep_job_indices) in self.jobs:
	properties = common_properties.copy()
	properties['lilybuild_job_prop'] = job.to_prop()
	properties['lilybuild_job_index'] = i
	properties['virtual_builder_name'] = 'lilybuild-job - ' + common_properties['lilybuild_repo'] + ' - ' + job.name
	properties['lilybuild_dependency_job_indices'] = dep_job_indices
	ret.append({
	'sched_name': self.lbc.triggerable_scheduler_name,
	'props_to_set': properties,
	'unimportant': False,
	})

	return ret

	class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
	ci_def_file = '.gitlab-ci.yml'
	build_target_prop_name = 'harbormaster_build_target_phid'

	def __init__(
	self,
	lbc,
	src_relative=None,
	src_dir=None,
	storage_dir=None,
	repo_id=None,
	result_relative=None,
	result_dir=None,
	artifact_stage_relative=None,
	artifact_stage_dir=None,
	**kwargs):
	kwargs['name'] = 'Analyze CI file'
	kwargs['command'] = ['cat', self.ci_def_file]
	self.lbc = lbc
	self.src_relative = src_relative
	self.src_dir = src_dir
	self.work_root_dir = kwargs['workdir']
	self.script_dir = 'script'
	self.storage_dir = storage_dir
	self.repo_id = repo_id
	self.artifact_stage_relative = artifact_stage_relative
	self.artifact_stage_dir = artifact_stage_dir
	self.result_relative = result_relative
	self.result_dir = result_dir
	kwargs['workdir'] = self.src_dir
	kwargs = self.setupShellMixin(kwargs)
	super().__init__(**kwargs)
	self.observer = logobserver.BufferLogObserver()
	self.addLogObserver('stdio', self.observer)

	def stage_to_step(self, stage_name, stage_jobs, job_name_to_index_map, ci_file):
	jobs_with_data = []
	for job in stage_jobs:
	dep_job_names = [
	jn
	for jn in ci_file.get_jobs_to_pull_artifacts_from(job.name)
	if ci_file.jobs[jn].has_artifacts_archive()
	]
	dep_job_indices = [job_name_to_index_map[jn] for jn in dep_job_names]
	jobs_with_data.append((job, job_name_to_index_map[job.name], dep_job_indices))

	trigger = TriggerMultipleJobsStep(
	name=stage_name,
	lbc=self.lbc,
	jobs_with_data=jobs_with_data,
	waitForFinish=True,
	doStepIf=on_success,
	)

	return trigger

	def get_steps(self, stdout):
	f = ci_file.CIFile(stdout)

	stages = f.get_grouped_jobs()
	jobs = [job for (stage, js) in f.get_grouped_jobs() for job in js]
	job_names = [job.name for job in jobs]
	job_name_to_index_map = {}
	for (i, j) in enumerate(jobs):
	job_name_to_index_map[j.name] = i

	steps = [self.stage_to_step(stage_name, stage_jobs, job_name_to_index_map, f) for (stage_name, stage_jobs) in stages]
	print('steps:', steps)
	return steps

	def get_is_phorge(self):
	return not not self.getProperty(self.build_target_prop_name)

	def get_ref_and_type(self):
	ref_type = 'branch'
	ref = self.getProperty('branch')

	if self.getProperty('category') == 'tag':
	ref_type = 'tag'

	if ref is not None:
	m = re.match(r'^refs/(heads\|tags)/(.+)$', ref)
	if m:
	ref = m.group(2)

	return (ref, ref_type)

	@defer.inlineCallbacks
	def get_pipeline_ci_vars(self):
	url = yield self.build.getUrl()
	res = {
	'CI_PIPELINE_ID': self.build.buildid,
	'CI_PIPELINE_IID': self.build.buildid,
	'CI_PIPELINE_URL': url,
	'CI_PROJECT_ID': self.getProperty('lilybuild_repo_id'),
	'CI_CONFIG_PATH': self.ci_def_file,
	}
	if not self.get_is_phorge():
	res['CI_COMMIT_SHA'] = self.getProperty('got_revision')
	res['CI_COMMIT_SHORT_SHA'] = res['CI_COMMIT_SHA'][:8]
	(ref, ref_type) = self.get_ref_and_type()
	res['CI_COMMIT_REF_NAME'] = ref
	res['CI_COMMIT_REF_SLUG'] = ci_file.ci_slugify(ref)
	res['CI_COMMIT_REF_PROTECTED'] = 'false'
	if ref_type == 'tag':
	res['CI_COMMIT_TAG'] = ref
	elif ref_type == 'branch':
	res['CI_COMMIT_BRANCH'] = ref
	return res

	@defer.inlineCallbacks
	def run(self):
	# run './build.sh --list-stages' to generate the list of stages
	cmd = yield self.makeRemoteShellCommand()
	yield self.runCommand(cmd)

	# if the command passes extract the list of stages
	result = cmd.results()
	if result == util.SUCCESS:
	pipeline_vars = yield self.get_pipeline_ci_vars()
	self.setProperty('lilybuild_pipeline_vars', pipeline_vars, self.__class__.__name__)
	# create a ShellCommand for each stage and add them to the build
	self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))

	return result
	diff --git a/lilybuild/lilybuild/safetar.py b/lilybuild/lilybuild/safetar.py
	new file mode 100755
	index 0000000..a7f4a1c
	--- /dev/null
	+++ b/lilybuild/lilybuild/safetar.py
	@@ -0,0 +1,239 @@
	+#!/usr/bin/env python3
	+
	+import os
	+import glob
	+import re
	+try:
	+ import tarfile
	+ tarfile.FilterError
	+except AttributeError:
	+ import backports.tarfile as tarfile
	+
	+try:
	+ glob_translate = glob.translate
	+except AttributeError:
	+ # Taken from python 3.13 library
	+ def py313_fnmatch_translate(pat, STAR, QUESTION_MARK):
	+ res = []
	+ add = res.append
	+ i, n = 0, len(pat)
	+ while i < n:
	+ c = pat[i]
	+ i = i+1
	+ if c == '*':
	+ # compress consecutive `*` into one
	+ if (not res) or res[-1] is not STAR:
	+ add(STAR)
	+ elif c == '?':
	+ add(QUESTION_MARK)
	+ elif c == '[':
	+ j = i
	+ if j < n and pat[j] == '!':
	+ j = j+1
	+ if j < n and pat[j] == ']':
	+ j = j+1
	+ while j < n and pat[j] != ']':
	+ j = j+1
	+ if j >= n:
	+ add('\\[')
	+ else:
	+ stuff = pat[i:j]
	+ if '-' not in stuff:
	+ stuff = stuff.replace('\\', r'\\')
	+ else:
	+ chunks = []
	+ k = i+2 if pat[i] == '!' else i+1
	+ while True:
	+ k = pat.find('-', k, j)
	+ if k < 0:
	+ break
	+ chunks.append(pat[i:k])
	+ i = k+1
	+ k = k+3
	+ chunk = pat[i:j]
	+ if chunk:
	+ chunks.append(chunk)
	+ else:
	+ chunks[-1] += '-'
	+ # Remove empty ranges -- invalid in RE.
	+ for k in range(len(chunks)-1, 0, -1):
	+ if chunks[k-1][-1] > chunks[k][0]:
	+ chunks[k-1] = chunks[k-1][:-1] + chunks[k][1:]
	+ del chunks[k]
	+ # Escape backslashes and hyphens for set difference (--).
	+ # Hyphens that create ranges shouldn't be escaped.
	+ stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
	+ for s in chunks)
	+ # Escape set operations (&&, ~~ and \|\|).
	+ stuff = re.sub(r'([&~\|])', r'\\\1', stuff)
	+ i = j+1
	+ if not stuff:
	+ # Empty range: never match.
	+ add('(?!)')
	+ elif stuff == '!':
	+ # Negated empty range: match any character.
	+ add('.')
	+ else:
	+ if stuff[0] == '!':
	+ stuff = '^' + stuff[1:]
	+ elif stuff[0] in ('^', '['):
	+ stuff = '\\' + stuff
	+ add(f'[{stuff}]')
	+ else:
	+ add(re.escape(c))
	+ assert i == n
	+ return res
	+
	+ def py313_translate(pat, *, recursive=False, include_hidden=False, seps=None):
	+ """Translate a pathname with shell wildcards to a regular expression.
	+
	+ If `recursive` is true, the pattern segment '**' will match any number of
	+ path segments.
	+
	+ If `include_hidden` is true, wildcards can match path segments beginning
	+ with a dot ('.').
	+
	+ If a sequence of separator characters is given to `seps`, they will be
	+ used to split the pattern into segments and match path separators. If not
	+ given, os.path.sep and os.path.altsep (where available) are used.
	+ """
	+ if not seps:
	+ if os.path.altsep:
	+ seps = (os.path.sep, os.path.altsep)
	+ else:
	+ seps = os.path.sep
	+ escaped_seps = ''.join(map(re.escape, seps))
	+ any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
	+ not_sep = f'[^{escaped_seps}]'
	+ if include_hidden:
	+ one_last_segment = f'{not_sep}+'
	+ one_segment = f'{one_last_segment}{any_sep}'
	+ any_segments = f'(?:.+{any_sep})?'
	+ any_last_segments = '.*'
	+ else:
	+ one_last_segment = f'[^{escaped_seps}.]{not_sep}*'
	+ one_segment = f'{one_last_segment}{any_sep}'
	+ any_segments = f'(?:{one_segment})*'
	+ any_last_segments = f'{any_segments}(?:{one_last_segment})?'
	+
	+ results = []
	+ parts = re.split(any_sep, pat)
	+ last_part_idx = len(parts) - 1
	+ for idx, part in enumerate(parts):
	+ if part == '*':
	+ results.append(one_segment if idx < last_part_idx else one_last_segment)
	+ elif recursive and part == '**':
	+ if idx < last_part_idx:
	+ if parts[idx + 1] != '**':
	+ results.append(any_segments)
	+ else:
	+ results.append(any_last_segments)
	+ else:
	+ if part:
	+ if not include_hidden and part[0] in '*?':
	+ results.append(r'(?!\.)')
	+ results.extend(py313_fnmatch_translate(part, f'{not_sep}*', not_sep))
	+ if idx < last_part_idx:
	+ results.append(any_sep)
	+ res = ''.join(results)
	+ return fr'(?s:{res})\Z'
	+
	+ glob_translate = py313_translate
	+
	+def extract(target_dir, archive_file):
	+ with tarfile.open(archive_file) as tf:
	+ tf.errorlevel = 1
	+ tf.extractall(target_dir, filter='data')
	+
	+class ArchiveFilter:
	+ def __init__(
	+ self,
	+ base_dir,
	+ limit_bytes=None,
	+ items_to_exclude=None
	+ ):
	+ self.base_dir = base_dir
	+ self.total_bytes_added = 0
	+ self.limit_bytes = limit_bytes or 10010241024 # 100 MiB
	+ self.exclude_re = [re.compile(glob_translate(i, recursive=True, include_hidden=True)) for i in (items_to_exclude or [])]
	+
	+ def __call__(self, member):
	+ member.name = os.path.relpath('/' + member.name, self.base_dir)
	+ filtered_member = tarfile.data_filter(member, self.base_dir)
	+ if not filtered_member:
	+ return None
	+ if self.total_bytes_added + member.size > self.limit_bytes:
	+ self.total_bytes_added = self.limit_bytes + 1
	+ raise RuntimeError('Limit exceeded')
	+ name = member.name
	+ for r in self.exclude_re:
	+ if r.match(name):
	+ return None
	+ if member.isdir() and r.match(name + '/'):
	+ return None
	+
	+ self.total_bytes_added += member.size
	+ # Assume that data_filter does not do anything else to it besides rejecting
	+ # Any remaining (permissions) will be stripped when the archive is extracted
	+ # Directly using filtered_member will cause errors in further processing,
	+ # as the data_filter seems intended only for extraction.
	+ return member
	+
	+def create(archive_file, base_dir, content, limit_bytes, items_to_exclude):
	+ base_dir = os.path.abspath(base_dir)
	+ try:
	+ with tarfile.open(archive_file, 'w') as tf:
	+ tf.errorlevel = 1
	+ archive_filter = ArchiveFilter(
	+ base_dir,
	+ limit_bytes,
	+ items_to_exclude=items_to_exclude
	+ )
	+ for g in content:
	+ # iglob is important because once we found one file, we
	+ # add it, and if it does not pass the data filter,
	+ # then we are done with the whole archive. Using glob
	+ # will make it hang here, resulting in DoS.
	+ for f in glob.iglob(g, root_dir=base_dir, recursive=True):
	+ # Specifying 'xxx/**' as the glob will probably make
	+ # this called multiple times on the parent and child
	+ # dirs, so best to avoid it. However, we aren't
	+ # good enough to sanitize this.
	+ tf.add(os.path.join(base_dir, f), filter=archive_filter)
	+ except tarfile.FilterError:
	+ # To prevent exploiting '/', '../../../' globs etc., we
	+ # cannot allow the filename to be exposed
	+ raise RuntimeError('Did not pass the data_filter')
	+
	+if __name__ == '__main__':
	+ import sys
	+ import json
	+ if len(sys.argv) > 1:
	+ a = json.loads(sys.argv[1])
	+ else:
	+ a = json.loads(sys.stdin.read())
	+ op = a.get('op')
	+ if op == 'create':
	+ # Do not remove this try-catch, or it will print out the original
	+ # FilterError, resulting in at least one file name being exposed.
	+ # The file name in the filter error should not be exposed,
	+ # or it allows the attacker to view arbitrary directory structure
	+ # inside the whole worker.
	+ try:
	+ create(
	+ a['archive_file'],
	+ a['base_dir'],
	+ a['content'],
	+ a.get('limit_bytes'),
	+ a.get('items_to_exclude')
	+ )
	+ except RuntimeError as e:
	+ print('Cannot create archive:', e)
	+ sys.exit(1)
	+ elif op == 'extract':
	+ # Does not need a try-catch, because only the things inside the archive
	+ # or the target dir can be exposed.
	+ extract(a['target_dir'], a['archive_file'])
	+ else:
	+ print('Unknown operation:', op)
	+ sys.exit(1)
	diff --git a/lilybuild/lilybuild/tests/safetar_test_worker.py b/lilybuild/lilybuild/tests/safetar_test_worker.py
	new file mode 100644
	index 0000000..95496f6
	--- /dev/null
	+++ b/lilybuild/lilybuild/tests/safetar_test_worker.py
	@@ -0,0 +1,208 @@
	+
	+import unittest
	+import tempfile
	+import os
	+import stat
	+from lilybuild.safetar import (
	+ create, extract
	+)
	+try:
	+ import tarfile
	+ tarfile.FilterError
	+except AttributeError:
	+ import backports.tarfile as tarfile
	+
	+def make_artifact_dir(root_dir):
	+ os.makedirs(os.path.join(root_dir, 'public'))
	+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	+ print('test', file=f)
	+ os.makedirs(os.path.join(root_dir, 'other'))
	+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
	+ print('another test', file=f)
	+
	+def make_artifact_dir_link(root_dir):
	+ os.makedirs(os.path.join(root_dir, 'public'))
	+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	+ print('test', file=f)
	+ os.makedirs(os.path.join(root_dir, 'other'))
	+ os.symlink('../public/a', os.path.join(root_dir, 'other', 'a'))
	+
	+def make_bad_artifact_archive(root_dir):
	+ os.makedirs(os.path.join(root_dir, 'public'))
	+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
	+ print('test', file=f)
	+ os.makedirs(os.path.join(root_dir, 'other'))
	+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
	+ print('should not be there', file=f)
	+ archive = os.path.join(root_dir, 'artifacts.tar')
	+ with tarfile.open(archive, 'w') as f:
	+ f.add(os.path.join(root_dir, 'public'), 'public')
	+ f.add(os.path.join(root_dir, 'other'), '../../../other')
	+ return archive
	+
	+class SafetarTest(unittest.TestCase):
	+ def test_create(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, None)
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other/a')
	+ target = os.path.join(dir_name, 'extracts')
	+ os.makedirs(target)
	+ extract(target, archive)
	+
	+ def test_create_glob(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['*'], None, None)
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['p*/a'], None, None)
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['**/a'], None, None)
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other/a')
	+
	+ def test_create_glob_not_exploitable(self):
	+ # This tests for any traversal attack
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ with self.assertRaises(RuntimeError):
	+ create(archive, dir_name, ['/**'], None, None)
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ with self.assertRaises(RuntimeError):
	+ create(archive, os.path.join(dir_name, 'public'), ['../**'], None, None)
	+
	+ def test_create_out_of_scope(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ with self.assertRaises(RuntimeError):
	+ create(archive, os.path.join(dir_name, 'public'), ['.', '../other'], None, None)
	+ with self.assertRaises(RuntimeError):
	+ create(archive, os.path.join(dir_name, 'public'), ['/home'], None, None)
	+
	+ def test_create_good_link(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir_link(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, None)
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other/a')
	+
	+ def test_create_bad_link(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir_link(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ with self.assertRaises(RuntimeError):
	+ create(archive, os.path.join(dir_name, 'other'), ['.'], None, None)
	+
	+ def test_create_limited(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ with self.assertRaises(RuntimeError):
	+ create(archive, dir_name, ['public', 'other'], 8, None)
	+
	+ def test_create_filtered(self):
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['other/a'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['other'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['other/'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['other/a/'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other')
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['oth'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public/a')
	+ f.getmember('other')
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['**/a'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public')
	+ f.getmember('other')
	+ with self.assertRaises(KeyError):
	+ f.getmember('public/a')
	+ with self.assertRaises(KeyError):
	+ f.getmember('other/a')
	+
	+ with tempfile.TemporaryDirectory() as dir_name:
	+ make_artifact_dir(dir_name)
	+ archive = os.path.join(dir_name, 'artifacts.tar')
	+ create(archive, dir_name, ['public', 'other'], None, ['a'])
	+ with tarfile.open(archive, 'r') as f:
	+ f.getmember('public')
	+ f.getmember('other')
	+ f.getmember('public/a')
	+ f.getmember('other/a')
	+
	+ def test_extract_bad(self):
	+ with tempfile.TemporaryDirectory() as root_dir:
	+ archive_file = make_bad_artifact_archive(root_dir)
	+ with self.assertRaises(tarfile.FilterError):
	+ extract(root_dir, archive_file)
	+
	+if __name__ == '__main__':
	+ unittest.main()
	diff --git a/lilybuild/run-tests.sh b/lilybuild/run-tests.sh
	index 31c5a8e..061fc9f 100755
	--- a/lilybuild/run-tests.sh
	+++ b/lilybuild/run-tests.sh
	@@ -1,5 +1,13 @@
	#!/bin/sh

	d="$(dirname "$(realpath "$0")")"

	-python -m unittest discover -s "$d"/lilybuild/tests -p '*_test.py' -t "$d"
	+if [ "$1" = "master" ]; then
	+ python -m unittest discover -s "$d"/lilybuild/tests -p '*_test.py' -t "$d"
	+elif [ "$1" = "worker" ]; then
	+ python -m unittest discover -s "$d"/lilybuild/tests -p '*_test_worker.py' -t "$d"
	+else
	+ echo 'Specify tests to run'
	+ echo "$0 [master\|worker]"
	+ exit 1
	+fi

File Metadata

Mime Type: text/x-diff
Expires: Fri, Oct 9, 4:16 AM (1 d, 7 h)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 1784435
Default Alt Text: (38 KB)

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions