Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F2598251
D221.1751452993.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Award Token
Flag For Later
Size
20 KB
Referenced Files
None
Subscribers
None
D221.1751452993.diff
View Options
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,14 +14,25 @@
script: &push
- if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman push "$IMAGE"; fi
-unit-test:
+unit-test-master:
stage: unit-test
image: docker.io/buildbot/buildbot-master:v4.2.1
before_script: []
script:
- /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
- . /buildbot_venv/bin/activate
- - ./lilybuild/run-tests.sh
+ - ./lilybuild/run-tests.sh master
+
+unit-test-worker:
+ stage: unit-test
+ image: alpine
+ before_script: []
+ script:
+ - apk add --no-cache python3 py3-virtualenv
+ - virtualenv --python=python3 /buildbot_venv
+ - /buildbot_venv/bin/pip3 install 'twisted[tls]' jsonschema backports.tarfile
+ - . /buildbot_venv/bin/activate
+ - ./lilybuild/run-tests.sh worker
build:master:
stage: build
diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -7,6 +7,9 @@
from .helpers import rsync_rules_from_artifacts, get_job_script
import re
import sys
+import json
+
+SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'
def on_success(step):
return step.build.results == util.SUCCESS
@@ -122,12 +125,13 @@
unarchive_job = steps.ShellCommand(
name=f'Unarchive artifacts from job #{i}',
command=[
- 'tar',
- '-xf',
- self.artifact_file_name,
- '-C',
- self.src_relative,
+ SAFETAR_EXEC,
],
+ initialStdin=json.dumps({
+ 'op': 'extract',
+ 'archive_file': self.artifact_file_name,
+ 'target_dir': self.src_relative,
+ }),
workdir=self.work_root_dir,
doStepIf=on_success,
)
@@ -160,40 +164,18 @@
steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
if 'paths' in job.artifacts:
- clean_stage_dir_step = steps.ShellCommand(
- name='Clean stage dir',
- command=[
- 'rm',
- '-rf',
- self.artifact_stage_relative,
- ],
- workdir=self.work_root_dir,
- doStepIf=on_success,
- )
- collect_artifact_step = steps.ShellCommand(
- name='Collect artifacts',
- command=[
- 'rsync',
- '-av',
- self.result_relative + '/',
- '--delete',
- '--prune-empty-dirs',
- ] + rsync_rules_from_artifacts(job.artifacts) + [
- self.artifact_stage_relative,
- ],
- workdir=self.work_root_dir,
- doStepIf=on_success,
- )
archive_artifact_step = steps.ShellCommand(
name='Archive artifacts',
command=[
- 'tar',
- '-cf',
- self.artifact_file_name,
- '-C',
- self.artifact_stage_relative,
- '.',
+ SAFETAR_EXEC,
],
+ initialStdin=json.dumps({
+ 'op': 'create',
+ 'archive_file': self.artifact_file_name,
+ 'base_dir': self.result_relative,
+ 'content': job.artifacts.get('paths', []),
+ 'items_to_exclude': job.artifacts.get('exclude', []),
+ }),
workdir=self.work_root_dir,
doStepIf=on_success,
)
@@ -211,13 +193,12 @@
workdir=self.work_root_dir,
doStepIf=on_success,
)
- steps_to_run += [clean_stage_dir_step, collect_artifact_step, archive_artifact_step, upload_artifact_step]
+ steps_to_run += [archive_artifact_step, upload_artifact_step]
clean_stage_dir_again_step = steps.ShellCommand(
name='Clean stage, result and artifact',
command=[
'rm',
'-rf',
- self.artifact_stage_relative,
self.result_relative,
self.artifact_file_name,
],
diff --git a/lilybuild/lilybuild/safetar.py b/lilybuild/lilybuild/safetar.py
new file mode 100755
--- /dev/null
+++ b/lilybuild/lilybuild/safetar.py
@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import re
+try:
+ import tarfile
+ tarfile.FilterError
+except AttributeError:
+ import backports.tarfile as tarfile
+
+# Taken from python 3.13 library
+def py313_translate(pat, *, recursive=False, include_hidden=False, seps=None):
+ """Translate a pathname with shell wildcards to a regular expression.
+
+ If `recursive` is true, the pattern segment '**' will match any number of
+ path segments.
+
+ If `include_hidden` is true, wildcards can match path segments beginning
+ with a dot ('.').
+
+ If a sequence of separator characters is given to `seps`, they will be
+ used to split the pattern into segments and match path separators. If not
+ given, os.path.sep and os.path.altsep (where available) are used.
+ """
+ if not seps:
+ if os.path.altsep:
+ seps = (os.path.sep, os.path.altsep)
+ else:
+ seps = os.path.sep
+ escaped_seps = ''.join(map(re.escape, seps))
+ any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
+ not_sep = f'[^{escaped_seps}]'
+ if include_hidden:
+ one_last_segment = f'{not_sep}+'
+ one_segment = f'{one_last_segment}{any_sep}'
+ any_segments = f'(?:.+{any_sep})?'
+ any_last_segments = '.*'
+ else:
+ one_last_segment = f'[^{escaped_seps}.]{not_sep}*'
+ one_segment = f'{one_last_segment}{any_sep}'
+ any_segments = f'(?:{one_segment})*'
+ any_last_segments = f'{any_segments}(?:{one_last_segment})?'
+
+ results = []
+ parts = re.split(any_sep, pat)
+ last_part_idx = len(parts) - 1
+ for idx, part in enumerate(parts):
+ if part == '*':
+ results.append(one_segment if idx < last_part_idx else one_last_segment)
+ elif recursive and part == '**':
+ if idx < last_part_idx:
+ if parts[idx + 1] != '**':
+ results.append(any_segments)
+ else:
+ results.append(any_last_segments)
+ else:
+ if part:
+ if not include_hidden and part[0] in '*?':
+ results.append(r'(?!\.)')
+ results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep))
+ if idx < last_part_idx:
+ results.append(any_sep)
+ res = ''.join(results)
+ return fr'(?s:{res})\Z'
+
+try:
+ glob_translate = glob.translate
+except AttributeError:
+ glob_translate = py313_translate
+
+def extract(target_dir, archive_file):
+ with tarfile.open(archive_file) as tf:
+ tf.errorlevel = 1
+ tf.extractall(target_dir, filter='data')
+
+class ArchiveFilter:
+ def __init__(
+ self,
+ base_dir,
+ limit_bytes=None,
+ items_to_exclude=None
+ ):
+ self.base_dir = base_dir
+ self.total_bytes_added = 0
+ self.limit_bytes = limit_bytes or 100*1024*1024 # 100 MiB
+ self.exclude_re = [re.compile(glob_translate(i, recursive=True, include_hidden=True)) for i in (items_to_exclude or [])]
+
+ def __call__(self, member):
+ member.name = os.path.relpath('/' + member.name, self.base_dir)
+ filtered_member = tarfile.data_filter(member, self.base_dir)
+ if not filtered_member:
+ return None
+ if self.total_bytes_added + member.size > self.limit_bytes:
+ self.total_bytes_added = self.limit_bytes + 1
+ raise RuntimeError('Limit exceeded')
+ name = member.name
+ for r in self.exclude_re:
+ if r.match(name):
+ return None
+ if member.isdir() and r.match(name + '/'):
+ return None
+
+ self.total_bytes_added += member.size
+ # Assume that data_filter does not do anything else to it besides rejecting
+ # Any remaining (permissions) will be stripped when the archive is extracted
+ # Directly using filtered_member will cause errors in further processing,
+ # as the data_filter seems intended only for extraction.
+ return member
+
+def create(archive_file, base_dir, content, limit_bytes, items_to_exclude):
+ base_dir = os.path.abspath(base_dir)
+ try:
+ with tarfile.open(archive_file, 'w') as tf:
+ tf.errorlevel = 1
+ archive_filter = ArchiveFilter(
+ base_dir,
+ limit_bytes,
+ items_to_exclude=items_to_exclude
+ )
+ for g in content:
+ # iglob is important because once we found one file, we
+ # add it, and if it does not pass the data filter,
+ # then we are done with the whole archive. Using glob
+ # will make it hang here, resulting in DoS.
+ for f in glob.iglob(g, root_dir=base_dir, recursive=True):
+ # Specifying 'xxx/**' as the glob will probably make
+ # this called multiple times on the parent and child
+ # dirs, so best to avoid it. However, we aren't
+ # good enough to sanitize this.
+ tf.add(os.path.join(base_dir, f), filter=archive_filter)
+ except tarfile.FilterError:
+ # To prevent exploiting '/**', '../../../**' globs etc., we
+ # cannot allow the filename to be exposed
+ raise RuntimeError('Did not pass the data_filter')
+
+if __name__ == '__main__':
+ import sys
+ import json
+ if len(sys.argv) > 1:
+ a = json.loads(sys.argv[1])
+ else:
+ a = json.loads(sys.stdin.read())
+ op = a.get('op')
+ if op == 'create':
+ # Do not remove this try-catch, or it will print out the original
+ # FilterError, resulting in at least one file name being exposed.
+ # The file name in the filter error should not be exposed,
+ # or it allows the attacker to view arbitrary directory structure
+ # inside the whole worker.
+ try:
+ create(
+ a['archive_file'],
+ a['base_dir'],
+ a['content'],
+ a.get('limit_bytes'),
+ a.get('items_to_exclude')
+ )
+ except RuntimeError as e:
+ print('Cannot create archive:', e)
+ sys.exit(1)
+ elif op == 'extract':
+ # Does not need a try-catch, because only the things inside the archive
+ # or the target dir can be exposed.
+ extract(a['target_dir'], a['archive_file'])
+ else:
+ print('Unknown operation:', op)
+ sys.exit(1)
diff --git a/lilybuild/lilybuild/tests/safetar_test_worker.py b/lilybuild/lilybuild/tests/safetar_test_worker.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/tests/safetar_test_worker.py
@@ -0,0 +1,208 @@
+
+import unittest
+import tempfile
+import os
+import stat
+from lilybuild.safetar import (
+ create, extract
+)
+try:
+ import tarfile
+ tarfile.FilterError
+except AttributeError:
+ import backports.tarfile as tarfile
+
+def make_artifact_dir(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
+ print('another test', file=f)
+
+def make_artifact_dir_link(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ os.symlink('../public/a', os.path.join(root_dir, 'other', 'a'))
+
+def make_bad_artifact_archive(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
+ print('should not be there', file=f)
+ archive = os.path.join(root_dir, 'artifacts.tar')
+ with tarfile.open(archive, 'w') as f:
+ f.add(os.path.join(root_dir, 'public'), 'public')
+ f.add(os.path.join(root_dir, 'other'), '../../../other')
+ return archive
+
+class SafetarTest(unittest.TestCase):
+ def test_create(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+ target = os.path.join(dir_name, 'extracts')
+ os.makedirs(target)
+ extract(target, archive)
+
+ def test_create_glob(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['*'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['p*/a'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['**/a'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ def test_create_glob_not_exploitable(self):
+ # This tests for any traversal attack
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, dir_name, ['/**'], None, None)
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'public'), ['../**'], None, None)
+
+ def test_create_out_of_scope(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'public'), ['.', '../other'], None, None)
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'public'), ['/home'], None, None)
+
+ def test_create_good_link(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir_link(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ def test_create_bad_link(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir_link(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'other'), ['.'], None, None)
+
+ def test_create_limited(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, dir_name, ['public', 'other'], 8, None)
+
+ def test_create_filtered(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other/a'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other/'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other/a/'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other')
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['oth'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other')
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['**/a'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public')
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['a'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public')
+ f.getmember('other')
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ def test_extract_bad(self):
+ with tempfile.TemporaryDirectory() as root_dir:
+ archive_file = make_bad_artifact_archive(root_dir)
+ with self.assertRaises(tarfile.FilterError):
+ extract(root_dir, archive_file)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/lilybuild/run-tests.sh b/lilybuild/run-tests.sh
--- a/lilybuild/run-tests.sh
+++ b/lilybuild/run-tests.sh
@@ -2,4 +2,12 @@
d="$(dirname "$(realpath "$0")")"
-python -m unittest discover -s "$d"/lilybuild/tests -p '*_test.py' -t "$d"
+if [ "$1" = "master" ]; then
+ python -m unittest discover -s "$d"/lilybuild/tests -p '*_test.py' -t "$d"
+elif [ "$1" = "worker" ]; then
+ python -m unittest discover -s "$d"/lilybuild/tests -p '*_test_worker.py' -t "$d"
+else
+ echo 'Specify tests to run'
+ echo "$0 [master|worker]"
+ exit 1
+fi
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Wed, Jul 2, 3:43 AM (16 h, 33 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
241988
Default Alt Text
D221.1751452993.diff (20 KB)
Attached To
Mode
D221: Use safetar in place of rsync for archiving
Attached
Detach File
Event Timeline
Log In to Comment