Page MenuHomePhorge
Files F2598244

D221.1751452678.diff
No OneTemporary
Actions

Size
23 KB
Referenced Files
None
Subscribers
None

D221.1751452678.diff
View Options

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -14,14 +14,25 @@
script: &push
- if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman push "$IMAGE"; fi
-unit-test:
+unit-test-master:
stage: unit-test
image: docker.io/buildbot/buildbot-master:v4.2.1
before_script: []
script:
- /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
- . /buildbot_venv/bin/activate
- - ./lilybuild/run-tests.sh
+ - ./lilybuild/run-tests.sh master
+
+unit-test-worker:
+ stage: unit-test
+ image: alpine
+ before_script: []
+ script:
+ - apk add --no-cache python3 py3-virtualenv
+ - virtualenv --python=python3 /buildbot_venv
+ - /buildbot_venv/bin/pip3 install 'twisted[tls]' jsonschema backports.tarfile
+ - . /buildbot_venv/bin/activate
+ - ./lilybuild/run-tests.sh worker
build:master:
stage: build
diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -7,6 +7,9 @@
from .helpers import rsync_rules_from_artifacts, get_job_script
import re
import sys
+import json
+
+SAFETAR_EXEC = '/lilybuild/lilybuild/safetar.py'
def on_success(step):
return step.build.results == util.SUCCESS
@@ -122,12 +125,13 @@
unarchive_job = steps.ShellCommand(
name=f'Unarchive artifacts from job #{i}',
command=[
- 'tar',
- '-xf',
- self.artifact_file_name,
- '-C',
- self.src_relative,
+ SAFETAR_EXEC,
],
+ initialStdin=json.dumps({
+ 'op': 'extract',
+ 'archive_file': self.artifact_file_name,
+ 'target_dir': self.src_relative,
+ }),
workdir=self.work_root_dir,
doStepIf=on_success,
)
@@ -160,40 +164,18 @@
steps_to_run = [script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
if 'paths' in job.artifacts:
- clean_stage_dir_step = steps.ShellCommand(
- name='Clean stage dir',
- command=[
- 'rm',
- '-rf',
- self.artifact_stage_relative,
- ],
- workdir=self.work_root_dir,
- doStepIf=on_success,
- )
- collect_artifact_step = steps.ShellCommand(
- name='Collect artifacts',
- command=[
- 'rsync',
- '-av',
- self.result_relative + '/',
- '--delete',
- '--prune-empty-dirs',
- ] + rsync_rules_from_artifacts(job.artifacts) + [
- self.artifact_stage_relative,
- ],
- workdir=self.work_root_dir,
- doStepIf=on_success,
- )
archive_artifact_step = steps.ShellCommand(
name='Archive artifacts',
command=[
- 'tar',
- '-cf',
- self.artifact_file_name,
- '-C',
- self.artifact_stage_relative,
- '.',
+ SAFETAR_EXEC,
],
+ initialStdin=json.dumps({
+ 'op': 'create',
+ 'archive_file': self.artifact_file_name,
+ 'base_dir': self.result_relative,
+ 'content': job.artifacts.get('paths', []),
+ 'items_to_exclude': job.artifacts.get('exclude', []),
+ }),
workdir=self.work_root_dir,
doStepIf=on_success,
)
@@ -211,13 +193,12 @@
workdir=self.work_root_dir,
doStepIf=on_success,
)
- steps_to_run += [clean_stage_dir_step, collect_artifact_step, archive_artifact_step, upload_artifact_step]
+ steps_to_run += [archive_artifact_step, upload_artifact_step]
clean_stage_dir_again_step = steps.ShellCommand(
name='Clean stage, result and artifact',
command=[
'rm',
'-rf',
- self.artifact_stage_relative,
self.result_relative,
self.artifact_file_name,
],
diff --git a/lilybuild/lilybuild/safetar.py b/lilybuild/lilybuild/safetar.py
new file mode 100755
--- /dev/null
+++ b/lilybuild/lilybuild/safetar.py
@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import re
+try:
+ import tarfile
+ tarfile.FilterError
+except AttributeError:
+ import backports.tarfile as tarfile
+
+try:
+ glob_translate = glob.translate
+except AttributeError:
+ # Taken from python 3.13 library
+ def py313_fnmatch_translate(pat, STAR, QUESTION_MARK):
+ res = []
+ add = res.append
+ i, n = 0, len(pat)
+ while i < n:
+ c = pat[i]
+ i = i+1
+ if c == '*':
+ # compress consecutive `*` into one
+ if (not res) or res[-1] is not STAR:
+ add(STAR)
+ elif c == '?':
+ add(QUESTION_MARK)
+ elif c == '[':
+ j = i
+ if j < n and pat[j] == '!':
+ j = j+1
+ if j < n and pat[j] == ']':
+ j = j+1
+ while j < n and pat[j] != ']':
+ j = j+1
+ if j >= n:
+ add('\\[')
+ else:
+ stuff = pat[i:j]
+ if '-' not in stuff:
+ stuff = stuff.replace('\\', r'\\')
+ else:
+ chunks = []
+ k = i+2 if pat[i] == '!' else i+1
+ while True:
+ k = pat.find('-', k, j)
+ if k < 0:
+ break
+ chunks.append(pat[i:k])
+ i = k+1
+ k = k+3
+ chunk = pat[i:j]
+ if chunk:
+ chunks.append(chunk)
+ else:
+ chunks[-1] += '-'
+ # Remove empty ranges -- invalid in RE.
+ for k in range(len(chunks)-1, 0, -1):
+ if chunks[k-1][-1] > chunks[k][0]:
+ chunks[k-1] = chunks[k-1][:-1] + chunks[k][1:]
+ del chunks[k]
+ # Escape backslashes and hyphens for set difference (--).
+ # Hyphens that create ranges shouldn't be escaped.
+ stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-')
+ for s in chunks)
+ # Escape set operations (&&, ~~ and ||).
+ stuff = re.sub(r'([&~|])', r'\\\1', stuff)
+ i = j+1
+ if not stuff:
+ # Empty range: never match.
+ add('(?!)')
+ elif stuff == '!':
+ # Negated empty range: match any character.
+ add('.')
+ else:
+ if stuff[0] == '!':
+ stuff = '^' + stuff[1:]
+ elif stuff[0] in ('^', '['):
+ stuff = '\\' + stuff
+ add(f'[{stuff}]')
+ else:
+ add(re.escape(c))
+ assert i == n
+ return res
+
+ def py313_translate(pat, *, recursive=False, include_hidden=False, seps=None):
+ """Translate a pathname with shell wildcards to a regular expression.
+
+ If `recursive` is true, the pattern segment '**' will match any number of
+ path segments.
+
+ If `include_hidden` is true, wildcards can match path segments beginning
+ with a dot ('.').
+
+ If a sequence of separator characters is given to `seps`, they will be
+ used to split the pattern into segments and match path separators. If not
+ given, os.path.sep and os.path.altsep (where available) are used.
+ """
+ if not seps:
+ if os.path.altsep:
+ seps = (os.path.sep, os.path.altsep)
+ else:
+ seps = os.path.sep
+ escaped_seps = ''.join(map(re.escape, seps))
+ any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
+ not_sep = f'[^{escaped_seps}]'
+ if include_hidden:
+ one_last_segment = f'{not_sep}+'
+ one_segment = f'{one_last_segment}{any_sep}'
+ any_segments = f'(?:.+{any_sep})?'
+ any_last_segments = '.*'
+ else:
+ one_last_segment = f'[^{escaped_seps}.]{not_sep}*'
+ one_segment = f'{one_last_segment}{any_sep}'
+ any_segments = f'(?:{one_segment})*'
+ any_last_segments = f'{any_segments}(?:{one_last_segment})?'
+
+ results = []
+ parts = re.split(any_sep, pat)
+ last_part_idx = len(parts) - 1
+ for idx, part in enumerate(parts):
+ if part == '*':
+ results.append(one_segment if idx < last_part_idx else one_last_segment)
+ elif recursive and part == '**':
+ if idx < last_part_idx:
+ if parts[idx + 1] != '**':
+ results.append(any_segments)
+ else:
+ results.append(any_last_segments)
+ else:
+ if part:
+ if not include_hidden and part[0] in '*?':
+ results.append(r'(?!\.)')
+ results.extend(py313_fnmatch_translate(part, f'{not_sep}*', not_sep))
+ if idx < last_part_idx:
+ results.append(any_sep)
+ res = ''.join(results)
+ return fr'(?s:{res})\Z'
+
+ glob_translate = py313_translate
+
+def extract(target_dir, archive_file):
+ with tarfile.open(archive_file) as tf:
+ tf.errorlevel = 1
+ tf.extractall(target_dir, filter='data')
+
+class ArchiveFilter:
+ def __init__(
+ self,
+ base_dir,
+ limit_bytes=None,
+ items_to_exclude=None
+ ):
+ self.base_dir = base_dir
+ self.total_bytes_added = 0
+ self.limit_bytes = limit_bytes or 100*1024*1024 # 100 MiB
+ self.exclude_re = [re.compile(glob_translate(i, recursive=True, include_hidden=True)) for i in (items_to_exclude or [])]
+
+ def __call__(self, member):
+ member.name = os.path.relpath('/' + member.name, self.base_dir)
+ filtered_member = tarfile.data_filter(member, self.base_dir)
+ if not filtered_member:
+ return None
+ if self.total_bytes_added + member.size > self.limit_bytes:
+ self.total_bytes_added = self.limit_bytes + 1
+ raise RuntimeError('Limit exceeded')
+ name = member.name
+ for r in self.exclude_re:
+ if r.match(name):
+ return None
+ if member.isdir() and r.match(name + '/'):
+ return None
+
+ self.total_bytes_added += member.size
+ # Assume that data_filter does not do anything else to it besides rejecting
+ # Any remaining (permissions) will be stripped when the archive is extracted
+ # Directly using filtered_member will cause errors in further processing,
+ # as the data_filter seems intended only for extraction.
+ return member
+
+def create(archive_file, base_dir, content, limit_bytes, items_to_exclude):
+ base_dir = os.path.abspath(base_dir)
+ try:
+ with tarfile.open(archive_file, 'w') as tf:
+ tf.errorlevel = 1
+ archive_filter = ArchiveFilter(
+ base_dir,
+ limit_bytes,
+ items_to_exclude=items_to_exclude
+ )
+ for g in content:
+ # iglob is important because once we found one file, we
+ # add it, and if it does not pass the data filter,
+ # then we are done with the whole archive. Using glob
+ # will make it hang here, resulting in DoS.
+ for f in glob.iglob(g, root_dir=base_dir, recursive=True):
+ # Specifying 'xxx/**' as the glob will probably make
+ # this called multiple times on the parent and child
+ # dirs, so best to avoid it. However, we aren't
+ # good enough to sanitize this.
+ tf.add(os.path.join(base_dir, f), filter=archive_filter)
+ except tarfile.FilterError:
+ # To prevent exploiting '/**', '../../../**' globs etc., we
+ # cannot allow the filename to be exposed
+ raise RuntimeError('Did not pass the data_filter')
+
+if __name__ == '__main__':
+ import sys
+ import json
+ if len(sys.argv) > 1:
+ a = json.loads(sys.argv[1])
+ else:
+ a = json.loads(sys.stdin.read())
+ op = a.get('op')
+ if op == 'create':
+ # Do not remove this try-catch, or it will print out the original
+ # FilterError, resulting in at least one file name being exposed.
+ # The file name in the filter error should not be exposed,
+ # or it allows the attacker to view arbitrary directory structure
+ # inside the whole worker.
+ try:
+ create(
+ a['archive_file'],
+ a['base_dir'],
+ a['content'],
+ a.get('limit_bytes'),
+ a.get('items_to_exclude')
+ )
+ except RuntimeError as e:
+ print('Cannot create archive:', e)
+ sys.exit(1)
+ elif op == 'extract':
+ # Does not need a try-catch, because only the things inside the archive
+ # or the target dir can be exposed.
+ extract(a['target_dir'], a['archive_file'])
+ else:
+ print('Unknown operation:', op)
+ sys.exit(1)
diff --git a/lilybuild/lilybuild/tests/safetar_test_worker.py b/lilybuild/lilybuild/tests/safetar_test_worker.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/tests/safetar_test_worker.py
@@ -0,0 +1,208 @@
+
+import unittest
+import tempfile
+import os
+import stat
+from lilybuild.safetar import (
+ create, extract
+)
+try:
+ import tarfile
+ tarfile.FilterError
+except AttributeError:
+ import backports.tarfile as tarfile
+
+def make_artifact_dir(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
+ print('another test', file=f)
+
+def make_artifact_dir_link(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ os.symlink('../public/a', os.path.join(root_dir, 'other', 'a'))
+
+def make_bad_artifact_archive(root_dir):
+ os.makedirs(os.path.join(root_dir, 'public'))
+ with open(os.path.join(root_dir, 'public', 'a'), 'w') as f:
+ print('test', file=f)
+ os.makedirs(os.path.join(root_dir, 'other'))
+ with open(os.path.join(root_dir, 'other', 'a'), 'w') as f:
+ print('should not be there', file=f)
+ archive = os.path.join(root_dir, 'artifacts.tar')
+ with tarfile.open(archive, 'w') as f:
+ f.add(os.path.join(root_dir, 'public'), 'public')
+ f.add(os.path.join(root_dir, 'other'), '../../../other')
+ return archive
+
+class SafetarTest(unittest.TestCase):
+ def test_create(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+ target = os.path.join(dir_name, 'extracts')
+ os.makedirs(target)
+ extract(target, archive)
+
+ def test_create_glob(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['*'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['p*/a'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['**/a'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ def test_create_glob_not_exploitable(self):
+ # This tests for any traversal attack
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, dir_name, ['/**'], None, None)
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'public'), ['../**'], None, None)
+
+ def test_create_out_of_scope(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'public'), ['.', '../other'], None, None)
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'public'), ['/home'], None, None)
+
+ def test_create_good_link(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir_link(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, None)
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ def test_create_bad_link(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir_link(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, os.path.join(dir_name, 'other'), ['.'], None, None)
+
+ def test_create_limited(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ with self.assertRaises(RuntimeError):
+ create(archive, dir_name, ['public', 'other'], 8, None)
+
+ def test_create_filtered(self):
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other/a'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other/'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['other/a/'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other')
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['oth'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public/a')
+ f.getmember('other')
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['**/a'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public')
+ f.getmember('other')
+ with self.assertRaises(KeyError):
+ f.getmember('public/a')
+ with self.assertRaises(KeyError):
+ f.getmember('other/a')
+
+ with tempfile.TemporaryDirectory() as dir_name:
+ make_artifact_dir(dir_name)
+ archive = os.path.join(dir_name, 'artifacts.tar')
+ create(archive, dir_name, ['public', 'other'], None, ['a'])
+ with tarfile.open(archive, 'r') as f:
+ f.getmember('public')
+ f.getmember('other')
+ f.getmember('public/a')
+ f.getmember('other/a')
+
+ def test_extract_bad(self):
+ with tempfile.TemporaryDirectory() as root_dir:
+ archive_file = make_bad_artifact_archive(root_dir)
+ with self.assertRaises(tarfile.FilterError):
+ extract(root_dir, archive_file)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/lilybuild/run-tests.sh b/lilybuild/run-tests.sh
--- a/lilybuild/run-tests.sh
+++ b/lilybuild/run-tests.sh
@@ -2,4 +2,12 @@
d="$(dirname "$(realpath "$0")")"
-python -m unittest discover -s "$d"/lilybuild/tests -p '*_test.py' -t "$d"
+if [ "$1" = "master" ]; then
+ python -m unittest discover -s "$d"/lilybuild/tests -p '*_test.py' -t "$d"
+elif [ "$1" = "worker" ]; then
+ python -m unittest discover -s "$d"/lilybuild/tests -p '*_test_worker.py' -t "$d"
+else
+ echo 'Specify tests to run'
+ echo "$0 [master|worker]"
+ exit 1
+fi

File Metadata

Mime Type
text/plain
Expires
Wed, Jul 2, 3:37 AM (16 h, 14 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
241984
Default Alt Text
D221.1751452678.diff (23 KB)

Event Timeline