Page MenuHomePhorge

No OneTemporary

Authored By
Unknown
Size
12 KB
Referenced Files
None
Subscribers
None
diff --git a/lilybuild/lilybuild/ci_syntax/ci_file.py b/lilybuild/lilybuild/ci_syntax/ci_file.py
index f437990..c61898f 100644
--- a/lilybuild/lilybuild/ci_syntax/ci_file.py
+++ b/lilybuild/lilybuild/ci_syntax/ci_file.py
@@ -1,82 +1,83 @@
import yaml
import jsonschema
import json
import os
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json
schema_file = os.path.join(os.path.dirname(__file__), 'ci.json')
schema = None
def get_schema():
global schema
if schema is not None:
return schema
else:
with open(schema_file, 'r') as f:
schema = json.loads(f.read())
return schema
def toplevel_entries():
return get_schema()['properties']
def normalize_script(script):
if script is None:
return []
elif isinstance(script, str):
return [script]
else:
res = []
for l in script:
if isinstance(l, str):
res.append(l)
else:
res += l
return res
class CIJob:
def __init__(self, job_name, job_struct, defaults):
self.name = job_name
self.image = job_struct.get('image', defaults.get('image'))
self.before_script = normalize_script(job_struct.get('before_script', defaults.get('before_script')))
self.script = normalize_script(job_struct.get('script', defaults.get('script')))
self.after_script = normalize_script(job_struct.get('after_script', defaults.get('after_script')))
+ self.artifacts = job_struct.get('artifacts', defaults.get('artifacts', {}))
OLD_TOPLEVEL_DEFAULTS = ['image', 'services', 'cache', 'before_script', 'after_script']
class CIFile:
'''
Class for parsing CI file.
'''
def __init__(self, file_content):
'''
Construct a CI File from its text content.
'''
f = yaml.safe_load(file_content)
if f is None:
self.stages = []
self.jobs = {}
return
jsonschema.validate(instance=f, schema=get_schema())
self.stages = f.get('stages', [])
self.jobs = {}
defaults = f.get('default', {})
if f.get('variables'):
defaults['variables'] = f['variables']
for kw in OLD_TOPLEVEL_DEFAULTS:
if kw not in defaults and kw in f:
defaults[kw] = f[kw]
for job_name, job_struct in f.items():
# 'pages' is a special job
if job_name != 'pages' and job_name in toplevel_entries():
continue
# jobs starting with . will only be used for base jobs of other jobs,
# they themselves are not run
if job_name.startswith('.'):
continue
self.jobs[job_name] = CIJob(job_name, job_struct, defaults)
diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
index b8a3123..8040261 100644
--- a/lilybuild/lilybuild/config.py
+++ b/lilybuild/lilybuild/config.py
@@ -1,118 +1,171 @@
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from twisted.internet import defer
from .ci_syntax import ci_file
+from .helpers import rsync_rules_from_artifacts
import yaml
DEFAULT_IMAGE = 'alpine'
+# 200 MiB
+ARTIFACT_MAX_SIZE = 200 * 1024 * 1024
ci_def_file = '.gitlab-ci.yml'
script_header = '''\
#!/bin/sh
set -e -x
cd /build
'''
class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
TOPLEVEL_ENTRIES = [
'image',
]
- def __init__(self, **kwargs):
+ def __init__(
+ self,
+ src_relative=None,
+ src_dir=None,
+ storage_dir=None,
+ repo_id=None,
+ artifact_stage_relative=None,
+ artifact_stage_dir=None,
+ **kwargs):
kwargs['name'] = 'Analyze CI file'
kwargs['command'] = ['cat', ci_def_file]
- self.src_dir = kwargs['src_dir']
+ self.src_relative = src_relative
+ self.src_dir = src_dir
self.work_root_dir = kwargs['workdir']
self.script_dir = 'script'
- del kwargs['src_dir']
+ self.storage_dir = storage_dir
+ self.repo_id = repo_id
+ self.artifact_stage_relative = artifact_stage_relative
+ self.artifact_stage_dir = artifact_stage_dir
kwargs['workdir'] = self.src_dir
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver('stdio', self.observer)
- def job_to_steps(self, job):
+ def job_to_steps(self, job, job_index):
script_name = self.script_dir + '/run.sh'
script_step = steps.StringDownload(
script_header +
'\n\n'.join(job.script),
- name='Set up script',
+ name=f'{job.name}: Set up script',
workerdest=script_name,
)
chmod_step = steps.ShellCommand(
- name='Make script executable',
+ name=f'{job.name}: Make script executable',
command=['chmod', '+x', script_name],
workdir=self.work_root_dir,
)
run_step = steps.ShellCommand(
- name='Run script in container',
+ name=f'{job.name}: Run script in container',
command=[
'/lilybuild/podman-helper',
job.image or DEFAULT_IMAGE,
'sources',
self.script_dir,
],
workdir=self.work_root_dir,
)
- return [script_step, chmod_step, run_step]
+ steps_to_run = [script_step, chmod_step, run_step]
+ if 'paths' in job.artifacts:
+ collect_artifact_step = steps.ShellCommand(
+ name=f'{job.name}: Collect artifacts',
+ command=[
+ 'rsync',
+ '-av',
+ self.src_relative + '/',
+ '--delete',
+ '--prune-empty-dirs',
+ ] + rsync_rules_from_artifacts(job.artifacts) + [
+ self.artifact_stage_relative,
+ ],
+ workdir=self.work_root_dir,
+ )
+ upload_artifact_step = steps.DirectoryUpload(
+ workersrc='.',
+ maxsize=ARTIFACT_MAX_SIZE,
+ name=f'{job.name}: Upload artifacts',
+ masterdest=f'{self.storage_dir}/repos/{self.repo_id}/builds/{self.build.buildid}/jobs/{job_index}/artifacts',
+ workdir=self.artifact_stage_dir,
+ )
+ steps_to_run += [collect_artifact_step, upload_artifact_step]
+
+ return steps_to_run
+
def get_steps(self, stdout):
f = ci_file.CIFile(stdout)
jobs = [f.jobs[j] for j in f.jobs]
- steps = [s for j in jobs for s in self.job_to_steps(j)]
+ steps = [s for (i, j) in enumerate(jobs) for s in self.job_to_steps(j, i)]
print('steps:', steps)
return steps
@defer.inlineCallbacks
def run(self):
# run './build.sh --list-stages' to generate the list of stages
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))
return result
def add_pipeline_defs(c, workernames, repo, branch):
factory = util.BuildFactory()
work_root = 'build'
- src_dir = work_root + '/sources'
+ src_relative = 'sources'
+ src_dir = work_root + '/' + src_relative
+ artifact_stage_relative = 'stage'
+ artifact_stage_dir = work_root + '/' + artifact_stage_relative
+ storage_dir = '/buildbot/storage'
# check out the source
factory.addStep(steps.Git(repourl=repo, mode='incremental', workdir=src_dir))
- factory.addStep(AnalyzeCIFileCommand(workdir=work_root, src_dir=src_dir))
+ factory.addStep(AnalyzeCIFileCommand(
+ workdir=work_root,
+ src_relative=src_relative,
+ src_dir=src_dir,
+ storage_dir=storage_dir,
+ artifact_stage_relative=artifact_stage_relative,
+ artifact_stage_dir=artifact_stage_dir,
+ # TODO: specify this
+ repo_id='1',
+ ))
builder_name = 'pipeline'
builder = util.BuilderConfig(
name=builder_name,
workernames=workernames,
factory=factory)
c['builders'].append(builder)
# Change sources
c['change_source'].append(changes.GitPoller(
repo,
workdir='gitpoller-workdir', branch=branch,
pollInterval=300))
# Schedulers
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="all",
change_filter=util.ChangeFilter(branch=branch),
treeStableTimer=None,
builderNames=[builder_name]))
c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=[builder_name]))
diff --git a/lilybuild/lilybuild/helpers.py b/lilybuild/lilybuild/helpers.py
new file mode 100644
index 0000000..0affef0
--- /dev/null
+++ b/lilybuild/lilybuild/helpers.py
@@ -0,0 +1,27 @@
+
+def normalize_path_for_rsync(path):
+ n = path
+ if n.startswith('./'):
+ n = n[2:]
+ if n.endswith('/'):
+ n = n[:-1]
+
+ return '/' + n
+
+def rsync_rules_from_artifacts(artifacts):
+ paths = artifacts.get('paths', [])
+ # Include all dirs
+ rules = ['--include', '*/']
+ for p in paths:
+ normalized_path = normalize_path_for_rsync(p)
+ rules += [
+ # If path already has /** at the end, the second will actually do nothing,
+ # but it's fine to add it anyway. The directory itself will still
+ # be visited because of the --include */ option we add at the beginning.
+ '--include', normalized_path,
+ '--include', normalized_path + '/**',
+ ]
+
+ # Exclude everything else
+ rules += ['--exclude', '*']
+ return rules
diff --git a/lilybuild/lilybuild/tests/helpers_test.py b/lilybuild/lilybuild/tests/helpers_test.py
new file mode 100644
index 0000000..22858dd
--- /dev/null
+++ b/lilybuild/lilybuild/tests/helpers_test.py
@@ -0,0 +1,59 @@
+
+import unittest
+from lilybuild.helpers import rsync_rules_from_artifacts
+
+class HelpersTest(unittest.TestCase):
+ def test_empty(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({}),
+ ['--include', '*/', '--exclude', '*']
+ )
+
+ def test_simple(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['public']}),
+ ['--include', '*/',
+ '--include', '/public',
+ '--include', '/public/**',
+ '--exclude', '*']
+ )
+
+ def test_dotslash(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['./public/']}),
+ ['--include', '*/',
+ '--include', '/public',
+ '--include', '/public/**',
+ '--exclude', '*']
+ )
+
+ def test_doublestar(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['./public/**']}),
+ ['--include', '*/',
+ '--include', '/public/**',
+ '--include', '/public/**/**',
+ '--exclude', '*']
+ )
+
+ def test_doublestar_middle(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['./public/**/*.html']}),
+ ['--include', '*/',
+ '--include', '/public/**/*.html',
+ '--include', '/public/**/*.html/**',
+ '--exclude', '*']
+ )
+
+ def test_dotdotslash(self):
+ # No exploit possible because rsync will not visit beyond the source root
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['../etc/passwd']}),
+ ['--include', '*/',
+ '--include', '/../etc/passwd',
+ '--include', '/../etc/passwd/**',
+ '--exclude', '*']
+ )
+
+if __name__ == '__main__':
+ unittest.main()

File Metadata

Mime Type
text/x-diff
Expires
Thu, Nov 19, 11:06 AM (1 d, 23 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1841191
Default Alt Text
(12 KB)

Event Timeline