Page Menu
Home
Phorge
Search
Configure Global Search
Log In
Files
F85951372
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Flag For Later
Award Token
Authored By
Unknown
Size
12 KB
Referenced Files
None
Subscribers
None
View Options
diff --git a/lilybuild/lilybuild/ci_syntax/ci_file.py b/lilybuild/lilybuild/ci_syntax/ci_file.py
index f437990..c61898f 100644
--- a/lilybuild/lilybuild/ci_syntax/ci_file.py
+++ b/lilybuild/lilybuild/ci_syntax/ci_file.py
@@ -1,82 +1,83 @@
import yaml
import jsonschema
import json
import os
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json
schema_file = os.path.join(os.path.dirname(__file__), 'ci.json')
schema = None
def get_schema():
global schema
if schema is not None:
return schema
else:
with open(schema_file, 'r') as f:
schema = json.loads(f.read())
return schema
def toplevel_entries():
return get_schema()['properties']
def normalize_script(script):
if script is None:
return []
elif isinstance(script, str):
return [script]
else:
res = []
for l in script:
if isinstance(l, str):
res.append(l)
else:
res += l
return res
class CIJob:
def __init__(self, job_name, job_struct, defaults):
self.name = job_name
self.image = job_struct.get('image', defaults.get('image'))
self.before_script = normalize_script(job_struct.get('before_script', defaults.get('before_script')))
self.script = normalize_script(job_struct.get('script', defaults.get('script')))
self.after_script = normalize_script(job_struct.get('after_script', defaults.get('after_script')))
+ self.artifacts = job_struct.get('artifacts', defaults.get('artifacts', {}))
OLD_TOPLEVEL_DEFAULTS = ['image', 'services', 'cache', 'before_script', 'after_script']
class CIFile:
'''
Class for parsing CI file.
'''
def __init__(self, file_content):
'''
Construct a CI File from its text content.
'''
f = yaml.safe_load(file_content)
if f is None:
self.stages = []
self.jobs = {}
return
jsonschema.validate(instance=f, schema=get_schema())
self.stages = f.get('stages', [])
self.jobs = {}
defaults = f.get('default', {})
if f.get('variables'):
defaults['variables'] = f['variables']
for kw in OLD_TOPLEVEL_DEFAULTS:
if kw not in defaults and kw in f:
defaults[kw] = f[kw]
for job_name, job_struct in f.items():
# 'pages' is a special job
if job_name != 'pages' and job_name in toplevel_entries():
continue
# jobs starting with . will only be used for base jobs of other jobs,
# they themselves are not run
if job_name.startswith('.'):
continue
self.jobs[job_name] = CIJob(job_name, job_struct, defaults)
diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
index b8a3123..8040261 100644
--- a/lilybuild/lilybuild/config.py
+++ b/lilybuild/lilybuild/config.py
@@ -1,118 +1,171 @@
from buildbot.plugins import *
from buildbot.process import buildstep, logobserver
from twisted.internet import defer
from .ci_syntax import ci_file
+from .helpers import rsync_rules_from_artifacts
import yaml
DEFAULT_IMAGE = 'alpine'
+# 200 MiB
+ARTIFACT_MAX_SIZE = 200 * 1024 * 1024
ci_def_file = '.gitlab-ci.yml'
script_header = '''\
#!/bin/sh
set -e -x
cd /build
'''
class AnalyzeCIFileCommand(buildstep.ShellMixin, steps.BuildStep):
TOPLEVEL_ENTRIES = [
'image',
]
- def __init__(self, **kwargs):
+ def __init__(
+ self,
+ src_relative=None,
+ src_dir=None,
+ storage_dir=None,
+ repo_id=None,
+ artifact_stage_relative=None,
+ artifact_stage_dir=None,
+ **kwargs):
kwargs['name'] = 'Analyze CI file'
kwargs['command'] = ['cat', ci_def_file]
- self.src_dir = kwargs['src_dir']
+ self.src_relative = src_relative
+ self.src_dir = src_dir
self.work_root_dir = kwargs['workdir']
self.script_dir = 'script'
- del kwargs['src_dir']
+ self.storage_dir = storage_dir
+ self.repo_id = repo_id
+ self.artifact_stage_relative = artifact_stage_relative
+ self.artifact_stage_dir = artifact_stage_dir
kwargs['workdir'] = self.src_dir
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
self.observer = logobserver.BufferLogObserver()
self.addLogObserver('stdio', self.observer)
- def job_to_steps(self, job):
+ def job_to_steps(self, job, job_index):
script_name = self.script_dir + '/run.sh'
script_step = steps.StringDownload(
script_header +
'\n\n'.join(job.script),
- name='Set up script',
+ name=f'{job.name}: Set up script',
workerdest=script_name,
)
chmod_step = steps.ShellCommand(
- name='Make script executable',
+ name=f'{job.name}: Make script executable',
command=['chmod', '+x', script_name],
workdir=self.work_root_dir,
)
run_step = steps.ShellCommand(
- name='Run script in container',
+ name=f'{job.name}: Run script in container',
command=[
'/lilybuild/podman-helper',
job.image or DEFAULT_IMAGE,
'sources',
self.script_dir,
],
workdir=self.work_root_dir,
)
- return [script_step, chmod_step, run_step]
+ steps_to_run = [script_step, chmod_step, run_step]
+ if 'paths' in job.artifacts:
+ collect_artifact_step = steps.ShellCommand(
+ name=f'{job.name}: Collect artifacts',
+ command=[
+ 'rsync',
+ '-av',
+ self.src_relative + '/',
+ '--delete',
+ '--prune-empty-dirs',
+ ] + rsync_rules_from_artifacts(job.artifacts) + [
+ self.artifact_stage_relative,
+ ],
+ workdir=self.work_root_dir,
+ )
+ upload_artifact_step = steps.DirectoryUpload(
+ workersrc='.',
+ maxsize=ARTIFACT_MAX_SIZE,
+ name=f'{job.name}: Upload artifacts',
+ masterdest=f'{self.storage_dir}/repos/{self.repo_id}/builds/{self.build.buildid}/jobs/{job_index}/artifacts',
+ workdir=self.artifact_stage_dir,
+ )
+ steps_to_run += [collect_artifact_step, upload_artifact_step]
+
+ return steps_to_run
+
def get_steps(self, stdout):
f = ci_file.CIFile(stdout)
jobs = [f.jobs[j] for j in f.jobs]
- steps = [s for j in jobs for s in self.job_to_steps(j)]
+ steps = [s for (i, j) in enumerate(jobs) for s in self.job_to_steps(j, i)]
print('steps:', steps)
return steps
@defer.inlineCallbacks
def run(self):
# run './build.sh --list-stages' to generate the list of stages
cmd = yield self.makeRemoteShellCommand()
yield self.runCommand(cmd)
# if the command passes extract the list of stages
result = cmd.results()
if result == util.SUCCESS:
# create a ShellCommand for each stage and add them to the build
self.build.addStepsAfterCurrentStep(self.get_steps(self.observer.getStdout()))
return result
def add_pipeline_defs(c, workernames, repo, branch):
factory = util.BuildFactory()
work_root = 'build'
- src_dir = work_root + '/sources'
+ src_relative = 'sources'
+ src_dir = work_root + '/' + src_relative
+ artifact_stage_relative = 'stage'
+ artifact_stage_dir = work_root + '/' + artifact_stage_relative
+ storage_dir = '/buildbot/storage'
# check out the source
factory.addStep(steps.Git(repourl=repo, mode='incremental', workdir=src_dir))
- factory.addStep(AnalyzeCIFileCommand(workdir=work_root, src_dir=src_dir))
+ factory.addStep(AnalyzeCIFileCommand(
+ workdir=work_root,
+ src_relative=src_relative,
+ src_dir=src_dir,
+ storage_dir=storage_dir,
+ artifact_stage_relative=artifact_stage_relative,
+ artifact_stage_dir=artifact_stage_dir,
+ # TODO: specify this
+ repo_id='1',
+ ))
builder_name = 'pipeline'
builder = util.BuilderConfig(
name=builder_name,
workernames=workernames,
factory=factory)
c['builders'].append(builder)
# Change sources
c['change_source'].append(changes.GitPoller(
repo,
workdir='gitpoller-workdir', branch=branch,
pollInterval=300))
# Schedulers
c['schedulers'].append(schedulers.SingleBranchScheduler(
name="all",
change_filter=util.ChangeFilter(branch=branch),
treeStableTimer=None,
builderNames=[builder_name]))
c['schedulers'].append(schedulers.ForceScheduler(
name="force",
builderNames=[builder_name]))
diff --git a/lilybuild/lilybuild/helpers.py b/lilybuild/lilybuild/helpers.py
new file mode 100644
index 0000000..0affef0
--- /dev/null
+++ b/lilybuild/lilybuild/helpers.py
@@ -0,0 +1,27 @@
+
+def normalize_path_for_rsync(path):
+ n = path
+ if n.startswith('./'):
+ n = n[2:]
+ if n.endswith('/'):
+ n = n[:-1]
+
+ return '/' + n
+
+def rsync_rules_from_artifacts(artifacts):
+ paths = artifacts.get('paths', [])
+ # Include all dirs
+ rules = ['--include', '*/']
+ for p in paths:
+ normalized_path = normalize_path_for_rsync(p)
+ rules += [
+ # If path already has /** at the end, the second will actually do nothing,
+ # but it's fine to add it anyway. The directory itself will still
+ # be visited because of the --include */ option we add at the beginning.
+ '--include', normalized_path,
+ '--include', normalized_path + '/**',
+ ]
+
+ # Exclude everything else
+ rules += ['--exclude', '*']
+ return rules
diff --git a/lilybuild/lilybuild/tests/helpers_test.py b/lilybuild/lilybuild/tests/helpers_test.py
new file mode 100644
index 0000000..22858dd
--- /dev/null
+++ b/lilybuild/lilybuild/tests/helpers_test.py
@@ -0,0 +1,59 @@
+
+import unittest
+from lilybuild.helpers import rsync_rules_from_artifacts
+
+class HelpersTest(unittest.TestCase):
+ def test_empty(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({}),
+ ['--include', '*/', '--exclude', '*']
+ )
+
+ def test_simple(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['public']}),
+ ['--include', '*/',
+ '--include', '/public',
+ '--include', '/public/**',
+ '--exclude', '*']
+ )
+
+ def test_dotslash(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['./public/']}),
+ ['--include', '*/',
+ '--include', '/public',
+ '--include', '/public/**',
+ '--exclude', '*']
+ )
+
+ def test_doublestar(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['./public/**']}),
+ ['--include', '*/',
+ '--include', '/public/**',
+ '--include', '/public/**/**',
+ '--exclude', '*']
+ )
+
+ def test_doublestar_middle(self):
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['./public/**/*.html']}),
+ ['--include', '*/',
+ '--include', '/public/**/*.html',
+ '--include', '/public/**/*.html/**',
+ '--exclude', '*']
+ )
+
+ def test_dotdotslash(self):
+ # No exploit possible because rsync will not visit beyond the source root
+ self.assertEqual(
+ rsync_rules_from_artifacts({'paths': ['../etc/passwd']}),
+ ['--include', '*/',
+ '--include', '/../etc/passwd',
+ '--include', '/../etc/passwd/**',
+ '--exclude', '*']
+ )
+
+if __name__ == '__main__':
+ unittest.main()
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Thu, Nov 19, 11:06 AM (1 d, 23 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1841191
Default Alt Text
(12 KB)
Attached To
Mode
rB lilybuild
Attached
Detach File
Event Timeline