Page MenuHomePhorge
Files F8199726

D244.1760328260.diff
No OneTemporary
Actions

Size
13 KB
Referenced Files
None
Subscribers
None

D244.1760328260.diff
View Options

diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,5 @@
/venv
.coverage
coverage.xml
+dist
+lilybuild.egg-info
diff --git a/Containerfile.master b/Containerfile.master
--- a/Containerfile.master
+++ b/Containerfile.master
@@ -2,3 +2,7 @@
FROM docker.io/buildbot/buildbot-master:v4.2.1
RUN /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
+
+COPY --from=lilybuild . /usr/src/lilybuild
+
+RUN /buildbot_venv/bin/pip3 install /usr/src/lilybuild
diff --git a/build-master.sh b/build-master.sh
--- a/build-master.sh
+++ b/build-master.sh
@@ -1,3 +1,3 @@
#!/bin/sh
-podman build -f Containerfile.master master "$@"
+podman build -f Containerfile.master --build-context=lilybuild=lilybuild master "$@"
diff --git a/lilybuild/lilybuild/artifacts/__init__.py b/lilybuild/lilybuild/artifacts/__init__.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/artifacts/__init__.py
@@ -0,0 +1,4 @@
+
+from .application import ArtifactsApplication
+
+ep = ArtifactsApplication()
diff --git a/lilybuild/lilybuild/artifacts/application.py b/lilybuild/lilybuild/artifacts/application.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/artifacts/application.py
@@ -0,0 +1,12 @@
+
+from buildbot.www.plugin import Application
+from .resource import Api
+
+class ArtifactsApplication(Application):
+ def __init__(self):
+ self.description = 'LilyBuild Artifacts support'
+ self.version = '0.0.0'
+ self.static_dir = ''
+ self.ui = True
+ self.api = Api(self)
+ self.resource = self.api.app.resource()
diff --git a/lilybuild/lilybuild/artifacts/authz_utils.py b/lilybuild/lilybuild/artifacts/authz_utils.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/artifacts/authz_utils.py
@@ -0,0 +1,24 @@
+
+from twisted.internet import defer
+from buildbot.www.authz.endpointmatchers import EndpointMatcherBase
+
+class AnyArtifactEndpointMatcher(EndpointMatcherBase):
+ def __init__(self, role, defaultDeny=True):
+ super().__init__(role, defaultDeny)
+
+ def match(self, ep, action='get', options=None):
+ if isinstance(ep, tuple) and ep[0] == 'lilybuild_artifacts':
+ return defer.secceed(Match(self.master))
+ return defer.succeed(None)
+
+class BuildArtifactEndpointMatcher(EndpointMatcherBase):
+ def __init__(self, role, defaultDeny=True):
+ super().__init__(role, defaultDeny)
+
+ @defer.inlineCallbacks
+ def match(self, ep, action='get', options=None):
+ if isinstance(ep, tuple) and ep[0] == 'lilybuild_artifacts' and ep[1] == 'builds':
+ build_id = ep[2]
+ build = yield self.master.data.get(('builds', build_id))
+ return Match(self.master, build=build)
+ return None
diff --git a/lilybuild/lilybuild/artifacts/resource.py b/lilybuild/lilybuild/artifacts/resource.py
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/artifacts/resource.py
@@ -0,0 +1,80 @@
+
+import json
+from klein import Klein
+from twisted.internet import defer
+from twisted.web.static import File
+from buildbot.data.resultspec import ResultSpec, Property
+import os
+
+POSSIBLE_ARTIFACT_TYPES = {
+ 'archive': 'artifacts.tar',
+ 'reports': 'reports.tar',
+}
+BAD_REQUEST = 'BAD_REQUEST'
+NOT_FOUND = 'NOT_FOUND'
+
+def validate_artifact_type(at):
+ return at in POSSIBLE_ARTIFACT_TYPES
+
+class Api:
+ app = Klein()
+
+ def __init__(self, ep):
+ self.ep = ep
+
+ def lbc(self):
+ return self.ep.config['lbc']
+
+ def www(self):
+ return self.ep.master.www
+
+ @app.route('/builds/<int:build_id>/jobs/<int:job_index>/artifacts/<artifact_type>', methods=['GET'])
+ @defer.inlineCallbacks
+ def getArtifact(self, request, build_id, job_index, artifact_type):
+ if not validate_artifact_type(artifact_type):
+ request.setResponseCode(400)
+ return json.dumps({'error': BAD_REQUEST})
+ storage_dir = self.lbc().storage_dir
+ try:
+ artifact_file = yield self.get_artifact_location(request, build_id, job_index, artifact_type)
+ return File(artifact_file)
+ except:
+ request.setResponseCode(404)
+ return json.dumps({'error': NOT_FOUND})
+
+ @defer.inlineCallbacks
+ def get_artifact_location(self, request, build_id, job_index, artifact_type):
+ result_spec = ResultSpec(
+ # see popProperties in ResultSpec
+ properties=[Property(b'property', 'eq', [
+ 'lilybuild_repo_id',
+ 'lilybuild_source',
+ ])]
+ )
+
+ build = yield self.ep.master.data.get_with_resultspec(
+ ('builds', build_id), result_spec
+ )
+ if not build:
+ raise FileNotFoundError('Not Found')
+ if 'lilybuild_repo_id' not in build['properties']:
+ raise FileNotFoundError('Not Found')
+ if 'lilybuild_source' not in build['properties']:
+ source = 'none'
+ else:
+ source = build['properties']['lilybuild_source'][0]
+ # Things not pushed to staging area is considered private
+ # and can only be seen by project members
+ if source == 'arc-patch':
+ # see ForgejoAuthz
+ yield self.www().assertUserAllowed(request, ('lilybuild_artifacts', 'builds', build_id, 'jobs', job_index, 'artifacts', artifact_type), 'get', {})
+ repo_id = build['properties']['lilybuild_repo_id'][0]
+ fn = POSSIBLE_ARTIFACT_TYPES[artifact_type]
+ res = os.path.join(
+ self.lbc().storage_dir,
+ 'repos', str(repo_id),
+ 'builds', str(build_id),
+ 'jobs', str(job_index),
+ 'artifacts', fn
+ )
+ return res
diff --git a/lilybuild/lilybuild/auth.py b/lilybuild/lilybuild/auth.py
--- a/lilybuild/lilybuild/auth.py
+++ b/lilybuild/lilybuild/auth.py
@@ -67,20 +67,34 @@
@defer.inlineCallbacks
def assertUserAllowed(self, ep, action, options, userDetails):
- try:
- (epobject, epdict) = self.master.data.getEndpoint(ep)
- if isinstance(epobject, BuildEndpoint):
- build_props = yield self.master.data.get(('builds', epdict['buildid'], 'properties'))
- ok = yield self.user_can_access_build(userDetails, build_props)
+ if isinstance(ep, tuple) and ep[0] == 'lilybuild_artifacts' and ep[1] == 'builds':
+ try:
+ build_id = ep[2]
+ ok = yield self.user_can_access_build_by_id(userDetails, build_id)
if ok:
return defer.succeed(None)
- except Exception as e:
- print('exception:', e)
- finally:
- pass
+ except Exception as e:
+ print('exception when checking artifact access', e)
+ else:
+ try:
+ (epobject, epdict) = self.master.data.getEndpoint(ep)
+ if isinstance(epobject, BuildEndpoint):
+ ok = yield self.user_can_access_build_by_id(userDetails, epdict['buildid'])
+ if ok:
+ return defer.succeed(None)
+ except Exception as e:
+ print('exception:', e)
+ finally:
+ pass
res = yield super().assertUserAllowed(ep, action, options, userDetails)
return res
+ @defer.inlineCallbacks
+ def user_can_access_build_by_id(self, userDetails, build_id):
+ build_props = yield self.master.data.get(('builds', build_id, 'properties'))
+ ok = yield self.user_can_access_build(userDetails, build_props)
+ return ok
+
@defer.inlineCallbacks
def user_can_access_build(self, userDetails, build_props):
if 'lilybuild_repo' not in build_props:
@@ -92,6 +106,8 @@
if repo_name.count('/') != 1:
return False
+ if 'username' not in userDetails:
+ return False
username = userDetails['username']
props = Properties()
props.master = self.master
diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -46,6 +46,7 @@
artifact_stage_relative=None,
artifact_stage_dir=None,
job_prop=None,
+ artifact_link_base=None,
**kwargs):
self.lbc = lbc
self.src_relative = src_relative
@@ -58,6 +59,7 @@
self.artifact_stage_dir = artifact_stage_dir
self.result_relative = result_relative
self.result_dir = result_dir
+ self.artifact_link_base = artifact_link_base
super().__init__(name='Run step', **kwargs)
@defer.inlineCallbacks
@@ -94,7 +96,7 @@
self.addCompleteLog('exception', f'{e}')
return res
- def get_upload_artifacts_jobs(self, short_name, artifact_name, base_dir, paths, exclude, master_pattern, job_index, doStepIf=on_success):
+ def get_upload_artifacts_jobs(self, short_name, artifact_type, artifact_name, base_dir, paths, exclude, master_pattern, job_index, doStepIf=on_success):
archive_artifact_step = steps.ShellCommand(
name=f'Archive artifacts: {short_name}',
command=[
@@ -116,12 +118,15 @@
job=job_index,
doStepIf=doStepIf,
)
+ parent_build_id = self.getProperty('lilybuild_pipeline_vars')['CI_PIPELINE_ID']
+ artifact_url = f'{self.artifact_link_base}/plugins/lilybuild_artifacts/builds/{parent_build_id}/jobs/{job_index}/artifacts/{artifact_type}' if self.artifact_link_base else None
upload_artifact_step = steps.FileUpload(
workersrc=artifact_name,
maxsize=self.artifact_max_size,
name=f'Upload artifacts: {short_name}',
masterdest=masterdest,
workdir=self.work_root_dir,
+ url=artifact_url,
doStepIf=doStepIf,
)
return [archive_artifact_step, upload_artifact_step]
@@ -206,6 +211,7 @@
if 'paths' in job.artifacts:
steps_to_run += self.get_upload_artifacts_jobs(
'files',
+ 'archive',
self.artifact_file_name,
self.result_relative,
job.artifacts.get('paths', []),
@@ -228,6 +234,7 @@
workdir=self.work_root_dir,
doStepIf=on_always,
)] + self.get_upload_artifacts_jobs(
+ 'reports',
'reports',
self.reports_file_name,
self.artifact_stage_relative,
diff --git a/lilybuild/lilybuild/config.py b/lilybuild/lilybuild/config.py
--- a/lilybuild/lilybuild/config.py
+++ b/lilybuild/lilybuild/config.py
@@ -22,7 +22,7 @@
result_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/result')
artifact_stage_relative = 'stage'
artifact_stage_dir = util.Interpolate('repos/%(prop:lilybuild_repo_id)s/stage')
-storage_dir = '/buildbot/storage'
+default_storage_dir = '/buildbot/storage'
report_phorge = 'phorge'
report_forgejo = 'forgejo'
@@ -35,7 +35,7 @@
triggerable_scheduler_name = 'lilybuild-triggerable'
force_triggerable_scheduler_name = 'lilybuild-force-triggerable'
- def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None):
+ def __init__(self, c, workernames, reports=None, phorge_base_url=None, phorge_token=None, ssh_priv_key=None, ssh_known_hosts=None, storage_dir=default_storage_dir, artifact_link_base=None):
self.c = c
self.poll_interval = 300
self.workernames = workernames
@@ -49,6 +49,8 @@
self.phorge_token = phorge_token
self.ssh_priv_key = ssh_priv_key
self.ssh_known_hosts = ssh_known_hosts
+ self.storage_dir = storage_dir
+ self.artifact_link_base = normalize_base_url(artifact_link_base)
def configure_factory_and_builder(self):
self.add_lilybuild_builder()
@@ -84,7 +86,7 @@
workdir=work_root,
src_relative=src_relative,
src_dir=src_dir,
- storage_dir=storage_dir,
+ storage_dir=self.storage_dir,
artifact_stage_relative=artifact_stage_relative,
artifact_stage_dir=artifact_stage_dir,
result_relative=result_relative,
@@ -111,11 +113,12 @@
workdir=work_root,
src_relative=src_relative,
src_dir=src_dir,
- storage_dir=storage_dir,
+ storage_dir=self.storage_dir,
artifact_stage_relative=artifact_stage_relative,
artifact_stage_dir=artifact_stage_dir,
result_relative=result_relative,
result_dir=result_dir,
+ artifact_link_base=self.artifact_link_base
))
builder_job = util.BuilderConfig(
diff --git a/lilybuild/pyproject.toml b/lilybuild/pyproject.toml
new file mode 100644
--- /dev/null
+++ b/lilybuild/pyproject.toml
@@ -0,0 +1,23 @@
+[build-system]
+requires = ["setuptools >= 75.8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "lilybuild"
+version = "0.0.0"
+authors = [
+ { name="tusooa", email="tusooa@kazv.moe" },
+]
+description = "A buildbot extension to build GitLab-style CI files"
+readme = "README.md"
+requires-python = ">=3.9"
+
+[project.urls]
+Homepage = "https://iron.lily-is.land/diffusion/B/"
+Issues = "https://iron.lily-is.land/project/view/13/"
+
+[project.entry-points."buildbot.util"]
+LilyBuildConfig = "lilybuild.config:LilyBuildConfig"
+
+[project.entry-points."buildbot.www"]
+lilybuild_artifacts = "lilybuild.artifacts:ep"

File Metadata

Mime Type
text/plain
Expires
Sun, Oct 12, 9:04 PM (17 h, 12 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
513968
Default Alt Text
D244.1760328260.diff (13 KB)

Event Timeline