Page MenuHomePhorge
Files F2704425

D223.1752802268.diff
No OneTemporary
Actions

Size
15 KB
Referenced Files
None
Subscribers
None

D223.1752802268.diff
View Options

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,10 +5,6 @@
default:
image: quay.io/podman/stable
- before_script:
- - 'if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman login -u "$REGISTRY_USER" --password-stdin "$REGISTRY" <<< "$REGISTRY_PASSWORD"; fi'
- - IMAGE_PREFIX="$REGISTRY/infra/lilybuild"
- - IMAGE_VER="${CI_COMMIT_REF_SLUG-none}"
.push:
script: &push
@@ -17,7 +13,6 @@
unit-test-master:
stage: unit-test
image: docker.io/buildbot/buildbot-master:v4.2.1
- before_script: []
script:
- /buildbot_venv/bin/pip3 install jsonschema backports.tarfile
- . /buildbot_venv/bin/activate
@@ -26,7 +21,6 @@
unit-test-worker:
stage: unit-test
image: alpine
- before_script: []
script:
- apk add --no-cache python3 py3-virtualenv
- virtualenv --python=python3 /buildbot_venv
@@ -34,22 +28,29 @@
- . /buildbot_venv/bin/activate
- ./lilybuild/run-tests.sh worker
-build:master:
+.build:
stage: build
+ before_script:
+ - 'if [ -n "${CI_COMMIT_REF_SLUG}" ]; then sudo -u podman podman login -u "$REGISTRY_USER" --password-stdin "$REGISTRY" <<< "$REGISTRY_PASSWORD"; fi'
+ - IMAGE_PREFIX="$REGISTRY/infra/lilybuild"
+ - IMAGE_VER="${CI_COMMIT_REF_SLUG-none}"
+
+build:master:
+ extends: .build
script:
- IMAGE="$IMAGE_PREFIX/buildbot-master:$IMAGE_VER"
- sudo -u podman ./build-master.sh -t "$IMAGE"
- *push
build:worker:
- stage: build
+ extends: .build
script:
- IMAGE="$IMAGE_PREFIX/buildbot-worker:$IMAGE_VER"
- sudo -u podman ./build-worker.sh -t "$IMAGE"
- *push
build:volume-helper:
- stage: build
+ extends: .build
script:
- IMAGE="$IMAGE_PREFIX/volume-helper:$IMAGE_VER"
- sudo -u podman ./build-volume-helper.sh -t "$IMAGE"
diff --git a/lilybuild/lilybuild/ci_syntax/ci_file.py b/lilybuild/lilybuild/ci_syntax/ci_file.py
--- a/lilybuild/lilybuild/ci_syntax/ci_file.py
+++ b/lilybuild/lilybuild/ci_syntax/ci_file.py
@@ -8,6 +8,10 @@
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/assets/javascripts/editor/schema/ci.json
schema_file = os.path.join(os.path.dirname(__file__), 'ci.json')
schema = None
+extend_limit = 11
+
+class CIValidationError(Exception):
+ pass
def get_schema():
global schema
@@ -39,17 +43,62 @@
def ci_slugify(s):
return re.sub(slug_re, '-', s.lower()[:63]).strip('-')
+def get_job_extend_seq(job_name, all_jobs, depth=0):
+ # DFS traversal; child-first, self-last order
+ res = []
+ if job_name not in all_jobs:
+ raise CIValidationError(f'Job "{job_name}" does not exist, but occurs in `extends`.')
+ job_struct = all_jobs.get(job_name) or {}
+ parents = job_struct.get('extends', [])
+ if depth > extend_limit:
+ raise CIValidationError(f'`extends` depth is over the limit of {extend_limit}.')
+ if isinstance(parents, str):
+ parents = [parents]
+ for p in parents:
+ res += get_job_extend_seq(p, all_jobs, depth + 1)
+ res.append(job_name)
+
+ return res
+
+depth_limit = 20
+def merge_job_deep(res, parent, depth=0):
+ res_keys = set(res.keys())
+ if depth > depth_limit:
+ raise CIValidationError(f'depth is over the limit when merging job.')
+ for k in parent:
+ if k not in res_keys:
+ res[k] = parent[k]
+ elif isinstance(parent[k], dict) and isinstance(res[k], dict):
+ child = res[k]
+ # Necessary to avoid changing anything inside res[k] that was shallow-copied
+ res[k] = {}
+ merge_job_deep(res[k], child, depth + 1)
+ merge_job_deep(res[k], parent[k], depth + 1)
+ else:
+ # Presenting in both parent and child, and it's not a dict,
+ # so the child should take preference.
+ pass
+
+def expand_job(job_name, all_jobs, defaults):
+ seq = get_job_extend_seq(job_name, all_jobs)
+ res = {}
+ for ancestor_name in reversed(seq):
+ merge_job_deep(res, all_jobs[ancestor_name])
+ merge_job_deep(res, defaults)
+ if 'extends' in res:
+ del res['extends']
+ return res
+
class CIJob:
- def __init__(self, job_name, job_stage, job_struct, defaults):
+ def __init__(self, job_name, job_stage, job_struct):
self.name = job_name
self.stage = job_stage
self.struct_raw = job_struct
- self.defaults_raw = defaults
- self.image = job_struct.get('image', defaults.get('image'))
- self.before_script = normalize_script(job_struct.get('before_script', defaults.get('before_script')))
- self.script = normalize_script(job_struct.get('script', defaults.get('script')))
- self.after_script = normalize_script(job_struct.get('after_script', defaults.get('after_script')))
- self.artifacts = job_struct.get('artifacts', defaults.get('artifacts', {}))
+ self.image = job_struct.get('image')
+ self.before_script = normalize_script(job_struct.get('before_script'))
+ self.script = normalize_script(job_struct.get('script'))
+ self.after_script = normalize_script(job_struct.get('after_script'))
+ self.artifacts = job_struct.get('artifacts') or {}
self.dependencies = job_struct.get('dependencies')
def get_predefined_ci_variables(self):
@@ -68,7 +117,6 @@
'name': self.name,
'stage': self.stage,
'struct_raw': self.struct_raw,
- 'defaults_raw': self.defaults_raw,
}
@classmethod
@@ -76,8 +124,7 @@
return cls(
prop['name'],
prop['stage'],
- prop['struct_raw'],
- prop['defaults_raw']
+ prop['struct_raw']
)
def is_pages(self):
@@ -87,9 +134,6 @@
DEFAULT_STAGES = ['.pre', 'build', 'test', 'deploy', '.post']
DEFAULT_JOB_STAGE = 'test'
-class CIValidationError(Exception):
- pass
-
class CIFile:
'''
Class for parsing CI file.
@@ -117,18 +161,23 @@
if kw not in defaults and kw in f:
defaults[kw] = f[kw]
+ all_jobs = {}
for job_name, job_struct in f.items():
# 'pages' is a special job
if job_name != 'pages' and job_name in toplevel_entries():
continue
+ all_jobs[job_name] = job_struct
+
+ for job_name in all_jobs:
# jobs starting with . will only be used for base jobs of other jobs,
# they themselves are not run
if job_name.startswith('.'):
continue
+ job_struct = expand_job(job_name, all_jobs, defaults)
job_stage = job_struct.get('stage', DEFAULT_JOB_STAGE)
if job_stage not in self.stages:
raise CIValidationError(f'Job "{job_name}": Stage "{job_stage}" is not specified in CI file')
- self.jobs[job_name] = CIJob(job_name, job_stage, job_struct, defaults)
+ self.jobs[job_name] = CIJob(job_name, job_stage, job_struct)
self.validate_logic()
diff --git a/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py b/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
--- a/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
+++ b/lilybuild/lilybuild/tests/ci_syntax/ci_file_test.py
@@ -1,7 +1,7 @@
import unittest
import yaml
-from lilybuild.ci_syntax.ci_file import CIFile, CIValidationError
+from lilybuild.ci_syntax.ci_file import CIFile, CIValidationError, get_job_extend_seq, merge_job_deep
from lilybuild.tests.resources import get_res
class CIFileTest(unittest.TestCase):
@@ -99,11 +99,206 @@
r = CIFile(get_res('late_dep'))
self.assertEqual(str(m.exception), 'Dependency "a2" of job "a" is not before the job in stage')
+ def test_artifacts(self):
+ r = CIFile(get_res('has_artifacts_archive'))
+ self.assertEqual(r.jobs['has_archive'].artifacts, { 'paths': ['a'] })
+ self.assertEqual(r.jobs['no_archive'].artifacts, {})
+ self.assertEqual(r.jobs['no_artifacts'].artifacts, {})
+
def test_has_artifacts_archive(self):
r = CIFile(get_res('has_artifacts_archive'))
self.assertTrue(r.jobs['has_archive'].has_artifacts_archive())
self.assertFalse(r.jobs['no_archive'].has_artifacts_archive())
self.assertFalse(r.jobs['no_artifacts'].has_artifacts_archive())
+ def test_extends(self):
+ r = CIFile(get_res('extends'))
+ self.assertEqual(r.jobs['build-a'].struct_raw, {
+ 'image': 'alpine',
+ 'variables': { 'SECRET': 'abcdef', 'CC': 'gcc' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'build',
+ 'before_script': ['apk add gcc'],
+ 'script': ['make a', 'make install'],
+ })
+ self.assertEqual(r.jobs['build-b'].struct_raw, {
+ 'image': 'alpine',
+ 'variables': { 'SECRET': 'abcdef', 'CC': 'cc' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'build',
+ 'before_script': ['apk add gcc'],
+ 'script': ['make b', 'make install'],
+ })
+ self.assertEqual(r.jobs['test'].struct_raw, {
+ 'image': 'tester',
+ 'variables': { 'SECRET': 'abcdef' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'test',
+ 'script': ['make test'],
+ })
+ self.assertEqual(r.jobs['container-amd64'].struct_raw, {
+ 'image': 'podman',
+ 'variables': { 'SECRET': 'abcdef', 'DOCKER': 'podman', 'PLATFORM': 'amd64', 'LOGIN': 'defghi' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'deploy',
+ 'script': ['podman login', 'make docker'],
+ 'tags': ['podman']
+ })
+ self.assertEqual(r.jobs['container-aarch64'].struct_raw, {
+ 'image': 'podman',
+ 'variables': { 'SECRET': 'abcdef', 'DOCKER': 'podman', 'PLATFORM': 'aarch64', 'LOGIN': 'defghi' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'deploy',
+ 'script': ['podman login', 'make docker'],
+ 'tags': ['podman']
+ })
+ self.assertEqual(r.jobs['appimage-amd64'].struct_raw, {
+ 'image': 'alpine',
+ 'variables': { 'SECRET': 'abcdef', 'PLATFORM': 'amd64', 'LOGIN': 'defghi' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'deploy',
+ 'script': ['make appimage'],
+ 'tags': ['amd64']
+ })
+ self.assertEqual(r.jobs['appimage-aarch64'].struct_raw, {
+ 'image': 'alpine',
+ 'variables': { 'SECRET': 'abcdef', 'PLATFORM': 'aarch64', 'LOGIN': 'defghi' },
+ 'after_script': ['rm -r cache'],
+ 'stage': 'deploy',
+ 'script': ['make appimage'],
+ 'tags': ['aarch64']
+ })
+
+class GetJobExtendSeqTest(unittest.TestCase):
+ def test_no_extends(self):
+ all_jobs = {
+ 'nothing': {}
+ }
+ self.assertEqual(get_job_extend_seq('nothing', all_jobs), ['nothing'])
+
+ def test_multi_extends(self):
+ all_jobs = {
+ 'a': { 'extends': ['b', 'c'] },
+ 'b': { 'extends': ['d', 'e'] },
+ 'c': { 'extends': ['f', 'g'] },
+ 'd': {},
+ 'e': {},
+ 'f': {},
+ 'g': {},
+ }
+ self.assertEqual(get_job_extend_seq('a', all_jobs), ['d', 'e', 'b', 'f', 'g', 'c', 'a'])
+
+ def test_common_ancestor(self):
+ all_jobs = {
+ 'a': { 'extends': ['b', 'c'] },
+ 'b': { 'extends': ['d', 'e'] },
+ 'c': { 'extends': ['e', 'd'] },
+ 'd': {},
+ 'e': {},
+ 'f': {},
+ 'g': {},
+ }
+ self.assertEqual(get_job_extend_seq('a', all_jobs), ['d', 'e', 'b', 'e', 'd', 'c', 'a'])
+
+ def test_self_cycle(self):
+ all_jobs = {
+ 'a': { 'extends': ['a'] },
+ }
+ with self.assertRaises(CIValidationError) as m:
+ r = get_job_extend_seq('a', all_jobs)
+ self.assertTrue(str(m.exception).startswith('`extends` depth is over the limit of'))
+
+ def test_complex_cycle(self):
+ all_jobs = {
+ 'a': { 'extends': ['b', 'c'] },
+ 'b': { 'extends': ['d', 'c'] },
+ 'c': { 'extends': ['d'] },
+ 'd': { 'extends': ['a'] },
+ }
+ with self.assertRaises(CIValidationError) as m:
+ r = get_job_extend_seq('a', all_jobs)
+ self.assertTrue(str(m.exception).startswith('`extends` depth is over the limit of'))
+
+ def test_nonexistent_parent(self):
+ all_jobs = {
+ 'a': { 'extends': ['b', 'c'] },
+ 'b': { 'extends': ['d', 'c'] },
+ 'c': { 'extends': ['d'] },
+ }
+ with self.assertRaises(CIValidationError) as m:
+ r = get_job_extend_seq('a', all_jobs)
+ self.assertEqual(str(m.exception), 'Job "d" does not exist, but occurs in `extends`.')
+
+ def test_null_parent(self):
+ all_jobs = {
+ 'a': { 'extends': ['b', 'c'] },
+ 'b': { 'extends': ['d', 'c'] },
+ 'c': { 'extends': ['d'] },
+ 'd': None,
+ }
+ self.assertEqual(get_job_extend_seq('a', all_jobs), ['d', 'd', 'c', 'b', 'd', 'c', 'a'])
+
+class MergeJobTest(unittest.TestCase):
+ def test_simple(self):
+ parent = {
+ 'a': {
+ 'a/1': {
+ 'a/1/1': '3',
+ 'a/1/2': 4,
+ },
+ 'a/2': {},
+ },
+ 'b': 1,
+ }
+ child = {
+ 'a': {
+ 'a/1': {
+ 'a/1/2': '6',
+ 'a/1/3': 5,
+ },
+ 'a/2': 'mew',
+ },
+ 'c': 'test',
+ }
+ merge_job_deep(child, parent)
+ self.assertEqual(parent, {
+ 'a': {
+ 'a/1': {
+ 'a/1/1': '3',
+ 'a/1/2': 4,
+ },
+ 'a/2': {},
+ },
+ 'b': 1,
+ })
+ self.assertEqual(child, {
+ 'a': {
+ 'a/1': {
+ 'a/1/1': '3',
+ 'a/1/2': '6',
+ 'a/1/3': 5,
+ },
+ 'a/2': 'mew',
+ },
+ 'b': 1,
+ 'c': 'test',
+ })
+
+ def test_no_overflow(self):
+ a = {
+ '1': '2',
+ }
+ b = {
+ '2': '4',
+ 'a': a,
+ }
+ a['a'] = b
+
+ res = {}
+ merge_job_deep(res, b)
+ with self.assertRaises(CIValidationError) as m:
+ merge_job_deep(res, a)
+ self.assertEqual(str(m.exception), 'depth is over the limit when merging job.')
+
if __name__ == '__main__':
unittest.main()
diff --git a/lilybuild/lilybuild/tests/ci_syntax/res/extends.yaml b/lilybuild/lilybuild/tests/ci_syntax/res/extends.yaml
new file mode 100644
--- /dev/null
+++ b/lilybuild/lilybuild/tests/ci_syntax/res/extends.yaml
@@ -0,0 +1,83 @@
+
+variables:
+ SECRET: 'abcdef'
+
+default:
+ image: alpine
+ after_script:
+ - rm -r cache
+
+stages:
+ - build
+ - test
+ - deploy
+
+.build:
+ stage: build
+ before_script:
+ - apk add gcc
+ variables:
+ CC: gcc
+
+build-a:
+ extends: .build
+ script:
+ - make a
+ - make install
+
+build-b:
+ extends: .build
+ variables:
+ CC: cc
+ script:
+ - make b
+ - make install
+
+test:
+ image: tester
+ stage: test
+ script:
+ - make test
+
+.deploy:
+ stage: deploy
+ variables:
+ LOGIN: 'defghi'
+
+.container:
+ variables:
+ DOCKER: 'podman'
+ image: podman
+ script:
+ - podman login
+ - make docker
+ tags:
+ - podman
+
+.appimage:
+ script:
+ - make appimage
+
+.deploy-amd64:
+ variables:
+ PLATFORM: amd64
+ extends: .deploy
+ tags: [amd64]
+
+.deploy-aarch64:
+ variables:
+ PLATFORM: aarch64
+ extends: .deploy
+ tags: [aarch64]
+
+container-amd64:
+ extends: [.deploy-amd64, .container]
+
+container-aarch64:
+ extends: [.deploy-aarch64, .container]
+
+appimage-amd64:
+ extends: [.deploy-amd64, .appimage]
+
+appimage-aarch64:
+ extends: [.deploy-aarch64, .appimage]

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 17, 6:31 PM (15 h, 58 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
260770
Default Alt Text
D223.1752802268.diff (15 KB)

Event Timeline