Page MenuHomePhorge
Files F29695648

D264.1768057084.diff
No OneTemporary
Actions

Size
7 KB
Referenced Files
None
Subscribers
None

D264.1768057084.diff
View Options

diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -5,7 +5,7 @@
from twisted.internet import defer
from .ci_syntax import ci_file
from .ci_syntax import rules as ci_rules
-from .helpers import rsync_rules_from_artifacts, get_job_script, normalize_image, normalize_services
+from .helpers import rsync_rules_from_artifacts, get_job_script, normalize_image, normalize_services, ci_vars_to_env_file
from .phorge import SendCoverageToPhorge
import re
import sys
@@ -189,17 +189,26 @@
def job_to_steps(self, job, job_index, variables):
script_name = self.script_dir + '/run.sh'
+ env_filename = self.script_dir + '/env'
source_step = self.lbc.create_source_step()
script_step = steps.StringDownload(
- get_job_script(variables, job),
+ get_job_script(job),
name='Set up script',
workerdest=script_name,
workdir=self.work_root_dir,
doStepIf=on_success,
)
+ env_step = steps.StringDownload(
+ ci_vars_to_env_file(variables),
+ name='Set up env file',
+ workerdest=env_filename,
+ workdir=self.work_root_dir,
+ doStepIf=on_success,
+ )
+
chmod_step = steps.ShellCommand(
name='Make script executable',
command=['chmod', '+x', script_name],
@@ -275,7 +284,7 @@
alwaysRun=True,
)
- steps_to_run = [source_step, script_step, chmod_step] + artifact_steps + [run_step, clean_script_step]
+ steps_to_run = [source_step, script_step, chmod_step, env_step] + artifact_steps + [run_step, clean_script_step]
if 'paths' in job.artifacts:
steps_to_run += self.get_upload_artifacts_jobs(
'files',
diff --git a/lilybuild/lilybuild/helpers.py b/lilybuild/lilybuild/helpers.py
--- a/lilybuild/lilybuild/helpers.py
+++ b/lilybuild/lilybuild/helpers.py
@@ -42,11 +42,15 @@
},
})
-def ci_vars_to_cmds(v):
+def ci_vars_to_env_file(v):
res = []
for name in v:
- value = shlex.quote('{}'.format(v[name]))
- res.append(f'export {name}={value}')
+ value = v[name]
+ if not isinstance(value, str):
+ value = str(value)
+ if '\n' not in value:
+ res.append(f'{name}={value}')
+ # Otherwise, ignore multiline variables because podman cannot pass it in env file
return '\n'.join(res)
DEFAULT_SCRIPT_HEADER = '''\
@@ -57,12 +61,9 @@
'''
-def get_job_script(variables, job):
- var_cmds = ci_vars_to_cmds(variables)
-
+def get_job_script(job):
return (
DEFAULT_SCRIPT_HEADER +
- '\n' + var_cmds + '\n\n' +
'\n\n'.join(job.before_script) + '\n\n' +
'\n\n'.join(job.script) +
'\n\nset +e\n\n' +
diff --git a/lilybuild/lilybuild/tests/helpers_test.py b/lilybuild/lilybuild/tests/helpers_test.py
--- a/lilybuild/lilybuild/tests/helpers_test.py
+++ b/lilybuild/lilybuild/tests/helpers_test.py
@@ -6,7 +6,7 @@
rsync_rules_from_artifacts,
normalize_base_url,
phorge_token_to_arcrc,
- ci_vars_to_cmds,
+ ci_vars_to_env_file,
get_job_script,
DEFAULT_SCRIPT_HEADER,
normalize_image,
@@ -88,20 +88,19 @@
class CiVarsTest(unittest.TestCase):
def test_simple(self):
- self.assertEqual(ci_vars_to_cmds({}), '')
- self.assertEqual(ci_vars_to_cmds({'VAR': 'val'}), 'export VAR=val')
- self.assertEqual(ci_vars_to_cmds({'VAR': 'foo bar'}), "export VAR='foo bar'")
+ self.assertEqual(ci_vars_to_env_file({}), '')
+ self.assertEqual(ci_vars_to_env_file({'VAR': 'val'}), 'VAR=val')
+ self.assertEqual(ci_vars_to_env_file({'VAR': 'foo bar'}), "VAR=foo bar")
+ self.assertEqual(ci_vars_to_env_file({'VAR': '\nbar', 'MEW': 'abc def'}), "MEW=abc def")
+ self.assertEqual(ci_vars_to_env_file({'VAR': 12345}), "VAR=12345")
class GetJobScriptTest(unittest.TestCase):
def test_only_script(self):
r = CIFile(get_res('pages_attr'))
- job_script = get_job_script({}, r.jobs['is-pages'])
+ job_script = get_job_script(r.jobs['is-pages'])
self.assertEqual(job_script, f'''\
{DEFAULT_SCRIPT_HEADER}
-
-
-
make docs
set +e
@@ -112,12 +111,9 @@
def test_before_and_after(self):
r = CIFile(get_res('defaults'))
- job_script = get_job_script({}, r.jobs['build-a'])
+ job_script = get_job_script(r.jobs['build-a'])
self.assertEqual(job_script, f'''\
-{DEFAULT_SCRIPT_HEADER}
-
-
-ls
+{DEFAULT_SCRIPT_HEADER}ls
make
diff --git a/lilybuild/podman-helper b/lilybuild/podman-helper
--- a/lilybuild/podman-helper
+++ b/lilybuild/podman-helper
@@ -12,6 +12,7 @@
work_vol_mount_dir = '/build'
script_vol_mount_dir = '/script'
script_name = script_vol_mount_dir + '/run.sh'
+env_file_basename = 'env'
volume_helper_image = os.environ.get('LILYBUILD_VOLUME_HELPER_IMAGE', 'r.lily-is.land/infra/lilybuild/volume-helper:servant')
key_file_pub = '/secrets/lilybuild-volume-helper-key.pub'
key_file_sub = '/secrets/lilybuild-volume-helper-key'
@@ -155,7 +156,7 @@
if res.returncode != 0:
perror('Cannot remove service network.')
-def start_service_container(service, network_id):
+def start_service_container(service, network_id, env_filename):
image = service['name']
ep_args = []
if service['entrypoint']:
@@ -172,6 +173,7 @@
cmd_args += service['command']
res = verbose_run([
'podman', 'run', '-d', '--label', 'lilybuild=job-service',
+ f'--env-file={env_filename}',
f'--network={network_id}',
] + [
f'--network-alias={alias}' for alias in service['aliases']
@@ -209,7 +211,7 @@
# -v removes anonymous volumes associated with the container
rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '-v', '--'] + container_ids)
-def run_in_container(image, work_volname, script_volname, network_id):
+def run_in_container(image, work_volname, script_volname, network_id, env_filename):
timeout = 60 * 60 * 2 # 2 hours by default
steady_deadline = time.monotonic() + timeout
network_args = []
@@ -220,6 +222,7 @@
'podman', 'run', '-d',
f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
+ f'--env-file={env_filename}',
] + network_args + image_to_podman_args(image) + [
script_name,
], capture_output=True, encoding='utf-8')
@@ -276,6 +279,9 @@
work_dir = sys.argv[2]
script_dir = sys.argv[3]
result_dir = sys.argv[4]
+
+ env_filename = os.path.join(script_dir, env_file_basename)
+
services = []
if len(sys.argv) >= 6:
services = json.loads(sys.argv[5])
@@ -299,7 +305,7 @@
pinfo('Starting job-defined services...')
global service_containers
for service in services:
- service_containers.append(start_service_container(service, service_network_id))
+ service_containers.append(start_service_container(service, service_network_id, env_filename))
pinfo('Waiting for job-defined services...')
ensure_service_containers_up(service_containers)
@@ -310,7 +316,7 @@
psuccess('Imported.')
pinfo('Running container...')
- retcode = run_in_container(image, work_vol, script_vol, service_network_id)
+ retcode = run_in_container(image, work_vol, script_vol, service_network_id, env_filename)
pinfo(f'Returned {retcode}.')

File Metadata

Mime Type
text/plain
Expires
Sat, Jan 10, 6:58 AM (7 h, 21 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
922306
Default Alt Text
D264.1768057084.diff (7 KB)

Event Timeline