Page MenuHomePhorge
Files F85803157

No OneTemporary
Actions

Size
8 KB
Referenced Files
None
Subscribers
None

View Options

diff --git a/lilybuild/podman-helper b/lilybuild/podman-helper
index d858284..a0dd99d 100755
--- a/lilybuild/podman-helper
+++ b/lilybuild/podman-helper
@@ -1,253 +1,255 @@
#!/usr/bin/env python3
import subprocess
import sys
import os
import json
import random
import traceback
import string
import time
work_vol_mount_dir = '/build'
script_vol_mount_dir = '/script'
script_name = script_vol_mount_dir + '/run.sh'
volume_helper_image = os.environ.get('LILYBUILD_VOLUME_HELPER_IMAGE', 'r.lily-is.land/infra/lilybuild/volume-helper:servant')
key_file_pub = '/secrets/lilybuild-volume-helper-key.pub'
key_file_sub = '/secrets/lilybuild-volume-helper-key'
ssh_port = '2222'
ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null'
worker_container_name = os.environ['HOSTNAME']
volumes_to_remove = []
helper_container_id = None
ssh_max_wait = 10
ssh_wait_interval_sec = 1
col_info = '\x1b[1;34m'
col_success = '\x1b[1;32m'
col_error = '\x1b[1;31m'
col_reset = '\x1b[0m'
def pinfo(*args, **kwargs):
print(col_info, *args, col_reset, **kwargs)
sys.stdout.flush()
def perror(*args, **kwargs):
print(col_error, *args, col_reset, **kwargs)
sys.stdout.flush()
def psuccess(*args, **kwargs):
print(col_success, *args, col_reset, **kwargs)
sys.stdout.flush()
def gen_random_id():
# https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits
return ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(10))
def verbose_run(*args, **kwargs):
print('run:', args, kwargs)
sys.stdout.flush()
return subprocess.run(*args, **kwargs)
def create_volume(t):
res = verbose_run([
'podman', 'volume', 'create',
'--label', 'lilybuild=' + t,
], check=True, capture_output=True, encoding='utf-8')
volname = res.stdout.strip()
volumes_to_remove.append(volname)
return volname
def clean_volumes():
verbose_run([
'podman', 'volume', 'rm', '-f', '--',
] + volumes_to_remove, capture_output=True)
def clean_helper_container():
verbose_run([
'podman', 'container', 'rm', '-f', helper_container_id,
], capture_output=True)
def start_helper_service(work_volname, script_volname):
res = verbose_run([
'podman', 'container', 'inspect', worker_container_name,
], check=True, capture_output=True, encoding='utf-8')
container_stat = json.loads(res.stdout)[0]
pod = container_stat.get('Pod')
networks = list(container_stat.get('NetworkSettings').get('Networks').keys())
alias = gen_random_id()
container_name = 'lilybuild-helper-' + alias
with open(key_file_pub) as f:
pub_key = f.readline().strip()
res = verbose_run([
'podman', 'run', '--rm', '-d', '--name', container_name,
f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
f'--pod={pod}',
f'--net={networks[0]}',
f'--network-alias={alias}',
+ '--image-volume=ignore',
'--label', 'lilybuild=helper',
'-e', 'PUID=0',
'-e', 'PGID=0',
'-e', f'PUBLIC_KEY={pub_key}',
'-e', 'USER_NAME=helper',
'-e', 'SUDO_ACCESS=true',
volume_helper_image,
], check=True, capture_output=True, encoding='utf-8')
pinfo('Waiting for ssh service to be up...')
service_up = False
for i in range(ssh_max_wait):
chk = verbose_run(['nc', alias, ssh_port], input=b'', capture_output=True)
if chk.returncode == 0 and chk.stdout is not None and chk.stdout.startswith(b'SSH'):
service_up = True
break
else:
time.sleep(ssh_wait_interval_sec)
if not service_up:
raise RuntimeError('Service is still not up!')
psuccess('Service is up.')
return (container_name, alias)
def import_volume(alias, local_dir, vol_mount_dir):
# I'll just use the shell instead of pipe2+fork+exec+wait, much easier
verbose_run([
'rsync', '-a', '--delete',
'--rsh', ssh_command,
f'{local_dir}/',
f'helper@{alias}:{vol_mount_dir}',
], check=True)
def export_volume(alias, local_dir, vol_mount_dir):
verbose_run([
'rsync', '-a', '--delete',
'--rsh', ssh_command,
f'helper@{alias}:{vol_mount_dir}/',
local_dir,
], check=True)
def image_to_podman_args(image):
name = image['name']
args = []
if 'entrypoint' in image:
# ci.json requires that the entrypoint is an array of strings
ep = json.dumps(image['entrypoint'])
args += ['--entrypoint', ep]
args += [name]
return args
def run_in_container(image, work_volname, script_volname):
timeout = 60 * 60 * 2 # 2 hours by default
steady_deadline = time.monotonic() + timeout
start_process = verbose_run([
'podman', 'run', '-d',
f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
] + image_to_podman_args(image) + [
script_name,
], capture_output=True, encoding='utf-8')
if start_process.returncode != 0:
perror('Cannot run container. Error message:')
print(start_process.stderr)
return start_process.returncode
container_id = start_process.stdout.strip()
steady_now = time.monotonic()
log_args = []
retcode = None
try:
while steady_deadline > steady_now:
log_process = verbose_run([
'podman', 'logs', '--follow'
] + log_args + ['--', container_id], timeout=steady_deadline - steady_now)
# Exited from `podman logs`: why? Is the container still running?
inspect_running = verbose_run([
'podman', 'container', 'inspect',
'--format', '{{.State.Status}}', '--', container_id,
], capture_output=True, encoding='utf-8', check=True)
if inspect_running.stdout.strip() == 'exited':
inspect_retcode = verbose_run([
'podman', 'container', 'inspect',
'--format', '{{.State.ExitCode}}', '--', container_id,
], capture_output=True, encoding='utf-8', check=True)
retcode = int(inspect_retcode.stdout.strip())
break
else:
perror('`podman logs` unexpectedly quits when the container is still running, resuming logs...')
log_args = ['--tail', '10']
steady_now = time.monotonic()
if retcode is None:
perror('Command timed out.')
retcode = 1
except subprocess.TimeoutExpired:
perror('Command timed out.')
retcode = 1
except subprocess.CalledProcessError as e:
perror('Cannot inspect container', e)
finally:
pinfo('Cleaning up container...')
stop_proc = verbose_run(['podman', 'container', 'stop', '--', container_id])
if stop_proc.returncode != 0:
perror('Cannot stop container.')
- rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '--', container_id])
+ # -v removes anonymous volumes associated with the container
+ rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '-v', '--', container_id])
pinfo('Cleaned.')
return retcode
def main():
image = json.loads(sys.argv[1])
work_dir = sys.argv[2]
script_dir = sys.argv[3]
result_dir = sys.argv[4]
pinfo('Creating volumes...')
work_vol = create_volume('work')
script_vol = create_volume('script')
psuccess('Created.')
pinfo('Starting helper service...')
global helper_container_id
(helper_container_id, alias) = start_helper_service(work_vol, script_vol)
psuccess('Started...')
pinfo('Importing volumes...')
import_volume(alias, work_dir, work_vol_mount_dir)
import_volume(alias, script_dir, script_vol_mount_dir)
psuccess('Imported.')
pinfo('Running container...')
retcode = run_in_container(image, work_vol, script_vol)
pinfo(f'Returned {retcode}.')
if retcode != 0:
perror('Job failed.')
else:
psuccess('Job succeeded.')
# We should collect the result regardless whether it succeeded
pinfo('Collecting build changes...')
export_volume(alias, result_dir, work_vol_mount_dir)
psuccess('Collected.')
return retcode
retcode = 1
try:
retcode = main()
except Exception as e:
perror('Error!', e)
print(traceback.format_exc())
raise
finally:
if helper_container_id:
pinfo('Cleaning helper container')
clean_helper_container()
psuccess('Cleaned.')
pinfo('Cleaning volumes...')
clean_volumes()
psuccess('Cleaned.')
sys.exit(retcode)

File Metadata

Mime Type
text/x-diff
Expires
Fri, Oct 9, 1:56 AM (1 d, 2 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
1784390
Default Alt Text
(8 KB)

Event Timeline