Changeset View
Changeset View
Standalone View
Standalone View
lilybuild/podman-helper
| #!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
| import subprocess | import subprocess | ||||
| import sys | import sys | ||||
| import os | import os | ||||
| import json | import json | ||||
| import random | import random | ||||
| import traceback | import traceback | ||||
| import string | import string | ||||
| import time | import time | ||||
| work_vol_mount_dir = '/build' | work_vol_mount_dir = '/build' | ||||
| script_vol_mount_dir = '/script' | script_vol_mount_dir = '/script' | ||||
| script_name = script_vol_mount_dir + '/run.sh' | script_name = script_vol_mount_dir + '/run.sh' | ||||
| env_file_basename = 'env' | |||||
| volume_helper_image = os.environ.get('LILYBUILD_VOLUME_HELPER_IMAGE', 'r.lily-is.land/infra/lilybuild/volume-helper:servant') | volume_helper_image = os.environ.get('LILYBUILD_VOLUME_HELPER_IMAGE', 'r.lily-is.land/infra/lilybuild/volume-helper:servant') | ||||
| key_file_pub = '/secrets/lilybuild-volume-helper-key.pub' | key_file_pub = '/secrets/lilybuild-volume-helper-key.pub' | ||||
| key_file_sub = '/secrets/lilybuild-volume-helper-key' | key_file_sub = '/secrets/lilybuild-volume-helper-key' | ||||
| ssh_port = '2222' | ssh_port = '2222' | ||||
| ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null' | ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null' | ||||
| worker_container_name = os.environ['HOSTNAME'] | worker_container_name = os.environ['HOSTNAME'] | ||||
| volumes_to_remove = [] | volumes_to_remove = [] | ||||
| helper_container_id = None | helper_container_id = None | ||||
| ▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines | |||||
| def clean_service_network(network_id): | def clean_service_network(network_id): | ||||
| res = verbose_run([ | res = verbose_run([ | ||||
| 'podman', 'network', 'rm', '-f', '--', network_id | 'podman', 'network', 'rm', '-f', '--', network_id | ||||
| ], capture_output=True, encoding='utf-8') | ], capture_output=True, encoding='utf-8') | ||||
| if res.returncode != 0: | if res.returncode != 0: | ||||
| perror('Cannot remove service network.') | perror('Cannot remove service network.') | ||||
| def start_service_container(service, network_id): | def start_service_container(service, network_id, env_filename): | ||||
| image = service['name'] | image = service['name'] | ||||
| ep_args = [] | ep_args = [] | ||||
| if service['entrypoint']: | if service['entrypoint']: | ||||
| if isinstance(service['entrypoint'], str): | if isinstance(service['entrypoint'], str): | ||||
| entrypoint = service['entrypoint'] | entrypoint = service['entrypoint'] | ||||
| else: | else: | ||||
| entrypoint = json.dumps(service['entrypoint']) | entrypoint = json.dumps(service['entrypoint']) | ||||
| ep_args += [f'--entrypoint={entrypoint}'] | ep_args += [f'--entrypoint={entrypoint}'] | ||||
| cmd_args = [] | cmd_args = [] | ||||
| if service['command']: | if service['command']: | ||||
| if isinstance(service['command'], str): | if isinstance(service['command'], str): | ||||
| cmd_args += [service['command']] | cmd_args += [service['command']] | ||||
| else: | else: | ||||
| cmd_args += service['command'] | cmd_args += service['command'] | ||||
| res = verbose_run([ | res = verbose_run([ | ||||
| 'podman', 'run', '-d', '--label', 'lilybuild=job-service', | 'podman', 'run', '-d', '--label', 'lilybuild=job-service', | ||||
| f'--env-file={env_filename}', | |||||
| f'--network={network_id}', | f'--network={network_id}', | ||||
| ] + [ | ] + [ | ||||
| f'--network-alias={alias}' for alias in service['aliases'] | f'--network-alias={alias}' for alias in service['aliases'] | ||||
| ] + ep_args + [ | ] + ep_args + [ | ||||
| '--', | '--', | ||||
| image, | image, | ||||
| ] + cmd_args, check=True, capture_output=True, encoding='utf-8') | ] + cmd_args, check=True, capture_output=True, encoding='utf-8') | ||||
| return res.stdout.strip() | return res.stdout.strip() | ||||
| Show All 21 Lines | |||||
| def prune_service_containers(container_ids): | def prune_service_containers(container_ids): | ||||
| stop_proc = verbose_run(['podman', 'container', 'stop', '--'] + container_ids) | stop_proc = verbose_run(['podman', 'container', 'stop', '--'] + container_ids) | ||||
| if stop_proc.returncode != 0: | if stop_proc.returncode != 0: | ||||
| perror('Cannot stop container.') | perror('Cannot stop container.') | ||||
| # -v removes anonymous volumes associated with the container | # -v removes anonymous volumes associated with the container | ||||
| rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '-v', '--'] + container_ids) | rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '-v', '--'] + container_ids) | ||||
| def run_in_container(image, work_volname, script_volname, network_id): | def run_in_container(image, work_volname, script_volname, network_id, env_filename): | ||||
| timeout = 60 * 60 * 2 # 2 hours by default | timeout = 60 * 60 * 2 # 2 hours by default | ||||
| steady_deadline = time.monotonic() + timeout | steady_deadline = time.monotonic() + timeout | ||||
| network_args = [] | network_args = [] | ||||
| if network_id: | if network_id: | ||||
| network_args += [f'--network={network_id}'] | network_args += [f'--network={network_id}'] | ||||
| start_process = verbose_run([ | start_process = verbose_run([ | ||||
| 'podman', 'run', '-d', | 'podman', 'run', '-d', | ||||
| f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}', | f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}', | ||||
| f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}', | f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}', | ||||
| f'--env-file={env_filename}', | |||||
| ] + network_args + image_to_podman_args(image) + [ | ] + network_args + image_to_podman_args(image) + [ | ||||
| script_name, | script_name, | ||||
| ], capture_output=True, encoding='utf-8') | ], capture_output=True, encoding='utf-8') | ||||
| if start_process.returncode != 0: | if start_process.returncode != 0: | ||||
| perror('Cannot run container. Error message:') | perror('Cannot run container. Error message:') | ||||
| print(start_process.stderr) | print(start_process.stderr) | ||||
| return start_process.returncode | return start_process.returncode | ||||
| container_id = start_process.stdout.strip() | container_id = start_process.stdout.strip() | ||||
| Show All 40 Lines | finally: | ||||
| pinfo('Cleaned.') | pinfo('Cleaned.') | ||||
| return retcode | return retcode | ||||
| def main(): | def main(): | ||||
| image = json.loads(sys.argv[1]) | image = json.loads(sys.argv[1]) | ||||
| work_dir = sys.argv[2] | work_dir = sys.argv[2] | ||||
| script_dir = sys.argv[3] | script_dir = sys.argv[3] | ||||
| result_dir = sys.argv[4] | result_dir = sys.argv[4] | ||||
| env_filename = os.path.join(script_dir, env_file_basename) | |||||
| services = [] | services = [] | ||||
| if len(sys.argv) >= 6: | if len(sys.argv) >= 6: | ||||
| services = json.loads(sys.argv[5]) | services = json.loads(sys.argv[5]) | ||||
| pinfo('Creating volumes...') | pinfo('Creating volumes...') | ||||
| work_vol = create_volume('work') | work_vol = create_volume('work') | ||||
| script_vol = create_volume('script') | script_vol = create_volume('script') | ||||
| psuccess('Created.') | psuccess('Created.') | ||||
| pinfo('Starting helper service...') | pinfo('Starting helper service...') | ||||
| global helper_container_id | global helper_container_id | ||||
| (helper_container_id, alias) = start_helper_service(work_vol, script_vol) | (helper_container_id, alias) = start_helper_service(work_vol, script_vol) | ||||
| psuccess('Started...') | psuccess('Started...') | ||||
| if services: | if services: | ||||
| pinfo('Creating service network...') | pinfo('Creating service network...') | ||||
| global service_network_id | global service_network_id | ||||
| service_network_id = create_service_network() | service_network_id = create_service_network() | ||||
| psuccess('Created.') | psuccess('Created.') | ||||
| pinfo('Starting job-defined services...') | pinfo('Starting job-defined services...') | ||||
| global service_containers | global service_containers | ||||
| for service in services: | for service in services: | ||||
| service_containers.append(start_service_container(service, service_network_id)) | service_containers.append(start_service_container(service, service_network_id, env_filename)) | ||||
| pinfo('Waiting for job-defined services...') | pinfo('Waiting for job-defined services...') | ||||
| ensure_service_containers_up(service_containers) | ensure_service_containers_up(service_containers) | ||||
| pinfo('Importing volumes...') | pinfo('Importing volumes...') | ||||
| import_volume(alias, work_dir, work_vol_mount_dir) | import_volume(alias, work_dir, work_vol_mount_dir) | ||||
| import_volume(alias, script_dir, script_vol_mount_dir) | import_volume(alias, script_dir, script_vol_mount_dir) | ||||
| psuccess('Imported.') | psuccess('Imported.') | ||||
| pinfo('Running container...') | pinfo('Running container...') | ||||
| retcode = run_in_container(image, work_vol, script_vol, service_network_id) | retcode = run_in_container(image, work_vol, script_vol, service_network_id, env_filename) | ||||
| pinfo(f'Returned {retcode}.') | pinfo(f'Returned {retcode}.') | ||||
| if retcode != 0: | if retcode != 0: | ||||
| perror('Job failed.') | perror('Job failed.') | ||||
| else: | else: | ||||
| psuccess('Job succeeded.') | psuccess('Job succeeded.') | ||||
| Show All 37 Lines | |||||