Changeset View
Changeset View
Standalone View
Standalone View
lilybuild/podman-helper
| Show All 16 Lines | |||||
| key_file_sub = '/secrets/lilybuild-volume-helper-key' | key_file_sub = '/secrets/lilybuild-volume-helper-key' | ||||
| ssh_port = '2222' | ssh_port = '2222' | ||||
| ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null' | ssh_command = f'ssh -p {ssh_port} -i {key_file_sub} -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null' | ||||
| worker_container_name = os.environ['HOSTNAME'] | worker_container_name = os.environ['HOSTNAME'] | ||||
| volumes_to_remove = [] | volumes_to_remove = [] | ||||
| helper_container_id = None | helper_container_id = None | ||||
| ssh_max_wait = 10 | ssh_max_wait = 10 | ||||
| ssh_wait_interval_sec = 1 | ssh_wait_interval_sec = 1 | ||||
| service_network_id = None | |||||
| service_containers = [] | |||||
| service_max_wait_sec = 60 * 5 | |||||
| service_wait_interval_sec = 10 | |||||
| col_info = '\x1b[1;34m' | col_info = '\x1b[1;34m' | ||||
| col_success = '\x1b[1;32m' | col_success = '\x1b[1;32m' | ||||
| col_error = '\x1b[1;31m' | col_error = '\x1b[1;31m' | ||||
| col_reset = '\x1b[0m' | col_reset = '\x1b[0m' | ||||
| def pinfo(*args, **kwargs): | def pinfo(*args, **kwargs): | ||||
| print(col_info, *args, col_reset, **kwargs) | print(col_info, *args, col_reset, **kwargs) | ||||
| ▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | |||||
| def image_to_podman_args(image): | def image_to_podman_args(image): | ||||
| name = image['name'] | name = image['name'] | ||||
| args = [] | args = [] | ||||
| if 'entrypoint' in image: | if 'entrypoint' in image: | ||||
| # ci.json requires that the entrypoint is an array of strings | # ci.json requires that the entrypoint is an array of strings | ||||
| ep = json.dumps(image['entrypoint']) | ep = json.dumps(image['entrypoint']) | ||||
| args += ['--entrypoint', ep] | args += ['--entrypoint', ep] | ||||
| args += [name] | args += ['--', name] | ||||
| return args | return args | ||||
| def run_in_container(image, work_volname, script_volname): | def create_service_network(): | ||||
| res = verbose_run([ | |||||
| 'podman', 'network', 'create', '--label', 'lilybuild=service-network' | |||||
| ], capture_output=True, check=True, encoding='utf-8') | |||||
| return res.stdout.strip() | |||||
| def clean_service_network(network_id): | |||||
| res = verbose_run([ | |||||
| 'podman', 'network', 'rm', '-f', '--', network_id | |||||
| ], capture_output=True, encoding='utf-8') | |||||
| if res.returncode != 0: | |||||
| perror('Cannot remove service network.') | |||||
| def start_service_container(service, network_id): | |||||
| image = service['name'] | |||||
| ep_args = [] | |||||
| if service['entrypoint']: | |||||
| if isinstance(service['entrypoint'], str): | |||||
| entrypoint = service['entrypoint'] | |||||
| else: | |||||
| entrypoint = json.dumps(service['entrypoint']) | |||||
| ep_args += [f'--entrypoint={entrypoint}'] | |||||
| cmd_args = [] | |||||
| if service['command']: | |||||
| if isinstance(service['command'], str): | |||||
| cmd_args += [service['command']] | |||||
| else: | |||||
| cmd_args += service['command'] | |||||
| res = verbose_run([ | |||||
| 'podman', 'run', '-d', '--label', 'lilybuild=job-service', | |||||
| f'--network={network_id}', | |||||
| ] + [ | |||||
| f'--network-alias={alias}' for alias in service['aliases'] | |||||
| ] + ep_args + [ | |||||
| '--', | |||||
| image, | |||||
| ] + cmd_args, check=True, capture_output=True, encoding='utf-8') | |||||
| return res.stdout.strip() | |||||
| def ensure_service_containers_up(container_ids): | |||||
| waiting_container_ids = container_ids[:] | |||||
| steady_deadline = time.monotonic() + service_max_wait_sec | |||||
| pinfo('Waiting for service containers...') | |||||
| while waiting_container_ids: | |||||
| for cid in waiting_container_ids[:]: | |||||
| res = verbose_run([ | |||||
| 'podman', 'container', 'inspect', '--', cid | |||||
| ], check=True, capture_output=True, encoding='utf-8') | |||||
| ins = json.loads(res.stdout)[0] | |||||
| if ins.get('State', {}).get('Status') == 'running': | |||||
| psuccess(f'Container {cid} is up') | |||||
| waiting_container_ids.remove(cid) | |||||
| if waiting_container_ids: | |||||
| if time.monotonic() > steady_deadline: | |||||
| perror('Containers are not yet up after deadline.') | |||||
| raise TimeoutError('Service containers startup timeout') | |||||
| pinfo('Some containers are not yet up. Waiting...') | |||||
| time.sleep(service_wait_interval_sec) | |||||
| psuccess('All service containers are up.') | |||||
| def prune_service_containers(container_ids): | |||||
| stop_proc = verbose_run(['podman', 'container', 'stop', '--'] + container_ids) | |||||
| if stop_proc.returncode != 0: | |||||
| perror('Cannot stop container.') | |||||
| # -v removes anonymous volumes associated with the container | |||||
| rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '-v', '--'] + container_ids) | |||||
| def run_in_container(image, work_volname, script_volname, network_id): | |||||
| timeout = 60 * 60 * 2 # 2 hours by default | timeout = 60 * 60 * 2 # 2 hours by default | ||||
| steady_deadline = time.monotonic() + timeout | steady_deadline = time.monotonic() + timeout | ||||
| network_args = [] | |||||
| if network_id: | |||||
| network_args += [f'--network={network_id}'] | |||||
| start_process = verbose_run([ | start_process = verbose_run([ | ||||
| 'podman', 'run', '-d', | 'podman', 'run', '-d', | ||||
| f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}', | f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}', | ||||
| f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}', | f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}', | ||||
| ] + image_to_podman_args(image) + [ | ] + network_args + image_to_podman_args(image) + [ | ||||
| script_name, | script_name, | ||||
| ], capture_output=True, encoding='utf-8') | ], capture_output=True, encoding='utf-8') | ||||
| if start_process.returncode != 0: | if start_process.returncode != 0: | ||||
| perror('Cannot run container. Error message:') | perror('Cannot run container. Error message:') | ||||
| print(start_process.stderr) | print(start_process.stderr) | ||||
| return start_process.returncode | return start_process.returncode | ||||
| container_id = start_process.stdout.strip() | container_id = start_process.stdout.strip() | ||||
| Show All 39 Lines | finally: | ||||
| pinfo('Cleaned.') | pinfo('Cleaned.') | ||||
| return retcode | return retcode | ||||
| def main(): | def main(): | ||||
| image = json.loads(sys.argv[1]) | image = json.loads(sys.argv[1]) | ||||
| work_dir = sys.argv[2] | work_dir = sys.argv[2] | ||||
| script_dir = sys.argv[3] | script_dir = sys.argv[3] | ||||
| result_dir = sys.argv[4] | result_dir = sys.argv[4] | ||||
| services = [] | |||||
| if len(sys.argv) >= 6: | |||||
| services = json.loads(sys.argv[5]) | |||||
| pinfo('Creating volumes...') | pinfo('Creating volumes...') | ||||
| work_vol = create_volume('work') | work_vol = create_volume('work') | ||||
| script_vol = create_volume('script') | script_vol = create_volume('script') | ||||
| psuccess('Created.') | psuccess('Created.') | ||||
| pinfo('Starting helper service...') | pinfo('Starting helper service...') | ||||
| global helper_container_id | global helper_container_id | ||||
| (helper_container_id, alias) = start_helper_service(work_vol, script_vol) | (helper_container_id, alias) = start_helper_service(work_vol, script_vol) | ||||
| psuccess('Started...') | psuccess('Started...') | ||||
| if services: | |||||
| pinfo('Creating service network...') | |||||
| global service_network_id | |||||
| service_network_id = create_service_network() | |||||
| psuccess('Created.') | |||||
| pinfo('Starting job-defined services...') | |||||
| global service_containers | |||||
| for service in services: | |||||
| service_containers.append(start_service_container(service, service_network_id)) | |||||
| pinfo('Waiting for job-defined services...') | |||||
| ensure_service_containers_up(service_containers) | |||||
| pinfo('Importing volumes...') | pinfo('Importing volumes...') | ||||
| import_volume(alias, work_dir, work_vol_mount_dir) | import_volume(alias, work_dir, work_vol_mount_dir) | ||||
| import_volume(alias, script_dir, script_vol_mount_dir) | import_volume(alias, script_dir, script_vol_mount_dir) | ||||
| psuccess('Imported.') | psuccess('Imported.') | ||||
| pinfo('Running container...') | pinfo('Running container...') | ||||
| retcode = run_in_container(image, work_vol, script_vol) | retcode = run_in_container(image, work_vol, script_vol, service_network_id) | ||||
| pinfo(f'Returned {retcode}.') | pinfo(f'Returned {retcode}.') | ||||
| if retcode != 0: | if retcode != 0: | ||||
| perror('Job failed.') | perror('Job failed.') | ||||
| else: | else: | ||||
| psuccess('Job succeeded.') | psuccess('Job succeeded.') | ||||
| # We should collect the result regardless whether it succeeded | # We should collect the result regardless whether it succeeded | ||||
| pinfo('Collecting build changes...') | pinfo('Collecting build changes...') | ||||
| export_volume(alias, result_dir, work_vol_mount_dir) | export_volume(alias, result_dir, work_vol_mount_dir) | ||||
| psuccess('Collected.') | psuccess('Collected.') | ||||
| return retcode | return retcode | ||||
| retcode = 1 | def cleanup_all(): | ||||
| if service_containers: | |||||
| pinfo('Cleaning service containers...') | |||||
| prune_service_containers(service_containers) | |||||
| psuccess('Cleaned.') | |||||
| if service_network_id: | |||||
| pinfo('Cleaning service network...') | |||||
| clean_service_network(service_network_id) | |||||
| psuccess('Cleaned.') | |||||
| try: | |||||
| retcode = main() | |||||
| except Exception as e: | |||||
| perror('Error!', e) | |||||
| print(traceback.format_exc()) | |||||
| raise | |||||
| finally: | |||||
| if helper_container_id: | if helper_container_id: | ||||
| pinfo('Cleaning helper container') | pinfo('Cleaning helper container') | ||||
| clean_helper_container() | clean_helper_container() | ||||
| psuccess('Cleaned.') | psuccess('Cleaned.') | ||||
| pinfo('Cleaning volumes...') | pinfo('Cleaning volumes...') | ||||
| clean_volumes() | clean_volumes() | ||||
| psuccess('Cleaned.') | psuccess('Cleaned.') | ||||
| retcode = 1 | |||||
| try: | |||||
| retcode = main() | |||||
| except Exception as e: | |||||
| perror('Error!', e) | |||||
| print(traceback.format_exc()) | |||||
| raise | |||||
| finally: | |||||
| cleanup_all() | |||||
| sys.exit(retcode) | sys.exit(retcode) | ||||