Page MenuHomePhorge
Files F13751356

D254.1765418809.diff
No OneTemporary
Actions

Size
3 KB
Referenced Files
None
Subscribers
None

D254.1765418809.diff
View Options

diff --git a/lilybuild/lilybuild/ci_steps.py b/lilybuild/lilybuild/ci_steps.py
--- a/lilybuild/lilybuild/ci_steps.py
+++ b/lilybuild/lilybuild/ci_steps.py
@@ -253,6 +253,9 @@
],
workdir=self.work_root_dir,
doStepIf=on_success,
+ # 2h timeout by default
+ # TODO support timeout by each job
+ timeout=60 * 60 * 2,
)
clean_script_step = steps.ShellCommand(
diff --git a/lilybuild/podman-helper b/lilybuild/podman-helper
--- a/lilybuild/podman-helper
+++ b/lilybuild/podman-helper
@@ -60,7 +60,7 @@
def clean_volumes():
verbose_run([
- 'podman', 'volume', 'rm', '--',
+ 'podman', 'volume', 'rm', '-f', '--',
] + volumes_to_remove, capture_output=True)
def clean_helper_container():
@@ -139,14 +139,60 @@
def run_in_container(image, work_volname, script_volname):
timeout = 60 * 60 * 2 # 2 hours by default
- p = verbose_run([
- 'podman', 'run', '--rm',
+ steady_deadline = time.monotonic() + timeout
+ start_process = verbose_run([
+ 'podman', 'run', '-d',
f'--mount=type=volume,source={work_volname},destination={work_vol_mount_dir}',
f'--mount=type=volume,source={script_volname},destination={script_vol_mount_dir}',
] + image_to_podman_args(image) + [
script_name,
- ], timeout=timeout)
- return p
+ ], capture_output=True, encoding='utf-8')
+ if start_process.returncode != 0:
+ perror('Cannot run container. Error message:')
+ print(start_process.stderr)
+ return start_process.returncode
+ container_id = start_process.stdout.strip()
+
+ steady_now = time.monotonic()
+ log_args = []
+ retcode = None
+ try:
+ while steady_deadline > steady_now:
+ log_process = verbose_run([
+ 'podman', 'logs', '--follow'
+ ] + log_args + ['--', container_id], timeout=steady_deadline - steady_now)
+ # Exited from `podman logs`: why? Is the container still running?
+ inspect_running = verbose_run([
+ 'podman', 'container', 'inspect',
+ '--format', '{{.State.Status}}', '--', container_id,
+ ], capture_output=True, encoding='utf-8', check=True)
+ if inspect_running.stdout.strip() == 'exited':
+ inspect_retcode = verbose_run([
+ 'podman', 'container', 'inspect',
+ '--format', '{{.State.ExitCode}}', '--', container_id,
+ ], capture_output=True, encoding='utf-8', check=True)
+ retcode = int(inspect_retcode.stdout.strip())
+ break
+ else:
+ perror('`podman logs` unexpectedly quits when the container is still running, resuming logs...')
+ log_args = ['--tail', '10']
+ steady_now = time.monotonic()
+ if retcode is None:
+ perror('Command timed out.')
+ retcode = 1
+ except subprocess.TimeoutExpired:
+ perror('Command timed out.')
+ retcode = 1
+ except subprocess.CalledProcessError as e:
+ perror('Cannot inspect container', e)
+ finally:
+ pinfo('Cleaning up container...')
+ stop_proc = verbose_run(['podman', 'container', 'stop', '--', container_id])
+ if stop_proc.returncode != 0:
+ perror('Cannot stop container.')
+ rm_proc = verbose_run(['podman', 'container', 'rm', '-f', '--', container_id])
+ pinfo('Cleaned.')
+ return retcode
def main():
image = json.loads(sys.argv[1])
@@ -170,12 +216,7 @@
psuccess('Imported.')
pinfo('Running container...')
- try:
- res = run_in_container(image, work_vol, script_vol)
- retcode = res.returncode
- except subprocess.TimeoutExpired:
- perror('Command timed out.')
- retcode = 1
+ retcode = run_in_container(image, work_vol, script_vol)
pinfo(f'Returned {retcode}.')

File Metadata

Mime Type
text/plain
Expires
Wed, Dec 10, 6:06 PM (15 h, 43 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
720201
Default Alt Text
D254.1765418809.diff (3 KB)

Event Timeline