[ROCm][CI] Stabilize ROCm shutdown and distributed compile CI (#41573)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-10 03:47:40 +00:00
committed by GitHub
parent 986edc858a
commit fb1ac806c5
2 changed files with 12 additions and 9 deletions
@@ -299,19 +299,22 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
start_time = time.time()
proc.send_signal(signal.SIGTERM)
# abort timeout (0) should exit promptly
for _ in range(20):
if proc.poll() is not None:
break
time.sleep(0.1)
# abort timeout (0) should stop the server promptly. On ROCm, process
# exit can spend extra time in HIP/RCCL/native extension teardown after
# the server and engine have already shut down.
max_exit_time = 4.0 if _IS_ROCM else 2.1
if proc.poll() is None:
try:
proc.wait(timeout=max_exit_time)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait(timeout=5)
pytest.fail("Process did not exit after SIGTERM with abort timeout")
exit_time = time.time() - start_time
assert exit_time < 2.1, f"Default shutdown took too long: {exit_time:.1f}s"
assert exit_time < max_exit_time, (
f"Default shutdown took too long: {exit_time:.1f}s"
)
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
await _assert_children_cleaned_up(child_pids)