From fb1ac806c55a6dc96fe92261b80c8550e9c39d2f Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sat, 9 May 2026 22:47:40 -0500 Subject: [PATCH] [ROCm][CI] Stabilize ROCm shutdown and distributed compile CI (#41573) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 4 ++-- .../openai/completion/test_shutdown.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index bad2796266..9fb3503c8d 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1100,13 +1100,13 @@ steps: - vllm/compilation/ - vllm/model_executor/layers - tests/compile/passes/distributed/ + - tests/compile/fusions_e2e/ - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - export VLLM_TEST_CLEAN_GPU_MEMORY=1 - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py - - pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py - - pytest -v -s tests/compile/passes/distributed/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions + - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions #----------------------------------------------------------- mi300 ยท cuda ------------------------------------------------------------# diff --git a/tests/entrypoints/openai/completion/test_shutdown.py b/tests/entrypoints/openai/completion/test_shutdown.py index 966c9f869c..d7fba364ab 100644 --- a/tests/entrypoints/openai/completion/test_shutdown.py +++ b/tests/entrypoints/openai/completion/test_shutdown.py @@ -299,19 +299,22 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float): start_time = time.time() proc.send_signal(signal.SIGTERM) - # abort timeout (0) should exit promptly - for _ in range(20): - if proc.poll() is not None: - break - time.sleep(0.1) + # abort timeout (0) should stop the server promptly. On ROCm, process + # exit can spend extra time in HIP/RCCL/native extension teardown after + # the server and engine have already shut down. + max_exit_time = 4.0 if _IS_ROCM else 2.1 - if proc.poll() is None: + try: + proc.wait(timeout=max_exit_time) + except subprocess.TimeoutExpired: proc.kill() proc.wait(timeout=5) pytest.fail("Process did not exit after SIGTERM with abort timeout") exit_time = time.time() - start_time - assert exit_time < 2.1, f"Default shutdown took too long: {exit_time:.1f}s" + assert exit_time < max_exit_time, ( + f"Default shutdown took too long: {exit_time:.1f}s" + ) assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}" await _assert_children_cleaned_up(child_pids)