[ROCm][CI] Stabilize ROCm shutdown and distributed compile CI (#41573)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -1100,13 +1100,13 @@ steps:
|
||||
- vllm/compilation/
|
||||
- vllm/model_executor/layers
|
||||
- tests/compile/passes/distributed/
|
||||
- tests/compile/fusions_e2e/
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- export VLLM_TEST_CLEAN_GPU_MEMORY=1
|
||||
- VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py
|
||||
- pytest -v -s tests/compile/passes/distributed/test_sequence_parallelism.py
|
||||
- pytest -v -s tests/compile/passes/distributed/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions
|
||||
- pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions
|
||||
|
||||
#----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
|
||||
|
||||
|
||||
@@ -299,19 +299,22 @@ async def test_abort_timeout_exits_quickly(wait_for_engine_idle: float):
|
||||
start_time = time.time()
|
||||
proc.send_signal(signal.SIGTERM)
|
||||
|
||||
# abort timeout (0) should exit promptly
|
||||
for _ in range(20):
|
||||
if proc.poll() is not None:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
# abort timeout (0) should stop the server promptly. On ROCm, process
|
||||
# exit can spend extra time in HIP/RCCL/native extension teardown after
|
||||
# the server and engine have already shut down.
|
||||
max_exit_time = 4.0 if _IS_ROCM else 2.1
|
||||
|
||||
if proc.poll() is None:
|
||||
try:
|
||||
proc.wait(timeout=max_exit_time)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait(timeout=5)
|
||||
pytest.fail("Process did not exit after SIGTERM with abort timeout")
|
||||
|
||||
exit_time = time.time() - start_time
|
||||
assert exit_time < 2.1, f"Default shutdown took too long: {exit_time:.1f}s"
|
||||
assert exit_time < max_exit_time, (
|
||||
f"Default shutdown took too long: {exit_time:.1f}s"
|
||||
)
|
||||
assert proc.returncode in (0, -15, None), f"Unexpected: {proc.returncode}"
|
||||
|
||||
await _assert_children_cleaned_up(child_pids)
|
||||
|
||||
Reference in New Issue
Block a user