[ROCm] Give EngineCore cleanup grace after request abort (#52281)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Andreas Karatzas
2026-08-19 11:39:59 -05:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent e9e1630e93
commit 3a386cfaf5
5 changed files with 221 additions and 3 deletions
@@ -242,6 +242,40 @@ def test_handles_shutdown_event():
assert supervisor.is_ready is False
@pytest.mark.asyncio
async def test_shutdown_children_uses_engine_process_timeout(
monkeypatch: pytest.MonkeyPatch,
):
supervisor = DPSupervisor(_make_unit_args(shutdown_timeout=0.0))
supervisor._processes = [
SimpleNamespace(name="APIServer_DPRank_4", pid=None, is_alive=lambda: False)
]
calls = []
timeout_calls = []
def get_process_timeout(request_timeout, manager_timeout):
timeout_calls.append((request_timeout, manager_timeout))
return 15.0
monkeypatch.setattr(
dp_sup,
"get_engine_process_shutdown_timeout",
get_process_timeout,
)
monkeypatch.setattr(
dp_sup,
"_join_processes_with_timeout",
lambda processes, timeout: calls.append((processes, timeout)),
)
await supervisor._shutdown_children()
assert timeout_calls == [(0.0, 0.0)]
assert calls == [
(supervisor._processes, 15.0 + CHILD_EXIT_GRACE_S),
]
@pytest.mark.asyncio
async def test_handles_child_exit(
monkeypatch: pytest.MonkeyPatch,
@@ -2,14 +2,20 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from multiprocessing import connection
from threading import Event
from types import SimpleNamespace
import pytest
import zmq
import vllm.platforms as platforms
from vllm.v1.engine import core as core_module
from vllm.v1.engine import utils as engine_utils
from vllm.v1.engine.core import EngineCoreProc, EngineShutdownState
from vllm.v1.engine.utils import (
CoreEngine,
CoreEngineLaunch,
CoreEngineProcManager,
EngineZmqAddresses,
wait_for_engine_startup,
)
@@ -17,6 +23,129 @@ from vllm.v1.engine.utils import (
pytestmark = pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
("is_rocm", "request_timeout", "manager_timeout", "process_timeout"),
[
(True, 0, 0, 15.0),
(True, 0, 7, 7),
(True, 0, None, None),
(False, 0, 0, 0),
(True, 7, 0, 0),
],
)
def test_engine_core_process_shutdown_timeout(
monkeypatch: pytest.MonkeyPatch,
is_rocm: bool,
request_timeout: float | None,
manager_timeout: float | None,
process_timeout: float | None,
):
manager = object.__new__(CoreEngineProcManager)
manager._request_shutdown_timeout = request_timeout
manager.manager_stopped = Event()
manager.processes = [object()]
detach_results = iter((object(), None))
manager._finalizer = SimpleNamespace(detach=lambda: next(detach_results))
shutdown_calls = []
monkeypatch.setattr(
engine_utils,
"current_platform",
SimpleNamespace(is_rocm=lambda: is_rocm),
)
monkeypatch.setattr(
engine_utils,
"shutdown",
lambda processes, timeout: shutdown_calls.append((processes, timeout)),
)
manager.shutdown(timeout=manager_timeout)
manager.shutdown(timeout=manager_timeout)
assert manager.manager_stopped.is_set()
assert shutdown_calls == [(manager.processes, process_timeout)]
@pytest.mark.parametrize(
(
"is_rocm",
"shutdown_state",
"has_work",
"shutdown_timeout",
"exit_code",
"expected_calls",
),
[
(
True,
EngineShutdownState.SHUTTING_DOWN,
False,
0,
None,
["shutdown", "freeze"],
),
(False, EngineShutdownState.SHUTTING_DOWN, False, 0, None, ["shutdown"]),
(True, EngineShutdownState.RUNNING, False, 0, None, ["shutdown"]),
(True, EngineShutdownState.SHUTTING_DOWN, True, 0, None, ["shutdown"]),
(True, EngineShutdownState.SHUTTING_DOWN, False, 7, None, ["shutdown"]),
(True, EngineShutdownState.SHUTTING_DOWN, False, 0, 1, ["shutdown"]),
],
)
def test_freeze_gc_after_clean_rocm_engine_core_shutdown(
monkeypatch: pytest.MonkeyPatch,
is_rocm: bool,
shutdown_state: EngineShutdownState,
has_work: bool,
shutdown_timeout: int,
exit_code: int | None,
expected_calls: list[str],
):
calls: list[str] = []
vllm_config = SimpleNamespace(shutdown_timeout=shutdown_timeout)
proc = SimpleNamespace(
shutdown_state=EngineShutdownState.RUNNING,
has_work=lambda: has_work,
vllm_config=vllm_config,
)
def run_busy_loop():
proc.shutdown_state = shutdown_state
raise SystemExit(exit_code)
proc.run_busy_loop = run_busy_loop
proc.shutdown = lambda: calls.append("shutdown")
parallel_config = SimpleNamespace(
data_parallel_size=1,
numa_bind=False,
reconfigure_for_independent_dp_rank=lambda: None,
)
vllm_config.parallel_config = parallel_config
for name in (
"maybe_register_config_serialize_by_value",
"set_process_title",
"maybe_init_worker_tracer",
"decorate_logs",
):
monkeypatch.setattr(core_module, name, lambda *args, **kwargs: None)
monkeypatch.setattr(core_module, "EngineCoreProc", lambda *args, **kwargs: proc)
monkeypatch.setattr(
core_module,
"SignalCallback",
lambda callback: SimpleNamespace(trigger=lambda: None, stop=lambda: None),
)
monkeypatch.setattr(core_module.signal, "signal", lambda *args: None)
monkeypatch.setattr(
platforms, "current_platform", SimpleNamespace(is_rocm=lambda: is_rocm)
)
monkeypatch.setattr(core_module.gc, "freeze", lambda: calls.append("freeze"))
with pytest.raises(SystemExit):
EngineCoreProc.run_engine_core(vllm_config=vllm_config)
assert calls == expected_calls
class _FinishedProcess:
name = "RustFrontend"