[Frontend] shutdown engine core and its sub processes asap when initializing (#52299)
Signed-off-by: Andy Xie <andy.xning@gmail.com> Co-authored-by: Simon Mo <simon.mo@hey.com>
This commit is contained in:
@@ -0,0 +1,224 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Regression tests for termination signals that arrive while EngineCoreProc is
|
||||
# still being constructed.
|
||||
#
|
||||
# When this fix matters
|
||||
# ---------------------
|
||||
# Engine shutdown is driven from the frontend. If a frontend process dies while
|
||||
# EngineCore is still starting up, the process manager sends SIGTERM to its
|
||||
# children and force kills whatever is still alive after a short timeout (see
|
||||
# shutdown in vllm/v1/utils.py). Construction is slow - it spawns the executor
|
||||
# processes, loads weights, profiles and captures CUDA graphs - so that
|
||||
# shutdown often lands inside the construction window. Bigger models and more
|
||||
# ranks widen the window. This is the case PR 52299 (follow up to 43417) has to
|
||||
# survive: 43417 stops EngineCore from exiting silently, but the request itself
|
||||
# still has to outlive the construction window.
|
||||
#
|
||||
# run_engine_core installs the signal handler before engine_core exists, so in
|
||||
# that window the handler runs with engine_core == None. Losing the request
|
||||
# there is permanent, not a delay: shutdown_state stays RUNNING, and
|
||||
# wakeup_engine returns early while consuming the one-shot SignalCallback. The
|
||||
# busy loop then starts RUNNING and blocks on input_queue forever, so no later
|
||||
# signal can wake it. Graceful shutdown is gone for good and the process only
|
||||
# stops when something force kills it, which skips executor shutdown and GPU
|
||||
# release. In a log this shows up as the signal being received, followed by a
|
||||
# force kill of EngineCore after the process manager timeout.
|
||||
#
|
||||
# What these tests pin
|
||||
# --------------------
|
||||
# A request arriving in the window must be remembered somewhere other than
|
||||
# engine_core, applied before run_busy_loop starts, and must not consume the
|
||||
# one-shot wakeup callback, so a later signal can still wake an idle loop.
|
||||
|
||||
import queue
|
||||
import signal as signal_module
|
||||
from typing import Any
|
||||
|
||||
from vllm.v1.engine import EngineCoreRequestType
|
||||
from vllm.v1.engine import core as core_module
|
||||
from vllm.v1.engine.core import EngineCoreProc, EngineShutdownState
|
||||
|
||||
WAKEUP_ITEM = (EngineCoreRequestType.WAKEUP, None)
|
||||
CONSTRUCTION_SIGNAL = signal_module.SIGTERM
|
||||
|
||||
|
||||
class _FakeSignal:
|
||||
# Minimal stand-in for the signal module. It records the installed handler
|
||||
# so the test can fire it by hand, and avoids touching the real process
|
||||
# signal disposition. Signals is reused from the real module so the handler
|
||||
# can still resolve signal.Signals(signum).name.
|
||||
SIGTERM = signal_module.SIGTERM
|
||||
SIGINT = signal_module.SIGINT
|
||||
SIG_DFL = signal_module.SIG_DFL
|
||||
Signals = signal_module.Signals
|
||||
|
||||
def __init__(self):
|
||||
self.handlers = {}
|
||||
|
||||
def signal(self, signum, handler):
|
||||
self.handlers[signum] = handler
|
||||
|
||||
def fire(self):
|
||||
self.handlers[CONSTRUCTION_SIGNAL](CONSTRUCTION_SIGNAL, None)
|
||||
|
||||
|
||||
class _FakeSignalCallback:
|
||||
# One-shot stand-in for SignalCallback. The real class runs the callback on
|
||||
# a dedicated thread and exits after a single trigger, so every later
|
||||
# trigger is a no-op. Running the callback inline keeps the test
|
||||
# deterministic instead of depending on thread scheduling.
|
||||
instances: list["_FakeSignalCallback"] = []
|
||||
|
||||
def __init__(self, callback):
|
||||
self.callback = callback
|
||||
self.trigger_count = 0
|
||||
self.consumed = False
|
||||
self.stopped = False
|
||||
_FakeSignalCallback.instances.append(self)
|
||||
|
||||
def trigger(self):
|
||||
self.trigger_count += 1
|
||||
if self.stopped or self.consumed:
|
||||
return
|
||||
self.consumed = True
|
||||
callback, self.callback = self.callback, None
|
||||
callback()
|
||||
|
||||
def stop(self):
|
||||
self.stopped = True
|
||||
|
||||
|
||||
class _StubParallelConfig:
|
||||
def __init__(self):
|
||||
self.data_parallel_size = 1
|
||||
self.data_parallel_rank_local = 0
|
||||
self.data_parallel_index = 0
|
||||
self.data_parallel_rank = 0
|
||||
self.numa_bind = False
|
||||
|
||||
def reconfigure_for_independent_dp_rank(self):
|
||||
pass
|
||||
|
||||
|
||||
class _StubModelConfig:
|
||||
is_moe = False
|
||||
|
||||
|
||||
class _StubVllmConfig:
|
||||
def __init__(self):
|
||||
self.parallel_config = _StubParallelConfig()
|
||||
self.model_config = _StubModelConfig()
|
||||
self.logging_config = None
|
||||
self.kv_transfer_config = None
|
||||
self.shutdown_timeout = 0
|
||||
|
||||
|
||||
def _run_engine_core(
|
||||
monkeypatch,
|
||||
*,
|
||||
signal_during_construction=False,
|
||||
signal_while_idle=False,
|
||||
):
|
||||
_FakeSignalCallback.instances.clear()
|
||||
fake_signal = _FakeSignal()
|
||||
trace = {}
|
||||
|
||||
class StubEngineCoreProc:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.shutdown_state = EngineShutdownState.RUNNING
|
||||
# Same queue type as EngineCore, so the wakeup path under test is
|
||||
# the real one.
|
||||
self.input_queue = queue.Queue[tuple[EngineCoreRequestType, Any]]()
|
||||
self.vllm_config = kwargs.get("vllm_config")
|
||||
if signal_during_construction:
|
||||
# The termination signal lands mid construction, while the
|
||||
# caller still sees engine_core == None.
|
||||
fake_signal.fire()
|
||||
trace["constructed"] = True
|
||||
|
||||
def run_busy_loop(self):
|
||||
# What the real loop sees on its first _handle_shutdown() check.
|
||||
trace["loop_entered"] = True
|
||||
trace["state_at_loop_entry"] = self.shutdown_state
|
||||
if signal_while_idle:
|
||||
fake_signal.fire()
|
||||
try:
|
||||
trace["wakeup"] = self.input_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
trace["wakeup"] = None
|
||||
|
||||
def shutdown(self):
|
||||
pass
|
||||
|
||||
def _send_engine_dead(self):
|
||||
pass
|
||||
|
||||
def has_work(self):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(core_module, "signal", fake_signal)
|
||||
monkeypatch.setattr(core_module, "SignalCallback", _FakeSignalCallback)
|
||||
monkeypatch.setattr(core_module, "EngineCoreProc", StubEngineCoreProc)
|
||||
monkeypatch.setattr(core_module, "DPEngineCoreProc", StubEngineCoreProc)
|
||||
monkeypatch.setattr(core_module, "configure_logging", lambda *a, **k: None)
|
||||
monkeypatch.setattr(
|
||||
core_module, "maybe_register_config_serialize_by_value", lambda *a, **k: None
|
||||
)
|
||||
monkeypatch.setattr(core_module, "set_process_title", lambda *a, **k: None)
|
||||
monkeypatch.setattr(core_module, "maybe_init_worker_tracer", lambda *a, **k: None)
|
||||
monkeypatch.setattr(core_module, "decorate_logs", lambda *a, **k: None)
|
||||
|
||||
EngineCoreProc.run_engine_core(vllm_config=_StubVllmConfig())
|
||||
return trace
|
||||
|
||||
|
||||
def test_signal_during_construction_is_applied_before_loop(monkeypatch):
|
||||
trace = _run_engine_core(monkeypatch, signal_during_construction=True)
|
||||
|
||||
assert trace["constructed"] is True
|
||||
assert trace["loop_entered"] is True
|
||||
# The request arrived while engine_core was None, so it could not be stored
|
||||
# there. It must still be applied before run_busy_loop starts, otherwise the
|
||||
# loop comes up as RUNNING, the shutdown is silently dropped, and force kill
|
||||
# becomes the only way out.
|
||||
assert trace["state_at_loop_entry"] == EngineShutdownState.REQUESTED, (
|
||||
"a signal received during construction must be applied before the loop "
|
||||
f"starts, got {trace['state_at_loop_entry']!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_signal_during_construction_keeps_later_wakeup_working(monkeypatch):
|
||||
trace = _run_engine_core(
|
||||
monkeypatch,
|
||||
signal_during_construction=True,
|
||||
signal_while_idle=True,
|
||||
)
|
||||
|
||||
assert trace["state_at_loop_entry"] == EngineShutdownState.REQUESTED, (
|
||||
"the construction time signal must be applied before the loop starts, "
|
||||
f"got {trace['state_at_loop_entry']!r}"
|
||||
)
|
||||
# SignalCallback is one-shot. If the construction time signal consumed it,
|
||||
# no wakeup could be delivered afterwards and the idle busy loop would block
|
||||
# on input_queue forever instead of shutting down, so the process would sit
|
||||
# there holding GPU memory until something force kills it.
|
||||
assert trace["wakeup"] == WAKEUP_ITEM, (
|
||||
"a signal after construction must still wake the idle loop, "
|
||||
f"got {trace['wakeup']!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_signal_while_idle_still_wakes_the_loop(monkeypatch):
|
||||
trace = _run_engine_core(monkeypatch, signal_while_idle=True)
|
||||
|
||||
assert trace["state_at_loop_entry"] == EngineShutdownState.RUNNING
|
||||
assert trace["wakeup"] == WAKEUP_ITEM
|
||||
|
||||
|
||||
def test_no_signal_leaves_the_loop_running(monkeypatch):
|
||||
trace = _run_engine_core(monkeypatch)
|
||||
|
||||
assert trace["state_at_loop_entry"] == EngineShutdownState.RUNNING
|
||||
assert trace["wakeup"] is None
|
||||
+51
-17
@@ -1410,6 +1410,50 @@ class EngineCoreProc(EngineCore):
|
||||
vllm_config.kv_transfer_config.engine_id,
|
||||
)
|
||||
|
||||
# declare engine core as None and register signal handler before
|
||||
# engine_core initialization in case engine core process is terminated
|
||||
# in the middle of initialization. For example, frontend process exits
|
||||
# unexpectedly, if signal handler is not registered, engine core process
|
||||
# will use default signal handler which will terminate the process
|
||||
# silently. If engine core model executor has been initialized, it will
|
||||
# not be terminated asap. Register signal handler early and only log a
|
||||
# message will make engine core process alive. engine core process and
|
||||
# its subprocesses(model executor processes) will be force killed in
|
||||
# CoreEngineProcManager.shutdown.
|
||||
engine_core = None
|
||||
# A termination signal can arrive while engine_core is still being
|
||||
# constructed. At that point the request cannot be recorded on
|
||||
# engine_core, so keep it in its own flag and apply it later.
|
||||
shutdown_requested = False
|
||||
|
||||
def wakeup_engine():
|
||||
# Wakes up idle engine via input_queue when shutdown is requested
|
||||
# Not safe in a signal handler - we may interrupt the main thread
|
||||
# while it is holding the non-reentrant input_queue.mutex
|
||||
assert engine_core is not None
|
||||
engine_core.input_queue.put_nowait((EngineCoreRequestType.WAKEUP, None))
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
nonlocal shutdown_requested
|
||||
signal_name = signal.Signals(signum).name
|
||||
logger.info(
|
||||
"[shutdown] EngineCore: trigger received signal=%s",
|
||||
signal_name,
|
||||
)
|
||||
# Record the request first so it survives the construction window.
|
||||
shutdown_requested = True
|
||||
if engine_core is not None:
|
||||
engine_core.shutdown_state = EngineShutdownState.REQUESTED
|
||||
# signal_callback is armed only after engine_core exists. Don't
|
||||
# trigger it earlier: it is one-shot, and a wakeup cannot be
|
||||
# delivered while there is no input_queue to wake.
|
||||
# For more info: https://github.com/vllm-project/vllm/pull/52299#pullrequestreview-5370255271
|
||||
if signal_callback is not None:
|
||||
signal_callback.trigger()
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
parallel_config.data_parallel_index = dp_rank
|
||||
if data_parallel and vllm_config.model_config.is_moe:
|
||||
# Set data parallel rank for this engine process.
|
||||
@@ -1424,25 +1468,15 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
assert engine_core is not None
|
||||
|
||||
def wakeup_engine():
|
||||
# Wakes up idle engine via input_queue when shutdown is requested
|
||||
# Not safe in a signal handler - we may interrupt the main thread
|
||||
# while it is holding the non-reentrant input_queue.mutex
|
||||
engine_core.input_queue.put_nowait((EngineCoreRequestType.WAKEUP, None))
|
||||
|
||||
signal_callback = SignalCallback(wakeup_engine)
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
signal_name = signal.Signals(signum).name
|
||||
logger.info(
|
||||
"[shutdown] EngineCore: trigger received signal=%s",
|
||||
signal_name,
|
||||
)
|
||||
# Apply a signal received during construction before entering the
|
||||
# loop. run_busy_loop calls _handle_shutdown first, so the loop exits
|
||||
# right away instead of blocking on an idle input_queue.
|
||||
if shutdown_requested:
|
||||
engine_core.shutdown_state = EngineShutdownState.REQUESTED
|
||||
signal_callback.trigger()
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
# Arm the wakeup callback only now that input_queue exists, so a
|
||||
# later signal can still wake an idle busy loop.
|
||||
signal_callback = SignalCallback(wakeup_engine)
|
||||
|
||||
engine_core.run_busy_loop()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user