diff --git a/tests/v1/engine/test_engine_core_shutdown_signal.py b/tests/v1/engine/test_engine_core_shutdown_signal.py new file mode 100644 index 0000000000..73bc794088 --- /dev/null +++ b/tests/v1/engine/test_engine_core_shutdown_signal.py @@ -0,0 +1,224 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Regression tests for termination signals that arrive while EngineCoreProc is +# still being constructed. +# +# When this fix matters +# --------------------- +# Engine shutdown is driven from the frontend. If a frontend process dies while +# EngineCore is still starting up, the process manager sends SIGTERM to its +# children and force kills whatever is still alive after a short timeout (see +# shutdown in vllm/v1/utils.py). Construction is slow - it spawns the executor +# processes, loads weights, profiles and captures CUDA graphs - so that +# shutdown often lands inside the construction window. Bigger models and more +# ranks widen the window. This is the case PR 52299 (follow up to 43417) has to +# survive: 43417 stops EngineCore from exiting silently, but the request itself +# still has to outlive the construction window. +# +# run_engine_core installs the signal handler before engine_core exists, so in +# that window the handler runs with engine_core == None. Losing the request +# there is permanent, not a delay: shutdown_state stays RUNNING, and +# wakeup_engine returns early while consuming the one-shot SignalCallback. The +# busy loop then starts RUNNING and blocks on input_queue forever, so no later +# signal can wake it. Graceful shutdown is gone for good and the process only +# stops when something force kills it, which skips executor shutdown and GPU +# release. In a log this shows up as the signal being received, followed by a +# force kill of EngineCore after the process manager timeout. +# +# What these tests pin +# -------------------- +# A request arriving in the window must be remembered somewhere other than +# engine_core, applied before run_busy_loop starts, and must not consume the +# one-shot wakeup callback, so a later signal can still wake an idle loop. + +import queue +import signal as signal_module +from typing import Any + +from vllm.v1.engine import EngineCoreRequestType +from vllm.v1.engine import core as core_module +from vllm.v1.engine.core import EngineCoreProc, EngineShutdownState + +WAKEUP_ITEM = (EngineCoreRequestType.WAKEUP, None) +CONSTRUCTION_SIGNAL = signal_module.SIGTERM + + +class _FakeSignal: + # Minimal stand-in for the signal module. It records the installed handler + # so the test can fire it by hand, and avoids touching the real process + # signal disposition. Signals is reused from the real module so the handler + # can still resolve signal.Signals(signum).name. + SIGTERM = signal_module.SIGTERM + SIGINT = signal_module.SIGINT + SIG_DFL = signal_module.SIG_DFL + Signals = signal_module.Signals + + def __init__(self): + self.handlers = {} + + def signal(self, signum, handler): + self.handlers[signum] = handler + + def fire(self): + self.handlers[CONSTRUCTION_SIGNAL](CONSTRUCTION_SIGNAL, None) + + +class _FakeSignalCallback: + # One-shot stand-in for SignalCallback. The real class runs the callback on + # a dedicated thread and exits after a single trigger, so every later + # trigger is a no-op. Running the callback inline keeps the test + # deterministic instead of depending on thread scheduling. + instances: list["_FakeSignalCallback"] = [] + + def __init__(self, callback): + self.callback = callback + self.trigger_count = 0 + self.consumed = False + self.stopped = False + _FakeSignalCallback.instances.append(self) + + def trigger(self): + self.trigger_count += 1 + if self.stopped or self.consumed: + return + self.consumed = True + callback, self.callback = self.callback, None + callback() + + def stop(self): + self.stopped = True + + +class _StubParallelConfig: + def __init__(self): + self.data_parallel_size = 1 + self.data_parallel_rank_local = 0 + self.data_parallel_index = 0 + self.data_parallel_rank = 0 + self.numa_bind = False + + def reconfigure_for_independent_dp_rank(self): + pass + + +class _StubModelConfig: + is_moe = False + + +class _StubVllmConfig: + def __init__(self): + self.parallel_config = _StubParallelConfig() + self.model_config = _StubModelConfig() + self.logging_config = None + self.kv_transfer_config = None + self.shutdown_timeout = 0 + + +def _run_engine_core( + monkeypatch, + *, + signal_during_construction=False, + signal_while_idle=False, +): + _FakeSignalCallback.instances.clear() + fake_signal = _FakeSignal() + trace = {} + + class StubEngineCoreProc: + def __init__(self, *args, **kwargs): + self.shutdown_state = EngineShutdownState.RUNNING + # Same queue type as EngineCore, so the wakeup path under test is + # the real one. + self.input_queue = queue.Queue[tuple[EngineCoreRequestType, Any]]() + self.vllm_config = kwargs.get("vllm_config") + if signal_during_construction: + # The termination signal lands mid construction, while the + # caller still sees engine_core == None. + fake_signal.fire() + trace["constructed"] = True + + def run_busy_loop(self): + # What the real loop sees on its first _handle_shutdown() check. + trace["loop_entered"] = True + trace["state_at_loop_entry"] = self.shutdown_state + if signal_while_idle: + fake_signal.fire() + try: + trace["wakeup"] = self.input_queue.get_nowait() + except queue.Empty: + trace["wakeup"] = None + + def shutdown(self): + pass + + def _send_engine_dead(self): + pass + + def has_work(self): + return False + + monkeypatch.setattr(core_module, "signal", fake_signal) + monkeypatch.setattr(core_module, "SignalCallback", _FakeSignalCallback) + monkeypatch.setattr(core_module, "EngineCoreProc", StubEngineCoreProc) + monkeypatch.setattr(core_module, "DPEngineCoreProc", StubEngineCoreProc) + monkeypatch.setattr(core_module, "configure_logging", lambda *a, **k: None) + monkeypatch.setattr( + core_module, "maybe_register_config_serialize_by_value", lambda *a, **k: None + ) + monkeypatch.setattr(core_module, "set_process_title", lambda *a, **k: None) + monkeypatch.setattr(core_module, "maybe_init_worker_tracer", lambda *a, **k: None) + monkeypatch.setattr(core_module, "decorate_logs", lambda *a, **k: None) + + EngineCoreProc.run_engine_core(vllm_config=_StubVllmConfig()) + return trace + + +def test_signal_during_construction_is_applied_before_loop(monkeypatch): + trace = _run_engine_core(monkeypatch, signal_during_construction=True) + + assert trace["constructed"] is True + assert trace["loop_entered"] is True + # The request arrived while engine_core was None, so it could not be stored + # there. It must still be applied before run_busy_loop starts, otherwise the + # loop comes up as RUNNING, the shutdown is silently dropped, and force kill + # becomes the only way out. + assert trace["state_at_loop_entry"] == EngineShutdownState.REQUESTED, ( + "a signal received during construction must be applied before the loop " + f"starts, got {trace['state_at_loop_entry']!r}" + ) + + +def test_signal_during_construction_keeps_later_wakeup_working(monkeypatch): + trace = _run_engine_core( + monkeypatch, + signal_during_construction=True, + signal_while_idle=True, + ) + + assert trace["state_at_loop_entry"] == EngineShutdownState.REQUESTED, ( + "the construction time signal must be applied before the loop starts, " + f"got {trace['state_at_loop_entry']!r}" + ) + # SignalCallback is one-shot. If the construction time signal consumed it, + # no wakeup could be delivered afterwards and the idle busy loop would block + # on input_queue forever instead of shutting down, so the process would sit + # there holding GPU memory until something force kills it. + assert trace["wakeup"] == WAKEUP_ITEM, ( + "a signal after construction must still wake the idle loop, " + f"got {trace['wakeup']!r}" + ) + + +def test_signal_while_idle_still_wakes_the_loop(monkeypatch): + trace = _run_engine_core(monkeypatch, signal_while_idle=True) + + assert trace["state_at_loop_entry"] == EngineShutdownState.RUNNING + assert trace["wakeup"] == WAKEUP_ITEM + + +def test_no_signal_leaves_the_loop_running(monkeypatch): + trace = _run_engine_core(monkeypatch) + + assert trace["state_at_loop_entry"] == EngineShutdownState.RUNNING + assert trace["wakeup"] is None diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 679bbbfd24..bfbaa38d94 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1410,6 +1410,50 @@ class EngineCoreProc(EngineCore): vllm_config.kv_transfer_config.engine_id, ) + # declare engine core as None and register signal handler before + # engine_core initialization in case engine core process is terminated + # in the middle of initialization. For example, frontend process exits + # unexpectedly, if signal handler is not registered, engine core process + # will use default signal handler which will terminate the process + # silently. If engine core model executor has been initialized, it will + # not be terminated asap. Register signal handler early and only log a + # message will make engine core process alive. engine core process and + # its subprocesses(model executor processes) will be force killed in + # CoreEngineProcManager.shutdown. + engine_core = None + # A termination signal can arrive while engine_core is still being + # constructed. At that point the request cannot be recorded on + # engine_core, so keep it in its own flag and apply it later. + shutdown_requested = False + + def wakeup_engine(): + # Wakes up idle engine via input_queue when shutdown is requested + # Not safe in a signal handler - we may interrupt the main thread + # while it is holding the non-reentrant input_queue.mutex + assert engine_core is not None + engine_core.input_queue.put_nowait((EngineCoreRequestType.WAKEUP, None)) + + def signal_handler(signum, frame): + nonlocal shutdown_requested + signal_name = signal.Signals(signum).name + logger.info( + "[shutdown] EngineCore: trigger received signal=%s", + signal_name, + ) + # Record the request first so it survives the construction window. + shutdown_requested = True + if engine_core is not None: + engine_core.shutdown_state = EngineShutdownState.REQUESTED + # signal_callback is armed only after engine_core exists. Don't + # trigger it earlier: it is one-shot, and a wakeup cannot be + # delivered while there is no input_queue to wake. + # For more info: https://github.com/vllm-project/vllm/pull/52299#pullrequestreview-5370255271 + if signal_callback is not None: + signal_callback.trigger() + + signal.signal(signal.SIGTERM, signal_handler) + signal.signal(signal.SIGINT, signal_handler) + parallel_config.data_parallel_index = dp_rank if data_parallel and vllm_config.model_config.is_moe: # Set data parallel rank for this engine process. @@ -1424,25 +1468,15 @@ class EngineCoreProc(EngineCore): assert engine_core is not None - def wakeup_engine(): - # Wakes up idle engine via input_queue when shutdown is requested - # Not safe in a signal handler - we may interrupt the main thread - # while it is holding the non-reentrant input_queue.mutex - engine_core.input_queue.put_nowait((EngineCoreRequestType.WAKEUP, None)) - - signal_callback = SignalCallback(wakeup_engine) - - def signal_handler(signum, frame): - signal_name = signal.Signals(signum).name - logger.info( - "[shutdown] EngineCore: trigger received signal=%s", - signal_name, - ) + # Apply a signal received during construction before entering the + # loop. run_busy_loop calls _handle_shutdown first, so the loop exits + # right away instead of blocking on an idle input_queue. + if shutdown_requested: engine_core.shutdown_state = EngineShutdownState.REQUESTED - signal_callback.trigger() - signal.signal(signal.SIGTERM, signal_handler) - signal.signal(signal.SIGINT, signal_handler) + # Arm the wakeup callback only now that input_queue exists, so a + # later signal can still wake an idle busy loop. + signal_callback = SignalCallback(wakeup_engine) engine_core.run_busy_loop()