From 036608b01271fde554c28396c9799455710817a4 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Thu, 24 Sep 2026 07:58:38 -0400 Subject: [PATCH] [Engine] Detect a fail-closed fault or death on any rank A fault on a non-zero TP rank could hang the engine: the worker's sentinel pipe was inherited by a helper process and stayed open, so the monitor never saw the exit. Earlier releases carry the same monitor and fail-closed exit path. - Worker exit is detected by pidfd, with the sentinel as fallback, before and after READY and for every worker through the whole initialisation window. - A per-worker fault pipe: ple_fail_closed writes one byte before os._exit and the monitor starts the failure ladder at once; the fd stays registered through worker teardown. - worker_main marks inherited pipes close-on-exec. - Any unexpected busy-loop failure exits 1; the termination ladder uses time.monotonic(); forced-exit paths no longer wait on logging. - Tests on real fork/spawn workers for rank 0 and rank 1, with and without an fd-inheriting helper. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/v1/engine/core.py | 53 +++- vllm/v1/executor/multiproc_executor.py | 410 ++++++++++++++++++++----- vllm/v1/ple_offload/connector.py | 3 +- vllm/v1/ple_offload/fatal.py | 35 +++ 4 files changed, 410 insertions(+), 91 deletions(-) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index e0fef97049..2fe72b4f8a 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -116,6 +116,24 @@ _R = TypeVar("_R") # Return type for collective_rpc _TEARDOWN_SIGNAL_BOUND_S = 30.0 +# How long the guard's forced-exit paths wait for their diagnostic line. The +# line is written from a helper thread; a blocked handler, a held logging lock +# or a full stderr pipe costs at most this, never the exit. +_TEARDOWN_EXIT_LOG_BOUND_S = 0.5 + + +def _log_bounded(emit: Callable[[], None], bound_s: float) -> None: + """Run ``emit`` (logging, flushes) on a daemon thread; wait <= bound_s.""" + try: + thread = threading.Thread( + target=emit, daemon=True, name="EngineCoreTeardownLog" + ) + thread.start() + thread.join(bound_s) + except BaseException: # noqa: BLE001 -- the caller's exit must still run + pass + + class _TeardownSignalGuard: """EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded. @@ -154,25 +172,34 @@ class _TeardownSignalGuard: ) self._watchdog.start() return - logger.warning( - "[shutdown] EngineCore: second late signal (%s) during resource " - "teardown; not deferring again", - name, - ) + # Default disposition first, so a third signal also ends the process + # while the (bounded) diagnostic line is written. signal.signal(signum, signal.SIG_DFL) + _log_bounded( + lambda: logger.warning( + "[shutdown] EngineCore: second late signal (%s) during " + "resource teardown; not deferring again", + name, + ), + _TEARDOWN_EXIT_LOG_BOUND_S, + ) os.kill(os.getpid(), signum) def _expire(self) -> None: if self._done.wait(self.bound_s): return - logger.error( - "[shutdown] EngineCore: resource teardown still running %.0f s " - "after a deferred signal; exiting (code 1)", - self.bound_s, - ) - for stream in (sys.stdout, sys.stderr): - with suppress(Exception): - stream.flush() + + def emit() -> None: + logger.error( + "[shutdown] EngineCore: resource teardown still running %.0f s " + "after a deferred signal; exiting (code 1)", + self.bound_s, + ) + for stream in (sys.stdout, sys.stderr): + with suppress(Exception): + stream.flush() + + _log_bounded(emit, _TEARDOWN_EXIT_LOG_BOUND_S) os._exit(1) def disarm(self) -> None: diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 2ce46b6407..f2d269c4a1 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -5,6 +5,7 @@ import os import pickle import queue import signal +import stat import sys import threading import time @@ -71,6 +72,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.executor.abstract import Executor, FailureCallback from vllm.v1.executor.vllm_net_devices import set_worker_net_device from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput +from vllm.v1.ple_offload.fatal import set_fatal_notify_fd from vllm.v1.worker.worker_base import WorkerWrapperBase logger = init_logger(__name__) @@ -97,13 +99,16 @@ def _install_worker_parent_death_signal(expected_parent_pid: int) -> None: ctypes.c_ulong, ] prctl.restype = ctypes.c_int - if prctl( - _PR_SET_PDEATHSIG, - int(_WORKER_PARENT_DEATH_SIGNAL), - 0, - 0, - 0, - ) != 0: + if ( + prctl( + _PR_SET_PDEATHSIG, + int(_WORKER_PARENT_DEATH_SIGNAL), + 0, + 0, + 0, + ) + != 0 + ): error_number = ctypes.get_errno() raise OSError(error_number, os.strerror(error_number)) @@ -173,6 +178,86 @@ def _exit_worker_process(exit_code: int) -> None: os._exit(exit_code) +def _worker_exit_waitables( + procs: Sequence[BaseProcess], +) -> list[tuple[Any, BaseProcess, int | None]]: + """What the worker monitor waits on to learn that a worker process exited. + + A pidfd (Linux >= 5.3) where possible: it becomes readable when the process + itself exits. proc.sentinel is only the read end of a pipe whose write end + the worker holds, so it reports the exit of the LAST process holding that + write end -- and anything the worker fork+execs inherits it. Every PLE- + owning worker does: registering its host buffers under the "file_system" + sharing strategy starts a torch_shm_manager child, which keeps the write + end, and which lives while any client -- rank 0's PLE child, for a TP1 + worker's buffers -- still maps them. In a rank-1 fault test PP0_TP1 died + in release_outputs, sat as a zombie, its sentinel never fired and the + engine hung; rank 0's death only looked detected because its PLE child + (and so both managers) died with it, ~8 s later. + + Returns (waitable, proc, pidfd-to-close-or-None) per process, falling back + to the sentinel where no pidfd can be opened. + """ + pidfd_open = getattr(os, "pidfd_open", None) + waitables: list[tuple[Any, BaseProcess, int | None]] = [] + try: + for proc in procs: + pidfd: int | None = None + pid = getattr(proc, "pid", None) + if pidfd_open is not None and pid is not None: + try: + pidfd = pidfd_open(pid) + except OSError: + # ESRCH (already reaped) or no pidfd support: the sentinel. + pidfd = None + try: + waitable = pidfd if pidfd is not None else proc.sentinel + waitables.append((waitable, proc, pidfd)) + except BaseException: + # Not yet in `waitables`: the outer handler would miss it. + if pidfd is not None: + os.close(pidfd) + raise + except BaseException: + # Nothing is handed out on failure: close what was opened. + for _, _, opened in waitables: + if opened is not None: + with suppress(OSError): + os.close(opened) + raise + return waitables + + +def _make_inherited_pipes_close_on_exec() -> None: + """Keep this worker's inherited pipe fds out of the programs it execs. + + Under the spawn start method multiprocessing hands the worker its end of + the sentinel pipe (and the ready / death pipes) as INHERITABLE fds, so any + program the worker fork+execs keeps them open: the torch_shm_manager that + the PLE host-buffer registration starts did, and outlived the worker, so + neither the sentinel nor the ready pipe reported its death (rank-1 fault injection). + Nothing this process execs needs them -- multiprocessing children get + their fds passed explicitly, subprocess closes them by default -- so mark + every inherited pipe above stderr close-on-exec. The executor's pidfd + wait is the primary detection; this only stops the leak at its source. + """ + if not sys.platform.startswith("linux"): + return + try: + fds = [int(name) for name in os.listdir("/proc/self/fd")] + except (OSError, ValueError): + return + for fd in fds: + if fd <= 2: + continue + try: + if stat.S_ISFIFO(os.fstat(fd).st_mode) and os.get_inheritable(fd): + os.set_inheritable(fd, False) + except OSError: + # The listing's own directory fd, or one closed since. + continue + + class FutureWrapper(Future): def __init__( self, @@ -304,6 +389,7 @@ class MultiprocExecutor(Executor): if inherited_fds is not None: inherited_fds.append(unready_worker_handle.death_writer.fileno()) inherited_fds.append(unready_worker_handle.ready_pipe.fileno()) + inherited_fds.append(unready_worker_handle.fault_reader.fileno()) # Workers must be created before wait_for_ready to avoid # deadlock, since worker.init_device() does a device sync. @@ -400,22 +486,78 @@ class MultiprocExecutor(Executor): # Monitors worker process liveness. If any die unexpectedly, # logs an error, shuts down the executor and invokes the failure # callback to inform the engine. + # + # Two wake sources per worker: + # - its exit: a pidfd, readable when the process itself has exited + # (fallback: proc.sentinel, which a fork+exec'd child holding the + # write end -- torch_shm_manager from the PLE registration -- keeps + # silent; see _worker_exit_waitables); + # - its fault pipe: one byte from the fail-closed path right before + # os._exit, so the failure ladder starts -- and ends the peers -- + # without waiting for the process teardown, which can itself wait + # on a live peer. A worker closing its end on the way out (EOF, no + # byte) is not a fault; its exit follows. + # In a rank-1 fault test a fault on PP0_TP1 was never detected and the + # engine hung on the response queue for 300 s. + # Opened here, before the thread starts, so a worker that dies early + # is still a zombie (pidfds work on zombies) rather than reaped. + exit_waitables = _worker_exit_waitables([h.proc for h in workers]) + def monitor_workers(): - sentinels = [h.proc.sentinel for h in workers] - died = multiprocessing.connection.wait(sentinels) + exit_to_handle = { + w: h for (w, _, _), h in zip(exit_waitables, workers) + } + fault_to_handle = { + h.fault_reader: h + for h in workers + if getattr(h, "fault_reader", None) is not None + } + waiting: list[Any] = list(exit_to_handle) + list(fault_to_handle) + handle = None + reported_fatal = False + try: + while handle is None: + ready = multiprocessing.connection.wait(waiting) + for obj in ready: + if obj in exit_to_handle: + handle = exit_to_handle[obj] + break + try: + data = os.read(obj.fileno(), 1) + except OSError: + data = b"" + if data: + handle = fault_to_handle[obj] + reported_fatal = True + break + waiting.remove(obj) + finally: + for _, _, pidfd in exit_waitables: + if pidfd is not None: + with suppress(OSError): + os.close(pidfd) _self = self_ref() if not _self or getattr(_self, "shutting_down", False): logger.debug("MultiprocWorkerMonitor: shutdown already initiated") return _self.is_failed = True - proc = next(h.proc for h in workers if h.proc.sentinel == died[0]) - logger.error( - "Worker proc %s died unexpectedly (exit code: %s), " - "shutting down executor. executor_shutdown_monotonic=%.9f", - proc.name, - proc.exitcode, - time.monotonic(), - ) + proc = handle.proc + if reported_fatal: + logger.error( + "Worker proc %s reported a fatal fault (fail-closed) and is " + "exiting; shutting down executor without waiting for its " + "process teardown. executor_shutdown_monotonic=%.9f", + proc.name, + time.monotonic(), + ) + else: + logger.error( + "Worker proc %s died unexpectedly (exit code: %s), " + "shutting down executor. executor_shutdown_monotonic=%.9f", + proc.name, + proc.exitcode, + time.monotonic(), + ) _self.shutdown() callback = _self.failure_callback if callback is not None: @@ -423,9 +565,16 @@ class MultiprocExecutor(Executor): callback() if not inline: - Thread( - target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor" - ).start() + try: + Thread( + target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor" + ).start() + except BaseException: + for _, _, pidfd in exit_waitables: + if pidfd is not None: + with suppress(OSError): + os.close(pidfd) + raise return monitor_workers() @@ -589,8 +738,9 @@ class MultiprocExecutor(Executor): # If we are in late stage shutdown, the interpreter may replace # `time` with `None`. return all(not proc.is_alive() for proc in procs) - start_time = time.time() - while time.time() - start_time < timeout: + # Monotonic: a wall-clock step must not cut or stretch the grace. + start_time = time.monotonic() + while time.monotonic() - start_time < timeout: if all(not proc.is_alive() for proc in procs): return True time.sleep(0.1) @@ -605,7 +755,7 @@ class MultiprocExecutor(Executor): initial_count, ) if after_failure and time: - start = time.time() + start = time.monotonic() states = list(shutdown_states or [None] * len(worker_procs)) def parked() -> list[BaseProcess]: @@ -637,7 +787,7 @@ class MultiprocExecutor(Executor): p.kill() # Workers that started their shutdown get a BOUNDED grace, counted # from the start of this ladder, then SIGTERM, then SIGKILL. - grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.time() - start)) + grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.monotonic() - start)) term_wait_s = _FAILED_WORKER_WAIT_S if wait_for_termination(active_procs(), timeout=grace_s): logger.info_once("[shutdown] Executor: all workers exited gracefully") @@ -738,6 +888,7 @@ class UnreadyWorkerProcHandle: ready_pipe: Connection death_writer: Connection | None = None shutdown_state: Any = None + fault_reader: Connection | None = None @dataclass @@ -753,6 +904,8 @@ class WorkerProcHandle: death_writer: Connection | None = None # Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes). shutdown_state: Any = None + # Read end of the worker's fault pipe (see start_worker_monitor). + fault_reader: Connection | None = None @classmethod def from_unready_handle( @@ -768,6 +921,7 @@ class WorkerProcHandle: peer_worker_response_mqs=peer_worker_response_mqs, death_writer=unready_handle.death_writer, shutdown_state=unready_handle.shutdown_state, + fault_reader=unready_handle.fault_reader, ) @@ -906,11 +1060,16 @@ class WorkerProc: ready_reader, ready_writer = context.Pipe(duplex=False) # Death pipe to let child detect parent process exit death_reader, death_writer = context.Pipe(duplex=False) + # Fault pipe: the worker's fail-closed path writes one byte before + # os._exit; the executor's monitor waits on it next to the sentinel. + fault_reader, fault_writer = context.Pipe(duplex=False) # One shared byte: has this worker started / finished its shutdown? shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED) if inherited_fds is not None: inherited_fds = inherited_fds.copy() - inherited_fds.extend((ready_reader.fileno(), death_writer.fileno())) + inherited_fds.extend( + (ready_reader.fileno(), death_writer.fileno(), fault_reader.fileno()) + ) process_kwargs = { "expected_parent_pid": os.getpid(), "vllm_config": vllm_config, @@ -920,6 +1079,7 @@ class WorkerProc: "input_shm_handle": input_shm_handle, "ready_pipe": ready_writer, "death_pipe": death_reader, + "fault_pipe": fault_writer, "shutdown_state": shutdown_state, "shared_worker_lock": shared_worker_lock, "is_driver_worker": is_driver_worker, @@ -943,10 +1103,11 @@ class WorkerProc: # Close child ends of pipes here in the parent ready_writer.close() death_reader.close() + fault_writer.close() # Keep death_writer open in parent - when parent exits, # death_reader in child will get EOFError return UnreadyWorkerProcHandle( - proc, rank, ready_reader, death_writer, shutdown_state + proc, rank, ready_reader, death_writer, shutdown_state, fault_reader ) @staticmethod @@ -983,10 +1144,59 @@ class WorkerProc: ready_proc_handles: list[WorkerProcHandle | None] = [None] * len( unready_proc_handles ) - while pipes: - ready = multiprocessing.connection.wait(pipes.keys()) - for pipe in ready: - assert isinstance(pipe, Connection) + # A worker that dies before READY need not close its ready pipe: a + # child it fork+exec'd (torch_shm_manager, started by the PLE + # registration) can hold the write end, so no EOF ever comes. Wait on + # each pending worker's exit (pidfd, else its sentinel -- see + # _worker_exit_waitables) and on its fault pipe (one byte from the + # fail-closed path; EOF is not a fault), as the post-READY monitor does. + exit_waitables = _worker_exit_waitables( + [h.proc for h in unready_proc_handles] + ) + exits = { + w: handle + for (w, _, _), handle in zip(exit_waitables, unready_proc_handles) + } + faults = { + handle.fault_reader: handle + for handle in unready_proc_handles + if getattr(handle, "fault_reader", None) is not None + } + try: + while pipes: + waiting: list[Any] = [*pipes.keys()] + # EVERY worker's exit and fault pipe, READY or not: the live + # monitor starts only after all are READY, so a READY worker + # that faults or dies while a sibling initialises must fail + # initialization here. + waiting += list(exits) + waiting += list(faults) + ready = multiprocessing.connection.wait(waiting) + WorkerProc._receive_ready( + ready, pipes, exits, faults, ready_proc_handles, e + ) + finally: + for _, _, pidfd in exit_waitables: + if pidfd is not None: + with suppress(OSError): + os.close(pidfd) + + return cast(list[WorkerProcHandle], ready_proc_handles) + + @staticmethod + def _receive_ready( + ready: list[Any], + pipes: dict[Connection, UnreadyWorkerProcHandle], + exits: dict[Any, UnreadyWorkerProcHandle], + faults: dict[Any, UnreadyWorkerProcHandle], + ready_proc_handles: list[WorkerProcHandle | None], + e: Exception, + ) -> None: + # Ready pipes first: a READY sent before an exit or fault still counts. + ordered = sorted(ready, key=lambda obj: 0 if obj in pipes else 1) + for obj in ordered: + if obj in pipes: + pipe = obj try: # Wait until the WorkerProc is ready. unready_proc_handle = pipes.pop(pipe) @@ -995,18 +1205,50 @@ class WorkerProc: raise e idx = unready_proc_handle.rank % len(ready_proc_handles) - ready_proc_handles[idx] = WorkerProc.wait_for_response_handle_ready( - response, unready_proc_handle + ready_proc_handles[idx] = ( + WorkerProc.wait_for_response_handle_ready( + response, unready_proc_handle + ) ) except EOFError: e.__suppress_context__ = True raise e from None - finally: # Close connection. pipe.close() - - return cast(list[WorkerProcHandle], ready_proc_handles) + continue + handle = exits.get(obj) if obj in exits else faults.get(obj) + if handle is None: + continue + pending = handle.ready_pipe in pipes + if pending and handle.ready_pipe.poll(0): + continue # its READY is queued: read it next round + phase = ( + "before READY" + if pending + else "after READY, while other workers were initializing" + ) + if obj in exits: + logger.error( + "Worker proc %s exited (code %s) %s", + handle.proc.name, + handle.proc.exitcode, + phase, + ) + raise e + try: + data = os.read(obj.fileno(), 1) + except OSError: + data = b"" + if data: + logger.error( + "Worker proc %s reported a fatal fault (fail-closed) %s", + handle.proc.name, + phase, + ) + raise e + # EOF on the fault pipe is not a fault; its exit follows. + del faults[obj] def shutdown(self, *, destroy_distributed: bool = True): if not destroy_distributed: @@ -1101,11 +1343,21 @@ class WorkerProc: # requested (death pipe EOF or SIGTERM/SIGINT): the process then exits # straight after WorkerProc.shutdown (see _exit_worker_process). orderly_shutdown = False - # Set when a READY worker's busy loop FAILED while a shutdown request - # was pending: full teardown, then exit code 1 (never the exit-0 path). - failed_during_shutdown = False + # Set when a READY worker's busy loop FAILED (anything but its own + # shutdown-request cancellation): full teardown, then exit code 1. + # Never keyed to shutdown_requested -- the death-pipe monitor can set + # it just after the check, which made a failure exit 0. + busy_loop_failed = False ready_writer = kwargs.pop("ready_pipe") death_pipe = kwargs.pop("death_pipe", None) + # Fault pipe: registered for the PLE fail-closed path + # (ple_offload.fatal.notify_parent_fatal); kept open for the life of + # the process so the byte written right before os._exit reaches the + # executor's monitor. Closed only on the orderly way out (EOF, which + # the monitor does not count as a fault). + fault_pipe = kwargs.pop("fault_pipe", None) + if fault_pipe is not None: + set_fatal_notify_fd(fault_pipe.fileno()) shutdown_state = kwargs.pop("shutdown_state", None) def mark_shutdown(value: int) -> None: @@ -1121,6 +1373,8 @@ class WorkerProc: os.close(fd) except Exception as e: logger.warning("Error closing inherited connection: %s: %s", type(e), e) + # Before anything below can fork+exec a helper (torch_shm_manager). + _make_inherited_pipes_close_on_exec() try: # Initialize tracer @@ -1174,13 +1428,14 @@ class WorkerProc: "[shutdown] WorkerProc: exiting after shutdown request" ) orderly_shutdown = True - elif shutdown_requested.is_set(): - # A real failure that raced a shutdown request: keep it a - # failure -- full teardown, then a non-zero exit. - logger.exception("WorkerProc failed during shutdown.") - failed_during_shutdown = True else: - logger.exception("WorkerProc failed.") + # A real failure, whether or not a shutdown request is pending + # (yet): full teardown, then a non-zero exit. + if shutdown_requested.is_set(): + logger.exception("WorkerProc failed during shutdown.") + else: + logger.exception("WorkerProc failed.") + busy_loop_failed = True # The parent sends a SIGTERM to all worker processes if # any worker dies. Set this value so we don't re-throw @@ -1205,36 +1460,43 @@ class WorkerProc: ready_writer.close() if death_pipe is not None: death_pipe.close() - # Clean up once worker exits busy loop - if worker is not None: - # Tells the executor's ladder this rank is draining, not parked - # in a collective (see _ensure_worker_termination). - mark_shutdown(_SHUTDOWN_STARTED) - if orderly_shutdown: - # Everything that must happen before exit (FINAL DRAIN, - # instrumentation flushes, PLE cleanup) is in - # worker.shutdown(); then exit instead of paying the NCCL - # destroy + finalization past the executor's grace. - exit_code = 0 - try: - worker.shutdown(destroy_distributed=False) - except BaseException: - logger.exception("WorkerProc shutdown failed") - exit_code = 1 + # The fault fd stays registered through the whole teardown: a PLE + # fail-closed inside worker.shutdown() must still reach the + # executor. Unregistered after it; on the orderly + # os._exit path the process exit closes it. + try: + # Clean up once worker exits busy loop + if worker is not None: + # Tells the executor's ladder this rank is draining, not parked + # in a collective (see _ensure_worker_termination). + mark_shutdown(_SHUTDOWN_STARTED) + if orderly_shutdown: + # Everything that must happen before exit (FINAL DRAIN, + # instrumentation flushes, PLE cleanup) is in + # worker.shutdown(); then exit instead of paying the NCCL + # destroy + finalization past the executor's grace. + exit_code = 0 + try: + worker.shutdown(destroy_distributed=False) + except BaseException: + logger.exception("WorkerProc shutdown failed") + exit_code = 1 + mark_shutdown(_SHUTDOWN_DONE) + _exit_worker_process(exit_code) + worker.shutdown() mark_shutdown(_SHUTDOWN_DONE) - _exit_worker_process(exit_code) - worker.shutdown() - mark_shutdown(_SHUTDOWN_DONE) - if failed_during_shutdown: - sys.exit(1) + if busy_loop_failed: + sys.exit(1) + finally: + if fault_pipe is not None: + set_fatal_notify_fd(None) + fault_pipe.close() class ResponseStatus(Enum): SUCCESS = auto() FAILURE = auto() - def _fail_ple_rpc_if_pending( - self, where: str, error: BaseException - ) -> None: + def _fail_ple_rpc_if_pending(self, where: str, error: BaseException) -> None: """Fail closed if RPC recovery would abandon queued PLE GPU work.""" owners = (self.worker, getattr(self.worker, "worker", None)) for owner in owners: @@ -1247,9 +1509,7 @@ class WorkerProc: try: pending = connector.has_pending_completions except BaseException as state_error: - connector._fail_closed( - f"{where}:pending-state", state_error - ) + connector._fail_closed(f"{where}:pending-state", state_error) if pending: connector._fail_closed(where, error) return @@ -1264,9 +1524,7 @@ class WorkerProc: output = output.get_output() except Exception as e: logger.exception("Error getting async model runner output") - self._fail_ple_rpc_if_pending( - "async-output-copy", e - ) + self._fail_ple_rpc_if_pending("async-output-copy", e) output = e if isinstance(output, Exception): @@ -1338,9 +1596,7 @@ class WorkerProc: if output_rank is None or self.rank == output_rank: self.handle_output(output) except Exception as e: - self._fail_ple_rpc_if_pending( - f"worker-rpc:{method}", e - ) + self._fail_ple_rpc_if_pending(f"worker-rpc:{method}", e) # Notes have been introduced in python 3.11 if hasattr(e, "add_note"): e.add_note(traceback.format_exc()) diff --git a/vllm/v1/ple_offload/connector.py b/vllm/v1/ple_offload/connector.py index 2cf05bb449..6aa4e046f4 100644 --- a/vllm/v1/ple_offload/connector.py +++ b/vllm/v1/ple_offload/connector.py @@ -25,7 +25,7 @@ from vllm.model_executor.layers.ple_offload_layer import ( flag_check_is_authoritative, stale_offload_flags, ) -from vllm.v1.ple_offload.fatal import ple_fail_closed +from vllm.v1.ple_offload.fatal import notify_parent_fatal, ple_fail_closed from vllm.v1.ple_offload.progress import PleProgressMonitor from vllm.v1.ple_offload.protocol import ( PleOffloadRegistration, @@ -469,6 +469,7 @@ class PleOffloadConnector: self._process_request(request, socket) except Exception: logger.exception("PLE request thread failed") + notify_parent_fatal() os._exit(1) finally: self._request_thread_ready.set() diff --git a/vllm/v1/ple_offload/fatal.py b/vllm/v1/ple_offload/fatal.py index a96515f25d..32831aa6b4 100644 --- a/vllm/v1/ple_offload/fatal.py +++ b/vllm/v1/ple_offload/fatal.py @@ -44,6 +44,40 @@ _TYPE_MODULE = type.__dict__["__module__"].__get__ _MAX_CHAIN = 16 _MAX_ARG_CHARS = 1024 +# Write end of the per-worker fault pipe (WorkerProc.make_worker_process); the +# executor's worker monitor waits on the read end next to the process +# sentinels. None outside a worker process (tests, the offload child). +_FATAL_NOTIFY_FD: int | None = None + + +def set_fatal_notify_fd(fd: int | None) -> None: + """Register the fault pipe worker_main received from the executor.""" + global _FATAL_NOTIFY_FD + _FATAL_NOTIFY_FD = fd + + +def notify_parent_fatal() -> None: + """Tell the executor this worker is dying, before the process is gone. + + The executor's only other signal is the process sentinel, which the kernel + closes when the LAST task of the worker leaves -- after the address-space + teardown, not at os._exit. On a rank whose GPU work is paired with a live + peer that teardown can wait for the peer's exit (in a rank-1 fault test, a + fail-closed fault on PP0_TP1 left its leader Z and unreapable for 300 s + while rank 0 waited on it in turn; the engine hung on the response + queue). One byte here wakes the monitor at once so the failure ladder ends + the peers, which is what frees the teardown. Nonblocking, never raises: + the caller os._exits right after, whatever happens here. + """ + fd = _FATAL_NOTIFY_FD + if fd is None: + return + try: + os.set_blocking(fd, False) + os.write(fd, b"F") + except BaseException: + pass + def _safe_type_label(cls: type) -> str: """``module.qualname`` of ``cls`` without running class-defined code.""" @@ -240,6 +274,7 @@ def ple_fail_closed( # runs a Python __exit__ between here and os._exit below; a bare # except keeps that window empty. pass + notify_parent_fatal() os._exit(1)