[Engine] Detect a fail-closed fault or death on any rank

A fault on a non-zero TP rank could hang the engine: the worker's sentinel pipe
was inherited by a helper process and stayed open, so the monitor never saw
the exit. Earlier releases carry the same monitor and fail-closed exit path.

- Worker exit is detected by pidfd, with the sentinel as fallback, before and
  after READY and for every worker through the whole initialisation window.
- A per-worker fault pipe: ple_fail_closed writes one byte before os._exit and
  the monitor starts the failure ladder at once; the fd stays registered
  through worker teardown.
- worker_main marks inherited pipes close-on-exec.
- Any unexpected busy-loop failure exits 1; the termination ladder uses
  time.monotonic(); forced-exit paths no longer wait on logging.
- Tests on real fork/spawn workers for rank 0 and rank 1, with and without an
  fd-inheriting helper.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:04:37 +07:00
co-authored by Hinata
parent 6c956ee3bd
commit 036608b012
4 changed files with 410 additions and 91 deletions
+40 -13
View File
@@ -116,6 +116,24 @@ _R = TypeVar("_R") # Return type for collective_rpc
_TEARDOWN_SIGNAL_BOUND_S = 30.0
# How long the guard's forced-exit paths wait for their diagnostic line. The
# line is written from a helper thread; a blocked handler, a held logging lock
# or a full stderr pipe costs at most this, never the exit.
_TEARDOWN_EXIT_LOG_BOUND_S = 0.5
def _log_bounded(emit: Callable[[], None], bound_s: float) -> None:
"""Run ``emit`` (logging, flushes) on a daemon thread; wait <= bound_s."""
try:
thread = threading.Thread(
target=emit, daemon=True, name="EngineCoreTeardownLog"
)
thread.start()
thread.join(bound_s)
except BaseException: # noqa: BLE001 -- the caller's exit must still run
pass
class _TeardownSignalGuard:
"""EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded.
@@ -154,25 +172,34 @@ class _TeardownSignalGuard:
)
self._watchdog.start()
return
logger.warning(
"[shutdown] EngineCore: second late signal (%s) during resource "
"teardown; not deferring again",
name,
)
# Default disposition first, so a third signal also ends the process
# while the (bounded) diagnostic line is written.
signal.signal(signum, signal.SIG_DFL)
_log_bounded(
lambda: logger.warning(
"[shutdown] EngineCore: second late signal (%s) during "
"resource teardown; not deferring again",
name,
),
_TEARDOWN_EXIT_LOG_BOUND_S,
)
os.kill(os.getpid(), signum)
def _expire(self) -> None:
if self._done.wait(self.bound_s):
return
logger.error(
"[shutdown] EngineCore: resource teardown still running %.0f s "
"after a deferred signal; exiting (code 1)",
self.bound_s,
)
for stream in (sys.stdout, sys.stderr):
with suppress(Exception):
stream.flush()
def emit() -> None:
logger.error(
"[shutdown] EngineCore: resource teardown still running %.0f s "
"after a deferred signal; exiting (code 1)",
self.bound_s,
)
for stream in (sys.stdout, sys.stderr):
with suppress(Exception):
stream.flush()
_log_bounded(emit, _TEARDOWN_EXIT_LOG_BOUND_S)
os._exit(1)
def disarm(self) -> None:
+333 -77
View File
@@ -5,6 +5,7 @@ import os
import pickle
import queue
import signal
import stat
import sys
import threading
import time
@@ -71,6 +72,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.executor.abstract import Executor, FailureCallback
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
from vllm.v1.ple_offload.fatal import set_fatal_notify_fd
from vllm.v1.worker.worker_base import WorkerWrapperBase
logger = init_logger(__name__)
@@ -97,13 +99,16 @@ def _install_worker_parent_death_signal(expected_parent_pid: int) -> None:
ctypes.c_ulong,
]
prctl.restype = ctypes.c_int
if prctl(
_PR_SET_PDEATHSIG,
int(_WORKER_PARENT_DEATH_SIGNAL),
0,
0,
0,
) != 0:
if (
prctl(
_PR_SET_PDEATHSIG,
int(_WORKER_PARENT_DEATH_SIGNAL),
0,
0,
0,
)
!= 0
):
error_number = ctypes.get_errno()
raise OSError(error_number, os.strerror(error_number))
@@ -173,6 +178,86 @@ def _exit_worker_process(exit_code: int) -> None:
os._exit(exit_code)
def _worker_exit_waitables(
procs: Sequence[BaseProcess],
) -> list[tuple[Any, BaseProcess, int | None]]:
"""What the worker monitor waits on to learn that a worker process exited.
A pidfd (Linux >= 5.3) where possible: it becomes readable when the process
itself exits. proc.sentinel is only the read end of a pipe whose write end
the worker holds, so it reports the exit of the LAST process holding that
write end -- and anything the worker fork+execs inherits it. Every PLE-
owning worker does: registering its host buffers under the "file_system"
sharing strategy starts a torch_shm_manager child, which keeps the write
end, and which lives while any client -- rank 0's PLE child, for a TP1
worker's buffers -- still maps them. In a rank-1 fault test PP0_TP1 died
in release_outputs, sat as a zombie, its sentinel never fired and the
engine hung; rank 0's death only looked detected because its PLE child
(and so both managers) died with it, ~8 s later.
Returns (waitable, proc, pidfd-to-close-or-None) per process, falling back
to the sentinel where no pidfd can be opened.
"""
pidfd_open = getattr(os, "pidfd_open", None)
waitables: list[tuple[Any, BaseProcess, int | None]] = []
try:
for proc in procs:
pidfd: int | None = None
pid = getattr(proc, "pid", None)
if pidfd_open is not None and pid is not None:
try:
pidfd = pidfd_open(pid)
except OSError:
# ESRCH (already reaped) or no pidfd support: the sentinel.
pidfd = None
try:
waitable = pidfd if pidfd is not None else proc.sentinel
waitables.append((waitable, proc, pidfd))
except BaseException:
# Not yet in `waitables`: the outer handler would miss it.
if pidfd is not None:
os.close(pidfd)
raise
except BaseException:
# Nothing is handed out on failure: close what was opened.
for _, _, opened in waitables:
if opened is not None:
with suppress(OSError):
os.close(opened)
raise
return waitables
def _make_inherited_pipes_close_on_exec() -> None:
"""Keep this worker's inherited pipe fds out of the programs it execs.
Under the spawn start method multiprocessing hands the worker its end of
the sentinel pipe (and the ready / death pipes) as INHERITABLE fds, so any
program the worker fork+execs keeps them open: the torch_shm_manager that
the PLE host-buffer registration starts did, and outlived the worker, so
neither the sentinel nor the ready pipe reported its death (rank-1 fault injection).
Nothing this process execs needs them -- multiprocessing children get
their fds passed explicitly, subprocess closes them by default -- so mark
every inherited pipe above stderr close-on-exec. The executor's pidfd
wait is the primary detection; this only stops the leak at its source.
"""
if not sys.platform.startswith("linux"):
return
try:
fds = [int(name) for name in os.listdir("/proc/self/fd")]
except (OSError, ValueError):
return
for fd in fds:
if fd <= 2:
continue
try:
if stat.S_ISFIFO(os.fstat(fd).st_mode) and os.get_inheritable(fd):
os.set_inheritable(fd, False)
except OSError:
# The listing's own directory fd, or one closed since.
continue
class FutureWrapper(Future):
def __init__(
self,
@@ -304,6 +389,7 @@ class MultiprocExecutor(Executor):
if inherited_fds is not None:
inherited_fds.append(unready_worker_handle.death_writer.fileno())
inherited_fds.append(unready_worker_handle.ready_pipe.fileno())
inherited_fds.append(unready_worker_handle.fault_reader.fileno())
# Workers must be created before wait_for_ready to avoid
# deadlock, since worker.init_device() does a device sync.
@@ -400,22 +486,78 @@ class MultiprocExecutor(Executor):
# Monitors worker process liveness. If any die unexpectedly,
# logs an error, shuts down the executor and invokes the failure
# callback to inform the engine.
#
# Two wake sources per worker:
# - its exit: a pidfd, readable when the process itself has exited
# (fallback: proc.sentinel, which a fork+exec'd child holding the
# write end -- torch_shm_manager from the PLE registration -- keeps
# silent; see _worker_exit_waitables);
# - its fault pipe: one byte from the fail-closed path right before
# os._exit, so the failure ladder starts -- and ends the peers --
# without waiting for the process teardown, which can itself wait
# on a live peer. A worker closing its end on the way out (EOF, no
# byte) is not a fault; its exit follows.
# In a rank-1 fault test a fault on PP0_TP1 was never detected and the
# engine hung on the response queue for 300 s.
# Opened here, before the thread starts, so a worker that dies early
# is still a zombie (pidfds work on zombies) rather than reaped.
exit_waitables = _worker_exit_waitables([h.proc for h in workers])
def monitor_workers():
sentinels = [h.proc.sentinel for h in workers]
died = multiprocessing.connection.wait(sentinels)
exit_to_handle = {
w: h for (w, _, _), h in zip(exit_waitables, workers)
}
fault_to_handle = {
h.fault_reader: h
for h in workers
if getattr(h, "fault_reader", None) is not None
}
waiting: list[Any] = list(exit_to_handle) + list(fault_to_handle)
handle = None
reported_fatal = False
try:
while handle is None:
ready = multiprocessing.connection.wait(waiting)
for obj in ready:
if obj in exit_to_handle:
handle = exit_to_handle[obj]
break
try:
data = os.read(obj.fileno(), 1)
except OSError:
data = b""
if data:
handle = fault_to_handle[obj]
reported_fatal = True
break
waiting.remove(obj)
finally:
for _, _, pidfd in exit_waitables:
if pidfd is not None:
with suppress(OSError):
os.close(pidfd)
_self = self_ref()
if not _self or getattr(_self, "shutting_down", False):
logger.debug("MultiprocWorkerMonitor: shutdown already initiated")
return
_self.is_failed = True
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
logger.error(
"Worker proc %s died unexpectedly (exit code: %s), "
"shutting down executor. executor_shutdown_monotonic=%.9f",
proc.name,
proc.exitcode,
time.monotonic(),
)
proc = handle.proc
if reported_fatal:
logger.error(
"Worker proc %s reported a fatal fault (fail-closed) and is "
"exiting; shutting down executor without waiting for its "
"process teardown. executor_shutdown_monotonic=%.9f",
proc.name,
time.monotonic(),
)
else:
logger.error(
"Worker proc %s died unexpectedly (exit code: %s), "
"shutting down executor. executor_shutdown_monotonic=%.9f",
proc.name,
proc.exitcode,
time.monotonic(),
)
_self.shutdown()
callback = _self.failure_callback
if callback is not None:
@@ -423,9 +565,16 @@ class MultiprocExecutor(Executor):
callback()
if not inline:
Thread(
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
).start()
try:
Thread(
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
).start()
except BaseException:
for _, _, pidfd in exit_waitables:
if pidfd is not None:
with suppress(OSError):
os.close(pidfd)
raise
return
monitor_workers()
@@ -589,8 +738,9 @@ class MultiprocExecutor(Executor):
# If we are in late stage shutdown, the interpreter may replace
# `time` with `None`.
return all(not proc.is_alive() for proc in procs)
start_time = time.time()
while time.time() - start_time < timeout:
# Monotonic: a wall-clock step must not cut or stretch the grace.
start_time = time.monotonic()
while time.monotonic() - start_time < timeout:
if all(not proc.is_alive() for proc in procs):
return True
time.sleep(0.1)
@@ -605,7 +755,7 @@ class MultiprocExecutor(Executor):
initial_count,
)
if after_failure and time:
start = time.time()
start = time.monotonic()
states = list(shutdown_states or [None] * len(worker_procs))
def parked() -> list[BaseProcess]:
@@ -637,7 +787,7 @@ class MultiprocExecutor(Executor):
p.kill()
# Workers that started their shutdown get a BOUNDED grace, counted
# from the start of this ladder, then SIGTERM, then SIGKILL.
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.time() - start))
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.monotonic() - start))
term_wait_s = _FAILED_WORKER_WAIT_S
if wait_for_termination(active_procs(), timeout=grace_s):
logger.info_once("[shutdown] Executor: all workers exited gracefully")
@@ -738,6 +888,7 @@ class UnreadyWorkerProcHandle:
ready_pipe: Connection
death_writer: Connection | None = None
shutdown_state: Any = None
fault_reader: Connection | None = None
@dataclass
@@ -753,6 +904,8 @@ class WorkerProcHandle:
death_writer: Connection | None = None
# Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes).
shutdown_state: Any = None
# Read end of the worker's fault pipe (see start_worker_monitor).
fault_reader: Connection | None = None
@classmethod
def from_unready_handle(
@@ -768,6 +921,7 @@ class WorkerProcHandle:
peer_worker_response_mqs=peer_worker_response_mqs,
death_writer=unready_handle.death_writer,
shutdown_state=unready_handle.shutdown_state,
fault_reader=unready_handle.fault_reader,
)
@@ -906,11 +1060,16 @@ class WorkerProc:
ready_reader, ready_writer = context.Pipe(duplex=False)
# Death pipe to let child detect parent process exit
death_reader, death_writer = context.Pipe(duplex=False)
# Fault pipe: the worker's fail-closed path writes one byte before
# os._exit; the executor's monitor waits on it next to the sentinel.
fault_reader, fault_writer = context.Pipe(duplex=False)
# One shared byte: has this worker started / finished its shutdown?
shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED)
if inherited_fds is not None:
inherited_fds = inherited_fds.copy()
inherited_fds.extend((ready_reader.fileno(), death_writer.fileno()))
inherited_fds.extend(
(ready_reader.fileno(), death_writer.fileno(), fault_reader.fileno())
)
process_kwargs = {
"expected_parent_pid": os.getpid(),
"vllm_config": vllm_config,
@@ -920,6 +1079,7 @@ class WorkerProc:
"input_shm_handle": input_shm_handle,
"ready_pipe": ready_writer,
"death_pipe": death_reader,
"fault_pipe": fault_writer,
"shutdown_state": shutdown_state,
"shared_worker_lock": shared_worker_lock,
"is_driver_worker": is_driver_worker,
@@ -943,10 +1103,11 @@ class WorkerProc:
# Close child ends of pipes here in the parent
ready_writer.close()
death_reader.close()
fault_writer.close()
# Keep death_writer open in parent - when parent exits,
# death_reader in child will get EOFError
return UnreadyWorkerProcHandle(
proc, rank, ready_reader, death_writer, shutdown_state
proc, rank, ready_reader, death_writer, shutdown_state, fault_reader
)
@staticmethod
@@ -983,10 +1144,59 @@ class WorkerProc:
ready_proc_handles: list[WorkerProcHandle | None] = [None] * len(
unready_proc_handles
)
while pipes:
ready = multiprocessing.connection.wait(pipes.keys())
for pipe in ready:
assert isinstance(pipe, Connection)
# A worker that dies before READY need not close its ready pipe: a
# child it fork+exec'd (torch_shm_manager, started by the PLE
# registration) can hold the write end, so no EOF ever comes. Wait on
# each pending worker's exit (pidfd, else its sentinel -- see
# _worker_exit_waitables) and on its fault pipe (one byte from the
# fail-closed path; EOF is not a fault), as the post-READY monitor does.
exit_waitables = _worker_exit_waitables(
[h.proc for h in unready_proc_handles]
)
exits = {
w: handle
for (w, _, _), handle in zip(exit_waitables, unready_proc_handles)
}
faults = {
handle.fault_reader: handle
for handle in unready_proc_handles
if getattr(handle, "fault_reader", None) is not None
}
try:
while pipes:
waiting: list[Any] = [*pipes.keys()]
# EVERY worker's exit and fault pipe, READY or not: the live
# monitor starts only after all are READY, so a READY worker
# that faults or dies while a sibling initialises must fail
# initialization here.
waiting += list(exits)
waiting += list(faults)
ready = multiprocessing.connection.wait(waiting)
WorkerProc._receive_ready(
ready, pipes, exits, faults, ready_proc_handles, e
)
finally:
for _, _, pidfd in exit_waitables:
if pidfd is not None:
with suppress(OSError):
os.close(pidfd)
return cast(list[WorkerProcHandle], ready_proc_handles)
@staticmethod
def _receive_ready(
ready: list[Any],
pipes: dict[Connection, UnreadyWorkerProcHandle],
exits: dict[Any, UnreadyWorkerProcHandle],
faults: dict[Any, UnreadyWorkerProcHandle],
ready_proc_handles: list[WorkerProcHandle | None],
e: Exception,
) -> None:
# Ready pipes first: a READY sent before an exit or fault still counts.
ordered = sorted(ready, key=lambda obj: 0 if obj in pipes else 1)
for obj in ordered:
if obj in pipes:
pipe = obj
try:
# Wait until the WorkerProc is ready.
unready_proc_handle = pipes.pop(pipe)
@@ -995,18 +1205,50 @@ class WorkerProc:
raise e
idx = unready_proc_handle.rank % len(ready_proc_handles)
ready_proc_handles[idx] = WorkerProc.wait_for_response_handle_ready(
response, unready_proc_handle
ready_proc_handles[idx] = (
WorkerProc.wait_for_response_handle_ready(
response, unready_proc_handle
)
)
except EOFError:
e.__suppress_context__ = True
raise e from None
finally:
# Close connection.
pipe.close()
return cast(list[WorkerProcHandle], ready_proc_handles)
continue
handle = exits.get(obj) if obj in exits else faults.get(obj)
if handle is None:
continue
pending = handle.ready_pipe in pipes
if pending and handle.ready_pipe.poll(0):
continue # its READY is queued: read it next round
phase = (
"before READY"
if pending
else "after READY, while other workers were initializing"
)
if obj in exits:
logger.error(
"Worker proc %s exited (code %s) %s",
handle.proc.name,
handle.proc.exitcode,
phase,
)
raise e
try:
data = os.read(obj.fileno(), 1)
except OSError:
data = b""
if data:
logger.error(
"Worker proc %s reported a fatal fault (fail-closed) %s",
handle.proc.name,
phase,
)
raise e
# EOF on the fault pipe is not a fault; its exit follows.
del faults[obj]
def shutdown(self, *, destroy_distributed: bool = True):
if not destroy_distributed:
@@ -1101,11 +1343,21 @@ class WorkerProc:
# requested (death pipe EOF or SIGTERM/SIGINT): the process then exits
# straight after WorkerProc.shutdown (see _exit_worker_process).
orderly_shutdown = False
# Set when a READY worker's busy loop FAILED while a shutdown request
# was pending: full teardown, then exit code 1 (never the exit-0 path).
failed_during_shutdown = False
# Set when a READY worker's busy loop FAILED (anything but its own
# shutdown-request cancellation): full teardown, then exit code 1.
# Never keyed to shutdown_requested -- the death-pipe monitor can set
# it just after the check, which made a failure exit 0.
busy_loop_failed = False
ready_writer = kwargs.pop("ready_pipe")
death_pipe = kwargs.pop("death_pipe", None)
# Fault pipe: registered for the PLE fail-closed path
# (ple_offload.fatal.notify_parent_fatal); kept open for the life of
# the process so the byte written right before os._exit reaches the
# executor's monitor. Closed only on the orderly way out (EOF, which
# the monitor does not count as a fault).
fault_pipe = kwargs.pop("fault_pipe", None)
if fault_pipe is not None:
set_fatal_notify_fd(fault_pipe.fileno())
shutdown_state = kwargs.pop("shutdown_state", None)
def mark_shutdown(value: int) -> None:
@@ -1121,6 +1373,8 @@ class WorkerProc:
os.close(fd)
except Exception as e:
logger.warning("Error closing inherited connection: %s: %s", type(e), e)
# Before anything below can fork+exec a helper (torch_shm_manager).
_make_inherited_pipes_close_on_exec()
try:
# Initialize tracer
@@ -1174,13 +1428,14 @@ class WorkerProc:
"[shutdown] WorkerProc: exiting after shutdown request"
)
orderly_shutdown = True
elif shutdown_requested.is_set():
# A real failure that raced a shutdown request: keep it a
# failure -- full teardown, then a non-zero exit.
logger.exception("WorkerProc failed during shutdown.")
failed_during_shutdown = True
else:
logger.exception("WorkerProc failed.")
# A real failure, whether or not a shutdown request is pending
# (yet): full teardown, then a non-zero exit.
if shutdown_requested.is_set():
logger.exception("WorkerProc failed during shutdown.")
else:
logger.exception("WorkerProc failed.")
busy_loop_failed = True
# The parent sends a SIGTERM to all worker processes if
# any worker dies. Set this value so we don't re-throw
@@ -1205,36 +1460,43 @@ class WorkerProc:
ready_writer.close()
if death_pipe is not None:
death_pipe.close()
# Clean up once worker exits busy loop
if worker is not None:
# Tells the executor's ladder this rank is draining, not parked
# in a collective (see _ensure_worker_termination).
mark_shutdown(_SHUTDOWN_STARTED)
if orderly_shutdown:
# Everything that must happen before exit (FINAL DRAIN,
# instrumentation flushes, PLE cleanup) is in
# worker.shutdown(); then exit instead of paying the NCCL
# destroy + finalization past the executor's grace.
exit_code = 0
try:
worker.shutdown(destroy_distributed=False)
except BaseException:
logger.exception("WorkerProc shutdown failed")
exit_code = 1
# The fault fd stays registered through the whole teardown: a PLE
# fail-closed inside worker.shutdown() must still reach the
# executor. Unregistered after it; on the orderly
# os._exit path the process exit closes it.
try:
# Clean up once worker exits busy loop
if worker is not None:
# Tells the executor's ladder this rank is draining, not parked
# in a collective (see _ensure_worker_termination).
mark_shutdown(_SHUTDOWN_STARTED)
if orderly_shutdown:
# Everything that must happen before exit (FINAL DRAIN,
# instrumentation flushes, PLE cleanup) is in
# worker.shutdown(); then exit instead of paying the NCCL
# destroy + finalization past the executor's grace.
exit_code = 0
try:
worker.shutdown(destroy_distributed=False)
except BaseException:
logger.exception("WorkerProc shutdown failed")
exit_code = 1
mark_shutdown(_SHUTDOWN_DONE)
_exit_worker_process(exit_code)
worker.shutdown()
mark_shutdown(_SHUTDOWN_DONE)
_exit_worker_process(exit_code)
worker.shutdown()
mark_shutdown(_SHUTDOWN_DONE)
if failed_during_shutdown:
sys.exit(1)
if busy_loop_failed:
sys.exit(1)
finally:
if fault_pipe is not None:
set_fatal_notify_fd(None)
fault_pipe.close()
class ResponseStatus(Enum):
SUCCESS = auto()
FAILURE = auto()
def _fail_ple_rpc_if_pending(
self, where: str, error: BaseException
) -> None:
def _fail_ple_rpc_if_pending(self, where: str, error: BaseException) -> None:
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
owners = (self.worker, getattr(self.worker, "worker", None))
for owner in owners:
@@ -1247,9 +1509,7 @@ class WorkerProc:
try:
pending = connector.has_pending_completions
except BaseException as state_error:
connector._fail_closed(
f"{where}:pending-state", state_error
)
connector._fail_closed(f"{where}:pending-state", state_error)
if pending:
connector._fail_closed(where, error)
return
@@ -1264,9 +1524,7 @@ class WorkerProc:
output = output.get_output()
except Exception as e:
logger.exception("Error getting async model runner output")
self._fail_ple_rpc_if_pending(
"async-output-copy", e
)
self._fail_ple_rpc_if_pending("async-output-copy", e)
output = e
if isinstance(output, Exception):
@@ -1338,9 +1596,7 @@ class WorkerProc:
if output_rank is None or self.rank == output_rank:
self.handle_output(output)
except Exception as e:
self._fail_ple_rpc_if_pending(
f"worker-rpc:{method}", e
)
self._fail_ple_rpc_if_pending(f"worker-rpc:{method}", e)
# Notes have been introduced in python 3.11
if hasattr(e, "add_note"):
e.add_note(traceback.format_exc())
+2 -1
View File
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.ple_offload_layer import (
flag_check_is_authoritative,
stale_offload_flags,
)
from vllm.v1.ple_offload.fatal import ple_fail_closed
from vllm.v1.ple_offload.fatal import notify_parent_fatal, ple_fail_closed
from vllm.v1.ple_offload.progress import PleProgressMonitor
from vllm.v1.ple_offload.protocol import (
PleOffloadRegistration,
@@ -469,6 +469,7 @@ class PleOffloadConnector:
self._process_request(request, socket)
except Exception:
logger.exception("PLE request thread failed")
notify_parent_fatal()
os._exit(1)
finally:
self._request_thread_ready.set()
+35
View File
@@ -44,6 +44,40 @@ _TYPE_MODULE = type.__dict__["__module__"].__get__
_MAX_CHAIN = 16
_MAX_ARG_CHARS = 1024
# Write end of the per-worker fault pipe (WorkerProc.make_worker_process); the
# executor's worker monitor waits on the read end next to the process
# sentinels. None outside a worker process (tests, the offload child).
_FATAL_NOTIFY_FD: int | None = None
def set_fatal_notify_fd(fd: int | None) -> None:
"""Register the fault pipe worker_main received from the executor."""
global _FATAL_NOTIFY_FD
_FATAL_NOTIFY_FD = fd
def notify_parent_fatal() -> None:
"""Tell the executor this worker is dying, before the process is gone.
The executor's only other signal is the process sentinel, which the kernel
closes when the LAST task of the worker leaves -- after the address-space
teardown, not at os._exit. On a rank whose GPU work is paired with a live
peer that teardown can wait for the peer's exit (in a rank-1 fault test, a
fail-closed fault on PP0_TP1 left its leader Z and unreapable for 300 s
while rank 0 waited on it in turn; the engine hung on the response
queue). One byte here wakes the monitor at once so the failure ladder ends
the peers, which is what frees the teardown. Nonblocking, never raises:
the caller os._exits right after, whatever happens here.
"""
fd = _FATAL_NOTIFY_FD
if fd is None:
return
try:
os.set_blocking(fd, False)
os.write(fd, b"F")
except BaseException:
pass
def _safe_type_label(cls: type) -> str:
"""``module.qualname`` of ``cls`` without running class-defined code."""
@@ -240,6 +274,7 @@ def ple_fail_closed(
# runs a Python __exit__ between here and os._exit below; a bare
# except keeps that window empty.
pass
notify_parent_fatal()
os._exit(1)