[Engine] Detect a fail-closed fault or death on any rank
A fault on a non-zero TP rank could hang the engine: the worker's sentinel pipe was inherited by a helper process and stayed open, so the monitor never saw the exit. Earlier releases carry the same monitor and fail-closed exit path. - Worker exit is detected by pidfd, with the sentinel as fallback, before and after READY and for every worker through the whole initialisation window. - A per-worker fault pipe: ple_fail_closed writes one byte before os._exit and the monitor starts the failure ladder at once; the fd stays registered through worker teardown. - worker_main marks inherited pipes close-on-exec. - Any unexpected busy-loop failure exits 1; the termination ladder uses time.monotonic(); forced-exit paths no longer wait on logging. - Tests on real fork/spawn workers for rank 0 and rank 1, with and without an fd-inheriting helper. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+40
-13
@@ -116,6 +116,24 @@ _R = TypeVar("_R") # Return type for collective_rpc
|
||||
_TEARDOWN_SIGNAL_BOUND_S = 30.0
|
||||
|
||||
|
||||
# How long the guard's forced-exit paths wait for their diagnostic line. The
|
||||
# line is written from a helper thread; a blocked handler, a held logging lock
|
||||
# or a full stderr pipe costs at most this, never the exit.
|
||||
_TEARDOWN_EXIT_LOG_BOUND_S = 0.5
|
||||
|
||||
|
||||
def _log_bounded(emit: Callable[[], None], bound_s: float) -> None:
|
||||
"""Run ``emit`` (logging, flushes) on a daemon thread; wait <= bound_s."""
|
||||
try:
|
||||
thread = threading.Thread(
|
||||
target=emit, daemon=True, name="EngineCoreTeardownLog"
|
||||
)
|
||||
thread.start()
|
||||
thread.join(bound_s)
|
||||
except BaseException: # noqa: BLE001 -- the caller's exit must still run
|
||||
pass
|
||||
|
||||
|
||||
class _TeardownSignalGuard:
|
||||
"""EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded.
|
||||
|
||||
@@ -154,25 +172,34 @@ class _TeardownSignalGuard:
|
||||
)
|
||||
self._watchdog.start()
|
||||
return
|
||||
logger.warning(
|
||||
"[shutdown] EngineCore: second late signal (%s) during resource "
|
||||
"teardown; not deferring again",
|
||||
name,
|
||||
)
|
||||
# Default disposition first, so a third signal also ends the process
|
||||
# while the (bounded) diagnostic line is written.
|
||||
signal.signal(signum, signal.SIG_DFL)
|
||||
_log_bounded(
|
||||
lambda: logger.warning(
|
||||
"[shutdown] EngineCore: second late signal (%s) during "
|
||||
"resource teardown; not deferring again",
|
||||
name,
|
||||
),
|
||||
_TEARDOWN_EXIT_LOG_BOUND_S,
|
||||
)
|
||||
os.kill(os.getpid(), signum)
|
||||
|
||||
def _expire(self) -> None:
|
||||
if self._done.wait(self.bound_s):
|
||||
return
|
||||
logger.error(
|
||||
"[shutdown] EngineCore: resource teardown still running %.0f s "
|
||||
"after a deferred signal; exiting (code 1)",
|
||||
self.bound_s,
|
||||
)
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with suppress(Exception):
|
||||
stream.flush()
|
||||
|
||||
def emit() -> None:
|
||||
logger.error(
|
||||
"[shutdown] EngineCore: resource teardown still running %.0f s "
|
||||
"after a deferred signal; exiting (code 1)",
|
||||
self.bound_s,
|
||||
)
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with suppress(Exception):
|
||||
stream.flush()
|
||||
|
||||
_log_bounded(emit, _TEARDOWN_EXIT_LOG_BOUND_S)
|
||||
os._exit(1)
|
||||
|
||||
def disarm(self) -> None:
|
||||
|
||||
@@ -5,6 +5,7 @@ import os
|
||||
import pickle
|
||||
import queue
|
||||
import signal
|
||||
import stat
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
@@ -71,6 +72,7 @@ from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.executor.abstract import Executor, FailureCallback
|
||||
from vllm.v1.executor.vllm_net_devices import set_worker_net_device
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, DraftTokenIds, ModelRunnerOutput
|
||||
from vllm.v1.ple_offload.fatal import set_fatal_notify_fd
|
||||
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -97,13 +99,16 @@ def _install_worker_parent_death_signal(expected_parent_pid: int) -> None:
|
||||
ctypes.c_ulong,
|
||||
]
|
||||
prctl.restype = ctypes.c_int
|
||||
if prctl(
|
||||
_PR_SET_PDEATHSIG,
|
||||
int(_WORKER_PARENT_DEATH_SIGNAL),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
) != 0:
|
||||
if (
|
||||
prctl(
|
||||
_PR_SET_PDEATHSIG,
|
||||
int(_WORKER_PARENT_DEATH_SIGNAL),
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
)
|
||||
!= 0
|
||||
):
|
||||
error_number = ctypes.get_errno()
|
||||
raise OSError(error_number, os.strerror(error_number))
|
||||
|
||||
@@ -173,6 +178,86 @@ def _exit_worker_process(exit_code: int) -> None:
|
||||
os._exit(exit_code)
|
||||
|
||||
|
||||
def _worker_exit_waitables(
|
||||
procs: Sequence[BaseProcess],
|
||||
) -> list[tuple[Any, BaseProcess, int | None]]:
|
||||
"""What the worker monitor waits on to learn that a worker process exited.
|
||||
|
||||
A pidfd (Linux >= 5.3) where possible: it becomes readable when the process
|
||||
itself exits. proc.sentinel is only the read end of a pipe whose write end
|
||||
the worker holds, so it reports the exit of the LAST process holding that
|
||||
write end -- and anything the worker fork+execs inherits it. Every PLE-
|
||||
owning worker does: registering its host buffers under the "file_system"
|
||||
sharing strategy starts a torch_shm_manager child, which keeps the write
|
||||
end, and which lives while any client -- rank 0's PLE child, for a TP1
|
||||
worker's buffers -- still maps them. In a rank-1 fault test PP0_TP1 died
|
||||
in release_outputs, sat as a zombie, its sentinel never fired and the
|
||||
engine hung; rank 0's death only looked detected because its PLE child
|
||||
(and so both managers) died with it, ~8 s later.
|
||||
|
||||
Returns (waitable, proc, pidfd-to-close-or-None) per process, falling back
|
||||
to the sentinel where no pidfd can be opened.
|
||||
"""
|
||||
pidfd_open = getattr(os, "pidfd_open", None)
|
||||
waitables: list[tuple[Any, BaseProcess, int | None]] = []
|
||||
try:
|
||||
for proc in procs:
|
||||
pidfd: int | None = None
|
||||
pid = getattr(proc, "pid", None)
|
||||
if pidfd_open is not None and pid is not None:
|
||||
try:
|
||||
pidfd = pidfd_open(pid)
|
||||
except OSError:
|
||||
# ESRCH (already reaped) or no pidfd support: the sentinel.
|
||||
pidfd = None
|
||||
try:
|
||||
waitable = pidfd if pidfd is not None else proc.sentinel
|
||||
waitables.append((waitable, proc, pidfd))
|
||||
except BaseException:
|
||||
# Not yet in `waitables`: the outer handler would miss it.
|
||||
if pidfd is not None:
|
||||
os.close(pidfd)
|
||||
raise
|
||||
except BaseException:
|
||||
# Nothing is handed out on failure: close what was opened.
|
||||
for _, _, opened in waitables:
|
||||
if opened is not None:
|
||||
with suppress(OSError):
|
||||
os.close(opened)
|
||||
raise
|
||||
return waitables
|
||||
|
||||
|
||||
def _make_inherited_pipes_close_on_exec() -> None:
|
||||
"""Keep this worker's inherited pipe fds out of the programs it execs.
|
||||
|
||||
Under the spawn start method multiprocessing hands the worker its end of
|
||||
the sentinel pipe (and the ready / death pipes) as INHERITABLE fds, so any
|
||||
program the worker fork+execs keeps them open: the torch_shm_manager that
|
||||
the PLE host-buffer registration starts did, and outlived the worker, so
|
||||
neither the sentinel nor the ready pipe reported its death (rank-1 fault injection).
|
||||
Nothing this process execs needs them -- multiprocessing children get
|
||||
their fds passed explicitly, subprocess closes them by default -- so mark
|
||||
every inherited pipe above stderr close-on-exec. The executor's pidfd
|
||||
wait is the primary detection; this only stops the leak at its source.
|
||||
"""
|
||||
if not sys.platform.startswith("linux"):
|
||||
return
|
||||
try:
|
||||
fds = [int(name) for name in os.listdir("/proc/self/fd")]
|
||||
except (OSError, ValueError):
|
||||
return
|
||||
for fd in fds:
|
||||
if fd <= 2:
|
||||
continue
|
||||
try:
|
||||
if stat.S_ISFIFO(os.fstat(fd).st_mode) and os.get_inheritable(fd):
|
||||
os.set_inheritable(fd, False)
|
||||
except OSError:
|
||||
# The listing's own directory fd, or one closed since.
|
||||
continue
|
||||
|
||||
|
||||
class FutureWrapper(Future):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -304,6 +389,7 @@ class MultiprocExecutor(Executor):
|
||||
if inherited_fds is not None:
|
||||
inherited_fds.append(unready_worker_handle.death_writer.fileno())
|
||||
inherited_fds.append(unready_worker_handle.ready_pipe.fileno())
|
||||
inherited_fds.append(unready_worker_handle.fault_reader.fileno())
|
||||
|
||||
# Workers must be created before wait_for_ready to avoid
|
||||
# deadlock, since worker.init_device() does a device sync.
|
||||
@@ -400,22 +486,78 @@ class MultiprocExecutor(Executor):
|
||||
# Monitors worker process liveness. If any die unexpectedly,
|
||||
# logs an error, shuts down the executor and invokes the failure
|
||||
# callback to inform the engine.
|
||||
#
|
||||
# Two wake sources per worker:
|
||||
# - its exit: a pidfd, readable when the process itself has exited
|
||||
# (fallback: proc.sentinel, which a fork+exec'd child holding the
|
||||
# write end -- torch_shm_manager from the PLE registration -- keeps
|
||||
# silent; see _worker_exit_waitables);
|
||||
# - its fault pipe: one byte from the fail-closed path right before
|
||||
# os._exit, so the failure ladder starts -- and ends the peers --
|
||||
# without waiting for the process teardown, which can itself wait
|
||||
# on a live peer. A worker closing its end on the way out (EOF, no
|
||||
# byte) is not a fault; its exit follows.
|
||||
# In a rank-1 fault test a fault on PP0_TP1 was never detected and the
|
||||
# engine hung on the response queue for 300 s.
|
||||
# Opened here, before the thread starts, so a worker that dies early
|
||||
# is still a zombie (pidfds work on zombies) rather than reaped.
|
||||
exit_waitables = _worker_exit_waitables([h.proc for h in workers])
|
||||
|
||||
def monitor_workers():
|
||||
sentinels = [h.proc.sentinel for h in workers]
|
||||
died = multiprocessing.connection.wait(sentinels)
|
||||
exit_to_handle = {
|
||||
w: h for (w, _, _), h in zip(exit_waitables, workers)
|
||||
}
|
||||
fault_to_handle = {
|
||||
h.fault_reader: h
|
||||
for h in workers
|
||||
if getattr(h, "fault_reader", None) is not None
|
||||
}
|
||||
waiting: list[Any] = list(exit_to_handle) + list(fault_to_handle)
|
||||
handle = None
|
||||
reported_fatal = False
|
||||
try:
|
||||
while handle is None:
|
||||
ready = multiprocessing.connection.wait(waiting)
|
||||
for obj in ready:
|
||||
if obj in exit_to_handle:
|
||||
handle = exit_to_handle[obj]
|
||||
break
|
||||
try:
|
||||
data = os.read(obj.fileno(), 1)
|
||||
except OSError:
|
||||
data = b""
|
||||
if data:
|
||||
handle = fault_to_handle[obj]
|
||||
reported_fatal = True
|
||||
break
|
||||
waiting.remove(obj)
|
||||
finally:
|
||||
for _, _, pidfd in exit_waitables:
|
||||
if pidfd is not None:
|
||||
with suppress(OSError):
|
||||
os.close(pidfd)
|
||||
_self = self_ref()
|
||||
if not _self or getattr(_self, "shutting_down", False):
|
||||
logger.debug("MultiprocWorkerMonitor: shutdown already initiated")
|
||||
return
|
||||
_self.is_failed = True
|
||||
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
|
||||
logger.error(
|
||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||
"shutting down executor. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
proc.exitcode,
|
||||
time.monotonic(),
|
||||
)
|
||||
proc = handle.proc
|
||||
if reported_fatal:
|
||||
logger.error(
|
||||
"Worker proc %s reported a fatal fault (fail-closed) and is "
|
||||
"exiting; shutting down executor without waiting for its "
|
||||
"process teardown. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
time.monotonic(),
|
||||
)
|
||||
else:
|
||||
logger.error(
|
||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||
"shutting down executor. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
proc.exitcode,
|
||||
time.monotonic(),
|
||||
)
|
||||
_self.shutdown()
|
||||
callback = _self.failure_callback
|
||||
if callback is not None:
|
||||
@@ -423,9 +565,16 @@ class MultiprocExecutor(Executor):
|
||||
callback()
|
||||
|
||||
if not inline:
|
||||
Thread(
|
||||
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
|
||||
).start()
|
||||
try:
|
||||
Thread(
|
||||
target=monitor_workers, daemon=True, name="MultiprocWorkerMonitor"
|
||||
).start()
|
||||
except BaseException:
|
||||
for _, _, pidfd in exit_waitables:
|
||||
if pidfd is not None:
|
||||
with suppress(OSError):
|
||||
os.close(pidfd)
|
||||
raise
|
||||
return
|
||||
|
||||
monitor_workers()
|
||||
@@ -589,8 +738,9 @@ class MultiprocExecutor(Executor):
|
||||
# If we are in late stage shutdown, the interpreter may replace
|
||||
# `time` with `None`.
|
||||
return all(not proc.is_alive() for proc in procs)
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
# Monotonic: a wall-clock step must not cut or stretch the grace.
|
||||
start_time = time.monotonic()
|
||||
while time.monotonic() - start_time < timeout:
|
||||
if all(not proc.is_alive() for proc in procs):
|
||||
return True
|
||||
time.sleep(0.1)
|
||||
@@ -605,7 +755,7 @@ class MultiprocExecutor(Executor):
|
||||
initial_count,
|
||||
)
|
||||
if after_failure and time:
|
||||
start = time.time()
|
||||
start = time.monotonic()
|
||||
states = list(shutdown_states or [None] * len(worker_procs))
|
||||
|
||||
def parked() -> list[BaseProcess]:
|
||||
@@ -637,7 +787,7 @@ class MultiprocExecutor(Executor):
|
||||
p.kill()
|
||||
# Workers that started their shutdown get a BOUNDED grace, counted
|
||||
# from the start of this ladder, then SIGTERM, then SIGKILL.
|
||||
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.time() - start))
|
||||
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.monotonic() - start))
|
||||
term_wait_s = _FAILED_WORKER_WAIT_S
|
||||
if wait_for_termination(active_procs(), timeout=grace_s):
|
||||
logger.info_once("[shutdown] Executor: all workers exited gracefully")
|
||||
@@ -738,6 +888,7 @@ class UnreadyWorkerProcHandle:
|
||||
ready_pipe: Connection
|
||||
death_writer: Connection | None = None
|
||||
shutdown_state: Any = None
|
||||
fault_reader: Connection | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -753,6 +904,8 @@ class WorkerProcHandle:
|
||||
death_writer: Connection | None = None
|
||||
# Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes).
|
||||
shutdown_state: Any = None
|
||||
# Read end of the worker's fault pipe (see start_worker_monitor).
|
||||
fault_reader: Connection | None = None
|
||||
|
||||
@classmethod
|
||||
def from_unready_handle(
|
||||
@@ -768,6 +921,7 @@ class WorkerProcHandle:
|
||||
peer_worker_response_mqs=peer_worker_response_mqs,
|
||||
death_writer=unready_handle.death_writer,
|
||||
shutdown_state=unready_handle.shutdown_state,
|
||||
fault_reader=unready_handle.fault_reader,
|
||||
)
|
||||
|
||||
|
||||
@@ -906,11 +1060,16 @@ class WorkerProc:
|
||||
ready_reader, ready_writer = context.Pipe(duplex=False)
|
||||
# Death pipe to let child detect parent process exit
|
||||
death_reader, death_writer = context.Pipe(duplex=False)
|
||||
# Fault pipe: the worker's fail-closed path writes one byte before
|
||||
# os._exit; the executor's monitor waits on it next to the sentinel.
|
||||
fault_reader, fault_writer = context.Pipe(duplex=False)
|
||||
# One shared byte: has this worker started / finished its shutdown?
|
||||
shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED)
|
||||
if inherited_fds is not None:
|
||||
inherited_fds = inherited_fds.copy()
|
||||
inherited_fds.extend((ready_reader.fileno(), death_writer.fileno()))
|
||||
inherited_fds.extend(
|
||||
(ready_reader.fileno(), death_writer.fileno(), fault_reader.fileno())
|
||||
)
|
||||
process_kwargs = {
|
||||
"expected_parent_pid": os.getpid(),
|
||||
"vllm_config": vllm_config,
|
||||
@@ -920,6 +1079,7 @@ class WorkerProc:
|
||||
"input_shm_handle": input_shm_handle,
|
||||
"ready_pipe": ready_writer,
|
||||
"death_pipe": death_reader,
|
||||
"fault_pipe": fault_writer,
|
||||
"shutdown_state": shutdown_state,
|
||||
"shared_worker_lock": shared_worker_lock,
|
||||
"is_driver_worker": is_driver_worker,
|
||||
@@ -943,10 +1103,11 @@ class WorkerProc:
|
||||
# Close child ends of pipes here in the parent
|
||||
ready_writer.close()
|
||||
death_reader.close()
|
||||
fault_writer.close()
|
||||
# Keep death_writer open in parent - when parent exits,
|
||||
# death_reader in child will get EOFError
|
||||
return UnreadyWorkerProcHandle(
|
||||
proc, rank, ready_reader, death_writer, shutdown_state
|
||||
proc, rank, ready_reader, death_writer, shutdown_state, fault_reader
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -983,10 +1144,59 @@ class WorkerProc:
|
||||
ready_proc_handles: list[WorkerProcHandle | None] = [None] * len(
|
||||
unready_proc_handles
|
||||
)
|
||||
while pipes:
|
||||
ready = multiprocessing.connection.wait(pipes.keys())
|
||||
for pipe in ready:
|
||||
assert isinstance(pipe, Connection)
|
||||
# A worker that dies before READY need not close its ready pipe: a
|
||||
# child it fork+exec'd (torch_shm_manager, started by the PLE
|
||||
# registration) can hold the write end, so no EOF ever comes. Wait on
|
||||
# each pending worker's exit (pidfd, else its sentinel -- see
|
||||
# _worker_exit_waitables) and on its fault pipe (one byte from the
|
||||
# fail-closed path; EOF is not a fault), as the post-READY monitor does.
|
||||
exit_waitables = _worker_exit_waitables(
|
||||
[h.proc for h in unready_proc_handles]
|
||||
)
|
||||
exits = {
|
||||
w: handle
|
||||
for (w, _, _), handle in zip(exit_waitables, unready_proc_handles)
|
||||
}
|
||||
faults = {
|
||||
handle.fault_reader: handle
|
||||
for handle in unready_proc_handles
|
||||
if getattr(handle, "fault_reader", None) is not None
|
||||
}
|
||||
try:
|
||||
while pipes:
|
||||
waiting: list[Any] = [*pipes.keys()]
|
||||
# EVERY worker's exit and fault pipe, READY or not: the live
|
||||
# monitor starts only after all are READY, so a READY worker
|
||||
# that faults or dies while a sibling initialises must fail
|
||||
# initialization here.
|
||||
waiting += list(exits)
|
||||
waiting += list(faults)
|
||||
ready = multiprocessing.connection.wait(waiting)
|
||||
WorkerProc._receive_ready(
|
||||
ready, pipes, exits, faults, ready_proc_handles, e
|
||||
)
|
||||
finally:
|
||||
for _, _, pidfd in exit_waitables:
|
||||
if pidfd is not None:
|
||||
with suppress(OSError):
|
||||
os.close(pidfd)
|
||||
|
||||
return cast(list[WorkerProcHandle], ready_proc_handles)
|
||||
|
||||
@staticmethod
|
||||
def _receive_ready(
|
||||
ready: list[Any],
|
||||
pipes: dict[Connection, UnreadyWorkerProcHandle],
|
||||
exits: dict[Any, UnreadyWorkerProcHandle],
|
||||
faults: dict[Any, UnreadyWorkerProcHandle],
|
||||
ready_proc_handles: list[WorkerProcHandle | None],
|
||||
e: Exception,
|
||||
) -> None:
|
||||
# Ready pipes first: a READY sent before an exit or fault still counts.
|
||||
ordered = sorted(ready, key=lambda obj: 0 if obj in pipes else 1)
|
||||
for obj in ordered:
|
||||
if obj in pipes:
|
||||
pipe = obj
|
||||
try:
|
||||
# Wait until the WorkerProc is ready.
|
||||
unready_proc_handle = pipes.pop(pipe)
|
||||
@@ -995,18 +1205,50 @@ class WorkerProc:
|
||||
raise e
|
||||
|
||||
idx = unready_proc_handle.rank % len(ready_proc_handles)
|
||||
ready_proc_handles[idx] = WorkerProc.wait_for_response_handle_ready(
|
||||
response, unready_proc_handle
|
||||
ready_proc_handles[idx] = (
|
||||
WorkerProc.wait_for_response_handle_ready(
|
||||
response, unready_proc_handle
|
||||
)
|
||||
)
|
||||
except EOFError:
|
||||
e.__suppress_context__ = True
|
||||
raise e from None
|
||||
|
||||
finally:
|
||||
# Close connection.
|
||||
pipe.close()
|
||||
|
||||
return cast(list[WorkerProcHandle], ready_proc_handles)
|
||||
continue
|
||||
handle = exits.get(obj) if obj in exits else faults.get(obj)
|
||||
if handle is None:
|
||||
continue
|
||||
pending = handle.ready_pipe in pipes
|
||||
if pending and handle.ready_pipe.poll(0):
|
||||
continue # its READY is queued: read it next round
|
||||
phase = (
|
||||
"before READY"
|
||||
if pending
|
||||
else "after READY, while other workers were initializing"
|
||||
)
|
||||
if obj in exits:
|
||||
logger.error(
|
||||
"Worker proc %s exited (code %s) %s",
|
||||
handle.proc.name,
|
||||
handle.proc.exitcode,
|
||||
phase,
|
||||
)
|
||||
raise e
|
||||
try:
|
||||
data = os.read(obj.fileno(), 1)
|
||||
except OSError:
|
||||
data = b""
|
||||
if data:
|
||||
logger.error(
|
||||
"Worker proc %s reported a fatal fault (fail-closed) %s",
|
||||
handle.proc.name,
|
||||
phase,
|
||||
)
|
||||
raise e
|
||||
# EOF on the fault pipe is not a fault; its exit follows.
|
||||
del faults[obj]
|
||||
|
||||
def shutdown(self, *, destroy_distributed: bool = True):
|
||||
if not destroy_distributed:
|
||||
@@ -1101,11 +1343,21 @@ class WorkerProc:
|
||||
# requested (death pipe EOF or SIGTERM/SIGINT): the process then exits
|
||||
# straight after WorkerProc.shutdown (see _exit_worker_process).
|
||||
orderly_shutdown = False
|
||||
# Set when a READY worker's busy loop FAILED while a shutdown request
|
||||
# was pending: full teardown, then exit code 1 (never the exit-0 path).
|
||||
failed_during_shutdown = False
|
||||
# Set when a READY worker's busy loop FAILED (anything but its own
|
||||
# shutdown-request cancellation): full teardown, then exit code 1.
|
||||
# Never keyed to shutdown_requested -- the death-pipe monitor can set
|
||||
# it just after the check, which made a failure exit 0.
|
||||
busy_loop_failed = False
|
||||
ready_writer = kwargs.pop("ready_pipe")
|
||||
death_pipe = kwargs.pop("death_pipe", None)
|
||||
# Fault pipe: registered for the PLE fail-closed path
|
||||
# (ple_offload.fatal.notify_parent_fatal); kept open for the life of
|
||||
# the process so the byte written right before os._exit reaches the
|
||||
# executor's monitor. Closed only on the orderly way out (EOF, which
|
||||
# the monitor does not count as a fault).
|
||||
fault_pipe = kwargs.pop("fault_pipe", None)
|
||||
if fault_pipe is not None:
|
||||
set_fatal_notify_fd(fault_pipe.fileno())
|
||||
shutdown_state = kwargs.pop("shutdown_state", None)
|
||||
|
||||
def mark_shutdown(value: int) -> None:
|
||||
@@ -1121,6 +1373,8 @@ class WorkerProc:
|
||||
os.close(fd)
|
||||
except Exception as e:
|
||||
logger.warning("Error closing inherited connection: %s: %s", type(e), e)
|
||||
# Before anything below can fork+exec a helper (torch_shm_manager).
|
||||
_make_inherited_pipes_close_on_exec()
|
||||
|
||||
try:
|
||||
# Initialize tracer
|
||||
@@ -1174,13 +1428,14 @@ class WorkerProc:
|
||||
"[shutdown] WorkerProc: exiting after shutdown request"
|
||||
)
|
||||
orderly_shutdown = True
|
||||
elif shutdown_requested.is_set():
|
||||
# A real failure that raced a shutdown request: keep it a
|
||||
# failure -- full teardown, then a non-zero exit.
|
||||
logger.exception("WorkerProc failed during shutdown.")
|
||||
failed_during_shutdown = True
|
||||
else:
|
||||
logger.exception("WorkerProc failed.")
|
||||
# A real failure, whether or not a shutdown request is pending
|
||||
# (yet): full teardown, then a non-zero exit.
|
||||
if shutdown_requested.is_set():
|
||||
logger.exception("WorkerProc failed during shutdown.")
|
||||
else:
|
||||
logger.exception("WorkerProc failed.")
|
||||
busy_loop_failed = True
|
||||
|
||||
# The parent sends a SIGTERM to all worker processes if
|
||||
# any worker dies. Set this value so we don't re-throw
|
||||
@@ -1205,36 +1460,43 @@ class WorkerProc:
|
||||
ready_writer.close()
|
||||
if death_pipe is not None:
|
||||
death_pipe.close()
|
||||
# Clean up once worker exits busy loop
|
||||
if worker is not None:
|
||||
# Tells the executor's ladder this rank is draining, not parked
|
||||
# in a collective (see _ensure_worker_termination).
|
||||
mark_shutdown(_SHUTDOWN_STARTED)
|
||||
if orderly_shutdown:
|
||||
# Everything that must happen before exit (FINAL DRAIN,
|
||||
# instrumentation flushes, PLE cleanup) is in
|
||||
# worker.shutdown(); then exit instead of paying the NCCL
|
||||
# destroy + finalization past the executor's grace.
|
||||
exit_code = 0
|
||||
try:
|
||||
worker.shutdown(destroy_distributed=False)
|
||||
except BaseException:
|
||||
logger.exception("WorkerProc shutdown failed")
|
||||
exit_code = 1
|
||||
# The fault fd stays registered through the whole teardown: a PLE
|
||||
# fail-closed inside worker.shutdown() must still reach the
|
||||
# executor. Unregistered after it; on the orderly
|
||||
# os._exit path the process exit closes it.
|
||||
try:
|
||||
# Clean up once worker exits busy loop
|
||||
if worker is not None:
|
||||
# Tells the executor's ladder this rank is draining, not parked
|
||||
# in a collective (see _ensure_worker_termination).
|
||||
mark_shutdown(_SHUTDOWN_STARTED)
|
||||
if orderly_shutdown:
|
||||
# Everything that must happen before exit (FINAL DRAIN,
|
||||
# instrumentation flushes, PLE cleanup) is in
|
||||
# worker.shutdown(); then exit instead of paying the NCCL
|
||||
# destroy + finalization past the executor's grace.
|
||||
exit_code = 0
|
||||
try:
|
||||
worker.shutdown(destroy_distributed=False)
|
||||
except BaseException:
|
||||
logger.exception("WorkerProc shutdown failed")
|
||||
exit_code = 1
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
_exit_worker_process(exit_code)
|
||||
worker.shutdown()
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
_exit_worker_process(exit_code)
|
||||
worker.shutdown()
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
if failed_during_shutdown:
|
||||
sys.exit(1)
|
||||
if busy_loop_failed:
|
||||
sys.exit(1)
|
||||
finally:
|
||||
if fault_pipe is not None:
|
||||
set_fatal_notify_fd(None)
|
||||
fault_pipe.close()
|
||||
|
||||
class ResponseStatus(Enum):
|
||||
SUCCESS = auto()
|
||||
FAILURE = auto()
|
||||
|
||||
def _fail_ple_rpc_if_pending(
|
||||
self, where: str, error: BaseException
|
||||
) -> None:
|
||||
def _fail_ple_rpc_if_pending(self, where: str, error: BaseException) -> None:
|
||||
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
|
||||
owners = (self.worker, getattr(self.worker, "worker", None))
|
||||
for owner in owners:
|
||||
@@ -1247,9 +1509,7 @@ class WorkerProc:
|
||||
try:
|
||||
pending = connector.has_pending_completions
|
||||
except BaseException as state_error:
|
||||
connector._fail_closed(
|
||||
f"{where}:pending-state", state_error
|
||||
)
|
||||
connector._fail_closed(f"{where}:pending-state", state_error)
|
||||
if pending:
|
||||
connector._fail_closed(where, error)
|
||||
return
|
||||
@@ -1264,9 +1524,7 @@ class WorkerProc:
|
||||
output = output.get_output()
|
||||
except Exception as e:
|
||||
logger.exception("Error getting async model runner output")
|
||||
self._fail_ple_rpc_if_pending(
|
||||
"async-output-copy", e
|
||||
)
|
||||
self._fail_ple_rpc_if_pending("async-output-copy", e)
|
||||
output = e
|
||||
|
||||
if isinstance(output, Exception):
|
||||
@@ -1338,9 +1596,7 @@ class WorkerProc:
|
||||
if output_rank is None or self.rank == output_rank:
|
||||
self.handle_output(output)
|
||||
except Exception as e:
|
||||
self._fail_ple_rpc_if_pending(
|
||||
f"worker-rpc:{method}", e
|
||||
)
|
||||
self._fail_ple_rpc_if_pending(f"worker-rpc:{method}", e)
|
||||
# Notes have been introduced in python 3.11
|
||||
if hasattr(e, "add_note"):
|
||||
e.add_note(traceback.format_exc())
|
||||
|
||||
@@ -25,7 +25,7 @@ from vllm.model_executor.layers.ple_offload_layer import (
|
||||
flag_check_is_authoritative,
|
||||
stale_offload_flags,
|
||||
)
|
||||
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||
from vllm.v1.ple_offload.fatal import notify_parent_fatal, ple_fail_closed
|
||||
from vllm.v1.ple_offload.progress import PleProgressMonitor
|
||||
from vllm.v1.ple_offload.protocol import (
|
||||
PleOffloadRegistration,
|
||||
@@ -469,6 +469,7 @@ class PleOffloadConnector:
|
||||
self._process_request(request, socket)
|
||||
except Exception:
|
||||
logger.exception("PLE request thread failed")
|
||||
notify_parent_fatal()
|
||||
os._exit(1)
|
||||
finally:
|
||||
self._request_thread_ready.set()
|
||||
|
||||
@@ -44,6 +44,40 @@ _TYPE_MODULE = type.__dict__["__module__"].__get__
|
||||
_MAX_CHAIN = 16
|
||||
_MAX_ARG_CHARS = 1024
|
||||
|
||||
# Write end of the per-worker fault pipe (WorkerProc.make_worker_process); the
|
||||
# executor's worker monitor waits on the read end next to the process
|
||||
# sentinels. None outside a worker process (tests, the offload child).
|
||||
_FATAL_NOTIFY_FD: int | None = None
|
||||
|
||||
|
||||
def set_fatal_notify_fd(fd: int | None) -> None:
|
||||
"""Register the fault pipe worker_main received from the executor."""
|
||||
global _FATAL_NOTIFY_FD
|
||||
_FATAL_NOTIFY_FD = fd
|
||||
|
||||
|
||||
def notify_parent_fatal() -> None:
|
||||
"""Tell the executor this worker is dying, before the process is gone.
|
||||
|
||||
The executor's only other signal is the process sentinel, which the kernel
|
||||
closes when the LAST task of the worker leaves -- after the address-space
|
||||
teardown, not at os._exit. On a rank whose GPU work is paired with a live
|
||||
peer that teardown can wait for the peer's exit (in a rank-1 fault test, a
|
||||
fail-closed fault on PP0_TP1 left its leader Z and unreapable for 300 s
|
||||
while rank 0 waited on it in turn; the engine hung on the response
|
||||
queue). One byte here wakes the monitor at once so the failure ladder ends
|
||||
the peers, which is what frees the teardown. Nonblocking, never raises:
|
||||
the caller os._exits right after, whatever happens here.
|
||||
"""
|
||||
fd = _FATAL_NOTIFY_FD
|
||||
if fd is None:
|
||||
return
|
||||
try:
|
||||
os.set_blocking(fd, False)
|
||||
os.write(fd, b"F")
|
||||
except BaseException:
|
||||
pass
|
||||
|
||||
|
||||
def _safe_type_label(cls: type) -> str:
|
||||
"""``module.qualname`` of ``cls`` without running class-defined code."""
|
||||
@@ -240,6 +274,7 @@ def ple_fail_closed(
|
||||
# runs a Python __exit__ between here and os._exit below; a bare
|
||||
# except keeps that window empty.
|
||||
pass
|
||||
notify_parent_fatal()
|
||||
os._exit(1)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user