[Engine] Drain-order shutdown
- EngineCore survives a late SIGTERM during teardown (bounded deferral) so every worker drains its guard telemetry. - A worker exits right after its orderly shutdown. - A worker failure racing shutdown stays a failure (exit 1). - Rank 0 waits for the PLE child's whole thread group, bounded, instead of its reap; drain-aware failure ladder with a bounded grace for survivors whose shutdown already started. - tests: the sorted-emit graph arm takes max_len on the host before capture. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+93
-5
@@ -4,12 +4,13 @@ import gc
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Callable, Generator, Sequence
|
||||
from concurrent.futures import Future
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from contextlib import ExitStack, contextmanager, suppress
|
||||
from enum import IntEnum
|
||||
from functools import partial
|
||||
from inspect import isclass, signature
|
||||
@@ -108,6 +109,76 @@ HANDSHAKE_TIMEOUT_MINS = 5
|
||||
_R = TypeVar("_R") # Return type for collective_rpc
|
||||
|
||||
|
||||
# How long EngineCore teardown may run after it deferred a late SIGTERM/SIGINT
|
||||
# before the process exits anyway. Covers the whole of engine_core.shutdown(),
|
||||
# not only the executor's worker ladder (grace + 4 s): a hang in later teardown
|
||||
# (scheduler, distributed cleanup) must not make the process unkillable.
|
||||
_TEARDOWN_SIGNAL_BOUND_S = 30.0
|
||||
|
||||
|
||||
class _TeardownSignalGuard:
|
||||
"""EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded.
|
||||
|
||||
Dying on the first late signal would SIGKILL every worker
|
||||
(PR_SET_PDEATHSIG) before its shutdown drain; see the finally block of
|
||||
EngineCoreProc.run_engine_core. The deferral is bounded two ways:
|
||||
- a SECOND late signal restores the default disposition and re-delivers
|
||||
itself, so the process dies of it (the maintainer's escalation);
|
||||
- after the first deferred signal a watchdog ends the process with
|
||||
exit code 1 if teardown is still running _TEARDOWN_SIGNAL_BOUND_S later.
|
||||
"""
|
||||
|
||||
def __init__(self, bound_s: float | None = None) -> None:
|
||||
self.bound_s = _TEARDOWN_SIGNAL_BOUND_S if bound_s is None else bound_s
|
||||
self.deferred = 0
|
||||
self._done = threading.Event()
|
||||
self._watchdog: threading.Thread | None = None
|
||||
|
||||
def __call__(self, signum, frame) -> None:
|
||||
name = signal.Signals(signum).name
|
||||
if self._done.is_set():
|
||||
return
|
||||
self.deferred += 1
|
||||
if self.deferred == 1:
|
||||
logger.warning(
|
||||
"[shutdown] EngineCore: %s during resource teardown deferred; "
|
||||
"workers finish their shutdown (FINAL DRAIN) first (a second "
|
||||
"signal, or %.0f s of teardown, ends the process)",
|
||||
name,
|
||||
self.bound_s,
|
||||
)
|
||||
self._watchdog = threading.Thread(
|
||||
target=self._expire,
|
||||
daemon=True,
|
||||
name="EngineCoreTeardownWatchdog",
|
||||
)
|
||||
self._watchdog.start()
|
||||
return
|
||||
logger.warning(
|
||||
"[shutdown] EngineCore: second late signal (%s) during resource "
|
||||
"teardown; not deferring again",
|
||||
name,
|
||||
)
|
||||
signal.signal(signum, signal.SIG_DFL)
|
||||
os.kill(os.getpid(), signum)
|
||||
|
||||
def _expire(self) -> None:
|
||||
if self._done.wait(self.bound_s):
|
||||
return
|
||||
logger.error(
|
||||
"[shutdown] EngineCore: resource teardown still running %.0f s "
|
||||
"after a deferred signal; exiting (code 1)",
|
||||
self.bound_s,
|
||||
)
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with suppress(Exception):
|
||||
stream.flush()
|
||||
os._exit(1)
|
||||
|
||||
def disarm(self) -> None:
|
||||
self._done.set()
|
||||
|
||||
|
||||
class EngineCore:
|
||||
"""Inner loop of vLLM's Engine."""
|
||||
|
||||
@@ -1441,12 +1512,29 @@ class EngineCoreProc(EngineCore):
|
||||
engine_core._send_engine_dead()
|
||||
raise e
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
# Workers carry PR_SET_PDEATHSIG=SIGKILL (fail-closed PLE teardown),
|
||||
# so if this process dies before model_executor.shutdown() has reaped
|
||||
# them, every worker is SIGKILLed mid-shutdown and its bounds-guard
|
||||
# FINAL DRAIN never runs. The API server's process manager sends a
|
||||
# second SIGTERM right after the signal that started this shutdown;
|
||||
# with SIG_DFL here it killed teardown before the executor waited for
|
||||
# the workers. Defer ONE late signal until teardown ends; a second
|
||||
# signal or _TEARDOWN_SIGNAL_BOUND_S of teardown after the deferral
|
||||
# still ends the process (_TeardownSignalGuard), as do the
|
||||
# executor's terminate/kill ladder and the process manager's timeout
|
||||
# (kill_process_tree -> SIGKILL).
|
||||
teardown_guard = _TeardownSignalGuard()
|
||||
signal.signal(signal.SIGTERM, teardown_guard)
|
||||
signal.signal(signal.SIGINT, teardown_guard)
|
||||
if signal_callback is not None:
|
||||
signal_callback.stop()
|
||||
if engine_core is not None:
|
||||
engine_core.shutdown()
|
||||
try:
|
||||
if engine_core is not None:
|
||||
engine_core.shutdown()
|
||||
finally:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
teardown_guard.disarm()
|
||||
if clean_shutdown:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -114,6 +114,65 @@ def _install_worker_parent_death_signal(expected_parent_pid: int) -> None:
|
||||
os._exit(1)
|
||||
|
||||
|
||||
# Grace and post-SIGTERM wait, after a worker's death, for survivors that never
|
||||
# started their shutdown (see MultiprocExecutor._ensure_worker_termination).
|
||||
_FAILED_WORKER_WAIT_S = 1.0
|
||||
# After a worker's death, the bounded grace for survivors whose shutdown has
|
||||
# STARTED (FINAL DRAIN in progress). Every FINAL DRAIN measured on the reference host took
|
||||
# < 0.1 s (drain-path tests: the drain line lands within ~0.05 s of the
|
||||
# shutdown request); 2 s is 20x that, so a draining rank is not cut
|
||||
# (a shorter cap did), while one blocked inside its shutdown on the dead rank is ended.
|
||||
_FAILED_DRAIN_GRACE_S = 2.0
|
||||
# Per-worker shared byte (WorkerProcHandle.shutdown_state): written by the
|
||||
# worker in worker_main's finally, read by the executor's termination ladder.
|
||||
_SHUTDOWN_NOT_STARTED = 0
|
||||
_SHUTDOWN_STARTED = 1
|
||||
_SHUTDOWN_DONE = 2
|
||||
|
||||
|
||||
class WorkerShutdownRequested(RuntimeError):
|
||||
"""The busy loop's dequeue was cancelled by a shutdown request.
|
||||
|
||||
Raised only by WorkerProc.worker_busy_loop when the RPC broadcast queue
|
||||
reports its own cancellation (shm_broadcast: RuntimeError("cancelled")
|
||||
after MessageQueue.shutdown(), which the death-pipe monitor calls once the
|
||||
parent closes the pipe). worker_main takes the orderly exit path on this
|
||||
type only; any other exception is a failure, even one that races a
|
||||
shutdown request. Subclasses RuntimeError so existing catchers still match.
|
||||
"""
|
||||
|
||||
|
||||
def _exit_worker_process(exit_code: int) -> None:
|
||||
"""End a worker process whose orderly shutdown has already run.
|
||||
|
||||
Called after WorkerProc.shutdown(destroy_distributed=False): the FINAL
|
||||
DRAIN, the instrumentation flushes and the PLE offload / shared-memory
|
||||
cleanup are done. What would remain is process-exit-only teardown --
|
||||
destroying the NCCL communicators (PyNcclCommunicator.destroy joins its
|
||||
ncclCommAbort thread for up to 5 s per communicator while CUDA graphs that
|
||||
captured NCCL ops are still alive) and interpreter finalization (CUDA
|
||||
context, allocator pools). The driver and kernel reclaim all of it at
|
||||
process exit, and without this the executor's 5 s grace expired and the
|
||||
worker was SIGKILLed at ~9 s anyway. Flush the logs,
|
||||
then leave.
|
||||
"""
|
||||
logger.info(
|
||||
"[shutdown] WorkerProc: shutdown complete; exiting process (code %d) "
|
||||
"without distributed teardown",
|
||||
exit_code,
|
||||
)
|
||||
try:
|
||||
import logging
|
||||
|
||||
logging.shutdown()
|
||||
except Exception: # noqa: BLE001 -- the exit must happen regardless
|
||||
pass
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
with suppress(Exception):
|
||||
stream.flush()
|
||||
os._exit(exit_code)
|
||||
|
||||
|
||||
class FutureWrapper(Future):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -494,10 +553,36 @@ class MultiprocExecutor(Executor):
|
||||
return future if non_block else future.result()
|
||||
|
||||
@staticmethod
|
||||
def _ensure_worker_termination(worker_procs: list[BaseProcess]):
|
||||
def _ensure_worker_termination(
|
||||
worker_procs: list[BaseProcess],
|
||||
*,
|
||||
after_failure: bool = False,
|
||||
shutdown_states: list[Any] | None = None,
|
||||
):
|
||||
"""Ensure that all worker processes are terminated. Assumes workers have
|
||||
received termination requests. Waits for processing, then sends
|
||||
termination and kill signals if needed."""
|
||||
termination and kill signals if needed.
|
||||
|
||||
after_failure: a worker already died. Survivors split in two. Those
|
||||
that never started WorkerProc.shutdown() are parked in a collective
|
||||
with the dead rank (NCCL / gloo, in C, where neither the grace nor a
|
||||
SIGTERM reaches them -- in a fault-injection test two ranks ran the full
|
||||
5 s + 4 s ladder to SIGKILL): they get _FAILED_WORKER_WAIT_S to start
|
||||
their shutdown, then SIGTERM, then SIGKILL after as long again. Those
|
||||
whose shutdown_states entry shows the shutdown started (FINAL DRAIN,
|
||||
flushes) get _FAILED_DRAIN_GRACE_S from the start of the ladder, then
|
||||
SIGTERM, then SIGKILL after _FAILED_WORKER_WAIT_S: a started shutdown
|
||||
can itself block on the dead rank (in a drain fault-injection test one
|
||||
survivor had started its shutdown and never logged its FINAL DRAIN --
|
||||
the drain's device synchronize waits on work paired with the dead
|
||||
rank -- and ran the full orderly ladder, workers gone 17.6 s after the
|
||||
fault). Before the drain-order fix the engine died on the late SIGTERM and
|
||||
PDEATHSIG killed all survivors at once. A worker with no state (None,
|
||||
or no shutdown_states) counts as parked. The orderly ladder (no
|
||||
failure) is unchanged.
|
||||
"""
|
||||
grace_s = float(envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS)
|
||||
term_wait_s = 4.0
|
||||
|
||||
def wait_for_termination(procs, timeout):
|
||||
if not time:
|
||||
@@ -519,9 +604,42 @@ class MultiprocExecutor(Executor):
|
||||
"[shutdown] Executor: waiting for worker exit count=%d",
|
||||
initial_count,
|
||||
)
|
||||
if wait_for_termination(
|
||||
active_procs(), timeout=envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS
|
||||
):
|
||||
if after_failure and time:
|
||||
start = time.time()
|
||||
states = list(shutdown_states or [None] * len(worker_procs))
|
||||
|
||||
def parked() -> list[BaseProcess]:
|
||||
return [
|
||||
proc
|
||||
for proc, state in zip(worker_procs, states)
|
||||
if proc.is_alive()
|
||||
and (state is None or state.value == _SHUTDOWN_NOT_STARTED)
|
||||
]
|
||||
|
||||
if not wait_for_termination(active_procs(), _FAILED_WORKER_WAIT_S):
|
||||
stuck = parked()
|
||||
if stuck:
|
||||
logger.warning(
|
||||
"[shutdown] Executor: workers parked after a worker "
|
||||
"failure (shutdown not started); sending SIGTERM count=%d",
|
||||
len(stuck),
|
||||
)
|
||||
for p in stuck:
|
||||
p.terminate()
|
||||
if not wait_for_termination(stuck, _FAILED_WORKER_WAIT_S):
|
||||
stuck = [p for p in stuck if p.is_alive()]
|
||||
logger.warning(
|
||||
"[shutdown] Executor: parked workers still running "
|
||||
"after SIGTERM; sending SIGKILL count=%d",
|
||||
len(stuck),
|
||||
)
|
||||
for p in stuck:
|
||||
p.kill()
|
||||
# Workers that started their shutdown get a BOUNDED grace, counted
|
||||
# from the start of this ladder, then SIGTERM, then SIGKILL.
|
||||
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.time() - start))
|
||||
term_wait_s = _FAILED_WORKER_WAIT_S
|
||||
if wait_for_termination(active_procs(), timeout=grace_s):
|
||||
logger.info_once("[shutdown] Executor: all workers exited gracefully")
|
||||
return
|
||||
|
||||
@@ -534,7 +652,7 @@ class MultiprocExecutor(Executor):
|
||||
)
|
||||
for p in remaining:
|
||||
p.terminate()
|
||||
if not wait_for_termination(active_procs(), 4):
|
||||
if not wait_for_termination(active_procs(), term_wait_s):
|
||||
# Send SIGKILL if still running
|
||||
remaining = active_procs()
|
||||
logger.warning(
|
||||
@@ -562,7 +680,13 @@ class MultiprocExecutor(Executor):
|
||||
if w.death_writer is not None:
|
||||
w.death_writer.close()
|
||||
w.death_writer = None
|
||||
self._ensure_worker_termination([w.proc for w in workers])
|
||||
self._ensure_worker_termination(
|
||||
[w.proc for w in workers],
|
||||
after_failure=getattr(self, "is_failed", False),
|
||||
shutdown_states=[
|
||||
getattr(w, "shutdown_state", None) for w in workers
|
||||
],
|
||||
)
|
||||
|
||||
for w in workers:
|
||||
# Shutdown response queues
|
||||
@@ -613,6 +737,7 @@ class UnreadyWorkerProcHandle:
|
||||
rank: int
|
||||
ready_pipe: Connection
|
||||
death_writer: Connection | None = None
|
||||
shutdown_state: Any = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -626,6 +751,8 @@ class WorkerProcHandle:
|
||||
# `peer_worker_response_mqs[i]`
|
||||
peer_worker_response_mqs: list[MessageQueue | None]
|
||||
death_writer: Connection | None = None
|
||||
# Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes).
|
||||
shutdown_state: Any = None
|
||||
|
||||
@classmethod
|
||||
def from_unready_handle(
|
||||
@@ -640,6 +767,7 @@ class WorkerProcHandle:
|
||||
worker_response_mq=worker_response_mq,
|
||||
peer_worker_response_mqs=peer_worker_response_mqs,
|
||||
death_writer=unready_handle.death_writer,
|
||||
shutdown_state=unready_handle.shutdown_state,
|
||||
)
|
||||
|
||||
|
||||
@@ -778,6 +906,8 @@ class WorkerProc:
|
||||
ready_reader, ready_writer = context.Pipe(duplex=False)
|
||||
# Death pipe to let child detect parent process exit
|
||||
death_reader, death_writer = context.Pipe(duplex=False)
|
||||
# One shared byte: has this worker started / finished its shutdown?
|
||||
shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED)
|
||||
if inherited_fds is not None:
|
||||
inherited_fds = inherited_fds.copy()
|
||||
inherited_fds.extend((ready_reader.fileno(), death_writer.fileno()))
|
||||
@@ -790,6 +920,7 @@ class WorkerProc:
|
||||
"input_shm_handle": input_shm_handle,
|
||||
"ready_pipe": ready_writer,
|
||||
"death_pipe": death_reader,
|
||||
"shutdown_state": shutdown_state,
|
||||
"shared_worker_lock": shared_worker_lock,
|
||||
"is_driver_worker": is_driver_worker,
|
||||
# Have the worker close parent end of this worker's pipes too
|
||||
@@ -814,7 +945,9 @@ class WorkerProc:
|
||||
death_reader.close()
|
||||
# Keep death_writer open in parent - when parent exits,
|
||||
# death_reader in child will get EOFError
|
||||
return UnreadyWorkerProcHandle(proc, rank, ready_reader, death_writer)
|
||||
return UnreadyWorkerProcHandle(
|
||||
proc, rank, ready_reader, death_writer, shutdown_state
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def wait_for_response_handle_ready(
|
||||
@@ -875,7 +1008,14 @@ class WorkerProc:
|
||||
|
||||
return cast(list[WorkerProcHandle], ready_proc_handles)
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self, *, destroy_distributed: bool = True):
|
||||
if not destroy_distributed:
|
||||
# worker_main os._exits right after this: the PLE offload handle
|
||||
# (rank 0 owns the child) must not wait for the child's reap,
|
||||
# which trails its exit by seconds for a large host table.
|
||||
ple_worker = sys.modules.get("vllm.v1.ple_offload.worker")
|
||||
if ple_worker is not None:
|
||||
ple_worker.mark_process_exit_imminent()
|
||||
if self.rpc_broadcast_mq is not None:
|
||||
self.rpc_broadcast_mq.shutdown()
|
||||
if self.worker_response_mq is not None:
|
||||
@@ -883,8 +1023,12 @@ class WorkerProc:
|
||||
self.worker.shutdown()
|
||||
self.rpc_broadcast_mq = None
|
||||
self.worker_response_mq = None
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
# worker_main passes False when the process exits right after this
|
||||
# (see _exit_worker_process): the NCCL destroy is exit-only work that
|
||||
# can block for 5 s per communicator.
|
||||
if destroy_distributed:
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
|
||||
def monitor_death_pipe(self, death_pipe, shutdown_requested: threading.Event):
|
||||
if death_pipe is None:
|
||||
@@ -953,8 +1097,20 @@ class WorkerProc:
|
||||
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
|
||||
|
||||
worker = None
|
||||
# Set once a READY worker leaves its busy loop because shutdown was
|
||||
# requested (death pipe EOF or SIGTERM/SIGINT): the process then exits
|
||||
# straight after WorkerProc.shutdown (see _exit_worker_process).
|
||||
orderly_shutdown = False
|
||||
# Set when a READY worker's busy loop FAILED while a shutdown request
|
||||
# was pending: full teardown, then exit code 1 (never the exit-0 path).
|
||||
failed_during_shutdown = False
|
||||
ready_writer = kwargs.pop("ready_pipe")
|
||||
death_pipe = kwargs.pop("death_pipe", None)
|
||||
shutdown_state = kwargs.pop("shutdown_state", None)
|
||||
|
||||
def mark_shutdown(value: int) -> None:
|
||||
if shutdown_state is not None:
|
||||
shutdown_state.value = value
|
||||
|
||||
# Close inherited pipes from parent (incl. other worker pipes)
|
||||
# Explicitly passing in existing pipes and closing them makes the pipe
|
||||
@@ -1001,7 +1157,7 @@ class WorkerProc:
|
||||
|
||||
worker.worker_busy_loop()
|
||||
|
||||
except Exception:
|
||||
except Exception as e:
|
||||
# NOTE: if an Exception arises in busy_loop, we send
|
||||
# a FAILURE message over the MQ RPC to notify the Executor,
|
||||
# which triggers system shutdown.
|
||||
@@ -1009,10 +1165,20 @@ class WorkerProc:
|
||||
|
||||
if ready_writer is not None:
|
||||
logger.exception("WorkerProc failed to start.")
|
||||
elif shutdown_requested.is_set():
|
||||
elif isinstance(e, WorkerShutdownRequested) and (
|
||||
shutdown_requested.is_set()
|
||||
):
|
||||
# The busy loop left BECAUSE of the shutdown request (its
|
||||
# dequeue was cancelled), not because something failed.
|
||||
logger.debug_once(
|
||||
"[shutdown] WorkerProc: exiting after shutdown request"
|
||||
)
|
||||
orderly_shutdown = True
|
||||
elif shutdown_requested.is_set():
|
||||
# A real failure that raced a shutdown request: keep it a
|
||||
# failure -- full teardown, then a non-zero exit.
|
||||
logger.exception("WorkerProc failed during shutdown.")
|
||||
failed_during_shutdown = True
|
||||
else:
|
||||
logger.exception("WorkerProc failed.")
|
||||
|
||||
@@ -1028,6 +1194,7 @@ class WorkerProc:
|
||||
logger.debug_once(
|
||||
"[shutdown] WorkerProc: terminated by shutdown signal"
|
||||
)
|
||||
orderly_shutdown = ready_writer is None
|
||||
else:
|
||||
logger.warning("WorkerProc was terminated")
|
||||
# SystemExit must never be ignored
|
||||
@@ -1040,7 +1207,26 @@ class WorkerProc:
|
||||
death_pipe.close()
|
||||
# Clean up once worker exits busy loop
|
||||
if worker is not None:
|
||||
# Tells the executor's ladder this rank is draining, not parked
|
||||
# in a collective (see _ensure_worker_termination).
|
||||
mark_shutdown(_SHUTDOWN_STARTED)
|
||||
if orderly_shutdown:
|
||||
# Everything that must happen before exit (FINAL DRAIN,
|
||||
# instrumentation flushes, PLE cleanup) is in
|
||||
# worker.shutdown(); then exit instead of paying the NCCL
|
||||
# destroy + finalization past the executor's grace.
|
||||
exit_code = 0
|
||||
try:
|
||||
worker.shutdown(destroy_distributed=False)
|
||||
except BaseException:
|
||||
logger.exception("WorkerProc shutdown failed")
|
||||
exit_code = 1
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
_exit_worker_process(exit_code)
|
||||
worker.shutdown()
|
||||
mark_shutdown(_SHUTDOWN_DONE)
|
||||
if failed_during_shutdown:
|
||||
sys.exit(1)
|
||||
|
||||
class ResponseStatus(Enum):
|
||||
SUCCESS = auto()
|
||||
@@ -1120,8 +1306,20 @@ class WorkerProc:
|
||||
def worker_busy_loop(self):
|
||||
"""Main busy loop for Multiprocessing Workers."""
|
||||
assert self.rpc_broadcast_mq is not None
|
||||
mq = self.rpc_broadcast_mq
|
||||
while True:
|
||||
self._execute_worker_rpc(self.rpc_broadcast_mq.dequeue(indefinite=True))
|
||||
# No local holds the request across the next dequeue: its payload
|
||||
# must be released first (test_multiproc_executor).
|
||||
self._execute_worker_rpc(self._dequeue_worker_rpc(mq))
|
||||
|
||||
@staticmethod
|
||||
def _dequeue_worker_rpc(mq: MessageQueue):
|
||||
try:
|
||||
return mq.dequeue(indefinite=True)
|
||||
except RuntimeError as e:
|
||||
if getattr(mq, "shutting_down", False) and str(e) == "cancelled":
|
||||
raise WorkerShutdownRequested("RPC broadcast queue shut down") from e
|
||||
raise
|
||||
|
||||
def _execute_worker_rpc(
|
||||
self,
|
||||
|
||||
@@ -102,6 +102,86 @@ def _unlink_ipc_socket(ipc_addr: str) -> None:
|
||||
logger.warning("Failed to unlink PLE IPC socket %s: %s", path, error)
|
||||
|
||||
|
||||
# Set once by a GPU worker process whose orderly shutdown is about to end in
|
||||
# os._exit (multiproc_executor._exit_worker_process). PleOffloadWorkerHandle
|
||||
# then waits for the offload child's whole thread group to be in the kernel's
|
||||
# exit path, not for its REAP. In a drain fault-injection test the ~51 GB
|
||||
# child's leader went Z at ~+2.7 s, but one thread stayed R -- PF_EXITING set,
|
||||
# in-kernel, freeing the address space -- until the reap at ~+8.6 s. Rank 0
|
||||
# waiting for that reap ran the executor's grace -> SIGTERM -> SIGKILL ladder on
|
||||
# every cycle. A read-only sample of one such shutdown showed
|
||||
# the same thing: from Z to reap (6.0 s), the only live task was R with
|
||||
# PF_EXITING, the leader's fd table was already closed (0 /dev/nvidia fds,
|
||||
# down from 13), and the child was never an nvidia-smi compute app, before or
|
||||
# after. It holds no GPU memory (host-mapped pull: no CUDA context). What is
|
||||
# left is kernel-side host-memory teardown, and nothing in this process
|
||||
# needs to wait for it.
|
||||
_PROCESS_EXIT_IMMINENT = False
|
||||
# Bound for the child's userspace cleanup (its proc_main finally) on that path;
|
||||
# it took ~2.7 s on the reference host. It has to fit inside the executor's 5 s
|
||||
# VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS with room for rank 0's own exit. After it,
|
||||
# the child is SIGKILLed (userspace ends at once) and not waited for.
|
||||
_EXIT_IMMINENT_CHILD_WAIT_S = 3.5
|
||||
# procfs mount the predicate reads (tests point it at a fake tree).
|
||||
_PROC_ROOT = "/proc"
|
||||
# task_struct flag: set by do_exit() -> exit_signals() before exit_mm(); a task
|
||||
# carrying it never returns to userspace.
|
||||
_PF_EXITING = 0x4
|
||||
|
||||
|
||||
def mark_process_exit_imminent() -> None:
|
||||
"""This process os._exits right after its shutdown; see close()."""
|
||||
global _PROCESS_EXIT_IMMINENT
|
||||
_PROCESS_EXIT_IMMINENT = True
|
||||
|
||||
|
||||
def _task_in_exit(stat: bytes) -> bool:
|
||||
"""One /proc/<pid>/task/<tid>/stat: Z/X, or PF_EXITING (kernel exit path)."""
|
||||
# Fields after the parenthesised comm (which may contain spaces/parens):
|
||||
# [0]=state ... [6]=flags (stat field 9).
|
||||
fields = stat[stat.rfind(b")") + 2 :].split()
|
||||
if not fields:
|
||||
return True
|
||||
if fields[0] in (b"Z", b"X"):
|
||||
return True
|
||||
try:
|
||||
return bool(int(fields[6]) & _PF_EXITING)
|
||||
except (IndexError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _child_thread_group_exiting(
|
||||
proc: subprocess.Popen, proc_root: str | None = None
|
||||
) -> bool:
|
||||
"""True once EVERY task of ``proc`` is exited or in the kernel exit path.
|
||||
|
||||
A zombie leader alone is not enough (a thread can still run userspace
|
||||
after the leader exits). Never reaps: the child is unreaped, so its PID
|
||||
cannot be reused and <proc_root>/<pid>/task is its own. Off Linux, only a
|
||||
reaped child counts.
|
||||
"""
|
||||
if proc.returncode is not None:
|
||||
return True
|
||||
task_dir = os.path.join(proc_root or _PROC_ROOT, str(proc.pid), "task")
|
||||
try:
|
||||
tids = os.listdir(task_dir)
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
except OSError:
|
||||
return proc.poll() is not None
|
||||
for tid in tids:
|
||||
try:
|
||||
with open(os.path.join(task_dir, tid, "stat"), "rb") as fh:
|
||||
stat = fh.read()
|
||||
except FileNotFoundError:
|
||||
continue # this task finished exiting between listdir and open
|
||||
except OSError:
|
||||
return False
|
||||
if not _task_in_exit(stat):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _publish_fence() -> None:
|
||||
with _PUBLISH_FENCE:
|
||||
pass
|
||||
@@ -448,6 +528,9 @@ class PleOffloadWorkerHandle:
|
||||
if self.ready_pipe_reader is not None:
|
||||
self.ready_pipe_reader.close()
|
||||
self.ready_pipe_reader = None
|
||||
if graceful and _PROCESS_EXIT_IMMINENT:
|
||||
self._close_without_reap()
|
||||
return
|
||||
if graceful and self.proc.poll() is None:
|
||||
# Keep parent-death supervision connected while SIGTERM lets
|
||||
# proc_main run its shutdown event and finally cleanup.
|
||||
@@ -472,6 +555,41 @@ class PleOffloadWorkerHandle:
|
||||
if self.proc.poll() is not None and self.ipc_addr is not None:
|
||||
_unlink_ipc_socket(self.ipc_addr)
|
||||
|
||||
def _close_without_reap(self) -> None:
|
||||
"""close() for a parent that os._exits next: wait (bounded) until every
|
||||
child task has left userspace, never for the reap.
|
||||
|
||||
SIGTERM lets proc_main's finally unlink the IPC socket, as in the
|
||||
graceful path; parent-death supervision stays connected meanwhile.
|
||||
Once the child has run exit, the address-space teardown that keeps it
|
||||
unreapable needs nothing from this process: the orphan is reaped by
|
||||
the subreaper/init after this process exits.
|
||||
"""
|
||||
if not _child_thread_group_exiting(self.proc):
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(self.proc.pid, signal.SIGTERM)
|
||||
deadline = time.monotonic() + _EXIT_IMMINENT_CHILD_WAIT_S
|
||||
while not _child_thread_group_exiting(self.proc):
|
||||
if time.monotonic() >= deadline:
|
||||
logger.warning(
|
||||
"PLE offload process %d still running %.1f s after "
|
||||
"SIGTERM; killing it without waiting (parent exits)",
|
||||
self.proc.pid,
|
||||
_EXIT_IMMINENT_CHILD_WAIT_S,
|
||||
)
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(self.proc.pid, signal.SIGKILL)
|
||||
break
|
||||
time.sleep(0.01)
|
||||
exited = _child_thread_group_exiting(self.proc)
|
||||
if self.death_writer is not None:
|
||||
self.death_writer.close()
|
||||
self.death_writer = None
|
||||
# Same fallback as close(): the child has run exit, so this exact
|
||||
# PLE-owned address is no longer in use.
|
||||
if exited and self.ipc_addr is not None:
|
||||
_unlink_ipc_socket(self.ipc_addr)
|
||||
|
||||
|
||||
def _init_offload_distributed() -> None:
|
||||
"""Initialize the single-rank Gloo world required by TP-aware layers."""
|
||||
|
||||
Reference in New Issue
Block a user