[Engine] Drain-order shutdown

- EngineCore survives a late SIGTERM during teardown (bounded deferral) so
  every worker drains its guard telemetry.
- A worker exits right after its orderly shutdown.
- A worker failure racing shutdown stays a failure (exit 1).
- Rank 0 waits for the PLE child's whole thread group, bounded, instead of its
  reap; drain-aware failure ladder with a bounded grace for survivors whose
  shutdown already started.
- tests: the sorted-emit graph arm takes max_len on the host before capture.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:04:37 +07:00
co-authored by Hinata
parent eeea036c12
commit 6c956ee3bd
3 changed files with 423 additions and 19 deletions
+93 -5
View File
@@ -4,12 +4,13 @@ import gc
import os
import queue
import signal
import sys
import threading
import time
from collections import defaultdict, deque
from collections.abc import Callable, Generator, Sequence
from concurrent.futures import Future
from contextlib import ExitStack, contextmanager
from contextlib import ExitStack, contextmanager, suppress
from enum import IntEnum
from functools import partial
from inspect import isclass, signature
@@ -108,6 +109,76 @@ HANDSHAKE_TIMEOUT_MINS = 5
_R = TypeVar("_R") # Return type for collective_rpc
# How long EngineCore teardown may run after it deferred a late SIGTERM/SIGINT
# before the process exits anyway. Covers the whole of engine_core.shutdown(),
# not only the executor's worker ladder (grace + 4 s): a hang in later teardown
# (scheduler, distributed cleanup) must not make the process unkillable.
_TEARDOWN_SIGNAL_BOUND_S = 30.0
class _TeardownSignalGuard:
"""EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded.
Dying on the first late signal would SIGKILL every worker
(PR_SET_PDEATHSIG) before its shutdown drain; see the finally block of
EngineCoreProc.run_engine_core. The deferral is bounded two ways:
- a SECOND late signal restores the default disposition and re-delivers
itself, so the process dies of it (the maintainer's escalation);
- after the first deferred signal a watchdog ends the process with
exit code 1 if teardown is still running _TEARDOWN_SIGNAL_BOUND_S later.
"""
def __init__(self, bound_s: float | None = None) -> None:
self.bound_s = _TEARDOWN_SIGNAL_BOUND_S if bound_s is None else bound_s
self.deferred = 0
self._done = threading.Event()
self._watchdog: threading.Thread | None = None
def __call__(self, signum, frame) -> None:
name = signal.Signals(signum).name
if self._done.is_set():
return
self.deferred += 1
if self.deferred == 1:
logger.warning(
"[shutdown] EngineCore: %s during resource teardown deferred; "
"workers finish their shutdown (FINAL DRAIN) first (a second "
"signal, or %.0f s of teardown, ends the process)",
name,
self.bound_s,
)
self._watchdog = threading.Thread(
target=self._expire,
daemon=True,
name="EngineCoreTeardownWatchdog",
)
self._watchdog.start()
return
logger.warning(
"[shutdown] EngineCore: second late signal (%s) during resource "
"teardown; not deferring again",
name,
)
signal.signal(signum, signal.SIG_DFL)
os.kill(os.getpid(), signum)
def _expire(self) -> None:
if self._done.wait(self.bound_s):
return
logger.error(
"[shutdown] EngineCore: resource teardown still running %.0f s "
"after a deferred signal; exiting (code 1)",
self.bound_s,
)
for stream in (sys.stdout, sys.stderr):
with suppress(Exception):
stream.flush()
os._exit(1)
def disarm(self) -> None:
self._done.set()
class EngineCore:
"""Inner loop of vLLM's Engine."""
@@ -1441,12 +1512,29 @@ class EngineCoreProc(EngineCore):
engine_core._send_engine_dead()
raise e
finally:
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
# Workers carry PR_SET_PDEATHSIG=SIGKILL (fail-closed PLE teardown),
# so if this process dies before model_executor.shutdown() has reaped
# them, every worker is SIGKILLed mid-shutdown and its bounds-guard
# FINAL DRAIN never runs. The API server's process manager sends a
# second SIGTERM right after the signal that started this shutdown;
# with SIG_DFL here it killed teardown before the executor waited for
# the workers. Defer ONE late signal until teardown ends; a second
# signal or _TEARDOWN_SIGNAL_BOUND_S of teardown after the deferral
# still ends the process (_TeardownSignalGuard), as do the
# executor's terminate/kill ladder and the process manager's timeout
# (kill_process_tree -> SIGKILL).
teardown_guard = _TeardownSignalGuard()
signal.signal(signal.SIGTERM, teardown_guard)
signal.signal(signal.SIGINT, teardown_guard)
if signal_callback is not None:
signal_callback.stop()
if engine_core is not None:
engine_core.shutdown()
try:
if engine_core is not None:
engine_core.shutdown()
finally:
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
teardown_guard.disarm()
if clean_shutdown:
from vllm.platforms import current_platform
+212 -14
View File
@@ -114,6 +114,65 @@ def _install_worker_parent_death_signal(expected_parent_pid: int) -> None:
os._exit(1)
# Grace and post-SIGTERM wait, after a worker's death, for survivors that never
# started their shutdown (see MultiprocExecutor._ensure_worker_termination).
_FAILED_WORKER_WAIT_S = 1.0
# After a worker's death, the bounded grace for survivors whose shutdown has
# STARTED (FINAL DRAIN in progress). Every FINAL DRAIN measured on the reference host took
# < 0.1 s (drain-path tests: the drain line lands within ~0.05 s of the
# shutdown request); 2 s is 20x that, so a draining rank is not cut
# (a shorter cap did), while one blocked inside its shutdown on the dead rank is ended.
_FAILED_DRAIN_GRACE_S = 2.0
# Per-worker shared byte (WorkerProcHandle.shutdown_state): written by the
# worker in worker_main's finally, read by the executor's termination ladder.
_SHUTDOWN_NOT_STARTED = 0
_SHUTDOWN_STARTED = 1
_SHUTDOWN_DONE = 2
class WorkerShutdownRequested(RuntimeError):
"""The busy loop's dequeue was cancelled by a shutdown request.
Raised only by WorkerProc.worker_busy_loop when the RPC broadcast queue
reports its own cancellation (shm_broadcast: RuntimeError("cancelled")
after MessageQueue.shutdown(), which the death-pipe monitor calls once the
parent closes the pipe). worker_main takes the orderly exit path on this
type only; any other exception is a failure, even one that races a
shutdown request. Subclasses RuntimeError so existing catchers still match.
"""
def _exit_worker_process(exit_code: int) -> None:
"""End a worker process whose orderly shutdown has already run.
Called after WorkerProc.shutdown(destroy_distributed=False): the FINAL
DRAIN, the instrumentation flushes and the PLE offload / shared-memory
cleanup are done. What would remain is process-exit-only teardown --
destroying the NCCL communicators (PyNcclCommunicator.destroy joins its
ncclCommAbort thread for up to 5 s per communicator while CUDA graphs that
captured NCCL ops are still alive) and interpreter finalization (CUDA
context, allocator pools). The driver and kernel reclaim all of it at
process exit, and without this the executor's 5 s grace expired and the
worker was SIGKILLed at ~9 s anyway. Flush the logs,
then leave.
"""
logger.info(
"[shutdown] WorkerProc: shutdown complete; exiting process (code %d) "
"without distributed teardown",
exit_code,
)
try:
import logging
logging.shutdown()
except Exception: # noqa: BLE001 -- the exit must happen regardless
pass
for stream in (sys.stdout, sys.stderr):
with suppress(Exception):
stream.flush()
os._exit(exit_code)
class FutureWrapper(Future):
def __init__(
self,
@@ -494,10 +553,36 @@ class MultiprocExecutor(Executor):
return future if non_block else future.result()
@staticmethod
def _ensure_worker_termination(worker_procs: list[BaseProcess]):
def _ensure_worker_termination(
worker_procs: list[BaseProcess],
*,
after_failure: bool = False,
shutdown_states: list[Any] | None = None,
):
"""Ensure that all worker processes are terminated. Assumes workers have
received termination requests. Waits for processing, then sends
termination and kill signals if needed."""
termination and kill signals if needed.
after_failure: a worker already died. Survivors split in two. Those
that never started WorkerProc.shutdown() are parked in a collective
with the dead rank (NCCL / gloo, in C, where neither the grace nor a
SIGTERM reaches them -- in a fault-injection test two ranks ran the full
5 s + 4 s ladder to SIGKILL): they get _FAILED_WORKER_WAIT_S to start
their shutdown, then SIGTERM, then SIGKILL after as long again. Those
whose shutdown_states entry shows the shutdown started (FINAL DRAIN,
flushes) get _FAILED_DRAIN_GRACE_S from the start of the ladder, then
SIGTERM, then SIGKILL after _FAILED_WORKER_WAIT_S: a started shutdown
can itself block on the dead rank (in a drain fault-injection test one
survivor had started its shutdown and never logged its FINAL DRAIN --
the drain's device synchronize waits on work paired with the dead
rank -- and ran the full orderly ladder, workers gone 17.6 s after the
fault). Before the drain-order fix the engine died on the late SIGTERM and
PDEATHSIG killed all survivors at once. A worker with no state (None,
or no shutdown_states) counts as parked. The orderly ladder (no
failure) is unchanged.
"""
grace_s = float(envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS)
term_wait_s = 4.0
def wait_for_termination(procs, timeout):
if not time:
@@ -519,9 +604,42 @@ class MultiprocExecutor(Executor):
"[shutdown] Executor: waiting for worker exit count=%d",
initial_count,
)
if wait_for_termination(
active_procs(), timeout=envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS
):
if after_failure and time:
start = time.time()
states = list(shutdown_states or [None] * len(worker_procs))
def parked() -> list[BaseProcess]:
return [
proc
for proc, state in zip(worker_procs, states)
if proc.is_alive()
and (state is None or state.value == _SHUTDOWN_NOT_STARTED)
]
if not wait_for_termination(active_procs(), _FAILED_WORKER_WAIT_S):
stuck = parked()
if stuck:
logger.warning(
"[shutdown] Executor: workers parked after a worker "
"failure (shutdown not started); sending SIGTERM count=%d",
len(stuck),
)
for p in stuck:
p.terminate()
if not wait_for_termination(stuck, _FAILED_WORKER_WAIT_S):
stuck = [p for p in stuck if p.is_alive()]
logger.warning(
"[shutdown] Executor: parked workers still running "
"after SIGTERM; sending SIGKILL count=%d",
len(stuck),
)
for p in stuck:
p.kill()
# Workers that started their shutdown get a BOUNDED grace, counted
# from the start of this ladder, then SIGTERM, then SIGKILL.
grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.time() - start))
term_wait_s = _FAILED_WORKER_WAIT_S
if wait_for_termination(active_procs(), timeout=grace_s):
logger.info_once("[shutdown] Executor: all workers exited gracefully")
return
@@ -534,7 +652,7 @@ class MultiprocExecutor(Executor):
)
for p in remaining:
p.terminate()
if not wait_for_termination(active_procs(), 4):
if not wait_for_termination(active_procs(), term_wait_s):
# Send SIGKILL if still running
remaining = active_procs()
logger.warning(
@@ -562,7 +680,13 @@ class MultiprocExecutor(Executor):
if w.death_writer is not None:
w.death_writer.close()
w.death_writer = None
self._ensure_worker_termination([w.proc for w in workers])
self._ensure_worker_termination(
[w.proc for w in workers],
after_failure=getattr(self, "is_failed", False),
shutdown_states=[
getattr(w, "shutdown_state", None) for w in workers
],
)
for w in workers:
# Shutdown response queues
@@ -613,6 +737,7 @@ class UnreadyWorkerProcHandle:
rank: int
ready_pipe: Connection
death_writer: Connection | None = None
shutdown_state: Any = None
@dataclass
@@ -626,6 +751,8 @@ class WorkerProcHandle:
# `peer_worker_response_mqs[i]`
peer_worker_response_mqs: list[MessageQueue | None]
death_writer: Connection | None = None
# Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes).
shutdown_state: Any = None
@classmethod
def from_unready_handle(
@@ -640,6 +767,7 @@ class WorkerProcHandle:
worker_response_mq=worker_response_mq,
peer_worker_response_mqs=peer_worker_response_mqs,
death_writer=unready_handle.death_writer,
shutdown_state=unready_handle.shutdown_state,
)
@@ -778,6 +906,8 @@ class WorkerProc:
ready_reader, ready_writer = context.Pipe(duplex=False)
# Death pipe to let child detect parent process exit
death_reader, death_writer = context.Pipe(duplex=False)
# One shared byte: has this worker started / finished its shutdown?
shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED)
if inherited_fds is not None:
inherited_fds = inherited_fds.copy()
inherited_fds.extend((ready_reader.fileno(), death_writer.fileno()))
@@ -790,6 +920,7 @@ class WorkerProc:
"input_shm_handle": input_shm_handle,
"ready_pipe": ready_writer,
"death_pipe": death_reader,
"shutdown_state": shutdown_state,
"shared_worker_lock": shared_worker_lock,
"is_driver_worker": is_driver_worker,
# Have the worker close parent end of this worker's pipes too
@@ -814,7 +945,9 @@ class WorkerProc:
death_reader.close()
# Keep death_writer open in parent - when parent exits,
# death_reader in child will get EOFError
return UnreadyWorkerProcHandle(proc, rank, ready_reader, death_writer)
return UnreadyWorkerProcHandle(
proc, rank, ready_reader, death_writer, shutdown_state
)
@staticmethod
def wait_for_response_handle_ready(
@@ -875,7 +1008,14 @@ class WorkerProc:
return cast(list[WorkerProcHandle], ready_proc_handles)
def shutdown(self):
def shutdown(self, *, destroy_distributed: bool = True):
if not destroy_distributed:
# worker_main os._exits right after this: the PLE offload handle
# (rank 0 owns the child) must not wait for the child's reap,
# which trails its exit by seconds for a large host table.
ple_worker = sys.modules.get("vllm.v1.ple_offload.worker")
if ple_worker is not None:
ple_worker.mark_process_exit_imminent()
if self.rpc_broadcast_mq is not None:
self.rpc_broadcast_mq.shutdown()
if self.worker_response_mq is not None:
@@ -883,8 +1023,12 @@ class WorkerProc:
self.worker.shutdown()
self.rpc_broadcast_mq = None
self.worker_response_mq = None
destroy_model_parallel()
destroy_distributed_environment()
# worker_main passes False when the process exits right after this
# (see _exit_worker_process): the NCCL destroy is exit-only work that
# can block for 5 s per communicator.
if destroy_distributed:
destroy_model_parallel()
destroy_distributed_environment()
def monitor_death_pipe(self, death_pipe, shutdown_requested: threading.Event):
if death_pipe is None:
@@ -953,8 +1097,20 @@ class WorkerProc:
set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"])
worker = None
# Set once a READY worker leaves its busy loop because shutdown was
# requested (death pipe EOF or SIGTERM/SIGINT): the process then exits
# straight after WorkerProc.shutdown (see _exit_worker_process).
orderly_shutdown = False
# Set when a READY worker's busy loop FAILED while a shutdown request
# was pending: full teardown, then exit code 1 (never the exit-0 path).
failed_during_shutdown = False
ready_writer = kwargs.pop("ready_pipe")
death_pipe = kwargs.pop("death_pipe", None)
shutdown_state = kwargs.pop("shutdown_state", None)
def mark_shutdown(value: int) -> None:
if shutdown_state is not None:
shutdown_state.value = value
# Close inherited pipes from parent (incl. other worker pipes)
# Explicitly passing in existing pipes and closing them makes the pipe
@@ -1001,7 +1157,7 @@ class WorkerProc:
worker.worker_busy_loop()
except Exception:
except Exception as e:
# NOTE: if an Exception arises in busy_loop, we send
# a FAILURE message over the MQ RPC to notify the Executor,
# which triggers system shutdown.
@@ -1009,10 +1165,20 @@ class WorkerProc:
if ready_writer is not None:
logger.exception("WorkerProc failed to start.")
elif shutdown_requested.is_set():
elif isinstance(e, WorkerShutdownRequested) and (
shutdown_requested.is_set()
):
# The busy loop left BECAUSE of the shutdown request (its
# dequeue was cancelled), not because something failed.
logger.debug_once(
"[shutdown] WorkerProc: exiting after shutdown request"
)
orderly_shutdown = True
elif shutdown_requested.is_set():
# A real failure that raced a shutdown request: keep it a
# failure -- full teardown, then a non-zero exit.
logger.exception("WorkerProc failed during shutdown.")
failed_during_shutdown = True
else:
logger.exception("WorkerProc failed.")
@@ -1028,6 +1194,7 @@ class WorkerProc:
logger.debug_once(
"[shutdown] WorkerProc: terminated by shutdown signal"
)
orderly_shutdown = ready_writer is None
else:
logger.warning("WorkerProc was terminated")
# SystemExit must never be ignored
@@ -1040,7 +1207,26 @@ class WorkerProc:
death_pipe.close()
# Clean up once worker exits busy loop
if worker is not None:
# Tells the executor's ladder this rank is draining, not parked
# in a collective (see _ensure_worker_termination).
mark_shutdown(_SHUTDOWN_STARTED)
if orderly_shutdown:
# Everything that must happen before exit (FINAL DRAIN,
# instrumentation flushes, PLE cleanup) is in
# worker.shutdown(); then exit instead of paying the NCCL
# destroy + finalization past the executor's grace.
exit_code = 0
try:
worker.shutdown(destroy_distributed=False)
except BaseException:
logger.exception("WorkerProc shutdown failed")
exit_code = 1
mark_shutdown(_SHUTDOWN_DONE)
_exit_worker_process(exit_code)
worker.shutdown()
mark_shutdown(_SHUTDOWN_DONE)
if failed_during_shutdown:
sys.exit(1)
class ResponseStatus(Enum):
SUCCESS = auto()
@@ -1120,8 +1306,20 @@ class WorkerProc:
def worker_busy_loop(self):
"""Main busy loop for Multiprocessing Workers."""
assert self.rpc_broadcast_mq is not None
mq = self.rpc_broadcast_mq
while True:
self._execute_worker_rpc(self.rpc_broadcast_mq.dequeue(indefinite=True))
# No local holds the request across the next dequeue: its payload
# must be released first (test_multiproc_executor).
self._execute_worker_rpc(self._dequeue_worker_rpc(mq))
@staticmethod
def _dequeue_worker_rpc(mq: MessageQueue):
try:
return mq.dequeue(indefinite=True)
except RuntimeError as e:
if getattr(mq, "shutting_down", False) and str(e) == "cancelled":
raise WorkerShutdownRequested("RPC broadcast queue shut down") from e
raise
def _execute_worker_rpc(
self,
+118
View File
@@ -102,6 +102,86 @@ def _unlink_ipc_socket(ipc_addr: str) -> None:
logger.warning("Failed to unlink PLE IPC socket %s: %s", path, error)
# Set once by a GPU worker process whose orderly shutdown is about to end in
# os._exit (multiproc_executor._exit_worker_process). PleOffloadWorkerHandle
# then waits for the offload child's whole thread group to be in the kernel's
# exit path, not for its REAP. In a drain fault-injection test the ~51 GB
# child's leader went Z at ~+2.7 s, but one thread stayed R -- PF_EXITING set,
# in-kernel, freeing the address space -- until the reap at ~+8.6 s. Rank 0
# waiting for that reap ran the executor's grace -> SIGTERM -> SIGKILL ladder on
# every cycle. A read-only sample of one such shutdown showed
# the same thing: from Z to reap (6.0 s), the only live task was R with
# PF_EXITING, the leader's fd table was already closed (0 /dev/nvidia fds,
# down from 13), and the child was never an nvidia-smi compute app, before or
# after. It holds no GPU memory (host-mapped pull: no CUDA context). What is
# left is kernel-side host-memory teardown, and nothing in this process
# needs to wait for it.
_PROCESS_EXIT_IMMINENT = False
# Bound for the child's userspace cleanup (its proc_main finally) on that path;
# it took ~2.7 s on the reference host. It has to fit inside the executor's 5 s
# VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS with room for rank 0's own exit. After it,
# the child is SIGKILLed (userspace ends at once) and not waited for.
_EXIT_IMMINENT_CHILD_WAIT_S = 3.5
# procfs mount the predicate reads (tests point it at a fake tree).
_PROC_ROOT = "/proc"
# task_struct flag: set by do_exit() -> exit_signals() before exit_mm(); a task
# carrying it never returns to userspace.
_PF_EXITING = 0x4
def mark_process_exit_imminent() -> None:
"""This process os._exits right after its shutdown; see close()."""
global _PROCESS_EXIT_IMMINENT
_PROCESS_EXIT_IMMINENT = True
def _task_in_exit(stat: bytes) -> bool:
"""One /proc/<pid>/task/<tid>/stat: Z/X, or PF_EXITING (kernel exit path)."""
# Fields after the parenthesised comm (which may contain spaces/parens):
# [0]=state ... [6]=flags (stat field 9).
fields = stat[stat.rfind(b")") + 2 :].split()
if not fields:
return True
if fields[0] in (b"Z", b"X"):
return True
try:
return bool(int(fields[6]) & _PF_EXITING)
except (IndexError, ValueError):
return False
def _child_thread_group_exiting(
proc: subprocess.Popen, proc_root: str | None = None
) -> bool:
"""True once EVERY task of ``proc`` is exited or in the kernel exit path.
A zombie leader alone is not enough (a thread can still run userspace
after the leader exits). Never reaps: the child is unreaped, so its PID
cannot be reused and <proc_root>/<pid>/task is its own. Off Linux, only a
reaped child counts.
"""
if proc.returncode is not None:
return True
task_dir = os.path.join(proc_root or _PROC_ROOT, str(proc.pid), "task")
try:
tids = os.listdir(task_dir)
except FileNotFoundError:
return True
except OSError:
return proc.poll() is not None
for tid in tids:
try:
with open(os.path.join(task_dir, tid, "stat"), "rb") as fh:
stat = fh.read()
except FileNotFoundError:
continue # this task finished exiting between listdir and open
except OSError:
return False
if not _task_in_exit(stat):
return False
return True
def _publish_fence() -> None:
with _PUBLISH_FENCE:
pass
@@ -448,6 +528,9 @@ class PleOffloadWorkerHandle:
if self.ready_pipe_reader is not None:
self.ready_pipe_reader.close()
self.ready_pipe_reader = None
if graceful and _PROCESS_EXIT_IMMINENT:
self._close_without_reap()
return
if graceful and self.proc.poll() is None:
# Keep parent-death supervision connected while SIGTERM lets
# proc_main run its shutdown event and finally cleanup.
@@ -472,6 +555,41 @@ class PleOffloadWorkerHandle:
if self.proc.poll() is not None and self.ipc_addr is not None:
_unlink_ipc_socket(self.ipc_addr)
def _close_without_reap(self) -> None:
"""close() for a parent that os._exits next: wait (bounded) until every
child task has left userspace, never for the reap.
SIGTERM lets proc_main's finally unlink the IPC socket, as in the
graceful path; parent-death supervision stays connected meanwhile.
Once the child has run exit, the address-space teardown that keeps it
unreapable needs nothing from this process: the orphan is reaped by
the subreaper/init after this process exits.
"""
if not _child_thread_group_exiting(self.proc):
with contextlib.suppress(ProcessLookupError):
os.kill(self.proc.pid, signal.SIGTERM)
deadline = time.monotonic() + _EXIT_IMMINENT_CHILD_WAIT_S
while not _child_thread_group_exiting(self.proc):
if time.monotonic() >= deadline:
logger.warning(
"PLE offload process %d still running %.1f s after "
"SIGTERM; killing it without waiting (parent exits)",
self.proc.pid,
_EXIT_IMMINENT_CHILD_WAIT_S,
)
with contextlib.suppress(ProcessLookupError):
os.kill(self.proc.pid, signal.SIGKILL)
break
time.sleep(0.01)
exited = _child_thread_group_exiting(self.proc)
if self.death_writer is not None:
self.death_writer.close()
self.death_writer = None
# Same fallback as close(): the child has run exit, so this exact
# PLE-owned address is no longer in use.
if exited and self.ipc_addr is not None:
_unlink_ipc_socket(self.ipc_addr)
def _init_offload_distributed() -> None:
"""Initialize the single-rank Gloo world required by TP-aware layers."""