From 6c956ee3bde519e648738f131ba6fe6c228373fd Mon Sep 17 00:00:00 2001 From: MOVZX Date: Thu, 24 Sep 2026 01:36:08 -0400 Subject: [PATCH] [Engine] Drain-order shutdown - EngineCore survives a late SIGTERM during teardown (bounded deferral) so every worker drains its guard telemetry. - A worker exits right after its orderly shutdown. - A worker failure racing shutdown stays a failure (exit 1). - Rank 0 waits for the PLE child's whole thread group, bounded, instead of its reap; drain-aware failure ladder with a bounded grace for survivors whose shutdown already started. - tests: the sorted-emit graph arm takes max_len on the host before capture. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/v1/engine/core.py | 98 ++++++++++- vllm/v1/executor/multiproc_executor.py | 226 +++++++++++++++++++++++-- vllm/v1/ple_offload/worker.py | 118 +++++++++++++ 3 files changed, 423 insertions(+), 19 deletions(-) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index f036207357..e0fef97049 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -4,12 +4,13 @@ import gc import os import queue import signal +import sys import threading import time from collections import defaultdict, deque from collections.abc import Callable, Generator, Sequence from concurrent.futures import Future -from contextlib import ExitStack, contextmanager +from contextlib import ExitStack, contextmanager, suppress from enum import IntEnum from functools import partial from inspect import isclass, signature @@ -108,6 +109,76 @@ HANDSHAKE_TIMEOUT_MINS = 5 _R = TypeVar("_R") # Return type for collective_rpc +# How long EngineCore teardown may run after it deferred a late SIGTERM/SIGINT +# before the process exits anyway. Covers the whole of engine_core.shutdown(), +# not only the executor's worker ladder (grace + 4 s): a hang in later teardown +# (scheduler, distributed cleanup) must not make the process unkillable. +_TEARDOWN_SIGNAL_BOUND_S = 30.0 + + +class _TeardownSignalGuard: + """EngineCore teardown: defer ONE late SIGTERM/SIGINT, bounded. + + Dying on the first late signal would SIGKILL every worker + (PR_SET_PDEATHSIG) before its shutdown drain; see the finally block of + EngineCoreProc.run_engine_core. The deferral is bounded two ways: + - a SECOND late signal restores the default disposition and re-delivers + itself, so the process dies of it (the maintainer's escalation); + - after the first deferred signal a watchdog ends the process with + exit code 1 if teardown is still running _TEARDOWN_SIGNAL_BOUND_S later. + """ + + def __init__(self, bound_s: float | None = None) -> None: + self.bound_s = _TEARDOWN_SIGNAL_BOUND_S if bound_s is None else bound_s + self.deferred = 0 + self._done = threading.Event() + self._watchdog: threading.Thread | None = None + + def __call__(self, signum, frame) -> None: + name = signal.Signals(signum).name + if self._done.is_set(): + return + self.deferred += 1 + if self.deferred == 1: + logger.warning( + "[shutdown] EngineCore: %s during resource teardown deferred; " + "workers finish their shutdown (FINAL DRAIN) first (a second " + "signal, or %.0f s of teardown, ends the process)", + name, + self.bound_s, + ) + self._watchdog = threading.Thread( + target=self._expire, + daemon=True, + name="EngineCoreTeardownWatchdog", + ) + self._watchdog.start() + return + logger.warning( + "[shutdown] EngineCore: second late signal (%s) during resource " + "teardown; not deferring again", + name, + ) + signal.signal(signum, signal.SIG_DFL) + os.kill(os.getpid(), signum) + + def _expire(self) -> None: + if self._done.wait(self.bound_s): + return + logger.error( + "[shutdown] EngineCore: resource teardown still running %.0f s " + "after a deferred signal; exiting (code 1)", + self.bound_s, + ) + for stream in (sys.stdout, sys.stderr): + with suppress(Exception): + stream.flush() + os._exit(1) + + def disarm(self) -> None: + self._done.set() + + class EngineCore: """Inner loop of vLLM's Engine.""" @@ -1441,12 +1512,29 @@ class EngineCoreProc(EngineCore): engine_core._send_engine_dead() raise e finally: - signal.signal(signal.SIGTERM, signal.SIG_DFL) - signal.signal(signal.SIGINT, signal.SIG_DFL) + # Workers carry PR_SET_PDEATHSIG=SIGKILL (fail-closed PLE teardown), + # so if this process dies before model_executor.shutdown() has reaped + # them, every worker is SIGKILLed mid-shutdown and its bounds-guard + # FINAL DRAIN never runs. The API server's process manager sends a + # second SIGTERM right after the signal that started this shutdown; + # with SIG_DFL here it killed teardown before the executor waited for + # the workers. Defer ONE late signal until teardown ends; a second + # signal or _TEARDOWN_SIGNAL_BOUND_S of teardown after the deferral + # still ends the process (_TeardownSignalGuard), as do the + # executor's terminate/kill ladder and the process manager's timeout + # (kill_process_tree -> SIGKILL). + teardown_guard = _TeardownSignalGuard() + signal.signal(signal.SIGTERM, teardown_guard) + signal.signal(signal.SIGINT, teardown_guard) if signal_callback is not None: signal_callback.stop() - if engine_core is not None: - engine_core.shutdown() + try: + if engine_core is not None: + engine_core.shutdown() + finally: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + teardown_guard.disarm() if clean_shutdown: from vllm.platforms import current_platform diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index fe11014b13..2ce46b6407 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -114,6 +114,65 @@ def _install_worker_parent_death_signal(expected_parent_pid: int) -> None: os._exit(1) +# Grace and post-SIGTERM wait, after a worker's death, for survivors that never +# started their shutdown (see MultiprocExecutor._ensure_worker_termination). +_FAILED_WORKER_WAIT_S = 1.0 +# After a worker's death, the bounded grace for survivors whose shutdown has +# STARTED (FINAL DRAIN in progress). Every FINAL DRAIN measured on the reference host took +# < 0.1 s (drain-path tests: the drain line lands within ~0.05 s of the +# shutdown request); 2 s is 20x that, so a draining rank is not cut +# (a shorter cap did), while one blocked inside its shutdown on the dead rank is ended. +_FAILED_DRAIN_GRACE_S = 2.0 +# Per-worker shared byte (WorkerProcHandle.shutdown_state): written by the +# worker in worker_main's finally, read by the executor's termination ladder. +_SHUTDOWN_NOT_STARTED = 0 +_SHUTDOWN_STARTED = 1 +_SHUTDOWN_DONE = 2 + + +class WorkerShutdownRequested(RuntimeError): + """The busy loop's dequeue was cancelled by a shutdown request. + + Raised only by WorkerProc.worker_busy_loop when the RPC broadcast queue + reports its own cancellation (shm_broadcast: RuntimeError("cancelled") + after MessageQueue.shutdown(), which the death-pipe monitor calls once the + parent closes the pipe). worker_main takes the orderly exit path on this + type only; any other exception is a failure, even one that races a + shutdown request. Subclasses RuntimeError so existing catchers still match. + """ + + +def _exit_worker_process(exit_code: int) -> None: + """End a worker process whose orderly shutdown has already run. + + Called after WorkerProc.shutdown(destroy_distributed=False): the FINAL + DRAIN, the instrumentation flushes and the PLE offload / shared-memory + cleanup are done. What would remain is process-exit-only teardown -- + destroying the NCCL communicators (PyNcclCommunicator.destroy joins its + ncclCommAbort thread for up to 5 s per communicator while CUDA graphs that + captured NCCL ops are still alive) and interpreter finalization (CUDA + context, allocator pools). The driver and kernel reclaim all of it at + process exit, and without this the executor's 5 s grace expired and the + worker was SIGKILLed at ~9 s anyway. Flush the logs, + then leave. + """ + logger.info( + "[shutdown] WorkerProc: shutdown complete; exiting process (code %d) " + "without distributed teardown", + exit_code, + ) + try: + import logging + + logging.shutdown() + except Exception: # noqa: BLE001 -- the exit must happen regardless + pass + for stream in (sys.stdout, sys.stderr): + with suppress(Exception): + stream.flush() + os._exit(exit_code) + + class FutureWrapper(Future): def __init__( self, @@ -494,10 +553,36 @@ class MultiprocExecutor(Executor): return future if non_block else future.result() @staticmethod - def _ensure_worker_termination(worker_procs: list[BaseProcess]): + def _ensure_worker_termination( + worker_procs: list[BaseProcess], + *, + after_failure: bool = False, + shutdown_states: list[Any] | None = None, + ): """Ensure that all worker processes are terminated. Assumes workers have received termination requests. Waits for processing, then sends - termination and kill signals if needed.""" + termination and kill signals if needed. + + after_failure: a worker already died. Survivors split in two. Those + that never started WorkerProc.shutdown() are parked in a collective + with the dead rank (NCCL / gloo, in C, where neither the grace nor a + SIGTERM reaches them -- in a fault-injection test two ranks ran the full + 5 s + 4 s ladder to SIGKILL): they get _FAILED_WORKER_WAIT_S to start + their shutdown, then SIGTERM, then SIGKILL after as long again. Those + whose shutdown_states entry shows the shutdown started (FINAL DRAIN, + flushes) get _FAILED_DRAIN_GRACE_S from the start of the ladder, then + SIGTERM, then SIGKILL after _FAILED_WORKER_WAIT_S: a started shutdown + can itself block on the dead rank (in a drain fault-injection test one + survivor had started its shutdown and never logged its FINAL DRAIN -- + the drain's device synchronize waits on work paired with the dead + rank -- and ran the full orderly ladder, workers gone 17.6 s after the + fault). Before the drain-order fix the engine died on the late SIGTERM and + PDEATHSIG killed all survivors at once. A worker with no state (None, + or no shutdown_states) counts as parked. The orderly ladder (no + failure) is unchanged. + """ + grace_s = float(envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS) + term_wait_s = 4.0 def wait_for_termination(procs, timeout): if not time: @@ -519,9 +604,42 @@ class MultiprocExecutor(Executor): "[shutdown] Executor: waiting for worker exit count=%d", initial_count, ) - if wait_for_termination( - active_procs(), timeout=envs.VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS - ): + if after_failure and time: + start = time.time() + states = list(shutdown_states or [None] * len(worker_procs)) + + def parked() -> list[BaseProcess]: + return [ + proc + for proc, state in zip(worker_procs, states) + if proc.is_alive() + and (state is None or state.value == _SHUTDOWN_NOT_STARTED) + ] + + if not wait_for_termination(active_procs(), _FAILED_WORKER_WAIT_S): + stuck = parked() + if stuck: + logger.warning( + "[shutdown] Executor: workers parked after a worker " + "failure (shutdown not started); sending SIGTERM count=%d", + len(stuck), + ) + for p in stuck: + p.terminate() + if not wait_for_termination(stuck, _FAILED_WORKER_WAIT_S): + stuck = [p for p in stuck if p.is_alive()] + logger.warning( + "[shutdown] Executor: parked workers still running " + "after SIGTERM; sending SIGKILL count=%d", + len(stuck), + ) + for p in stuck: + p.kill() + # Workers that started their shutdown get a BOUNDED grace, counted + # from the start of this ladder, then SIGTERM, then SIGKILL. + grace_s = max(0.0, _FAILED_DRAIN_GRACE_S - (time.time() - start)) + term_wait_s = _FAILED_WORKER_WAIT_S + if wait_for_termination(active_procs(), timeout=grace_s): logger.info_once("[shutdown] Executor: all workers exited gracefully") return @@ -534,7 +652,7 @@ class MultiprocExecutor(Executor): ) for p in remaining: p.terminate() - if not wait_for_termination(active_procs(), 4): + if not wait_for_termination(active_procs(), term_wait_s): # Send SIGKILL if still running remaining = active_procs() logger.warning( @@ -562,7 +680,13 @@ class MultiprocExecutor(Executor): if w.death_writer is not None: w.death_writer.close() w.death_writer = None - self._ensure_worker_termination([w.proc for w in workers]) + self._ensure_worker_termination( + [w.proc for w in workers], + after_failure=getattr(self, "is_failed", False), + shutdown_states=[ + getattr(w, "shutdown_state", None) for w in workers + ], + ) for w in workers: # Shutdown response queues @@ -613,6 +737,7 @@ class UnreadyWorkerProcHandle: rank: int ready_pipe: Connection death_writer: Connection | None = None + shutdown_state: Any = None @dataclass @@ -626,6 +751,8 @@ class WorkerProcHandle: # `peer_worker_response_mqs[i]` peer_worker_response_mqs: list[MessageQueue | None] death_writer: Connection | None = None + # Shared byte: _SHUTDOWN_NOT_STARTED / _STARTED / _DONE (worker writes). + shutdown_state: Any = None @classmethod def from_unready_handle( @@ -640,6 +767,7 @@ class WorkerProcHandle: worker_response_mq=worker_response_mq, peer_worker_response_mqs=peer_worker_response_mqs, death_writer=unready_handle.death_writer, + shutdown_state=unready_handle.shutdown_state, ) @@ -778,6 +906,8 @@ class WorkerProc: ready_reader, ready_writer = context.Pipe(duplex=False) # Death pipe to let child detect parent process exit death_reader, death_writer = context.Pipe(duplex=False) + # One shared byte: has this worker started / finished its shutdown? + shutdown_state = context.RawValue("b", _SHUTDOWN_NOT_STARTED) if inherited_fds is not None: inherited_fds = inherited_fds.copy() inherited_fds.extend((ready_reader.fileno(), death_writer.fileno())) @@ -790,6 +920,7 @@ class WorkerProc: "input_shm_handle": input_shm_handle, "ready_pipe": ready_writer, "death_pipe": death_reader, + "shutdown_state": shutdown_state, "shared_worker_lock": shared_worker_lock, "is_driver_worker": is_driver_worker, # Have the worker close parent end of this worker's pipes too @@ -814,7 +945,9 @@ class WorkerProc: death_reader.close() # Keep death_writer open in parent - when parent exits, # death_reader in child will get EOFError - return UnreadyWorkerProcHandle(proc, rank, ready_reader, death_writer) + return UnreadyWorkerProcHandle( + proc, rank, ready_reader, death_writer, shutdown_state + ) @staticmethod def wait_for_response_handle_ready( @@ -875,7 +1008,14 @@ class WorkerProc: return cast(list[WorkerProcHandle], ready_proc_handles) - def shutdown(self): + def shutdown(self, *, destroy_distributed: bool = True): + if not destroy_distributed: + # worker_main os._exits right after this: the PLE offload handle + # (rank 0 owns the child) must not wait for the child's reap, + # which trails its exit by seconds for a large host table. + ple_worker = sys.modules.get("vllm.v1.ple_offload.worker") + if ple_worker is not None: + ple_worker.mark_process_exit_imminent() if self.rpc_broadcast_mq is not None: self.rpc_broadcast_mq.shutdown() if self.worker_response_mq is not None: @@ -883,8 +1023,12 @@ class WorkerProc: self.worker.shutdown() self.rpc_broadcast_mq = None self.worker_response_mq = None - destroy_model_parallel() - destroy_distributed_environment() + # worker_main passes False when the process exits right after this + # (see _exit_worker_process): the NCCL destroy is exit-only work that + # can block for 5 s per communicator. + if destroy_distributed: + destroy_model_parallel() + destroy_distributed_environment() def monitor_death_pipe(self, death_pipe, shutdown_requested: threading.Event): if death_pipe is None: @@ -953,8 +1097,20 @@ class WorkerProc: set_worker_net_device(kwargs.get("local_rank", 0), kwargs["vllm_config"]) worker = None + # Set once a READY worker leaves its busy loop because shutdown was + # requested (death pipe EOF or SIGTERM/SIGINT): the process then exits + # straight after WorkerProc.shutdown (see _exit_worker_process). + orderly_shutdown = False + # Set when a READY worker's busy loop FAILED while a shutdown request + # was pending: full teardown, then exit code 1 (never the exit-0 path). + failed_during_shutdown = False ready_writer = kwargs.pop("ready_pipe") death_pipe = kwargs.pop("death_pipe", None) + shutdown_state = kwargs.pop("shutdown_state", None) + + def mark_shutdown(value: int) -> None: + if shutdown_state is not None: + shutdown_state.value = value # Close inherited pipes from parent (incl. other worker pipes) # Explicitly passing in existing pipes and closing them makes the pipe @@ -1001,7 +1157,7 @@ class WorkerProc: worker.worker_busy_loop() - except Exception: + except Exception as e: # NOTE: if an Exception arises in busy_loop, we send # a FAILURE message over the MQ RPC to notify the Executor, # which triggers system shutdown. @@ -1009,10 +1165,20 @@ class WorkerProc: if ready_writer is not None: logger.exception("WorkerProc failed to start.") - elif shutdown_requested.is_set(): + elif isinstance(e, WorkerShutdownRequested) and ( + shutdown_requested.is_set() + ): + # The busy loop left BECAUSE of the shutdown request (its + # dequeue was cancelled), not because something failed. logger.debug_once( "[shutdown] WorkerProc: exiting after shutdown request" ) + orderly_shutdown = True + elif shutdown_requested.is_set(): + # A real failure that raced a shutdown request: keep it a + # failure -- full teardown, then a non-zero exit. + logger.exception("WorkerProc failed during shutdown.") + failed_during_shutdown = True else: logger.exception("WorkerProc failed.") @@ -1028,6 +1194,7 @@ class WorkerProc: logger.debug_once( "[shutdown] WorkerProc: terminated by shutdown signal" ) + orderly_shutdown = ready_writer is None else: logger.warning("WorkerProc was terminated") # SystemExit must never be ignored @@ -1040,7 +1207,26 @@ class WorkerProc: death_pipe.close() # Clean up once worker exits busy loop if worker is not None: + # Tells the executor's ladder this rank is draining, not parked + # in a collective (see _ensure_worker_termination). + mark_shutdown(_SHUTDOWN_STARTED) + if orderly_shutdown: + # Everything that must happen before exit (FINAL DRAIN, + # instrumentation flushes, PLE cleanup) is in + # worker.shutdown(); then exit instead of paying the NCCL + # destroy + finalization past the executor's grace. + exit_code = 0 + try: + worker.shutdown(destroy_distributed=False) + except BaseException: + logger.exception("WorkerProc shutdown failed") + exit_code = 1 + mark_shutdown(_SHUTDOWN_DONE) + _exit_worker_process(exit_code) worker.shutdown() + mark_shutdown(_SHUTDOWN_DONE) + if failed_during_shutdown: + sys.exit(1) class ResponseStatus(Enum): SUCCESS = auto() @@ -1120,8 +1306,20 @@ class WorkerProc: def worker_busy_loop(self): """Main busy loop for Multiprocessing Workers.""" assert self.rpc_broadcast_mq is not None + mq = self.rpc_broadcast_mq while True: - self._execute_worker_rpc(self.rpc_broadcast_mq.dequeue(indefinite=True)) + # No local holds the request across the next dequeue: its payload + # must be released first (test_multiproc_executor). + self._execute_worker_rpc(self._dequeue_worker_rpc(mq)) + + @staticmethod + def _dequeue_worker_rpc(mq: MessageQueue): + try: + return mq.dequeue(indefinite=True) + except RuntimeError as e: + if getattr(mq, "shutting_down", False) and str(e) == "cancelled": + raise WorkerShutdownRequested("RPC broadcast queue shut down") from e + raise def _execute_worker_rpc( self, diff --git a/vllm/v1/ple_offload/worker.py b/vllm/v1/ple_offload/worker.py index 3d294073da..608f3644a5 100644 --- a/vllm/v1/ple_offload/worker.py +++ b/vllm/v1/ple_offload/worker.py @@ -102,6 +102,86 @@ def _unlink_ipc_socket(ipc_addr: str) -> None: logger.warning("Failed to unlink PLE IPC socket %s: %s", path, error) +# Set once by a GPU worker process whose orderly shutdown is about to end in +# os._exit (multiproc_executor._exit_worker_process). PleOffloadWorkerHandle +# then waits for the offload child's whole thread group to be in the kernel's +# exit path, not for its REAP. In a drain fault-injection test the ~51 GB +# child's leader went Z at ~+2.7 s, but one thread stayed R -- PF_EXITING set, +# in-kernel, freeing the address space -- until the reap at ~+8.6 s. Rank 0 +# waiting for that reap ran the executor's grace -> SIGTERM -> SIGKILL ladder on +# every cycle. A read-only sample of one such shutdown showed +# the same thing: from Z to reap (6.0 s), the only live task was R with +# PF_EXITING, the leader's fd table was already closed (0 /dev/nvidia fds, +# down from 13), and the child was never an nvidia-smi compute app, before or +# after. It holds no GPU memory (host-mapped pull: no CUDA context). What is +# left is kernel-side host-memory teardown, and nothing in this process +# needs to wait for it. +_PROCESS_EXIT_IMMINENT = False +# Bound for the child's userspace cleanup (its proc_main finally) on that path; +# it took ~2.7 s on the reference host. It has to fit inside the executor's 5 s +# VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS with room for rank 0's own exit. After it, +# the child is SIGKILLed (userspace ends at once) and not waited for. +_EXIT_IMMINENT_CHILD_WAIT_S = 3.5 +# procfs mount the predicate reads (tests point it at a fake tree). +_PROC_ROOT = "/proc" +# task_struct flag: set by do_exit() -> exit_signals() before exit_mm(); a task +# carrying it never returns to userspace. +_PF_EXITING = 0x4 + + +def mark_process_exit_imminent() -> None: + """This process os._exits right after its shutdown; see close().""" + global _PROCESS_EXIT_IMMINENT + _PROCESS_EXIT_IMMINENT = True + + +def _task_in_exit(stat: bytes) -> bool: + """One /proc//task//stat: Z/X, or PF_EXITING (kernel exit path).""" + # Fields after the parenthesised comm (which may contain spaces/parens): + # [0]=state ... [6]=flags (stat field 9). + fields = stat[stat.rfind(b")") + 2 :].split() + if not fields: + return True + if fields[0] in (b"Z", b"X"): + return True + try: + return bool(int(fields[6]) & _PF_EXITING) + except (IndexError, ValueError): + return False + + +def _child_thread_group_exiting( + proc: subprocess.Popen, proc_root: str | None = None +) -> bool: + """True once EVERY task of ``proc`` is exited or in the kernel exit path. + + A zombie leader alone is not enough (a thread can still run userspace + after the leader exits). Never reaps: the child is unreaped, so its PID + cannot be reused and //task is its own. Off Linux, only a + reaped child counts. + """ + if proc.returncode is not None: + return True + task_dir = os.path.join(proc_root or _PROC_ROOT, str(proc.pid), "task") + try: + tids = os.listdir(task_dir) + except FileNotFoundError: + return True + except OSError: + return proc.poll() is not None + for tid in tids: + try: + with open(os.path.join(task_dir, tid, "stat"), "rb") as fh: + stat = fh.read() + except FileNotFoundError: + continue # this task finished exiting between listdir and open + except OSError: + return False + if not _task_in_exit(stat): + return False + return True + + def _publish_fence() -> None: with _PUBLISH_FENCE: pass @@ -448,6 +528,9 @@ class PleOffloadWorkerHandle: if self.ready_pipe_reader is not None: self.ready_pipe_reader.close() self.ready_pipe_reader = None + if graceful and _PROCESS_EXIT_IMMINENT: + self._close_without_reap() + return if graceful and self.proc.poll() is None: # Keep parent-death supervision connected while SIGTERM lets # proc_main run its shutdown event and finally cleanup. @@ -472,6 +555,41 @@ class PleOffloadWorkerHandle: if self.proc.poll() is not None and self.ipc_addr is not None: _unlink_ipc_socket(self.ipc_addr) + def _close_without_reap(self) -> None: + """close() for a parent that os._exits next: wait (bounded) until every + child task has left userspace, never for the reap. + + SIGTERM lets proc_main's finally unlink the IPC socket, as in the + graceful path; parent-death supervision stays connected meanwhile. + Once the child has run exit, the address-space teardown that keeps it + unreapable needs nothing from this process: the orphan is reaped by + the subreaper/init after this process exits. + """ + if not _child_thread_group_exiting(self.proc): + with contextlib.suppress(ProcessLookupError): + os.kill(self.proc.pid, signal.SIGTERM) + deadline = time.monotonic() + _EXIT_IMMINENT_CHILD_WAIT_S + while not _child_thread_group_exiting(self.proc): + if time.monotonic() >= deadline: + logger.warning( + "PLE offload process %d still running %.1f s after " + "SIGTERM; killing it without waiting (parent exits)", + self.proc.pid, + _EXIT_IMMINENT_CHILD_WAIT_S, + ) + with contextlib.suppress(ProcessLookupError): + os.kill(self.proc.pid, signal.SIGKILL) + break + time.sleep(0.01) + exited = _child_thread_group_exiting(self.proc) + if self.death_writer is not None: + self.death_writer.close() + self.death_writer = None + # Same fallback as close(): the child has run exit, so this exact + # PLE-owned address is no longer in use. + if exited and self.ipc_addr is not None: + _unlink_ipc_socket(self.ipc_addr) + def _init_offload_distributed() -> None: """Initialize the single-rank Gloo world required by TP-aware layers."""