[PLE] Carry the host-mapped pull transport onto v0.30.0

Restores the fork's PLE offload package (not upstream) and its pull
transport: the offload process holds no CUDA context; the captured op waits on
a device-mapped host flag and copies host->GPU on the model stream; publication
is fenced; releases are synchronous across dummy/capture work; failures close
the rank. Includes the test-only fault-injection cuts for the transport.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:49:16 +07:00
co-authored by Hinata
parent bc0a1cee82
commit 025fd08603
13 changed files with 3755 additions and 5 deletions
@@ -0,0 +1,433 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base class for PLE layers that support cross-process CPU offload.
Synchronization between the offload process and the GPU main stream is
handled by :class:`CpuGpuSemaphore`, which uses CUDA's
``cuStreamWaitValue32`` / ``cuStreamWriteValue32`` stream memory operations.
Decode-step timeline, host-mapped PULL design
--------------------------------------------------------
Offload process (CPU, no CUDA) GPU main stream (forward)
------------------------------ ------------------------------
forward_impl() on CPU ... other GPU ops ...
wait host flag == 0 WaitValue32(host flag == 1) <- in graph
rows -> shared pinned host slot cuMemcpyHtoDAsync(slot -> GPU) <- in graph
publish fence; host flag = 1 return _gpu_output_buffer[:n]
... model consumes output ...
WriteValue32(host flag = 0) <- release_outputs
``release_outputs`` performs the sole reset. Safety requires serialized local
forwards on one ordered model stream: a new transaction cannot begin until the
prior release has been enqueued. If any prepared forward is abandoned or its
correspondence becomes uncertain, the worker must terminate; resetting,
retrying, or continuing could reuse stale rows or overwrite live input.
"""
import contextlib
import functools
import threading
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterator
from typing import Any, cast
import torch
from cuda.bindings import driver as cuda_driver
from cuda.bindings.driver import CUstreamWaitValue_flags
from torch import nn
import vllm.envs as envs
from vllm.utils.torch_utils import direct_register_custom_op
# Module-level flag set to True inside the offload subprocess.
# Because the offload process and GPU worker processes are separate OS
# processes (spawned via multiprocessing), each has its own memory space.
# A plain module-level bool is sufficient -- no thread-local storage needed.
_offload_worker_flag = False
# Native FULL capture runs on one Python thread. The custom-op implementation
# appends the exact arguments from every wait-copy invocation while that thread
# is inside the graph manager's capture scope.
_ple_copy_capture = threading.local()
@contextlib.contextmanager
def observe_ple_copy_geometry() -> Iterator[list[tuple[str, int, int]]]:
"""Collect ``(layer, rows, bytes)`` from actual wait-copy executions."""
if getattr(_ple_copy_capture, "records", None) is not None:
raise RuntimeError("nested PLE copy-geometry observation is unsupported")
records: list[tuple[str, int, int]] = []
_ple_copy_capture.records = records
try:
yield records
finally:
del _ple_copy_capture.records
def _record_ple_copy_geometry(
layer_name: str,
num_tokens: int,
row_bytes: int,
) -> None:
records = getattr(_ple_copy_capture, "records", None)
if records is not None:
rows = int(num_tokens)
records.append((str(layer_name), rows, rows * int(row_bytes)))
def is_offload_process() -> bool:
"""Return True inside the dedicated PLE CPU-offload subprocess."""
return _offload_worker_flag
def mark_as_offload_worker() -> None:
"""Mark the current process as the dedicated PLE offload worker."""
global _offload_worker_flag
_offload_worker_flag = True
def _cuda_check(result: Any, operation: str) -> Any:
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
error = result[0] if isinstance(result, tuple) else result
if error.value != 0:
raise RuntimeError(f"{operation} failed: {error}")
return result
class CpuGpuSemaphore:
"""Cross-process semaphore backed by a one-element int32 CUDA tensor.
The flag is stored in a regular GPU tensor that is shared through
PyTorch's CUDA IPC mechanism. Both processes obtain device pointers that
map to the same physical GPU memory, so a stream-memory wait in the GPU
worker observes the write issued by the offload process.
"""
RESET_VALUE = 0
DONE_VALUE = 1
def __init__(self, device: torch.device) -> None:
self._flag_tensor = torch.zeros(1, dtype=torch.int32, device=device)
self._flag_ptr: int | None = None
@classmethod
def from_ipc_tensor(cls, flag_tensor: torch.Tensor) -> "CpuGpuSemaphore":
"""Construct a semaphore from a CUDA tensor received through IPC."""
semaphore = cls.__new__(cls)
semaphore._flag_tensor = flag_tensor
semaphore._flag_ptr = None
return semaphore
@classmethod
def from_host_mapped(
cls, host_flag_tensor: torch.Tensor, device_ptr: int
) -> "CpuGpuSemaphore":
"""Semaphore on a page-locked, device-mapped shared host int32.
``host_flag_tensor`` keeps the shared storage alive; ``device_ptr`` is
its device address (``cuMemHostGetDevicePointer``). Stream memory ops
on it are executed by the GPU against host memory, so a CPU process
with no CUDA context can signal it with a plain store.
"""
semaphore = cls.__new__(cls)
semaphore._flag_tensor = host_flag_tensor
semaphore._flag_ptr = int(device_ptr)
return semaphore
@property
def flag_tensor(self) -> torch.Tensor:
"""Return the tensor that backs the semaphore flag."""
return self._flag_tensor
@property
def flag_ptr(self) -> int:
"""Device address used by the stream memory operations."""
if self._flag_ptr is not None:
return self._flag_ptr
return self._flag_tensor.data_ptr()
def reset(self, stream: torch.cuda.Stream | None = None) -> None:
"""Enqueue ``WriteValue32(flag=0)`` on ``stream``."""
if stream is None:
stream = torch.cuda.current_stream()
_cuda_check(
cuda_driver.cuStreamWriteValue32(
cuda_driver.CUstream(stream.cuda_stream),
cuda_driver.CUdeviceptr(self.flag_ptr),
self.RESET_VALUE,
0,
),
"CpuGpuSemaphore.reset",
)
def signal(self, stream: torch.cuda.Stream | None = None) -> None:
"""Enqueue ``WriteValue32(flag=1)`` on ``stream``."""
if stream is None:
stream = torch.cuda.current_stream()
_cuda_check(
cuda_driver.cuStreamWriteValue32(
cuda_driver.CUstream(stream.cuda_stream),
cuda_driver.CUdeviceptr(self.flag_ptr),
self.DONE_VALUE,
0,
),
"CpuGpuSemaphore.signal",
)
def wait_reset(self, stream: torch.cuda.Stream | None = None) -> None:
"""Enqueue ``WaitValue32(flag==0)`` on ``stream``."""
if stream is None:
stream = torch.cuda.current_stream()
_cuda_check(
cuda_driver.cuStreamWaitValue32(
cuda_driver.CUstream(stream.cuda_stream),
cuda_driver.CUdeviceptr(self.flag_ptr),
self.RESET_VALUE,
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
),
"CpuGpuSemaphore.wait_reset",
)
# ---------------------------------------------------------------------------
# Custom op: vllm::ple_offload_wait
# ---------------------------------------------------------------------------
# Wraps cuStreamWaitValue32 as a torch custom op so that
# torch.compile(fullgraph=True) treats the CUDA Driver call as an opaque node.
# The hidden_states argument creates a dynamo data-dependency edge that prevents
# the wait from being reordered before preceding GPU work.
# ---------------------------------------------------------------------------
def _ple_offload_wait_impl(
sem_flag_tensor: torch.Tensor,
gpu_output_buffer: torch.Tensor,
hidden_states: torch.Tensor,
) -> None:
"""Wait for the CPU result without releasing its output buffer."""
stream = torch.cuda.current_stream()
cuda_stream = cuda_driver.CUstream(stream.cuda_stream)
dev_ptr = cuda_driver.CUdeviceptr(sem_flag_tensor.data_ptr())
_cuda_check(
cuda_driver.cuStreamWaitValue32(
cuda_stream,
dev_ptr,
CpuGpuSemaphore.DONE_VALUE,
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
),
"cuStreamWaitValue32(done)",
)
def _ple_offload_wait_fake(
sem_flag_tensor: torch.Tensor,
gpu_output_buffer: torch.Tensor,
hidden_states: torch.Tensor,
) -> None:
"""Represent the side-effect-only wait during dynamo tracing."""
pass
direct_register_custom_op(
op_name="ple_offload_wait",
op_func=_ple_offload_wait_impl,
mutates_args=["gpu_output_buffer"],
fake_impl=_ple_offload_wait_fake,
)
# ---------------------------------------------------------------------------
# Custom op: vllm::ple_offload_wait_copy (host-mapped pull design)
# ---------------------------------------------------------------------------
# The offload process writes the result into a page-locked shared host buffer
# and stores flag=1 in a page-locked shared host int32; it holds no CUDA
# context. On the CURRENT stream (captured into the graph like the wait above):
# cuStreamWaitValue32(host flag == 1) -> GPU polls host memory
# cuMemcpyAsync(gpu buffer <- host buffer, num_tokens * row_bytes)
# The copy engine reads host memory coherently after the flag was observed,
# and the consumers read device memory as before. The flag is NOT reset here:
# ``release_outputs`` resets it after the forward (unchanged semantics), so
# capture warm-ups and dummy runs behave exactly as with ``ple_offload_wait``.
# Pointers are passed as ints so no CPU tensor enters the compiled graph.
# ---------------------------------------------------------------------------
def _ple_offload_wait_copy_impl(
gpu_output_buffer: torch.Tensor,
hidden_states: torch.Tensor,
flag_ptr: int,
host_ptr: int,
row_bytes: int,
num_tokens: int,
layer_name: str,
) -> None:
"""Wait for the host-side result, then DMA it into the GPU buffer."""
if not 0 <= int(num_tokens) <= gpu_output_buffer.shape[0] or int(row_bytes) != (
gpu_output_buffer.shape[1] * gpu_output_buffer.element_size()
):
raise RuntimeError(
f"PLE wait_copy out of bounds: {num_tokens} rows x {row_bytes} B vs "
f"GPU buffer {tuple(gpu_output_buffer.shape)} {gpu_output_buffer.dtype}"
)
nbytes = int(num_tokens) * int(row_bytes)
_record_ple_copy_geometry(layer_name, num_tokens, row_bytes)
stream = cuda_driver.CUstream(torch.cuda.current_stream().cuda_stream)
_cuda_check(
cuda_driver.cuStreamWaitValue32(
stream,
cuda_driver.CUdeviceptr(flag_ptr),
CpuGpuSemaphore.DONE_VALUE,
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
),
"cuStreamWaitValue32(host flag done)",
)
if nbytes > 0:
# host_ptr is this worker's host address of page-locked (registered)
# memory, so the copy is a true async DMA and is capturable.
_cuda_check(
cuda_driver.cuMemcpyHtoDAsync(
cuda_driver.CUdeviceptr(gpu_output_buffer.data_ptr()),
host_ptr,
nbytes,
stream,
),
"cuMemcpyHtoDAsync(PLE host -> GPU)",
)
def _ple_offload_wait_copy_fake(
gpu_output_buffer: torch.Tensor,
hidden_states: torch.Tensor,
flag_ptr: int,
host_ptr: int,
row_bytes: int,
num_tokens: int,
layer_name: str,
) -> None:
"""Represent the side-effect-only wait + copy during dynamo tracing."""
pass
direct_register_custom_op(
op_name="ple_offload_wait_copy",
op_func=_ple_offload_wait_copy_impl,
mutates_args=["gpu_output_buffer"],
fake_impl=_ple_offload_wait_copy_fake,
)
class PleOffloadLayer(nn.Module, ABC):
"""Base class for embedding-like PLE layers that can run on CPU.
Subclasses implement :meth:`forward_impl`, which contains the regular
on-device computation. In a GPU worker with PLE offload enabled, subclass
constructors are skipped so their large weights are never allocated. The
CPU process constructs a complete copy and owns those weights instead.
"""
_is_cpu_offloaded = False
_gpu_output_buffer: torch.Tensor
_sem: CpuGpuSemaphore
# Host-mapped pull design: host address of the shared output buffer and
# the bytes per output row. ``None`` keeps the IPC push behaviour.
_host_output_ptr: int | None = None
_host_row_bytes: int = 0
_ple_offload_name: str = "<unbound>"
def __init_subclass__(cls, **kwargs: object) -> None:
"""Skip subclass initialization in cross-process GPU-worker mode."""
super().__init_subclass__(**kwargs)
original_init_obj = cls.__dict__.get("__init__")
if original_init_obj is None or not callable(original_init_obj):
return
original_init = cast(Callable[..., None], original_init_obj)
@functools.wraps(original_init)
def guarded_init(
self: "PleOffloadLayer", *args: object, **kwargs: object
) -> None:
if envs.VLLM_PLE_CPU_OFFLOAD and not is_offload_process():
nn.Module.__init__(self)
return
original_init(self, *args, **kwargs)
cls.__init__ = guarded_init # type: ignore[method-assign, assignment]
@classmethod
def get_target_device(cls) -> torch.device:
"""Return CPU for the offload process and the active GPU otherwise."""
if envs.VLLM_PLE_CPU_OFFLOAD:
return torch.device("cpu")
return torch.device("cuda", torch.accelerator.current_device_index())
@abstractmethod
def forward_impl(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
*args: object,
**kwargs: object,
) -> torch.Tensor:
"""Execute the actual embedding computation on the owning device."""
raise NotImplementedError
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
"""Return the dtype used by cross-process output buffers."""
return default_dtype
def get_offload_output_dim(self, default_dim: int) -> int:
"""Return the last dimension used by cross-process output buffers."""
return default_dim
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
"""Initialize metadata needed by dummy offload execution."""
def setup_cross_process_offload(
self,
gpu_output_buffer: torch.Tensor,
semaphore: CpuGpuSemaphore,
host_output_ptr: int | None = None,
host_row_bytes: int = 0,
) -> None:
"""Configure the GPU-worker placeholder with its offload resources."""
self._is_cpu_offloaded = True
self._gpu_output_buffer = gpu_output_buffer
self._sem = semaphore
self._host_output_ptr = host_output_ptr
self._host_row_bytes = int(host_row_bytes)
def forward(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
*args: object,
**kwargs: object,
) -> torch.Tensor:
"""Wait for an offloaded result or delegate to ``forward_impl``."""
if self._is_cpu_offloaded:
if self._host_output_ptr is not None:
torch.ops.vllm.ple_offload_wait_copy(
self._gpu_output_buffer,
hidden_states,
self._sem.flag_ptr,
self._host_output_ptr,
self._host_row_bytes,
input_ids.shape[0],
self._ple_offload_name,
)
else:
torch.ops.vllm.ple_offload_wait(
self._sem.flag_tensor,
self._gpu_output_buffer,
hidden_states,
)
return self._gpu_output_buffer[: input_ids.shape[0]]
return self.forward_impl(hidden_states, input_ids, *args, **kwargs)
def release_offloaded_output(
self,
stream: torch.cuda.Stream | None = None,
) -> None:
"""Mark the cross-process output buffer reusable on ``stream``."""
if self._is_cpu_offloaded:
self._sem.reset(stream)
+30 -1
View File
@@ -352,9 +352,10 @@ class MultiprocExecutor(Executor):
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
logger.error(
"Worker proc %s died unexpectedly (exit code: %s), "
"shutting down executor.",
"shutting down executor. executor_shutdown_monotonic=%.9f",
proc.name,
proc.exitcode,
time.monotonic(),
)
_self.shutdown()
callback = _self.failure_callback
@@ -1039,6 +1040,28 @@ class WorkerProc:
SUCCESS = auto()
FAILURE = auto()
def _fail_ple_rpc_if_pending(
self, where: str, error: BaseException
) -> None:
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
owners = (self.worker, getattr(self.worker, "worker", None))
for owner in owners:
if owner is None:
continue
runner = getattr(owner, "model_runner", None)
connector = getattr(runner, "_ple_offload_connector", None)
if connector is None:
continue
try:
pending = connector.has_pending_completions
except BaseException as state_error:
connector._fail_closed(
f"{where}:pending-state", state_error
)
if pending:
connector._fail_closed(where, error)
return
def enqueue_output(self, output: Any):
"""Prepares output from the worker and enqueues it to the
worker_response_mq. If the output is an Exception, it is
@@ -1049,6 +1072,9 @@ class WorkerProc:
output = output.get_output()
except Exception as e:
logger.exception("Error getting async model runner output")
self._fail_ple_rpc_if_pending(
"async-output-copy", e
)
output = e
if isinstance(output, Exception):
@@ -1108,6 +1134,9 @@ class WorkerProc:
if output_rank is None or self.rank == output_rank:
self.handle_output(output)
except Exception as e:
self._fail_ple_rpc_if_pending(
f"worker-rpc:{method}", e
)
# Notes have been introduced in python 3.11
if hasattr(e, "add_note"):
e.add_note(traceback.format_exc())
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+855
View File
@@ -0,0 +1,855 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Exchange PLE data between a GPU worker and the CPU-offload process."""
import os
import queue
import threading
from dataclasses import dataclass
from multiprocessing.reduction import ForkingPickler
from typing import Any
import msgspec
import torch
from vllm.v1.ple_offload import rowdump as _rowdump
from vllm.v1.ple_offload.fatal import ple_fail_closed
from vllm.v1.ple_offload.progress import PleProgressMonitor
import torch.nn as nn
import zmq
from cuda.bindings import driver as cuda_driver
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.ple_offload_layer import (
CpuGpuSemaphore,
PleOffloadLayer,
)
from vllm.v1.ple_offload.protocol import (
PleOffloadRegistration,
PleOffloadRequest,
)
from vllm.v1.utils import record_function_or_nullcontext
from vllm_ple_test_fault import _FAULT
logger = init_logger(__name__)
# CUDA registration alone does not own Python storage. Captured graphs retain
# raw addresses, so every successfully registered host allocation remains
# strongly referenced until process exit.
_PROCESS_LIFETIME_REGISTERED_STORAGE: list[torch.Tensor] = []
def _unlink_filename_shared_storage(
tensor: torch.Tensor,
reserved_filenames: set[bytes],
) -> None:
"""Reserve and unlink one already filename-shared CPU storage."""
try:
storage = tensor.untyped_storage()
if storage.device.type != "cpu" or not storage.is_shared():
return
try:
storage._get_shared_fd()
except RuntimeError:
# Filename-shared storages do not expose a shared fd. Only now is
# it safe to query the filename: doing so on ordinary or fd-shared
# storage would convert it to a new filename-backed allocation.
pass
else:
return
_, filename, _ = storage._share_filename_cpu_()
if not isinstance(filename, bytes):
filename = os.fsencode(filename)
path = os.path.join("/dev/shm", os.fsdecode(filename).lstrip("/"))
except Exception:
logger.warning(
"Failed to inspect a PLE shared-memory storage during teardown",
exc_info=True,
)
return
# RefcountedMapAllocator unlinks the name when its shared counter reaches
# zero. Reserve one count before manual unlink so later storage destruction
# cannot try to unlink the already-removed name. The filename identifies
# the shared counter and also deduplicates tensor aliases and repeated close.
if filename in reserved_filenames:
return
try:
storage._shared_incref()
except Exception:
logger.warning(
"Failed to reserve the PLE shared-memory counter for %s",
path,
exc_info=True,
)
return
reserved_filenames.add(filename)
try:
os.unlink(path)
except FileNotFoundError:
# Another teardown participant already removed the name. Retain the
# reservation because this allocator must still avoid a later unlink.
pass
except OSError as error:
# The name remains available, so restore the allocator's original
# counter and permit a later release attempt to retry.
try:
storage._shared_decref()
except Exception:
# The reservation state is uncertain after a failed private call.
# Keep it marked to avoid adding another reservation on retry.
logger.warning(
"Failed to roll back the PLE shared-memory counter for %s",
path,
exc_info=True,
)
else:
reserved_filenames.discard(filename)
logger.warning("Failed to unlink PLE shared memory %s: %s", path, error)
@dataclass(frozen=True)
class _PendingPleOffloadRequest:
"""Bind request metadata to its MRV2 D2H completion event."""
request: PleOffloadRequest
d2h_done_event: torch.cuda.Event
eligible: bool = False
transaction_id: str | None = None
correlation_id: str | None = None
def _cuda_check(result: Any, operation: str) -> Any:
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
error = result[0] if isinstance(result, tuple) else result
if error.value != 0:
raise RuntimeError(f"{operation} failed: {error}")
return result
class PleOffloadConnector:
"""Connect a model runner V2 GPU worker to the PLE CPU worker."""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
device: torch.device,
ipc_addr: str,
*,
input_ids_source: torch.Tensor,
query_start_loc_source: torch.Tensor,
ngram_context_source: torch.Tensor | None,
) -> None:
self.device = device
self.rank = int(vllm_config.parallel_config.rank)
self.dp_rank = get_dp_group().rank_in_group
self.tp_rank = get_tp_group().rank_in_group
self.pp_rank = get_pp_group().rank_in_group
# Host-mapped pull design: per layer a shared host output buffer and a
# shared host int32 flag (element 0), filled/stored by the offload
# process with plain CPU writes. Page-locked + device-mapped here only
# after registration, because serialising for registration may move
# the shared storage to a new address.
self._host_output_bufs: dict[str, torch.Tensor] = {}
self._host_flags: dict[str, torch.Tensor] = {}
self._reserved_host_storage_filenames: set[bytes] = set()
self._gpu_output_bufs: dict[str, torch.Tensor] = {}
self._layers = self._setup_layers(vllm_config, model)
self._progress = PleProgressMonitor.from_environment(
rank=self.rank,
layer_names=tuple(self._layers),
)
# TP0 registers shared buffers with CUDA for asynchronous D2H copies.
scheduler_config = vllm_config.scheduler_config
self._input_ids_buf = torch.empty(
scheduler_config.max_num_batched_tokens,
dtype=torch.int32,
device="cpu",
).share_memory_()
self._query_start_loc_buf = torch.empty(
scheduler_config.max_num_seqs + 1,
dtype=torch.int32,
device="cpu",
).share_memory_()
self._ngram_context_buf = None
config = vllm_config.model_config.hf_text_config
ngram_context_len = int(config.ngram_size) - 1
if ngram_context_len > 0:
self._ngram_context_buf = torch.empty(
scheduler_config.max_num_seqs,
ngram_context_len,
dtype=torch.int32,
device="cpu",
).share_memory_()
# Runner input allocations are address-stable, so bind them once and
# pass only batch sizes through the per-forward request queue.
self._input_ids_source = input_ids_source
self._query_start_loc_source = query_start_loc_source
self._ngram_context_source = ngram_context_source
self._validate_input_sources()
self._pinned_input_buffers: list[torch.Tensor] = []
# Device-mapped host outputs/flags referenced by raw address from
# captured graphs: never unregistered while the process lives
# (unregistering under a pending graph wait unmaps it).
self._mapped_host_buffers: list[torch.Tensor] = []
# True while dummy/capture work (seeds, dummy forwards, capture
# cleanup resets) has run since the last real forward. Releases in
# that state complete synchronously, and the next real forward first
# passes a TP barrier, so no rank can publish a real request while
# another rank still has a dummy/cleanup reset pending.
# In this tree real-path kernel warm-ups run BEFORE
# capture and capture adds seeds/resets after them, so this must hold
# at every dummy->real transition, not only at the first real one.
self._after_dummy = True
self._request_queue: queue.Queue[_PendingPleOffloadRequest | None] = (
queue.Queue(maxsize=vllm_config.max_concurrent_batches)
)
self._dropped_events: list[torch.cuda.Event] = []
self._request_thread: threading.Thread | None = None
self._request_thread_ready = threading.Event()
self._zmq_ctx: zmq.Context | None = None
self._registration_socket: zmq.Socket | None = None
self._d2h_event_pool: queue.Queue[torch.cuda.Event] | None = None
try:
self._zmq_ctx = zmq.Context()
self._registration_socket = self._zmq_ctx.socket(zmq.PUSH)
self._registration_socket.connect(ipc_addr)
payload = self._serialize_registration(vllm_config)
# Serialising moved the shared storages to their final addresses.
# Page-lock/map exactly those, THEN publish, so a received
# registration always means "storage usable".
with torch.accelerator.device_index(self.device.index):
self._map_host_outputs(vllm_config)
if self.tp_rank == 0:
self._pin_input_buffers()
self._send_registration(payload, ipc_addr)
if self.tp_rank == 0:
with torch.accelerator.device_index(self.device.index):
self._d2h_event_pool = queue.Queue(
maxsize=vllm_config.max_concurrent_batches
)
for _ in range(vllm_config.max_concurrent_batches):
self._d2h_event_pool.put_nowait(torch.cuda.Event())
self._start_request_thread(ipc_addr)
except Exception:
self.close()
raise
def _setup_layers(
self,
vllm_config: VllmConfig,
model: nn.Module,
) -> dict[str, PleOffloadLayer]:
"""Attach output buffers and semaphores to GPU PLE placeholders."""
layers = {
name: module
for name, module in model.named_modules()
if isinstance(module, PleOffloadLayer)
}
if not layers:
raise RuntimeError(
"VLLM_PLE_CPU_OFFLOAD is enabled, but the model has no PleOffloadLayer"
)
config = vllm_config.model_config.hf_text_config
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
for name, layer in layers.items():
if vllm_config.load_config.load_format == "dummy":
layer.initialize_dummy_offload_metadata(self.device)
out_dim = layer.get_offload_output_dim(int(config.ple_embed_dim))
out_dtype = layer.get_offload_output_dtype(vllm_config.model_config.dtype)
# The captured forward DMAs the host result into this device
# buffer after the host flag is set; consumers read it as before.
self._gpu_output_bufs[name] = torch.empty(
max_num_tokens, out_dim, dtype=out_dtype, device=self.device
)
# Zero-initialised so dummy/profile forwards copy finite values.
self._host_output_bufs[name] = torch.zeros(
max_num_tokens, out_dim, dtype=out_dtype, device="cpu"
).share_memory_()
# 16 x int32 = one 64-byte line; element 0 is the flag.
self._host_flags[name] = torch.zeros(
16, dtype=torch.int32, device="cpu"
).share_memory_()
return layers
def _map_host_outputs(self, vllm_config: VllmConfig) -> None:
"""Page-lock + device-map host outputs/flags and arm the placeholders."""
for name, layer in self._layers.items():
layer._ple_offload_name = name
host_out = self._host_output_bufs[name]
host_flag = self._host_flags[name]
for buffer in (host_out, host_flag):
self._register_host_buffer(buffer, device_map=True)
flag_dev_ptr = _cuda_check(
cuda_driver.cuMemHostGetDevicePointer(host_flag.data_ptr(), 0),
"cuMemHostGetDevicePointer(PLE host flag)",
)[1]
out_dev_ptr = _cuda_check(
cuda_driver.cuMemHostGetDevicePointer(host_out.data_ptr(), 0),
"cuMemHostGetDevicePointer(PLE host output)",
)[1]
row_bytes = host_out.shape[1] * host_out.element_size()
gpu_buf = self._gpu_output_bufs[name]
if (
gpu_buf.shape[0] != host_out.shape[0]
or gpu_buf.shape[1] * gpu_buf.element_size() != row_bytes
):
raise RuntimeError(f"PLE host/GPU buffer geometry mismatch for {name}")
# Stream memops use the flag's DEVICE pointer; the H2D copy uses
# this worker's HOST pointer (no raw addresses cross
# processes; the offload writes through its own mapping).
layer.setup_cross_process_offload(
gpu_buf,
CpuGpuSemaphore.from_host_mapped(host_flag, int(flag_dev_ptr)),
host_output_ptr=host_out.data_ptr(),
host_row_bytes=row_bytes,
)
logger.info(
"PleOffload: host-mapped pull for %s (rows %d x %d B, flag dev "
"0x%x, out dev 0x%x, host==dev %s)",
name,
host_out.shape[0],
row_bytes,
int(flag_dev_ptr),
int(out_dev_ptr),
int(out_dev_ptr) == host_out.data_ptr(),
)
def _pin_input_buffers(self) -> None:
"""Page-lock shared input allocations without replacing their storage."""
buffers = [self._input_ids_buf, self._query_start_loc_buf]
if self._ngram_context_buf is not None:
buffers.append(self._ngram_context_buf)
for buffer in buffers:
self._register_host_buffer(buffer, device_map=False)
def _register_host_buffer(self, buffer: torch.Tensor, device_map: bool) -> None:
"""Page-lock (and optionally device-map) a shared CPU allocation."""
if buffer.device.type != "cpu" or not buffer.is_shared():
raise RuntimeError("PLE host buffers must be shared CPU tensors")
if not buffer.is_contiguous():
raise RuntimeError("PLE host buffers must be contiguous")
flags = cuda_driver.CU_MEMHOSTREGISTER_PORTABLE
if device_map:
flags |= cuda_driver.CU_MEMHOSTREGISTER_DEVICEMAP
_cuda_check(
cuda_driver.cuMemHostRegister(
buffer.data_ptr(),
buffer.numel() * buffer.element_size(),
flags,
),
"cuMemHostRegister(PLE host buffer)",
)
_PROCESS_LIFETIME_REGISTERED_STORAGE.append(buffer)
if device_map:
self._mapped_host_buffers.append(buffer)
else:
self._pinned_input_buffers.append(buffer)
if not buffer.is_pinned():
raise RuntimeError("CUDA did not page-lock a PLE host buffer")
def _unpin_input_buffers(self) -> None:
"""Release CUDA registrations after the request thread has stopped."""
for buffer in reversed(self._pinned_input_buffers):
try:
_cuda_check(
cuda_driver.cuMemHostUnregister(buffer.data_ptr()),
"cuMemHostUnregister(PLE input buffer)",
)
except RuntimeError:
logger.exception("Failed to unregister a PLE input buffer")
self._pinned_input_buffers.clear()
def _serialize_registration(self, vllm_config: VllmConfig) -> Any:
"""Build and serialise the registration (may move shared storages)."""
# Each GPU worker owns distinct host output buffers and flags, while
# TP0's shared inputs become the request source for its DP rank.
registration = PleOffloadRegistration(
worker_id=(
self.dp_rank * vllm_config.parallel_config.world_size
+ vllm_config.parallel_config.rank
),
tp_rank=self.tp_rank,
dp_rank=self.dp_rank,
host_output_buffers=dict(self._host_output_bufs),
host_flag_tensors=dict(self._host_flags),
input_ids_buf=self._input_ids_buf,
query_start_loc_buf=self._query_start_loc_buf,
ngram_context_buf=self._ngram_context_buf,
)
# ForkingPickler transmits tensors through shared-memory and CUDA IPC.
import torch.multiprocessing as torch_mp
original_strategy = torch_mp.get_sharing_strategy()
torch_mp.set_sharing_strategy("file_system")
try:
payload = ForkingPickler.dumps(registration)
finally:
torch_mp.set_sharing_strategy(original_strategy)
return payload
def _send_registration(self, payload: Any, ipc_addr: str) -> None:
"""Publish the serialised registration (storages already mapped)."""
assert self._registration_socket is not None
self._registration_socket.send(payload)
logger.info(
"PleOffload: registered %d PleOffloadLayer(s) "
"(dp_rank=%d, tp_rank=%d, ipc_addr=%s): %s",
len(self._layers),
self.dp_rank,
self.tp_rank,
ipc_addr,
sorted(self._layers),
)
def _start_request_thread(self, ipc_addr: str) -> None:
"""Start the thread that publishes batches after inputs are ready."""
self._request_thread = threading.Thread(
target=self._request_loop,
args=(ipc_addr,),
name=f"ple-offload-dp{self.dp_rank}",
daemon=True,
)
self._request_thread.start()
if not self._request_thread_ready.wait(timeout=10):
raise RuntimeError("Timed out starting the PLE request thread")
def _request_loop(self, ipc_addr: str) -> None:
"""Wait for staged inputs, then notify the CPU worker."""
socket: zmq.Socket | None = None
try:
if self._zmq_ctx is None:
raise RuntimeError("PLE ZMQ context closed before thread startup")
socket = self._zmq_ctx.socket(zmq.PUSH)
socket.connect(ipc_addr)
self._request_thread_ready.set()
while True:
request = self._request_queue.get()
if request is None:
return
self._process_request(request, socket)
except Exception:
logger.exception("PLE request thread failed")
os._exit(1)
finally:
self._request_thread_ready.set()
if socket is not None:
socket.close(linger=0)
def _process_request(
self, pending: _PendingPleOffloadRequest, socket: zmq.Socket
) -> None:
"""Wait for one staged batch and publish its request."""
request = pending.request
event_pool = self._d2h_event_pool
event = pending.d2h_done_event
assert event_pool is not None, "PLE D2H event pool is not initialized"
if _FAULT is not None and _FAULT.fires(
"block_input_completion",
role="gpu",
unit="connector-request",
eligible=pending.eligible,
tp_rank=self.tp_rank,
pp_rank=self.pp_rank,
dp_rank=self.dp_rank,
transaction_id=pending.transaction_id,
correlation_id=pending.correlation_id,
):
threading.Event().wait()
with torch.accelerator.device_index(self.device.index):
event.synchronize()
payload = msgspec.msgpack.encode(request)
if _FAULT is None:
socket.send(payload)
else:
# Armed runs only: correlation metadata rides as a second frame.
metadata = msgspec.msgpack.encode(
{
"serving": pending.eligible,
"transaction": pending.transaction_id,
"correlation": pending.correlation_id,
}
)
socket.send_multipart([payload, metadata])
event_pool.put_nowait(event)
def _validate_input_sources(self) -> None:
"""Validate fixed runner sources against shared input buffers."""
sources = [
("input_ids", self._input_ids_source, self._input_ids_buf),
(
"query_start_loc",
self._query_start_loc_source,
self._query_start_loc_buf,
),
]
if (self._ngram_context_source is None) != (self._ngram_context_buf is None):
raise ValueError("PLE ngram_context source and buffer must match")
if self._ngram_context_source is not None:
assert self._ngram_context_buf is not None
sources.append(
(
"ngram_context",
self._ngram_context_source,
self._ngram_context_buf,
)
)
for name, source, buffer in sources:
if (
source.device != self.device
or source.dtype != buffer.dtype
or source.ndim != buffer.ndim
or source.shape[0] < buffer.shape[0]
or source.shape[1:] != buffer.shape[1:]
):
raise ValueError(f"PLE {name} source is incompatible")
def _enqueue_cuda_inputs(
self,
request: PleOffloadRequest,
d2h_done_event: torch.cuda.Event,
) -> None:
"""Stage inputs on the model stream and record D2H completion."""
with (
record_function_or_nullcontext("ple_offload: enqueue_cuda_inputs"),
torch.accelerator.device_index(self.device.index),
):
stream = torch.cuda.current_stream(self.device)
self._input_ids_buf[: request.num_tokens].copy_(
self._input_ids_source[: request.num_tokens],
non_blocking=True,
)
self._query_start_loc_buf[: request.num_reqs + 1].copy_(
self._query_start_loc_source[: request.num_reqs + 1],
non_blocking=True,
)
if self._ngram_context_buf is not None:
assert self._ngram_context_source is not None
self._ngram_context_buf[: request.num_reqs].copy_(
self._ngram_context_source[: request.num_reqs],
non_blocking=True,
)
d2h_done_event.record(stream)
def _launch(
self,
num_reqs: int,
num_tokens: int,
*,
serving: bool = False,
transaction_id: str | None = None,
correlation_id: str | None = None,
) -> None:
"""Stage or queue one batch for request publication."""
# Inputs are replicated across TP ranks. One request per DP rank drives
# the CPU result fan-out to every registered TP output buffer.
if self.tp_rank != 0:
return
request = PleOffloadRequest(
dp_rank=self.dp_rank,
num_tokens=num_tokens,
num_reqs=num_reqs,
)
assert self._d2h_event_pool is not None, "PLE D2H event pool is not initialized"
try:
d2h_done_event = self._d2h_event_pool.get_nowait()
except queue.Empty as exc:
raise RuntimeError(
"PLE has more requests than configured concurrent batches"
) from exc
self._enqueue_cuda_inputs(request, d2h_done_event)
if _FAULT is not None and _FAULT.fires(
"drop_request",
role="gpu",
unit="connector-request",
eligible=serving,
tp_rank=self.tp_rank,
pp_rank=self.pp_rank,
dp_rank=self.dp_rank,
transaction_id=transaction_id,
correlation_id=correlation_id,
):
# Deliberately retain the event: the fault is loss after D2H, not
# premature event-pool reuse while that D2H may still be in flight.
self._dropped_events.append(d2h_done_event)
return
self._request_queue.put_nowait(
_PendingPleOffloadRequest(
request,
d2h_done_event,
eligible=serving,
transaction_id=transaction_id,
correlation_id=correlation_id,
)
)
def _fail_closed(self, where: str, error: BaseException) -> None:
ple_fail_closed(
where=where,
rank=self.rank,
layer=",".join(self._layers),
sequence=self._progress.sequence,
error=error,
)
@property
def has_pending_completions(self) -> bool:
return self._progress.has_pending_completions
def begin_forward(self, *, dummy_run: bool, num_tokens: int) -> int:
"""Begin the bounded transaction before any runner fault boundary."""
try:
return self._progress.begin(
startup=dummy_run,
num_tokens=num_tokens,
)
except Exception as error:
self._fail_closed("prepare_forward", error)
def fault_transaction(self, sequence: int) -> str:
"""Read diagnostic state only when the caller has established arming."""
return self._progress.describe_submission(sequence)
def completion_is_pending(self, sequence: int) -> bool:
"""Return exact-sequence incomplete completion state for cut 6."""
return self._progress.completion_is_pending(sequence)
def prepare_forward(
self,
num_reqs: int,
num_tokens: int,
dummy_run: bool,
*,
consumer_num_tokens: int | None = None,
serving: bool = False,
transaction_id: str | None = None,
correlation_id: str | None = None,
) -> None:
"""Prepare one PLE transaction; any ordinary exception is fatal."""
try:
self._prepare_forward_impl(
num_reqs,
num_tokens,
dummy_run,
consumer_num_tokens=consumer_num_tokens,
serving=serving,
transaction_id=transaction_id,
correlation_id=correlation_id,
)
except Exception as error:
self._fail_closed("prepare_forward", error)
def _prepare_forward_impl(
self,
num_reqs: int,
num_tokens: int,
dummy_run: bool,
*,
consumer_num_tokens: int | None = None,
serving: bool = False,
transaction_id: str | None = None,
correlation_id: str | None = None,
) -> None:
"""Submit real inputs or satisfy the PLE wait for a dummy forward."""
if consumer_num_tokens is None:
consumer_num_tokens = num_tokens
if int(num_tokens) != int(consumer_num_tokens):
raise RuntimeError(
"PLE request/consumer token-count mismatch: "
f"request={num_tokens}, consumer={consumer_num_tokens}"
)
if not 0 <= int(num_tokens) <= self._input_ids_buf.shape[0]:
raise RuntimeError(
f"PLE token count {num_tokens} exceeds connector capacity "
f"{self._input_ids_buf.shape[0]}"
)
if not 0 <= int(num_reqs) < self._query_start_loc_buf.shape[0]:
raise RuntimeError(
f"PLE request count {num_reqs} exceeds connector capacity "
f"{self._query_start_loc_buf.shape[0] - 1}"
)
self._rowdump_n = None if dummy_run else num_tokens
if dummy_run:
self._signal_dummy_outputs_impl(num_tokens)
return
if self._after_dummy:
# Every dummy/cleanup reset on THIS rank already completed
# (synchronous releases); the barrier extends that to every
# stage-0 TP rank before TP0 can publish (CPU group, no GPU work;
# all TP ranks run the same forwards in the same order).
get_tp_group().barrier()
self._after_dummy = False
logger.info(
"PleOffload: dummy->real transition (dp_rank=%d, tp_rank=%d): "
"releases were synchronous, TP barrier passed",
self.dp_rank,
self.tp_rank,
)
self._launch(
num_reqs,
num_tokens,
serving=serving,
transaction_id=transaction_id,
correlation_id=correlation_id,
)
def signal_dummy_outputs(self, num_tokens: int) -> None:
"""Locally satisfy PLE waits for dummy and capture forwards."""
self._progress.begin(startup=True, num_tokens=num_tokens)
self._signal_dummy_outputs_impl(num_tokens)
def _signal_dummy_outputs_impl(self, num_tokens: int) -> None:
"""Signal dummy outputs after the progress transaction is armed."""
# Dummy and capture forwards do not send CPU requests, but every PLE
# placeholder still waits for a completed output semaphore.
self._after_dummy = True
stream = torch.cuda.current_stream(self.device)
for layer in self._layers.values():
layer._gpu_output_buffer[:num_tokens].zero_()
layer._sem.signal(stream)
def release_outputs(
self,
*,
serving: bool = False,
transaction_id: str | None = None,
correlation_id: str | None = None,
) -> None:
"""Release one PLE transaction; any ordinary exception is fatal."""
try:
self._release_outputs_impl(
serving=serving,
transaction_id=transaction_id,
correlation_id=correlation_id,
)
self._progress.complete()
except Exception as error:
self._fail_closed("release_outputs", error)
def _release_outputs_impl(
self,
*,
serving: bool = False,
transaction_id: str | None = None,
correlation_id: str | None = None,
) -> None:
"""Mark GPU output buffers reusable after the model consumes them."""
if _FAULT is not None and _FAULT.fires(
"exc_in_release",
role="gpu",
unit="connector-release",
eligible=serving,
tp_rank=self.tp_rank,
pp_rank=self.pp_rank,
dp_rank=self.dp_rank,
transaction_id=transaction_id,
correlation_id=correlation_id,
):
raise RuntimeError("PLE test fault: exc_in_release")
n = getattr(self, "_rowdump_n", None)
self._rowdump_n = None
if _rowdump.enabled() and n is not None and not torch.cuda.is_current_stream_capturing():
torch.cuda.current_stream(self.device).synchronize()
seq = getattr(self, "_rowdump_seq", 0)
self._rowdump_seq = seq + 1
for layer in self._layers.values():
_rowdump.write(f"gpu-tp{self.tp_rank}", 0, self.tp_rank, seq,
self._input_ids_source[:n], layer._gpu_output_buffer[:n])
# Reset only after the consumer forward so the CPU worker cannot
# overwrite an output that a GPU PLE placeholder may still read.
stream = torch.cuda.current_stream(self.device)
for layer in self._layers.values():
layer.release_offloaded_output(stream)
# Retain the serving deadline until the model stream has actually
# executed every reset above. The monitor thread is the sole query site.
completion_event = torch.cuda.Event()
completion_event.record(stream)
self._progress.track_completion(completion_event)
# After dummy/capture work (profile, warm-up sizes, capture seed,
# adaptive-verification dummy replays, capture cleanup): complete the
# reset before returning, so it cannot still be pending when a real
# request lets the offload publish. Real-forward releases stay async.
# Never inside a stream capture.
if self._after_dummy and not torch.cuda.is_current_stream_capturing():
stream.synchronize()
def release_host_storage(self) -> None:
"""Unlink host-storage names while retaining mappings and tensors."""
buffers = [
*self._host_output_bufs.values(),
*self._host_flags.values(),
self._input_ids_buf,
self._query_start_loc_buf,
]
if self._ngram_context_buf is not None:
buffers.append(self._ngram_context_buf)
for buffer in buffers:
_unlink_filename_shared_storage(
buffer,
self._reserved_host_storage_filenames,
)
def close(self) -> None:
"""Stop transport and unlink names, retaining host mappings."""
request_thread = self._request_thread
if request_thread is not None and request_thread.is_alive():
try:
self._request_queue.put(None, timeout=5)
except queue.Full:
logger.error("Timed out stopping the PLE request thread")
request_thread.join(timeout=5)
if request_thread is not None and request_thread.is_alive():
# The thread may still access the registered buffers or ZMQ context.
# It may also still race another process opening a shared name, so
# this deferred-cleanup branch must not unlink host storage.
logger.error("PLE request thread did not stop; deferring resource cleanup")
self._progress.close()
return
self._request_thread = None
# Inputs, outputs, and flags remain registered and strongly referenced.
# A captured graph or delayed request can still contain their addresses;
# the CUDA driver releases registrations when the process exits.
registered = len(self._pinned_input_buffers) + len(self._mapped_host_buffers)
if registered:
logger.debug(
"PleOffload: %d registered host buffers retained until exit",
registered,
)
self._d2h_event_pool = None
if self._registration_socket is not None:
self._registration_socket.close(linger=0)
self._registration_socket = None
if self._zmq_ctx is not None:
self._zmq_ctx.term()
self._zmq_ctx = None
self._progress.close()
# __init__'s failure handler can reach here before the child has rebuilt
# a registration. Startup is already failing in that case; if a payload
# was sent, a late rebuild may also observe the removed name. Successful
# startup accepts every registration before an external close begins.
#
# NOTE: Graceful shutdown unlinks the PLE shared-memory names and the
# PLE ipc socket. The remaining 17 /tmp UUID sockets per boot are
# upstream vLLM VLLM_RPC_BASE_PATH behaviour; a tmpfiles/launch-profile
# sweep is recommended.
# Keep this last: unlinking preserves every existing CPU/CUDA mapping,
# but no transport user may still need to open a name at this point.
self.release_host_storage()
+153
View File
@@ -0,0 +1,153 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fail-closed helpers for the PLE single-slot transport."""
import functools
import os
import time
import traceback
from collections.abc import Callable
from typing import Any, NoReturn, TypeVar
_F = TypeVar("_F", bound=Callable[..., Any])
_TRUSTED_ERROR_LABELS = (
(RuntimeError, "RuntimeError"),
(TimeoutError, "TimeoutError"),
(ValueError, "ValueError"),
(OSError, "OSError"),
(EOFError, "EOFError"),
(BrokenPipeError, "BrokenPipeError"),
(AssertionError, "AssertionError"),
(KeyError, "KeyError"),
(IndexError, "IndexError"),
(TypeError, "TypeError"),
(MemoryError, "MemoryError"),
(KeyboardInterrupt, "KeyboardInterrupt"),
)
def ple_fail_closed(
*,
where: str,
rank: int,
layer: str,
sequence: int,
error: BaseException,
) -> NoReturn:
"""Emit one best-effort nonblocking diagnostic and terminate immediately."""
try:
os.set_blocking(2, False)
error_type = type(error)
error_name = "Exception"
trusted_error = False
for trusted_type, trusted_name in _TRUSTED_ERROR_LABELS:
if error_type is trusted_type:
error_name = trusted_name
trusted_error = True
break
safe_where = where[:128] if type(where) is str else "unknown"
safe_layer = layer[:256] if type(layer) is str else "unknown"
safe_rank = rank if type(rank) is int else -1
safe_sequence = sequence if type(sequence) is int else -1
detail_suffix = ""
if trusted_error:
args = BaseException.args.__get__(error, error_type)
if (
type(args) is tuple
and len(args) == 1
and type(args[0]) is str
):
detail = args[0][:512].replace("\n", "\\n")
detail_suffix = f":{detail}"
message = (
f"PLE fatal: where={safe_where} rank={safe_rank} layer={safe_layer} "
f"sequence={safe_sequence} error={error_name}{detail_suffix} "
f"pid={os.getpid()} "
f"monotonic={time.monotonic():.9f} wall={time.time():.9f}\n"
)
os.write(2, message.encode("utf-8", "backslashreplace")[:2048])
except BaseException:
pass
# THE TRACEBACK, unconditionally, in its own try so it cannot cost us the
# line above.
#
# This path os._exit(1)s, which runs no handlers and flushes nothing, so
# whatever is not written here is gone. The line above names the TYPE only
# when the type is one of _TRUSTED_ERROR_LABELS and flattens everything
# else to the literal string "Exception" -- and the flattened set includes
# the two that matter most here: torch.AcceleratorError (a RuntimeError
# SUBCLASS, so the `is` comparison misses it) and Triton CompilationError.
# An earlier boot died exactly that way: both PP0 ranks
# reported "error=Exception" and there was no traceback anywhere in the
# log, in any rank, so the cause could not be named at all and the next
# step was a re-run rather than a fix.
#
# A fail-closed path is where evidence matters most, not least: it fires
# once, it kills the process, and it is the only witness. Printing the type
# of a wrapped exception without the frames that raised it is not a
# diagnostic.
try:
# Three-arg form: it does not depend on being inside an active except
# block, and this function is called from one frame further out than
# the raise.
tb = "".join(
traceback.format_exception(type(error), error, error.__traceback__)
)
raw = tb.encode("utf-8", "backslashreplace")
# Truncate from the END, not the start. format_exception puts the
# frames first and the exception TYPE AND MESSAGE last, so a head-side
# cut throws away the one line that names what went wrong and keeps the
# import-time frames nobody needs. A long chained traceback is exactly
# the case where that matters.
clipped = len(raw) > 8192
raw = raw[-8192:]
os.write(
2,
b"PLE fatal traceback (tail) follows:\n" if clipped
else b"PLE fatal traceback follows:\n",
)
os.write(2, raw)
if not raw.endswith(b"\n"):
os.write(2, b"\n")
except BaseException:
# Never silent HERE of all places: without this the reader cannot tell
# "the wrapper predates the traceback fix" from "the traceback itself
# could not be produced", and those call for different next steps.
# Still guarded, because os._exit below must run whatever happens: an
# escape from this handler would leave the worker alive in the state
# the fail-closed path exists to end.
try:
os.write(2, b"PLE fatal: traceback unavailable\n")
except BaseException:
pass
os._exit(1)
def fail_ple_execute_model(func: _F) -> _F:
"""Fail a non-shutdown worker-step escape only on PLE-owning ranks.
This includes ``sample_tokens``, the second half of the split execute-model
step. A rank without a PLE connector rethrows the original exception.
"""
@functools.wraps(func)
def wrapped(self: Any, *args: Any, **kwargs: Any) -> Any:
try:
return func(self, *args, **kwargs)
except SystemExit:
# WorkerProc's signal handler uses SystemExit for clean shutdown.
raise
except BaseException as error:
connector = getattr(self, "_ple_offload_connector", None)
if connector is None:
model_runner = getattr(self, "model_runner", None)
connector = getattr(
model_runner, "_ple_offload_connector", None
)
if connector is None:
raise
connector._fail_closed(func.__name__, error)
return wrapped # type: ignore[return-value]
+314
View File
@@ -0,0 +1,314 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Independent progress deadline for PLE transactions."""
import math
import os
import threading
import time
from collections import deque
from dataclasses import dataclass
from typing import Any
from vllm.v1.ple_offload.fatal import ple_fail_closed
_DEFAULT_SERVING_DEADLINE_S = 300.0
_DEFAULT_STARTUP_DEADLINE_S = 1800.0
_MAX_PROGRESS_DEADLINE_S = 86400.0
_POLL_INTERVAL_S = 0.1
def _validate_deadline(value: float, name: str) -> float:
value = float(value)
if (
not math.isfinite(value)
or value <= 0
or value > _MAX_PROGRESS_DEADLINE_S
):
raise ValueError(
f"{name} must be finite, greater than zero, and no greater than "
f"{_MAX_PROGRESS_DEADLINE_S:g} seconds"
)
return value
def _environment_deadline(name: str, default: float) -> float:
value = float(os.environ.get(name, str(default)))
return _validate_deadline(value, name)
@dataclass(frozen=True)
class _Submission:
sequence: int
started: float
startup: bool
num_tokens: int
deadline: float
depth: int
@dataclass(frozen=True)
class _PendingCompletion:
sequence: int
started: float
startup: bool
num_tokens: int
deadline: float
event: Any
class PleProgressMonitor:
"""Bound host submission and the resulting ordered GPU completions."""
def __init__(
self,
*,
rank: int,
layer_names: tuple[str, ...],
serving_deadline_s: float,
startup_deadline_s: float,
) -> None:
serving_deadline_s = _validate_deadline(
serving_deadline_s, "serving_deadline_s"
)
startup_deadline_s = _validate_deadline(
startup_deadline_s, "startup_deadline_s"
)
self._rank = rank
self._layer = ",".join(layer_names)
self._serving_deadline_s = serving_deadline_s
self._startup_deadline_s = startup_deadline_s
self._condition = threading.Condition()
self._submission: _Submission | None = None
self._pending: deque[_PendingCompletion] = deque()
self._sequence = 0
self._stopping = False
self._thread = threading.Thread(
target=self._run,
name=f"ple-progress-rank{rank}",
daemon=True,
)
self._thread.start()
@classmethod
def from_environment(
cls, *, rank: int, layer_names: tuple[str, ...]
) -> "PleProgressMonitor":
return cls(
rank=rank,
layer_names=layer_names,
serving_deadline_s=_environment_deadline(
"VLLM_PLE_PROGRESS_DEADLINE_S",
_DEFAULT_SERVING_DEADLINE_S,
),
startup_deadline_s=_environment_deadline(
"VLLM_PLE_STARTUP_DEADLINE_S",
_DEFAULT_STARTUP_DEADLINE_S,
),
)
@property
def sequence(self) -> int:
with self._condition:
return self._sequence
@property
def has_pending_completions(self) -> bool:
with self._condition:
return bool(self._pending)
def describe_submission(self, sequence: int) -> str:
"""Return the armed transaction descriptor for the active sequence."""
with self._condition:
active = self._submission
if active is None or active.sequence != sequence:
raise RuntimeError("PLE fault correlation has no active submission")
return (
f"sequence:{active.sequence}"
f"@begin:{active.started:.9f}"
)
def completion_is_pending(self, sequence: int) -> bool:
"""Whether this serving sequence has a retained, incomplete event.
This method is called only behind the armed fault guard. An event that
has completed on the GPU but has not yet been dequeued is not counted.
"""
with self._condition:
pending = next(
(
item
for item in self._pending
if item.sequence == sequence and not item.startup
),
None,
)
if pending is None:
return False
return not bool(pending.event.query())
def begin(self, *, startup: bool, num_tokens: int) -> int:
now = time.monotonic()
budget = (
self._startup_deadline_s if startup else self._serving_deadline_s
)
with self._condition:
active = self._submission
if active is not None:
if not startup or not active.startup:
raise RuntimeError(
"Overlapping PLE host submissions violate the serialized "
"single-slot protocol"
)
sequence = active.sequence
self._submission = _Submission(
sequence=sequence,
started=active.started,
startup=True,
num_tokens=num_tokens,
deadline=now + budget,
depth=active.depth + 1,
)
else:
self._sequence += 1
sequence = self._sequence
self._submission = _Submission(
sequence=sequence,
started=now,
startup=startup,
num_tokens=num_tokens,
deadline=now + budget,
depth=1,
)
self._condition.notify_all()
return sequence
def track_completion(self, event: Any) -> None:
"""Retain a release event until the monitor observes GPU completion."""
if not callable(getattr(event, "query", None)):
raise TypeError("PLE completion event must provide query()")
with self._condition:
active = self._submission
if active is None:
raise RuntimeError("PLE completion has no matching submission")
self._pending.append(
_PendingCompletion(
sequence=active.sequence,
started=active.started,
startup=active.startup,
num_tokens=active.num_tokens,
deadline=active.deadline,
event=event,
)
)
self._condition.notify_all()
def complete(self) -> None:
with self._condition:
active = self._submission
if active is None or active.depth <= 0:
raise RuntimeError("PLE release has no matching transaction")
if active.depth == 1:
self._submission = None
else:
self._submission = _Submission(
sequence=active.sequence,
started=active.started,
startup=active.startup,
num_tokens=active.num_tokens,
deadline=active.deadline,
depth=active.depth - 1,
)
self._condition.notify_all()
def close(self) -> None:
with self._condition:
self._stopping = True
self._condition.notify_all()
self._thread.join(timeout=1)
def _run(self) -> None:
try:
self._monitor()
except BaseException as error:
ple_fail_closed(
where="progress-monitor-thread",
rank=self._rank,
layer=self._layer,
sequence=self._sequence,
error=error,
)
def _monitor(self) -> None:
while True:
with self._condition:
if self._stopping:
return
submission = self._submission
pending = self._pending[0] if self._pending else None
if submission is None and pending is None:
self._condition.wait(timeout=_POLL_INTERVAL_S)
continue
# CUDA completion is observed only here. Never synchronize and never
# use the host flag as evidence that the queued reset completed.
if pending is not None and pending.event.query():
with self._condition:
if self._pending and self._pending[0] is pending:
self._pending.popleft()
self._condition.notify_all()
continue
now = time.monotonic()
if pending is not None and now >= pending.deadline:
with self._condition:
if not self._pending or self._pending[0] is not pending:
continue
mode = "startup" if pending.startup else "serving"
ple_fail_closed(
where=(
f"progress-deadline:{mode}:gpu-completion:"
f"tokens={pending.num_tokens}"
),
rank=self._rank,
layer=self._layer,
sequence=pending.sequence,
error=TimeoutError("PLE GPU completion made no bounded progress"),
)
if submission is not None and now >= submission.deadline:
with self._condition:
if self._submission is not submission:
continue
mode = "startup" if submission.startup else "serving"
ple_fail_closed(
where=(
f"progress-deadline:{mode}:host-submission:"
f"tokens={submission.num_tokens}"
),
rank=self._rank,
layer=self._layer,
sequence=submission.sequence,
error=TimeoutError("PLE host submission made no bounded progress"),
)
deadlines = []
if pending is not None:
deadlines.append(pending.deadline)
if submission is not None:
deadlines.append(submission.deadline)
timeout = _POLL_INTERVAL_S
if deadlines:
timeout = min(
timeout,
max(0.0, min(deadlines) - time.monotonic()),
)
with self._condition:
if self._stopping:
return
if (
self._submission is not submission
or (self._pending[0] if self._pending else None) is not pending
):
continue
self._condition.wait(timeout=timeout)
+50
View File
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""IPC message definitions for PLE CPU offload (host-mapped pull design).
The offload process owns no CUDA context. Every GPU worker owns, per PLE
layer, a shared host output buffer and a shared host int32 flag, both
page-locked and device-mapped in the worker's context. The offload process
waits for ``flag == 0``, writes the layer result into the host buffer and
stores ``flag = 1`` with plain CPU stores; the GPU's captured forward waits on
the host flag, DMAs host -> GPU on the model stream, and ``release_outputs``
resets the flag after the forward.
"""
from dataclasses import dataclass
import msgspec
import torch
# ---------------------------------------------------------------------------
# IPC message dataclasses
# ---------------------------------------------------------------------------
@dataclass
class PleOffloadRegistration:
"""Sent once from each GPU worker during offload setup."""
worker_id: int
tp_rank: int
dp_rank: int
# Shared CPU tensors per PLE layer, filled by the offload process.
host_output_buffers: dict[str, torch.Tensor]
# Shared CPU int32 tensors per PLE layer; element 0 is the ready flag.
host_flag_tensors: dict[str, torch.Tensor]
# CPU tensors are allocated in shared memory and registered once.
input_ids_buf: torch.Tensor
query_start_loc_buf: torch.Tensor
ngram_context_buf: torch.Tensor | None
@dataclass
class PleOffloadRequest:
"""Sent by each DP rank's TP rank zero at every inference step."""
dp_rank: int
num_tokens: int
num_reqs: int
_PLE_OFFLOAD_REQUEST_DECODER = msgspec.msgpack.Decoder(PleOffloadRequest)
File diff suppressed because it is too large Load Diff
+118 -4
View File
@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import gc
import itertools
from collections import defaultdict
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Iterator
from contextlib import ExitStack
from dataclasses import dataclass, replace
from itertools import groupby, product
@@ -33,6 +34,9 @@ from vllm.distributed.parallel_state import (
)
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.ple_offload_layer import (
observe_ple_copy_geometry,
)
from vllm.model_executor.offloader.base import get_offloader
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
@@ -173,6 +177,14 @@ class CudaGraphManager:
self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map()
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
# FULL graphs bake every PLE wait/copy into their native graph. Metadata
# comes from the actual custom-op implementation during native capture.
self._ple_full_copy_geometry: dict[
BatchExecutionDescriptor, tuple[tuple[str, int, int], ...]
] = {}
self._expected_ple_copy_layers: tuple[str, ...] = ()
# Only the target-model manager is allowed to capture PLE operations.
self._allow_ple_full_graph = False
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
self._graphs_captured = False
@@ -488,6 +500,7 @@ class CudaGraphManager:
torch.accelerator.synchronize()
free_before = torch.accelerator.get_memory_info()[0]
with (
self.capture_ple_copy_geometry(desc),
capture_pool(self.pool, self.vllm_config) as pool,
torch.cuda.graph(
graph, pool, stream=self._capture_stream(desc)
@@ -514,6 +527,78 @@ class CudaGraphManager:
{desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0}
)
@contextlib.contextmanager
def capture_ple_copy_geometry(
self, desc: BatchExecutionDescriptor
) -> Iterator[None]:
"""Production capture callback for actual PLE custom-op geometry."""
with observe_ple_copy_geometry() as geometry:
yield
if geometry and not self._allow_ple_full_graph:
raise RuntimeError(
"PLE wait-copy reached an unsupported FULL graph path; "
"speculator/draft FULL graphs may not contain PLE operations"
)
if geometry:
self.record_ple_copy_geometry(desc, geometry)
elif self._expected_ple_copy_layers:
raise RuntimeError(
f"PLE copy geometry was not observed while capturing FULL graph {desc}"
)
def record_ple_copy_geometry(
self,
desc: BatchExecutionDescriptor,
geometry: Iterable[tuple[str, int, int]],
) -> None:
if desc.cg_mode != CUDAGraphMode.FULL:
raise ValueError("PLE capture geometry is recorded only for FULL graphs")
observed = tuple(
(str(layer), int(rows), int(nbytes))
for layer, rows, nbytes in geometry
)
if not observed:
raise RuntimeError(f"PLE FULL graph {desc} observed no wait-copy operations")
observed_layers = tuple(layer for layer, _, _ in observed)
expected_layers = self._expected_ple_copy_layers
if expected_layers and sorted(observed_layers) != sorted(expected_layers):
raise RuntimeError(
f"PLE FULL graph {desc} observed layers {observed_layers}, "
f"expected {expected_layers}"
)
inconsistent = [
(layer, rows, nbytes)
for layer, rows, nbytes in observed
if rows != desc.num_tokens or nbytes < 0
]
if inconsistent:
raise RuntimeError(
f"PLE FULL graph {desc} actual PLE copy geometry "
f"{inconsistent} does not match descriptor rows={desc.num_tokens}"
)
previous = self._ple_full_copy_geometry.get(desc)
if previous is not None and previous != observed:
raise RuntimeError(
f"PLE FULL graph {desc} changed copy geometry from "
f"{previous} to {observed}"
)
self._ple_full_copy_geometry[desc] = observed
def ple_copy_rows(self, desc: BatchExecutionDescriptor) -> int:
try:
geometry = self._ple_full_copy_geometry[desc]
except KeyError as error:
raise RuntimeError(
f"PLE copy geometry was not recorded for FULL graph {desc}"
) from error
rows = {num_rows for _, num_rows, _ in geometry}
if len(rows) != 1:
raise RuntimeError(
f"PLE FULL graph {desc} has inconsistent per-layer rows: {geometry}"
)
return rows.pop()
def dispatch(
self,
num_reqs: int,
@@ -552,6 +637,11 @@ class CudaGraphManager:
f"Expected FULL mode, got {desc.cg_mode}"
)
assert desc in self.graphs, f"No cudagraph for {desc}"
if (
desc in self._ple_full_copy_geometry
and not self._allow_ple_full_graph
):
raise RuntimeError("refusing unsupported PLE speculator FULL replay")
# Sync offloader before replay - needed when transitioning from
# eager/piecewise to full cudagraph (e.g., prefill → decode).
# The previous eager iteration's start_prefetch may have queued
@@ -597,11 +687,32 @@ class ModelCudaGraphManager(CudaGraphManager):
varlen_decode=varlen_decode,
ubatch_runner=ubatch_runner,
)
self._allow_ple_full_graph = True
self.hidden_states: torch.Tensor | None = None
self.aux_hidden_states: list[torch.Tensor] = []
self.use_aux_hidden_state_outputs = False
self.intermediate_tensors: IntermediateTensors | None = None
def _prepare_ple_capture(
self, model: nn.Module, *, initialize_breakable: bool
) -> None:
expected_ple_copy_layers = tuple(
str(getattr(module, "_ple_offload_name", name))
for name, module in model.named_modules()
if getattr(module, "_is_cpu_offloaded", False)
and getattr(module, "_host_output_ptr", None) is not None
)
if (
"<unbound>" in expected_ple_copy_layers
or len(set(expected_ple_copy_layers)) != len(expected_ple_copy_layers)
):
raise RuntimeError(
"PLE FULL capture requires distinct, bound connector layer identities"
)
self._expected_ple_copy_layers = expected_ple_copy_layers
if initialize_breakable:
self.init_breakable_cg_runner(model)
def capture(
self,
model: nn.Module,
@@ -619,8 +730,9 @@ class ModelCudaGraphManager(CudaGraphManager):
) -> None:
"""Capture CUDA graphs for model forward pass."""
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
if self.use_breakable_cg:
self.init_breakable_cg_runner(model)
self._prepare_ple_capture(
model, initialize_breakable=self.use_breakable_cg
)
if self.cudagraph_mode.has_piecewise_cudagraphs() and not (
self.use_breakable_cg or has_compiled_submodule(model)
@@ -926,7 +1038,9 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
if manager.use_breakable_cg:
# Create the breakable runner before the wrapper pool swap so
# its pool is covered as well.
manager.init_breakable_cg_runner(runner.model)
manager._prepare_ple_capture(
runner.model, initialize_breakable=True
)
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
BreakableCUDAGraphWrapper._all_instances
)
+140
View File
@@ -74,6 +74,9 @@ from vllm.v1.outputs import (
ECConnectorOutput,
ModelRunnerOutput,
)
from vllm.v1.ple_offload.connector import PleOffloadConnector
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
from vllm.v1.ple_offload.test_fault import _FAULT
from vllm.v1.watermarking import create_watermarker
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.watermarking.spec_decode import (
@@ -1757,6 +1760,45 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.ec_connector.no_forward(scheduler_output).ec_connector_output,
)
def _prepare_ple_forward(
self,
input_batch: InputBatch,
batch_desc: BatchExecutionDescriptor,
dummy_run: bool,
*,
serving: bool = False,
transaction_id: str | None = None,
correlation_id: str | None = None,
) -> None:
connector = self._ple_offload_connector
assert connector is not None
request_num_tokens = int(input_batch.num_tokens_after_padding)
if batch_desc.cg_mode == CUDAGraphMode.FULL:
manager = self.cudagraph_manager
if manager is None:
raise RuntimeError("PLE FULL replay has no CUDA graph manager")
copy_num_tokens = manager.ple_copy_rows(batch_desc)
else:
# PIECEWISE/eager execute the Python op with this actual padded view.
copy_num_tokens = int(input_batch.input_ids.shape[0])
if request_num_tokens != copy_num_tokens:
raise RuntimeError(
"PLE request/selected-forward copy-count mismatch: "
f"request={request_num_tokens}, copy={copy_num_tokens}, "
f"mode={batch_desc.cg_mode}"
)
connector.prepare_forward(
input_batch.num_reqs,
request_num_tokens,
dummy_run,
consumer_num_tokens=copy_num_tokens,
serving=serving,
transaction_id=transaction_id,
correlation_id=correlation_id,
)
@fail_ple_execute_model
@torch.inference_mode()
def execute_model(
self,
@@ -2038,6 +2080,54 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
self.step_timing.forward_start()
# prepare_inputs has finalized MRV2's GPU buffers. Stage D2H on this
# stream before the PLE placeholder can wait for CPU output.
connector = self._ple_offload_connector
ple_serving = False
ple_sequence = None
ple_transaction = None
ple_correlation = None
if connector is not None:
ple_serving = not dummy_run and not is_profile
ple_sequence = connector.begin_forward(
dummy_run=dummy_run,
num_tokens=int(input_batch.num_tokens_after_padding),
)
if _FAULT is not None and ple_serving:
ple_correlation = f"sequence:{ple_sequence}"
ple_transaction = connector.fault_transaction(ple_sequence)
if _FAULT is not None and _FAULT.fires(
"exc_before_prepare",
role="gpu",
unit="runner-forward",
eligible=ple_serving,
tp_rank=connector.tp_rank,
pp_rank=connector.pp_rank,
dp_rank=connector.dp_rank,
transaction_id=ple_transaction,
correlation_id=ple_correlation,
):
raise RuntimeError("PLE test fault: exc_before_prepare")
self._prepare_ple_forward(
input_batch,
batch_desc,
dummy_run,
serving=ple_serving,
transaction_id=ple_transaction,
correlation_id=ple_correlation,
)
if _FAULT is not None and _FAULT.fires(
"exc_after_prepare",
role="gpu",
unit="runner-forward",
eligible=ple_serving,
tp_rank=connector.tp_rank,
pp_rank=connector.pp_rank,
dp_rank=connector.dp_rank,
transaction_id=ple_transaction,
correlation_id=ple_correlation,
):
raise RuntimeError("PLE test fault: exc_after_prepare")
connector_kwargs = dict(
scheduler_output=scheduler_output,
request_state_indices=input_batch.idx_mapping,
@@ -2095,6 +2185,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Eager (NONE): call the raw model directly.
model_output = self.model(**model_inputs)
# The model has consumed every PLE output, so the CPU worker may reuse
# the registered buffers for the next request.
if connector is not None:
connector.release_outputs(
serving=ple_serving,
transaction_id=ple_transaction,
correlation_id=ple_correlation,
)
if _FAULT is not None and _FAULT.fires(
"exc_after_release",
role="gpu",
unit="runner-forward",
eligible=ple_serving,
tp_rank=connector.tp_rank,
pp_rank=connector.pp_rank,
dp_rank=connector.dp_rank,
transaction_id=ple_transaction,
correlation_id=ple_correlation,
):
raise RuntimeError("PLE test fault: exc_after_release")
self.kv_connector.finish_forward()
if self.is_last_pp_rank:
@@ -2123,6 +2233,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids=finished_req_ids,
ec_connector_output=ec_connector_output,
cudagraph_stats=cudagraph_stats,
ple_serving=ple_serving,
ple_sequence=ple_sequence,
ple_transaction=ple_transaction,
)
if not self.is_last_pp_rank:
@@ -2134,6 +2247,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
)
return None
@fail_ple_execute_model
@torch.inference_mode()
@step_eplb_after()
def sample_tokens(
@@ -2152,6 +2266,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
cudagraph_stats = self.execute_model_state.cudagraph_stats
ple_serving = self.execute_model_state.ple_serving
ple_sequence = self.execute_model_state.ple_sequence
ple_transaction = self.execute_model_state.ple_transaction
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -2161,6 +2278,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
if self.pcp_manager is not None:
input_batch = self.pcp_manager.global_batch
assert self.pp_handler is not None
connector = self._ple_offload_connector
if (
connector is not None
and _FAULT is not None
and ple_serving
and ple_sequence is not None
and connector.completion_is_pending(ple_sequence)
and _FAULT.fires(
"exc_async_output",
role="gpu",
unit="runner-output-relay",
eligible=True,
tp_rank=connector.tp_rank,
pp_rank=connector.pp_rank,
dp_rank=connector.dp_rank,
transaction_id=ple_transaction,
correlation_id=f"sequence:{ple_sequence}",
)
):
raise RuntimeError("PLE test fault: exc_async_output")
all_decode_next = self.pp_handler.receive(input_batch)
# Optimistically update num_computed_tokens for entire batch here.
# Will be adjusted for rejections if necessary in update_requests.
@@ -2462,6 +2599,9 @@ class ExecuteModelState(NamedTuple):
finished_req_ids: set[str]
ec_connector_output: ECConnectorOutput | None
cudagraph_stats: CUDAGraphStat | None
ple_serving: bool
ple_sequence: int | None
ple_transaction: str | None
class BatchReqState(NamedTuple):
+145
View File
@@ -91,6 +91,7 @@ from vllm.v1.outputs import (
DraftTokenIds,
ModelRunnerOutput,
)
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
from vllm.v1.utils import compute_iteration_details, report_usage_stats
from vllm.v1.worker import guard_telemetry
from vllm.v1.worker.sentinel.gpu_worker_sentinel import WorkerSentinel
@@ -238,6 +239,148 @@ class Worker(WorkerBase):
# Resolved lazily on first sleep/wake; persists worker-process state.
self._sleep_mode_backend: SleepModeBackend | None = None
def _has_ple_layers(self) -> bool:
"""Return whether this model configuration constructs PLE layers."""
if not envs.VLLM_PLE_CPU_OFFLOAD:
return False
text_config = self.model_config.hf_text_config
ple_layer_ids = getattr(text_config, "ple_layer_ids", None)
if not ple_layer_ids:
return False
# PP (vllm#54709, carried from the earlier overlay): only the pipeline rank
# that owns a PLE layer takes part in the offload (rank 0 with the
# default partition). Other ranks build PPMissingLayer there and must
# skip the connector entirely.
pp_size = self.parallel_config.pipeline_parallel_size
if pp_size > 1:
from vllm.distributed.utils import get_pp_indices
tp_size = self.parallel_config.tensor_parallel_size
pp_rank = (self.rank // tp_size) % pp_size
start, end = get_pp_indices(
int(text_config.num_hidden_layers), pp_rank, pp_size
)
return any(start <= int(i) - 1 < end for i in ple_layer_ids)
return True
def _validate_ple_offload_config(self) -> None:
"""Reject unsupported PLE offload execution modes."""
parallel_config = self.parallel_config
unsupported = []
if not self.use_v2_model_runner:
unsupported.append("model runner V1")
if not current_platform.is_cuda():
unsupported.append(f"device={current_platform.device_type}")
if parallel_config.nnodes != 1:
unsupported.append(f"nnodes={parallel_config.nnodes}")
if parallel_config.data_parallel_backend != "mp":
unsupported.append(f"DP backend={parallel_config.data_parallel_backend}")
if (
parallel_config.data_parallel_size_local
!= parallel_config.data_parallel_size
):
unsupported.append(
"non-local DP "
f"({parallel_config.data_parallel_size_local}/"
f"{parallel_config.data_parallel_size} local ranks)"
)
# PP allowed (vllm#54709): PLE layers must live on pipeline rank 0
# (enforced placement-based in models/config.py); num_workers stays
# dp_size * tp_size = the ranks of the owning stage.
if parallel_config.prefill_context_parallel_size != 1:
unsupported.append(f"PCP={parallel_config.prefill_context_parallel_size}")
if parallel_config.decode_context_parallel_size != 1:
unsupported.append(f"DCP={parallel_config.decode_context_parallel_size}")
if parallel_config.use_ubatching:
unsupported.append("ubatching/DBO")
if self.model_config.architecture not in {
"Qwen4ExpForCausalLM",
"Qwen4ExpForConditionalGeneration",
}:
unsupported.append("architecture")
if self.vllm_config.weight_transfer_config is not None:
unsupported.append("weight transfer")
if unsupported:
raise ValueError(
"VLLM_PLE_CPU_OFFLOAD does not support the requested "
"configuration. Unsupported settings: "
f"{', '.join(unsupported)}"
)
def spawn_ple_offload(self) -> None:
"""Spawn one node-local PLE CPU worker from DP0/TP0."""
if (
not self._ple_offload_enabled
or self.rank != 0
or self.parallel_config.data_parallel_rank != 0
):
return
from vllm.v1.ple_offload.worker import PleOffloadWorker
ipc_addr = self.parallel_config._ple_offload_ipc_path
if not ipc_addr:
raise RuntimeError("PLE offload IPC address was not initialized")
dp_size = self.parallel_config.data_parallel_size
tp_size = self.parallel_config.tensor_parallel_size
num_workers = dp_size * tp_size
logger.info(
"PleOffload: spawning worker "
"(rank=%d, local_rank=%d, dp_size=%d, tp_size=%d, "
"num_workers=%d, ipc_addr=%s).",
self.rank,
self.local_rank,
dp_size,
tp_size,
num_workers,
ipc_addr,
)
self._ple_offload_worker_handle = PleOffloadWorker.make_process(
self.vllm_config,
num_workers,
ipc_addr,
)
self._start_ple_offload_watchdog()
def _start_ple_offload_watchdog(self) -> None:
"""Fail this worker if the PLE offload process exits unexpectedly.
Host-mapped pull: a dead producer would leave the captured forward
waiting on its host flag forever; turn that into a fast, visible
engine failure instead of a hang. ``shutdown`` sets the stop marker
before closing the handle, so a normal close is not a failure.
"""
import threading
handle = self._ple_offload_worker_handle
self._ple_offload_stopping = False
def watch() -> None:
returncode = handle.proc.wait()
if getattr(self, "_ple_offload_stopping", False):
return
logger.error(
"PLE offload process %s exited unexpectedly (exitcode %s); "
"failing this worker so the engine does not hang on a PLE wait",
handle.proc.pid,
returncode,
)
os._exit(1)
threading.Thread(
target=watch, name="ple-offload-watchdog", daemon=True
).start()
def wait_ple_offload_ready(self) -> None:
"""Wait until weights and every TP registration are ready."""
if self._ple_offload_worker_handle is None:
return
from vllm.v1.ple_offload.worker import PleOffloadWorker
PleOffloadWorker.wait_for_ready(self._ple_offload_worker_handle)
@property
def sleep_mode_backend(self) -> "SleepModeBackend":
if self._sleep_mode_backend is None:
@@ -1263,6 +1406,7 @@ class Worker(WorkerBase):
) -> ModelRunnerOutput | AsyncModelRunnerOutput:
return self.model_runner.sample_tokens(grammar_output)
@fail_ple_execute_model
@torch.inference_mode()
@with_gpu_sync_check
def execute_model(
@@ -1587,6 +1731,7 @@ class Worker(WorkerBase):
self.model_runner.reset_lora_state()
def shutdown(self) -> None:
self._ple_offload_stopping = True
gc.unfreeze()
# Bounds-guard instrumentation: synchronous final read of the guard
+210
View File
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Minimal PLE child bootstrap; intentionally imports no torch or vLLM."""
import importlib.abc
import importlib.machinery
import os
import pickle
import struct
import sys
import threading
from multiprocessing.connection import Connection
from types import ModuleType
from typing import Any
_FAULT = None
if os.environ.get("VLLM_PLE_TEST_FAULT") is not None:
from vllm_ple_test_fault import (
_FAULT as _FAULT,
_block_on_harness_gate,
)
def _send_partial_ready_frame(fd: int) -> tuple[int, int]:
"""Write the complete header and an incomplete body in one syscall."""
body = pickle.dumps(
{"status": "READY", "layer_names": []},
protocol=pickle.HIGHEST_PROTOCOL,
)
frame = struct.pack("!i", len(body)) + body
fragment = frame[: 4 + max(1, len(body) // 2)]
try:
return os.write(fd, fragment), len(frame)
except OSError:
return 0, len(frame)
def _send_partial_ready_header(fd: int) -> tuple[int, int]:
"""Write fewer than the four required header bytes in one syscall."""
header = struct.pack("!i", 1)
try:
return os.write(fd, header[:2]), len(header)
except OSError:
return 0, len(header)
def _read_exact(fd: int, size: int) -> bytes:
chunks = bytearray()
while len(chunks) < size:
chunk = os.read(fd, size - len(chunks))
if not chunk:
raise EOFError("PLE launch payload was truncated")
chunks.extend(chunk)
return bytes(chunks)
def _recv_launch_payload(fd: int) -> dict:
size = struct.unpack("!Q", _read_exact(fd, 8))[0]
payload = pickle.loads(_read_exact(fd, size))
if not isinstance(payload, dict):
raise RuntimeError("invalid PLE bootstrap payload")
return payload
def _fires(
cut: str,
*,
unit: str = "bootstrap-launch",
transaction: str = "launch:1",
bytes_written: int | None = None,
frame_size: int | None = None,
) -> bool:
return _FAULT is not None and _FAULT.fires(
cut,
role="bootstrap",
unit=unit,
eligible=True,
transaction_id=transaction,
correlation_id="launch:1",
bytes_written=bytes_written,
frame_size=frame_size,
)
def _write_and_ack_partial(
fd: int,
cut: str,
sender: Any,
) -> bool:
"""A short or failed prefix write cannot acknowledge or enter the park."""
if _FAULT is None or _FAULT.cut != cut:
# Disarmed or armed for another cut: the control stream is untouched.
return False
written, frame_size = sender(fd)
if written <= 0 or written >= frame_size:
return False
expected = 2 if cut == "launch_partial_ready_header" else None
if expected is not None and written != expected:
return False
if expected is None and written <= 4:
return False
return _fires(
cut,
bytes_written=written,
frame_size=frame_size,
)
class _FaultImportLoader(importlib.abc.Loader):
def __init__(self, delegate: importlib.abc.Loader) -> None:
self._delegate = delegate
def create_module(self, spec):
create = getattr(self._delegate, "create_module", None)
return None if create is None else create(spec)
def exec_module(self, module: ModuleType) -> None:
if _fires(
"launch_block_worker_import",
unit="bootstrap-import",
transaction="module:vllm.v1.ple_offload.worker",
):
_block_on_harness_gate()
self._delegate.exec_module(module)
class _FaultImportFinder(importlib.abc.MetaPathFinder):
_TARGET = "vllm.v1.ple_offload.worker"
def find_spec(self, fullname: str, path, target=None):
if fullname != self._TARGET:
return None
spec = importlib.machinery.PathFinder.find_spec(fullname, path, target)
if spec is None or spec.loader is None:
raise ImportError(f"cannot find {fullname}")
if not isinstance(spec.loader, importlib.abc.Loader):
raise ImportError(f"{fullname} has no executable loader")
spec.loader = _FaultImportLoader(spec.loader)
return spec
def _install_import_gate() -> None:
target = _FaultImportFinder._TARGET
if target in sys.modules:
raise RuntimeError("PLE worker import gate was a cache hit")
sys.meta_path.insert(0, _FaultImportFinder())
def main() -> None:
"""Receive launch state before importing torch or vLLM."""
if len(sys.argv) != 3:
raise SystemExit("expected inherited PLE control and parent-death fds")
control_fd = int(sys.argv[1])
death_fd = int(sys.argv[2])
if control_fd == death_fd:
raise RuntimeError("PLE control and parent-death fds must be distinct")
os.set_inheritable(control_fd, False)
os.set_inheritable(death_fd, False)
death_pipe = Connection(death_fd)
def monitor_parent() -> None:
try:
death_pipe.recv()
except (EOFError, OSError):
pass
os._exit(1)
threading.Thread(
target=monitor_parent,
daemon=True,
name="PleOffloadBootstrapDeathMonitor",
).start()
if _fires("launch_park_before_payload_read"):
threading.Event().wait()
# An armed reconstruction payload invokes its reducer inside this loads().
payload = _recv_launch_payload(control_fd)
if _write_and_ack_partial(
control_fd,
"launch_partial_ready",
_send_partial_ready_frame,
):
threading.Event().wait()
if _write_and_ack_partial(
control_fd,
"launch_partial_ready_header",
_send_partial_ready_header,
):
threading.Event().wait()
if _fires("launch_silent_ready"):
threading.Event().wait()
ready_pipe = Connection(control_fd)
vllm_config = pickle.loads(payload["vllm_config_payload"])
if _FAULT is not None and _FAULT.cut == "launch_block_worker_import":
_install_import_gate()
from vllm.v1.ple_offload.worker import PleOffloadWorker
PleOffloadWorker.proc_main(
vllm_config=vllm_config,
num_workers=int(payload["num_workers"]),
ipc_addr=str(payload["ipc_addr"]),
ready_pipe=ready_pipe,
)
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fresh-interpreter entry point for the PLE CPU-offload worker."""
from vllm_ple_offload_bootstrap import main
main()