[PLE] Carry the host-mapped pull transport onto v0.30.0
Restores the fork's PLE offload package (not upstream) and its pull transport: the offload process holds no CUDA context; the captured op waits on a device-mapped host flag and copies host->GPU on the model stream; publication is fenced; releases are synchronous across dummy/capture work; failures close the rank. Includes the test-only fault-injection cuts for the transport. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,433 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base class for PLE layers that support cross-process CPU offload.
|
||||
|
||||
Synchronization between the offload process and the GPU main stream is
|
||||
handled by :class:`CpuGpuSemaphore`, which uses CUDA's
|
||||
``cuStreamWaitValue32`` / ``cuStreamWriteValue32`` stream memory operations.
|
||||
|
||||
Decode-step timeline, host-mapped PULL design
|
||||
--------------------------------------------------------
|
||||
Offload process (CPU, no CUDA) GPU main stream (forward)
|
||||
------------------------------ ------------------------------
|
||||
forward_impl() on CPU ... other GPU ops ...
|
||||
wait host flag == 0 WaitValue32(host flag == 1) <- in graph
|
||||
rows -> shared pinned host slot cuMemcpyHtoDAsync(slot -> GPU) <- in graph
|
||||
publish fence; host flag = 1 return _gpu_output_buffer[:n]
|
||||
... model consumes output ...
|
||||
WriteValue32(host flag = 0) <- release_outputs
|
||||
|
||||
``release_outputs`` performs the sole reset. Safety requires serialized local
|
||||
forwards on one ordered model stream: a new transaction cannot begin until the
|
||||
prior release has been enqueued. If any prepared forward is abandoned or its
|
||||
correspondence becomes uncertain, the worker must terminate; resetting,
|
||||
retrying, or continuing could reuse stale rows or overwrite live input.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterator
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
from cuda.bindings import driver as cuda_driver
|
||||
from cuda.bindings.driver import CUstreamWaitValue_flags
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
# Module-level flag set to True inside the offload subprocess.
|
||||
# Because the offload process and GPU worker processes are separate OS
|
||||
# processes (spawned via multiprocessing), each has its own memory space.
|
||||
# A plain module-level bool is sufficient -- no thread-local storage needed.
|
||||
_offload_worker_flag = False
|
||||
|
||||
# Native FULL capture runs on one Python thread. The custom-op implementation
|
||||
# appends the exact arguments from every wait-copy invocation while that thread
|
||||
# is inside the graph manager's capture scope.
|
||||
_ple_copy_capture = threading.local()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def observe_ple_copy_geometry() -> Iterator[list[tuple[str, int, int]]]:
|
||||
"""Collect ``(layer, rows, bytes)`` from actual wait-copy executions."""
|
||||
if getattr(_ple_copy_capture, "records", None) is not None:
|
||||
raise RuntimeError("nested PLE copy-geometry observation is unsupported")
|
||||
records: list[tuple[str, int, int]] = []
|
||||
_ple_copy_capture.records = records
|
||||
try:
|
||||
yield records
|
||||
finally:
|
||||
del _ple_copy_capture.records
|
||||
|
||||
|
||||
def _record_ple_copy_geometry(
|
||||
layer_name: str,
|
||||
num_tokens: int,
|
||||
row_bytes: int,
|
||||
) -> None:
|
||||
records = getattr(_ple_copy_capture, "records", None)
|
||||
if records is not None:
|
||||
rows = int(num_tokens)
|
||||
records.append((str(layer_name), rows, rows * int(row_bytes)))
|
||||
|
||||
|
||||
def is_offload_process() -> bool:
|
||||
"""Return True inside the dedicated PLE CPU-offload subprocess."""
|
||||
return _offload_worker_flag
|
||||
|
||||
|
||||
def mark_as_offload_worker() -> None:
|
||||
"""Mark the current process as the dedicated PLE offload worker."""
|
||||
global _offload_worker_flag
|
||||
_offload_worker_flag = True
|
||||
|
||||
|
||||
def _cuda_check(result: Any, operation: str) -> Any:
|
||||
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
|
||||
error = result[0] if isinstance(result, tuple) else result
|
||||
if error.value != 0:
|
||||
raise RuntimeError(f"{operation} failed: {error}")
|
||||
return result
|
||||
|
||||
|
||||
class CpuGpuSemaphore:
|
||||
"""Cross-process semaphore backed by a one-element int32 CUDA tensor.
|
||||
|
||||
The flag is stored in a regular GPU tensor that is shared through
|
||||
PyTorch's CUDA IPC mechanism. Both processes obtain device pointers that
|
||||
map to the same physical GPU memory, so a stream-memory wait in the GPU
|
||||
worker observes the write issued by the offload process.
|
||||
"""
|
||||
|
||||
RESET_VALUE = 0
|
||||
DONE_VALUE = 1
|
||||
|
||||
def __init__(self, device: torch.device) -> None:
|
||||
self._flag_tensor = torch.zeros(1, dtype=torch.int32, device=device)
|
||||
self._flag_ptr: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_ipc_tensor(cls, flag_tensor: torch.Tensor) -> "CpuGpuSemaphore":
|
||||
"""Construct a semaphore from a CUDA tensor received through IPC."""
|
||||
semaphore = cls.__new__(cls)
|
||||
semaphore._flag_tensor = flag_tensor
|
||||
semaphore._flag_ptr = None
|
||||
return semaphore
|
||||
|
||||
@classmethod
|
||||
def from_host_mapped(
|
||||
cls, host_flag_tensor: torch.Tensor, device_ptr: int
|
||||
) -> "CpuGpuSemaphore":
|
||||
"""Semaphore on a page-locked, device-mapped shared host int32.
|
||||
|
||||
``host_flag_tensor`` keeps the shared storage alive; ``device_ptr`` is
|
||||
its device address (``cuMemHostGetDevicePointer``). Stream memory ops
|
||||
on it are executed by the GPU against host memory, so a CPU process
|
||||
with no CUDA context can signal it with a plain store.
|
||||
"""
|
||||
semaphore = cls.__new__(cls)
|
||||
semaphore._flag_tensor = host_flag_tensor
|
||||
semaphore._flag_ptr = int(device_ptr)
|
||||
return semaphore
|
||||
|
||||
@property
|
||||
def flag_tensor(self) -> torch.Tensor:
|
||||
"""Return the tensor that backs the semaphore flag."""
|
||||
return self._flag_tensor
|
||||
|
||||
@property
|
||||
def flag_ptr(self) -> int:
|
||||
"""Device address used by the stream memory operations."""
|
||||
if self._flag_ptr is not None:
|
||||
return self._flag_ptr
|
||||
return self._flag_tensor.data_ptr()
|
||||
|
||||
def reset(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||
"""Enqueue ``WriteValue32(flag=0)`` on ``stream``."""
|
||||
if stream is None:
|
||||
stream = torch.cuda.current_stream()
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWriteValue32(
|
||||
cuda_driver.CUstream(stream.cuda_stream),
|
||||
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||
self.RESET_VALUE,
|
||||
0,
|
||||
),
|
||||
"CpuGpuSemaphore.reset",
|
||||
)
|
||||
|
||||
def signal(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||
"""Enqueue ``WriteValue32(flag=1)`` on ``stream``."""
|
||||
if stream is None:
|
||||
stream = torch.cuda.current_stream()
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWriteValue32(
|
||||
cuda_driver.CUstream(stream.cuda_stream),
|
||||
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||
self.DONE_VALUE,
|
||||
0,
|
||||
),
|
||||
"CpuGpuSemaphore.signal",
|
||||
)
|
||||
|
||||
def wait_reset(self, stream: torch.cuda.Stream | None = None) -> None:
|
||||
"""Enqueue ``WaitValue32(flag==0)`` on ``stream``."""
|
||||
if stream is None:
|
||||
stream = torch.cuda.current_stream()
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWaitValue32(
|
||||
cuda_driver.CUstream(stream.cuda_stream),
|
||||
cuda_driver.CUdeviceptr(self.flag_ptr),
|
||||
self.RESET_VALUE,
|
||||
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||
),
|
||||
"CpuGpuSemaphore.wait_reset",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Custom op: vllm::ple_offload_wait
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wraps cuStreamWaitValue32 as a torch custom op so that
|
||||
# torch.compile(fullgraph=True) treats the CUDA Driver call as an opaque node.
|
||||
# The hidden_states argument creates a dynamo data-dependency edge that prevents
|
||||
# the wait from being reordered before preceding GPU work.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ple_offload_wait_impl(
|
||||
sem_flag_tensor: torch.Tensor,
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> None:
|
||||
"""Wait for the CPU result without releasing its output buffer."""
|
||||
stream = torch.cuda.current_stream()
|
||||
cuda_stream = cuda_driver.CUstream(stream.cuda_stream)
|
||||
dev_ptr = cuda_driver.CUdeviceptr(sem_flag_tensor.data_ptr())
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWaitValue32(
|
||||
cuda_stream,
|
||||
dev_ptr,
|
||||
CpuGpuSemaphore.DONE_VALUE,
|
||||
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||
),
|
||||
"cuStreamWaitValue32(done)",
|
||||
)
|
||||
|
||||
|
||||
def _ple_offload_wait_fake(
|
||||
sem_flag_tensor: torch.Tensor,
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> None:
|
||||
"""Represent the side-effect-only wait during dynamo tracing."""
|
||||
pass
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="ple_offload_wait",
|
||||
op_func=_ple_offload_wait_impl,
|
||||
mutates_args=["gpu_output_buffer"],
|
||||
fake_impl=_ple_offload_wait_fake,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Custom op: vllm::ple_offload_wait_copy (host-mapped pull design)
|
||||
# ---------------------------------------------------------------------------
|
||||
# The offload process writes the result into a page-locked shared host buffer
|
||||
# and stores flag=1 in a page-locked shared host int32; it holds no CUDA
|
||||
# context. On the CURRENT stream (captured into the graph like the wait above):
|
||||
# cuStreamWaitValue32(host flag == 1) -> GPU polls host memory
|
||||
# cuMemcpyAsync(gpu buffer <- host buffer, num_tokens * row_bytes)
|
||||
# The copy engine reads host memory coherently after the flag was observed,
|
||||
# and the consumers read device memory as before. The flag is NOT reset here:
|
||||
# ``release_outputs`` resets it after the forward (unchanged semantics), so
|
||||
# capture warm-ups and dummy runs behave exactly as with ``ple_offload_wait``.
|
||||
# Pointers are passed as ints so no CPU tensor enters the compiled graph.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ple_offload_wait_copy_impl(
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
flag_ptr: int,
|
||||
host_ptr: int,
|
||||
row_bytes: int,
|
||||
num_tokens: int,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Wait for the host-side result, then DMA it into the GPU buffer."""
|
||||
if not 0 <= int(num_tokens) <= gpu_output_buffer.shape[0] or int(row_bytes) != (
|
||||
gpu_output_buffer.shape[1] * gpu_output_buffer.element_size()
|
||||
):
|
||||
raise RuntimeError(
|
||||
f"PLE wait_copy out of bounds: {num_tokens} rows x {row_bytes} B vs "
|
||||
f"GPU buffer {tuple(gpu_output_buffer.shape)} {gpu_output_buffer.dtype}"
|
||||
)
|
||||
nbytes = int(num_tokens) * int(row_bytes)
|
||||
_record_ple_copy_geometry(layer_name, num_tokens, row_bytes)
|
||||
stream = cuda_driver.CUstream(torch.cuda.current_stream().cuda_stream)
|
||||
_cuda_check(
|
||||
cuda_driver.cuStreamWaitValue32(
|
||||
stream,
|
||||
cuda_driver.CUdeviceptr(flag_ptr),
|
||||
CpuGpuSemaphore.DONE_VALUE,
|
||||
CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value,
|
||||
),
|
||||
"cuStreamWaitValue32(host flag done)",
|
||||
)
|
||||
if nbytes > 0:
|
||||
# host_ptr is this worker's host address of page-locked (registered)
|
||||
# memory, so the copy is a true async DMA and is capturable.
|
||||
_cuda_check(
|
||||
cuda_driver.cuMemcpyHtoDAsync(
|
||||
cuda_driver.CUdeviceptr(gpu_output_buffer.data_ptr()),
|
||||
host_ptr,
|
||||
nbytes,
|
||||
stream,
|
||||
),
|
||||
"cuMemcpyHtoDAsync(PLE host -> GPU)",
|
||||
)
|
||||
|
||||
|
||||
def _ple_offload_wait_copy_fake(
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
flag_ptr: int,
|
||||
host_ptr: int,
|
||||
row_bytes: int,
|
||||
num_tokens: int,
|
||||
layer_name: str,
|
||||
) -> None:
|
||||
"""Represent the side-effect-only wait + copy during dynamo tracing."""
|
||||
pass
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="ple_offload_wait_copy",
|
||||
op_func=_ple_offload_wait_copy_impl,
|
||||
mutates_args=["gpu_output_buffer"],
|
||||
fake_impl=_ple_offload_wait_copy_fake,
|
||||
)
|
||||
|
||||
|
||||
class PleOffloadLayer(nn.Module, ABC):
|
||||
"""Base class for embedding-like PLE layers that can run on CPU.
|
||||
|
||||
Subclasses implement :meth:`forward_impl`, which contains the regular
|
||||
on-device computation. In a GPU worker with PLE offload enabled, subclass
|
||||
constructors are skipped so their large weights are never allocated. The
|
||||
CPU process constructs a complete copy and owns those weights instead.
|
||||
"""
|
||||
|
||||
_is_cpu_offloaded = False
|
||||
_gpu_output_buffer: torch.Tensor
|
||||
_sem: CpuGpuSemaphore
|
||||
# Host-mapped pull design: host address of the shared output buffer and
|
||||
# the bytes per output row. ``None`` keeps the IPC push behaviour.
|
||||
_host_output_ptr: int | None = None
|
||||
_host_row_bytes: int = 0
|
||||
_ple_offload_name: str = "<unbound>"
|
||||
|
||||
def __init_subclass__(cls, **kwargs: object) -> None:
|
||||
"""Skip subclass initialization in cross-process GPU-worker mode."""
|
||||
super().__init_subclass__(**kwargs)
|
||||
original_init_obj = cls.__dict__.get("__init__")
|
||||
if original_init_obj is None or not callable(original_init_obj):
|
||||
return
|
||||
original_init = cast(Callable[..., None], original_init_obj)
|
||||
|
||||
@functools.wraps(original_init)
|
||||
def guarded_init(
|
||||
self: "PleOffloadLayer", *args: object, **kwargs: object
|
||||
) -> None:
|
||||
if envs.VLLM_PLE_CPU_OFFLOAD and not is_offload_process():
|
||||
nn.Module.__init__(self)
|
||||
return
|
||||
original_init(self, *args, **kwargs)
|
||||
|
||||
cls.__init__ = guarded_init # type: ignore[method-assign, assignment]
|
||||
|
||||
@classmethod
|
||||
def get_target_device(cls) -> torch.device:
|
||||
"""Return CPU for the offload process and the active GPU otherwise."""
|
||||
if envs.VLLM_PLE_CPU_OFFLOAD:
|
||||
return torch.device("cpu")
|
||||
return torch.device("cuda", torch.accelerator.current_device_index())
|
||||
|
||||
@abstractmethod
|
||||
def forward_impl(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
*args: object,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor:
|
||||
"""Execute the actual embedding computation on the owning device."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
|
||||
"""Return the dtype used by cross-process output buffers."""
|
||||
return default_dtype
|
||||
|
||||
def get_offload_output_dim(self, default_dim: int) -> int:
|
||||
"""Return the last dimension used by cross-process output buffers."""
|
||||
return default_dim
|
||||
|
||||
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
|
||||
"""Initialize metadata needed by dummy offload execution."""
|
||||
|
||||
def setup_cross_process_offload(
|
||||
self,
|
||||
gpu_output_buffer: torch.Tensor,
|
||||
semaphore: CpuGpuSemaphore,
|
||||
host_output_ptr: int | None = None,
|
||||
host_row_bytes: int = 0,
|
||||
) -> None:
|
||||
"""Configure the GPU-worker placeholder with its offload resources."""
|
||||
self._is_cpu_offloaded = True
|
||||
self._gpu_output_buffer = gpu_output_buffer
|
||||
self._sem = semaphore
|
||||
self._host_output_ptr = host_output_ptr
|
||||
self._host_row_bytes = int(host_row_bytes)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
*args: object,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor:
|
||||
"""Wait for an offloaded result or delegate to ``forward_impl``."""
|
||||
if self._is_cpu_offloaded:
|
||||
if self._host_output_ptr is not None:
|
||||
torch.ops.vllm.ple_offload_wait_copy(
|
||||
self._gpu_output_buffer,
|
||||
hidden_states,
|
||||
self._sem.flag_ptr,
|
||||
self._host_output_ptr,
|
||||
self._host_row_bytes,
|
||||
input_ids.shape[0],
|
||||
self._ple_offload_name,
|
||||
)
|
||||
else:
|
||||
torch.ops.vllm.ple_offload_wait(
|
||||
self._sem.flag_tensor,
|
||||
self._gpu_output_buffer,
|
||||
hidden_states,
|
||||
)
|
||||
return self._gpu_output_buffer[: input_ids.shape[0]]
|
||||
return self.forward_impl(hidden_states, input_ids, *args, **kwargs)
|
||||
|
||||
def release_offloaded_output(
|
||||
self,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
) -> None:
|
||||
"""Mark the cross-process output buffer reusable on ``stream``."""
|
||||
if self._is_cpu_offloaded:
|
||||
self._sem.reset(stream)
|
||||
@@ -352,9 +352,10 @@ class MultiprocExecutor(Executor):
|
||||
proc = next(h.proc for h in workers if h.proc.sentinel == died[0])
|
||||
logger.error(
|
||||
"Worker proc %s died unexpectedly (exit code: %s), "
|
||||
"shutting down executor.",
|
||||
"shutting down executor. executor_shutdown_monotonic=%.9f",
|
||||
proc.name,
|
||||
proc.exitcode,
|
||||
time.monotonic(),
|
||||
)
|
||||
_self.shutdown()
|
||||
callback = _self.failure_callback
|
||||
@@ -1039,6 +1040,28 @@ class WorkerProc:
|
||||
SUCCESS = auto()
|
||||
FAILURE = auto()
|
||||
|
||||
def _fail_ple_rpc_if_pending(
|
||||
self, where: str, error: BaseException
|
||||
) -> None:
|
||||
"""Fail closed if RPC recovery would abandon queued PLE GPU work."""
|
||||
owners = (self.worker, getattr(self.worker, "worker", None))
|
||||
for owner in owners:
|
||||
if owner is None:
|
||||
continue
|
||||
runner = getattr(owner, "model_runner", None)
|
||||
connector = getattr(runner, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
continue
|
||||
try:
|
||||
pending = connector.has_pending_completions
|
||||
except BaseException as state_error:
|
||||
connector._fail_closed(
|
||||
f"{where}:pending-state", state_error
|
||||
)
|
||||
if pending:
|
||||
connector._fail_closed(where, error)
|
||||
return
|
||||
|
||||
def enqueue_output(self, output: Any):
|
||||
"""Prepares output from the worker and enqueues it to the
|
||||
worker_response_mq. If the output is an Exception, it is
|
||||
@@ -1049,6 +1072,9 @@ class WorkerProc:
|
||||
output = output.get_output()
|
||||
except Exception as e:
|
||||
logger.exception("Error getting async model runner output")
|
||||
self._fail_ple_rpc_if_pending(
|
||||
"async-output-copy", e
|
||||
)
|
||||
output = e
|
||||
|
||||
if isinstance(output, Exception):
|
||||
@@ -1108,6 +1134,9 @@ class WorkerProc:
|
||||
if output_rank is None or self.rank == output_rank:
|
||||
self.handle_output(output)
|
||||
except Exception as e:
|
||||
self._fail_ple_rpc_if_pending(
|
||||
f"worker-rpc:{method}", e
|
||||
)
|
||||
# Notes have been introduced in python 3.11
|
||||
if hasattr(e, "add_note"):
|
||||
e.add_note(traceback.format_exc())
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,855 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Exchange PLE data between a GPU worker and the CPU-offload process."""
|
||||
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from multiprocessing.reduction import ForkingPickler
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
from vllm.v1.ple_offload import rowdump as _rowdump
|
||||
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||
from vllm.v1.ple_offload.progress import PleProgressMonitor
|
||||
import torch.nn as nn
|
||||
import zmq
|
||||
from cuda.bindings import driver as cuda_driver
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.ple_offload_layer import (
|
||||
CpuGpuSemaphore,
|
||||
PleOffloadLayer,
|
||||
)
|
||||
from vllm.v1.ple_offload.protocol import (
|
||||
PleOffloadRegistration,
|
||||
PleOffloadRequest,
|
||||
)
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
from vllm_ple_test_fault import _FAULT
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# CUDA registration alone does not own Python storage. Captured graphs retain
|
||||
# raw addresses, so every successfully registered host allocation remains
|
||||
# strongly referenced until process exit.
|
||||
_PROCESS_LIFETIME_REGISTERED_STORAGE: list[torch.Tensor] = []
|
||||
|
||||
|
||||
def _unlink_filename_shared_storage(
|
||||
tensor: torch.Tensor,
|
||||
reserved_filenames: set[bytes],
|
||||
) -> None:
|
||||
"""Reserve and unlink one already filename-shared CPU storage."""
|
||||
try:
|
||||
storage = tensor.untyped_storage()
|
||||
if storage.device.type != "cpu" or not storage.is_shared():
|
||||
return
|
||||
try:
|
||||
storage._get_shared_fd()
|
||||
except RuntimeError:
|
||||
# Filename-shared storages do not expose a shared fd. Only now is
|
||||
# it safe to query the filename: doing so on ordinary or fd-shared
|
||||
# storage would convert it to a new filename-backed allocation.
|
||||
pass
|
||||
else:
|
||||
return
|
||||
_, filename, _ = storage._share_filename_cpu_()
|
||||
if not isinstance(filename, bytes):
|
||||
filename = os.fsencode(filename)
|
||||
path = os.path.join("/dev/shm", os.fsdecode(filename).lstrip("/"))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to inspect a PLE shared-memory storage during teardown",
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
|
||||
# RefcountedMapAllocator unlinks the name when its shared counter reaches
|
||||
# zero. Reserve one count before manual unlink so later storage destruction
|
||||
# cannot try to unlink the already-removed name. The filename identifies
|
||||
# the shared counter and also deduplicates tensor aliases and repeated close.
|
||||
if filename in reserved_filenames:
|
||||
return
|
||||
try:
|
||||
storage._shared_incref()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to reserve the PLE shared-memory counter for %s",
|
||||
path,
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
reserved_filenames.add(filename)
|
||||
|
||||
try:
|
||||
os.unlink(path)
|
||||
except FileNotFoundError:
|
||||
# Another teardown participant already removed the name. Retain the
|
||||
# reservation because this allocator must still avoid a later unlink.
|
||||
pass
|
||||
except OSError as error:
|
||||
# The name remains available, so restore the allocator's original
|
||||
# counter and permit a later release attempt to retry.
|
||||
try:
|
||||
storage._shared_decref()
|
||||
except Exception:
|
||||
# The reservation state is uncertain after a failed private call.
|
||||
# Keep it marked to avoid adding another reservation on retry.
|
||||
logger.warning(
|
||||
"Failed to roll back the PLE shared-memory counter for %s",
|
||||
path,
|
||||
exc_info=True,
|
||||
)
|
||||
else:
|
||||
reserved_filenames.discard(filename)
|
||||
logger.warning("Failed to unlink PLE shared memory %s: %s", path, error)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PendingPleOffloadRequest:
|
||||
"""Bind request metadata to its MRV2 D2H completion event."""
|
||||
|
||||
request: PleOffloadRequest
|
||||
d2h_done_event: torch.cuda.Event
|
||||
eligible: bool = False
|
||||
transaction_id: str | None = None
|
||||
correlation_id: str | None = None
|
||||
|
||||
|
||||
def _cuda_check(result: Any, operation: str) -> Any:
|
||||
"""Check the ``(CUresult, ...)`` tuple returned by cuda-python calls."""
|
||||
error = result[0] if isinstance(result, tuple) else result
|
||||
if error.value != 0:
|
||||
raise RuntimeError(f"{operation} failed: {error}")
|
||||
return result
|
||||
|
||||
|
||||
class PleOffloadConnector:
|
||||
"""Connect a model runner V2 GPU worker to the PLE CPU worker."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
device: torch.device,
|
||||
ipc_addr: str,
|
||||
*,
|
||||
input_ids_source: torch.Tensor,
|
||||
query_start_loc_source: torch.Tensor,
|
||||
ngram_context_source: torch.Tensor | None,
|
||||
) -> None:
|
||||
self.device = device
|
||||
self.rank = int(vllm_config.parallel_config.rank)
|
||||
self.dp_rank = get_dp_group().rank_in_group
|
||||
self.tp_rank = get_tp_group().rank_in_group
|
||||
self.pp_rank = get_pp_group().rank_in_group
|
||||
# Host-mapped pull design: per layer a shared host output buffer and a
|
||||
# shared host int32 flag (element 0), filled/stored by the offload
|
||||
# process with plain CPU writes. Page-locked + device-mapped here only
|
||||
# after registration, because serialising for registration may move
|
||||
# the shared storage to a new address.
|
||||
self._host_output_bufs: dict[str, torch.Tensor] = {}
|
||||
self._host_flags: dict[str, torch.Tensor] = {}
|
||||
self._reserved_host_storage_filenames: set[bytes] = set()
|
||||
self._gpu_output_bufs: dict[str, torch.Tensor] = {}
|
||||
self._layers = self._setup_layers(vllm_config, model)
|
||||
self._progress = PleProgressMonitor.from_environment(
|
||||
rank=self.rank,
|
||||
layer_names=tuple(self._layers),
|
||||
)
|
||||
|
||||
# TP0 registers shared buffers with CUDA for asynchronous D2H copies.
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
self._input_ids_buf = torch.empty(
|
||||
scheduler_config.max_num_batched_tokens,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
).share_memory_()
|
||||
self._query_start_loc_buf = torch.empty(
|
||||
scheduler_config.max_num_seqs + 1,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
).share_memory_()
|
||||
self._ngram_context_buf = None
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
ngram_context_len = int(config.ngram_size) - 1
|
||||
if ngram_context_len > 0:
|
||||
self._ngram_context_buf = torch.empty(
|
||||
scheduler_config.max_num_seqs,
|
||||
ngram_context_len,
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
).share_memory_()
|
||||
|
||||
# Runner input allocations are address-stable, so bind them once and
|
||||
# pass only batch sizes through the per-forward request queue.
|
||||
self._input_ids_source = input_ids_source
|
||||
self._query_start_loc_source = query_start_loc_source
|
||||
self._ngram_context_source = ngram_context_source
|
||||
self._validate_input_sources()
|
||||
|
||||
self._pinned_input_buffers: list[torch.Tensor] = []
|
||||
# Device-mapped host outputs/flags referenced by raw address from
|
||||
# captured graphs: never unregistered while the process lives
|
||||
# (unregistering under a pending graph wait unmaps it).
|
||||
self._mapped_host_buffers: list[torch.Tensor] = []
|
||||
# True while dummy/capture work (seeds, dummy forwards, capture
|
||||
# cleanup resets) has run since the last real forward. Releases in
|
||||
# that state complete synchronously, and the next real forward first
|
||||
# passes a TP barrier, so no rank can publish a real request while
|
||||
# another rank still has a dummy/cleanup reset pending.
|
||||
# In this tree real-path kernel warm-ups run BEFORE
|
||||
# capture and capture adds seeds/resets after them, so this must hold
|
||||
# at every dummy->real transition, not only at the first real one.
|
||||
self._after_dummy = True
|
||||
self._request_queue: queue.Queue[_PendingPleOffloadRequest | None] = (
|
||||
queue.Queue(maxsize=vllm_config.max_concurrent_batches)
|
||||
)
|
||||
self._dropped_events: list[torch.cuda.Event] = []
|
||||
self._request_thread: threading.Thread | None = None
|
||||
self._request_thread_ready = threading.Event()
|
||||
self._zmq_ctx: zmq.Context | None = None
|
||||
self._registration_socket: zmq.Socket | None = None
|
||||
self._d2h_event_pool: queue.Queue[torch.cuda.Event] | None = None
|
||||
|
||||
try:
|
||||
self._zmq_ctx = zmq.Context()
|
||||
self._registration_socket = self._zmq_ctx.socket(zmq.PUSH)
|
||||
self._registration_socket.connect(ipc_addr)
|
||||
payload = self._serialize_registration(vllm_config)
|
||||
|
||||
# Serialising moved the shared storages to their final addresses.
|
||||
# Page-lock/map exactly those, THEN publish, so a received
|
||||
# registration always means "storage usable".
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self._map_host_outputs(vllm_config)
|
||||
if self.tp_rank == 0:
|
||||
self._pin_input_buffers()
|
||||
self._send_registration(payload, ipc_addr)
|
||||
|
||||
if self.tp_rank == 0:
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
self._d2h_event_pool = queue.Queue(
|
||||
maxsize=vllm_config.max_concurrent_batches
|
||||
)
|
||||
for _ in range(vllm_config.max_concurrent_batches):
|
||||
self._d2h_event_pool.put_nowait(torch.cuda.Event())
|
||||
self._start_request_thread(ipc_addr)
|
||||
except Exception:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def _setup_layers(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
) -> dict[str, PleOffloadLayer]:
|
||||
"""Attach output buffers and semaphores to GPU PLE placeholders."""
|
||||
layers = {
|
||||
name: module
|
||||
for name, module in model.named_modules()
|
||||
if isinstance(module, PleOffloadLayer)
|
||||
}
|
||||
if not layers:
|
||||
raise RuntimeError(
|
||||
"VLLM_PLE_CPU_OFFLOAD is enabled, but the model has no PleOffloadLayer"
|
||||
)
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
for name, layer in layers.items():
|
||||
if vllm_config.load_config.load_format == "dummy":
|
||||
layer.initialize_dummy_offload_metadata(self.device)
|
||||
out_dim = layer.get_offload_output_dim(int(config.ple_embed_dim))
|
||||
out_dtype = layer.get_offload_output_dtype(vllm_config.model_config.dtype)
|
||||
# The captured forward DMAs the host result into this device
|
||||
# buffer after the host flag is set; consumers read it as before.
|
||||
self._gpu_output_bufs[name] = torch.empty(
|
||||
max_num_tokens, out_dim, dtype=out_dtype, device=self.device
|
||||
)
|
||||
# Zero-initialised so dummy/profile forwards copy finite values.
|
||||
self._host_output_bufs[name] = torch.zeros(
|
||||
max_num_tokens, out_dim, dtype=out_dtype, device="cpu"
|
||||
).share_memory_()
|
||||
# 16 x int32 = one 64-byte line; element 0 is the flag.
|
||||
self._host_flags[name] = torch.zeros(
|
||||
16, dtype=torch.int32, device="cpu"
|
||||
).share_memory_()
|
||||
return layers
|
||||
|
||||
def _map_host_outputs(self, vllm_config: VllmConfig) -> None:
|
||||
"""Page-lock + device-map host outputs/flags and arm the placeholders."""
|
||||
for name, layer in self._layers.items():
|
||||
layer._ple_offload_name = name
|
||||
host_out = self._host_output_bufs[name]
|
||||
host_flag = self._host_flags[name]
|
||||
for buffer in (host_out, host_flag):
|
||||
self._register_host_buffer(buffer, device_map=True)
|
||||
flag_dev_ptr = _cuda_check(
|
||||
cuda_driver.cuMemHostGetDevicePointer(host_flag.data_ptr(), 0),
|
||||
"cuMemHostGetDevicePointer(PLE host flag)",
|
||||
)[1]
|
||||
out_dev_ptr = _cuda_check(
|
||||
cuda_driver.cuMemHostGetDevicePointer(host_out.data_ptr(), 0),
|
||||
"cuMemHostGetDevicePointer(PLE host output)",
|
||||
)[1]
|
||||
row_bytes = host_out.shape[1] * host_out.element_size()
|
||||
gpu_buf = self._gpu_output_bufs[name]
|
||||
if (
|
||||
gpu_buf.shape[0] != host_out.shape[0]
|
||||
or gpu_buf.shape[1] * gpu_buf.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError(f"PLE host/GPU buffer geometry mismatch for {name}")
|
||||
# Stream memops use the flag's DEVICE pointer; the H2D copy uses
|
||||
# this worker's HOST pointer (no raw addresses cross
|
||||
# processes; the offload writes through its own mapping).
|
||||
layer.setup_cross_process_offload(
|
||||
gpu_buf,
|
||||
CpuGpuSemaphore.from_host_mapped(host_flag, int(flag_dev_ptr)),
|
||||
host_output_ptr=host_out.data_ptr(),
|
||||
host_row_bytes=row_bytes,
|
||||
)
|
||||
logger.info(
|
||||
"PleOffload: host-mapped pull for %s (rows %d x %d B, flag dev "
|
||||
"0x%x, out dev 0x%x, host==dev %s)",
|
||||
name,
|
||||
host_out.shape[0],
|
||||
row_bytes,
|
||||
int(flag_dev_ptr),
|
||||
int(out_dev_ptr),
|
||||
int(out_dev_ptr) == host_out.data_ptr(),
|
||||
)
|
||||
|
||||
def _pin_input_buffers(self) -> None:
|
||||
"""Page-lock shared input allocations without replacing their storage."""
|
||||
buffers = [self._input_ids_buf, self._query_start_loc_buf]
|
||||
if self._ngram_context_buf is not None:
|
||||
buffers.append(self._ngram_context_buf)
|
||||
for buffer in buffers:
|
||||
self._register_host_buffer(buffer, device_map=False)
|
||||
|
||||
def _register_host_buffer(self, buffer: torch.Tensor, device_map: bool) -> None:
|
||||
"""Page-lock (and optionally device-map) a shared CPU allocation."""
|
||||
if buffer.device.type != "cpu" or not buffer.is_shared():
|
||||
raise RuntimeError("PLE host buffers must be shared CPU tensors")
|
||||
if not buffer.is_contiguous():
|
||||
raise RuntimeError("PLE host buffers must be contiguous")
|
||||
flags = cuda_driver.CU_MEMHOSTREGISTER_PORTABLE
|
||||
if device_map:
|
||||
flags |= cuda_driver.CU_MEMHOSTREGISTER_DEVICEMAP
|
||||
_cuda_check(
|
||||
cuda_driver.cuMemHostRegister(
|
||||
buffer.data_ptr(),
|
||||
buffer.numel() * buffer.element_size(),
|
||||
flags,
|
||||
),
|
||||
"cuMemHostRegister(PLE host buffer)",
|
||||
)
|
||||
_PROCESS_LIFETIME_REGISTERED_STORAGE.append(buffer)
|
||||
if device_map:
|
||||
self._mapped_host_buffers.append(buffer)
|
||||
else:
|
||||
self._pinned_input_buffers.append(buffer)
|
||||
if not buffer.is_pinned():
|
||||
raise RuntimeError("CUDA did not page-lock a PLE host buffer")
|
||||
|
||||
def _unpin_input_buffers(self) -> None:
|
||||
"""Release CUDA registrations after the request thread has stopped."""
|
||||
for buffer in reversed(self._pinned_input_buffers):
|
||||
try:
|
||||
_cuda_check(
|
||||
cuda_driver.cuMemHostUnregister(buffer.data_ptr()),
|
||||
"cuMemHostUnregister(PLE input buffer)",
|
||||
)
|
||||
except RuntimeError:
|
||||
logger.exception("Failed to unregister a PLE input buffer")
|
||||
self._pinned_input_buffers.clear()
|
||||
|
||||
def _serialize_registration(self, vllm_config: VllmConfig) -> Any:
|
||||
"""Build and serialise the registration (may move shared storages)."""
|
||||
# Each GPU worker owns distinct host output buffers and flags, while
|
||||
# TP0's shared inputs become the request source for its DP rank.
|
||||
registration = PleOffloadRegistration(
|
||||
worker_id=(
|
||||
self.dp_rank * vllm_config.parallel_config.world_size
|
||||
+ vllm_config.parallel_config.rank
|
||||
),
|
||||
tp_rank=self.tp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
host_output_buffers=dict(self._host_output_bufs),
|
||||
host_flag_tensors=dict(self._host_flags),
|
||||
input_ids_buf=self._input_ids_buf,
|
||||
query_start_loc_buf=self._query_start_loc_buf,
|
||||
ngram_context_buf=self._ngram_context_buf,
|
||||
)
|
||||
|
||||
# ForkingPickler transmits tensors through shared-memory and CUDA IPC.
|
||||
import torch.multiprocessing as torch_mp
|
||||
|
||||
original_strategy = torch_mp.get_sharing_strategy()
|
||||
torch_mp.set_sharing_strategy("file_system")
|
||||
try:
|
||||
payload = ForkingPickler.dumps(registration)
|
||||
finally:
|
||||
torch_mp.set_sharing_strategy(original_strategy)
|
||||
return payload
|
||||
|
||||
def _send_registration(self, payload: Any, ipc_addr: str) -> None:
|
||||
"""Publish the serialised registration (storages already mapped)."""
|
||||
assert self._registration_socket is not None
|
||||
self._registration_socket.send(payload)
|
||||
|
||||
logger.info(
|
||||
"PleOffload: registered %d PleOffloadLayer(s) "
|
||||
"(dp_rank=%d, tp_rank=%d, ipc_addr=%s): %s",
|
||||
len(self._layers),
|
||||
self.dp_rank,
|
||||
self.tp_rank,
|
||||
ipc_addr,
|
||||
sorted(self._layers),
|
||||
)
|
||||
|
||||
def _start_request_thread(self, ipc_addr: str) -> None:
|
||||
"""Start the thread that publishes batches after inputs are ready."""
|
||||
self._request_thread = threading.Thread(
|
||||
target=self._request_loop,
|
||||
args=(ipc_addr,),
|
||||
name=f"ple-offload-dp{self.dp_rank}",
|
||||
daemon=True,
|
||||
)
|
||||
self._request_thread.start()
|
||||
if not self._request_thread_ready.wait(timeout=10):
|
||||
raise RuntimeError("Timed out starting the PLE request thread")
|
||||
|
||||
def _request_loop(self, ipc_addr: str) -> None:
|
||||
"""Wait for staged inputs, then notify the CPU worker."""
|
||||
socket: zmq.Socket | None = None
|
||||
try:
|
||||
if self._zmq_ctx is None:
|
||||
raise RuntimeError("PLE ZMQ context closed before thread startup")
|
||||
socket = self._zmq_ctx.socket(zmq.PUSH)
|
||||
socket.connect(ipc_addr)
|
||||
self._request_thread_ready.set()
|
||||
while True:
|
||||
request = self._request_queue.get()
|
||||
if request is None:
|
||||
return
|
||||
self._process_request(request, socket)
|
||||
except Exception:
|
||||
logger.exception("PLE request thread failed")
|
||||
os._exit(1)
|
||||
finally:
|
||||
self._request_thread_ready.set()
|
||||
if socket is not None:
|
||||
socket.close(linger=0)
|
||||
|
||||
def _process_request(
|
||||
self, pending: _PendingPleOffloadRequest, socket: zmq.Socket
|
||||
) -> None:
|
||||
"""Wait for one staged batch and publish its request."""
|
||||
request = pending.request
|
||||
event_pool = self._d2h_event_pool
|
||||
event = pending.d2h_done_event
|
||||
assert event_pool is not None, "PLE D2H event pool is not initialized"
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"block_input_completion",
|
||||
role="gpu",
|
||||
unit="connector-request",
|
||||
eligible=pending.eligible,
|
||||
tp_rank=self.tp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
transaction_id=pending.transaction_id,
|
||||
correlation_id=pending.correlation_id,
|
||||
):
|
||||
threading.Event().wait()
|
||||
|
||||
with torch.accelerator.device_index(self.device.index):
|
||||
event.synchronize()
|
||||
|
||||
payload = msgspec.msgpack.encode(request)
|
||||
if _FAULT is None:
|
||||
socket.send(payload)
|
||||
else:
|
||||
# Armed runs only: correlation metadata rides as a second frame.
|
||||
metadata = msgspec.msgpack.encode(
|
||||
{
|
||||
"serving": pending.eligible,
|
||||
"transaction": pending.transaction_id,
|
||||
"correlation": pending.correlation_id,
|
||||
}
|
||||
)
|
||||
socket.send_multipart([payload, metadata])
|
||||
event_pool.put_nowait(event)
|
||||
|
||||
def _validate_input_sources(self) -> None:
|
||||
"""Validate fixed runner sources against shared input buffers."""
|
||||
sources = [
|
||||
("input_ids", self._input_ids_source, self._input_ids_buf),
|
||||
(
|
||||
"query_start_loc",
|
||||
self._query_start_loc_source,
|
||||
self._query_start_loc_buf,
|
||||
),
|
||||
]
|
||||
if (self._ngram_context_source is None) != (self._ngram_context_buf is None):
|
||||
raise ValueError("PLE ngram_context source and buffer must match")
|
||||
if self._ngram_context_source is not None:
|
||||
assert self._ngram_context_buf is not None
|
||||
sources.append(
|
||||
(
|
||||
"ngram_context",
|
||||
self._ngram_context_source,
|
||||
self._ngram_context_buf,
|
||||
)
|
||||
)
|
||||
|
||||
for name, source, buffer in sources:
|
||||
if (
|
||||
source.device != self.device
|
||||
or source.dtype != buffer.dtype
|
||||
or source.ndim != buffer.ndim
|
||||
or source.shape[0] < buffer.shape[0]
|
||||
or source.shape[1:] != buffer.shape[1:]
|
||||
):
|
||||
raise ValueError(f"PLE {name} source is incompatible")
|
||||
|
||||
def _enqueue_cuda_inputs(
|
||||
self,
|
||||
request: PleOffloadRequest,
|
||||
d2h_done_event: torch.cuda.Event,
|
||||
) -> None:
|
||||
"""Stage inputs on the model stream and record D2H completion."""
|
||||
with (
|
||||
record_function_or_nullcontext("ple_offload: enqueue_cuda_inputs"),
|
||||
torch.accelerator.device_index(self.device.index),
|
||||
):
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
self._input_ids_buf[: request.num_tokens].copy_(
|
||||
self._input_ids_source[: request.num_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
self._query_start_loc_buf[: request.num_reqs + 1].copy_(
|
||||
self._query_start_loc_source[: request.num_reqs + 1],
|
||||
non_blocking=True,
|
||||
)
|
||||
if self._ngram_context_buf is not None:
|
||||
assert self._ngram_context_source is not None
|
||||
self._ngram_context_buf[: request.num_reqs].copy_(
|
||||
self._ngram_context_source[: request.num_reqs],
|
||||
non_blocking=True,
|
||||
)
|
||||
d2h_done_event.record(stream)
|
||||
|
||||
def _launch(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Stage or queue one batch for request publication."""
|
||||
# Inputs are replicated across TP ranks. One request per DP rank drives
|
||||
# the CPU result fan-out to every registered TP output buffer.
|
||||
if self.tp_rank != 0:
|
||||
return
|
||||
|
||||
request = PleOffloadRequest(
|
||||
dp_rank=self.dp_rank,
|
||||
num_tokens=num_tokens,
|
||||
num_reqs=num_reqs,
|
||||
)
|
||||
assert self._d2h_event_pool is not None, "PLE D2H event pool is not initialized"
|
||||
try:
|
||||
d2h_done_event = self._d2h_event_pool.get_nowait()
|
||||
except queue.Empty as exc:
|
||||
raise RuntimeError(
|
||||
"PLE has more requests than configured concurrent batches"
|
||||
) from exc
|
||||
self._enqueue_cuda_inputs(request, d2h_done_event)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"drop_request",
|
||||
role="gpu",
|
||||
unit="connector-request",
|
||||
eligible=serving,
|
||||
tp_rank=self.tp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
):
|
||||
# Deliberately retain the event: the fault is loss after D2H, not
|
||||
# premature event-pool reuse while that D2H may still be in flight.
|
||||
self._dropped_events.append(d2h_done_event)
|
||||
return
|
||||
self._request_queue.put_nowait(
|
||||
_PendingPleOffloadRequest(
|
||||
request,
|
||||
d2h_done_event,
|
||||
eligible=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
)
|
||||
|
||||
def _fail_closed(self, where: str, error: BaseException) -> None:
|
||||
ple_fail_closed(
|
||||
where=where,
|
||||
rank=self.rank,
|
||||
layer=",".join(self._layers),
|
||||
sequence=self._progress.sequence,
|
||||
error=error,
|
||||
)
|
||||
|
||||
@property
|
||||
def has_pending_completions(self) -> bool:
|
||||
return self._progress.has_pending_completions
|
||||
|
||||
def begin_forward(self, *, dummy_run: bool, num_tokens: int) -> int:
|
||||
"""Begin the bounded transaction before any runner fault boundary."""
|
||||
try:
|
||||
return self._progress.begin(
|
||||
startup=dummy_run,
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
except Exception as error:
|
||||
self._fail_closed("prepare_forward", error)
|
||||
|
||||
def fault_transaction(self, sequence: int) -> str:
|
||||
"""Read diagnostic state only when the caller has established arming."""
|
||||
return self._progress.describe_submission(sequence)
|
||||
|
||||
def completion_is_pending(self, sequence: int) -> bool:
|
||||
"""Return exact-sequence incomplete completion state for cut 6."""
|
||||
return self._progress.completion_is_pending(sequence)
|
||||
|
||||
def prepare_forward(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
dummy_run: bool,
|
||||
*,
|
||||
consumer_num_tokens: int | None = None,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Prepare one PLE transaction; any ordinary exception is fatal."""
|
||||
try:
|
||||
self._prepare_forward_impl(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
dummy_run,
|
||||
consumer_num_tokens=consumer_num_tokens,
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
except Exception as error:
|
||||
self._fail_closed("prepare_forward", error)
|
||||
|
||||
def _prepare_forward_impl(
|
||||
self,
|
||||
num_reqs: int,
|
||||
num_tokens: int,
|
||||
dummy_run: bool,
|
||||
*,
|
||||
consumer_num_tokens: int | None = None,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Submit real inputs or satisfy the PLE wait for a dummy forward."""
|
||||
if consumer_num_tokens is None:
|
||||
consumer_num_tokens = num_tokens
|
||||
if int(num_tokens) != int(consumer_num_tokens):
|
||||
raise RuntimeError(
|
||||
"PLE request/consumer token-count mismatch: "
|
||||
f"request={num_tokens}, consumer={consumer_num_tokens}"
|
||||
)
|
||||
if not 0 <= int(num_tokens) <= self._input_ids_buf.shape[0]:
|
||||
raise RuntimeError(
|
||||
f"PLE token count {num_tokens} exceeds connector capacity "
|
||||
f"{self._input_ids_buf.shape[0]}"
|
||||
)
|
||||
if not 0 <= int(num_reqs) < self._query_start_loc_buf.shape[0]:
|
||||
raise RuntimeError(
|
||||
f"PLE request count {num_reqs} exceeds connector capacity "
|
||||
f"{self._query_start_loc_buf.shape[0] - 1}"
|
||||
)
|
||||
self._rowdump_n = None if dummy_run else num_tokens
|
||||
if dummy_run:
|
||||
self._signal_dummy_outputs_impl(num_tokens)
|
||||
return
|
||||
if self._after_dummy:
|
||||
# Every dummy/cleanup reset on THIS rank already completed
|
||||
# (synchronous releases); the barrier extends that to every
|
||||
# stage-0 TP rank before TP0 can publish (CPU group, no GPU work;
|
||||
# all TP ranks run the same forwards in the same order).
|
||||
get_tp_group().barrier()
|
||||
self._after_dummy = False
|
||||
logger.info(
|
||||
"PleOffload: dummy->real transition (dp_rank=%d, tp_rank=%d): "
|
||||
"releases were synchronous, TP barrier passed",
|
||||
self.dp_rank,
|
||||
self.tp_rank,
|
||||
)
|
||||
self._launch(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
def signal_dummy_outputs(self, num_tokens: int) -> None:
|
||||
"""Locally satisfy PLE waits for dummy and capture forwards."""
|
||||
self._progress.begin(startup=True, num_tokens=num_tokens)
|
||||
self._signal_dummy_outputs_impl(num_tokens)
|
||||
|
||||
def _signal_dummy_outputs_impl(self, num_tokens: int) -> None:
|
||||
"""Signal dummy outputs after the progress transaction is armed."""
|
||||
# Dummy and capture forwards do not send CPU requests, but every PLE
|
||||
# placeholder still waits for a completed output semaphore.
|
||||
self._after_dummy = True
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
for layer in self._layers.values():
|
||||
layer._gpu_output_buffer[:num_tokens].zero_()
|
||||
layer._sem.signal(stream)
|
||||
|
||||
def release_outputs(
|
||||
self,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Release one PLE transaction; any ordinary exception is fatal."""
|
||||
try:
|
||||
self._release_outputs_impl(
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
self._progress.complete()
|
||||
except Exception as error:
|
||||
self._fail_closed("release_outputs", error)
|
||||
|
||||
def _release_outputs_impl(
|
||||
self,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
"""Mark GPU output buffers reusable after the model consumes them."""
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_in_release",
|
||||
role="gpu",
|
||||
unit="connector-release",
|
||||
eligible=serving,
|
||||
tp_rank=self.tp_rank,
|
||||
pp_rank=self.pp_rank,
|
||||
dp_rank=self.dp_rank,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_in_release")
|
||||
n = getattr(self, "_rowdump_n", None)
|
||||
self._rowdump_n = None
|
||||
if _rowdump.enabled() and n is not None and not torch.cuda.is_current_stream_capturing():
|
||||
torch.cuda.current_stream(self.device).synchronize()
|
||||
seq = getattr(self, "_rowdump_seq", 0)
|
||||
self._rowdump_seq = seq + 1
|
||||
for layer in self._layers.values():
|
||||
_rowdump.write(f"gpu-tp{self.tp_rank}", 0, self.tp_rank, seq,
|
||||
self._input_ids_source[:n], layer._gpu_output_buffer[:n])
|
||||
# Reset only after the consumer forward so the CPU worker cannot
|
||||
# overwrite an output that a GPU PLE placeholder may still read.
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
for layer in self._layers.values():
|
||||
layer.release_offloaded_output(stream)
|
||||
# Retain the serving deadline until the model stream has actually
|
||||
# executed every reset above. The monitor thread is the sole query site.
|
||||
completion_event = torch.cuda.Event()
|
||||
completion_event.record(stream)
|
||||
self._progress.track_completion(completion_event)
|
||||
# After dummy/capture work (profile, warm-up sizes, capture seed,
|
||||
# adaptive-verification dummy replays, capture cleanup): complete the
|
||||
# reset before returning, so it cannot still be pending when a real
|
||||
# request lets the offload publish. Real-forward releases stay async.
|
||||
# Never inside a stream capture.
|
||||
if self._after_dummy and not torch.cuda.is_current_stream_capturing():
|
||||
stream.synchronize()
|
||||
|
||||
def release_host_storage(self) -> None:
|
||||
"""Unlink host-storage names while retaining mappings and tensors."""
|
||||
buffers = [
|
||||
*self._host_output_bufs.values(),
|
||||
*self._host_flags.values(),
|
||||
self._input_ids_buf,
|
||||
self._query_start_loc_buf,
|
||||
]
|
||||
if self._ngram_context_buf is not None:
|
||||
buffers.append(self._ngram_context_buf)
|
||||
for buffer in buffers:
|
||||
_unlink_filename_shared_storage(
|
||||
buffer,
|
||||
self._reserved_host_storage_filenames,
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Stop transport and unlink names, retaining host mappings."""
|
||||
request_thread = self._request_thread
|
||||
if request_thread is not None and request_thread.is_alive():
|
||||
try:
|
||||
self._request_queue.put(None, timeout=5)
|
||||
except queue.Full:
|
||||
logger.error("Timed out stopping the PLE request thread")
|
||||
request_thread.join(timeout=5)
|
||||
if request_thread is not None and request_thread.is_alive():
|
||||
# The thread may still access the registered buffers or ZMQ context.
|
||||
# It may also still race another process opening a shared name, so
|
||||
# this deferred-cleanup branch must not unlink host storage.
|
||||
logger.error("PLE request thread did not stop; deferring resource cleanup")
|
||||
self._progress.close()
|
||||
return
|
||||
self._request_thread = None
|
||||
|
||||
# Inputs, outputs, and flags remain registered and strongly referenced.
|
||||
# A captured graph or delayed request can still contain their addresses;
|
||||
# the CUDA driver releases registrations when the process exits.
|
||||
registered = len(self._pinned_input_buffers) + len(self._mapped_host_buffers)
|
||||
if registered:
|
||||
logger.debug(
|
||||
"PleOffload: %d registered host buffers retained until exit",
|
||||
registered,
|
||||
)
|
||||
self._d2h_event_pool = None
|
||||
if self._registration_socket is not None:
|
||||
self._registration_socket.close(linger=0)
|
||||
self._registration_socket = None
|
||||
if self._zmq_ctx is not None:
|
||||
self._zmq_ctx.term()
|
||||
self._zmq_ctx = None
|
||||
self._progress.close()
|
||||
|
||||
# __init__'s failure handler can reach here before the child has rebuilt
|
||||
# a registration. Startup is already failing in that case; if a payload
|
||||
# was sent, a late rebuild may also observe the removed name. Successful
|
||||
# startup accepts every registration before an external close begins.
|
||||
#
|
||||
# NOTE: Graceful shutdown unlinks the PLE shared-memory names and the
|
||||
# PLE ipc socket. The remaining 17 /tmp UUID sockets per boot are
|
||||
# upstream vLLM VLLM_RPC_BASE_PATH behaviour; a tmpfiles/launch-profile
|
||||
# sweep is recommended.
|
||||
# Keep this last: unlinking preserves every existing CPU/CUDA mapping,
|
||||
# but no transport user may still need to open a name at this point.
|
||||
self.release_host_storage()
|
||||
@@ -0,0 +1,153 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fail-closed helpers for the PLE single-slot transport."""
|
||||
|
||||
import functools
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Any, NoReturn, TypeVar
|
||||
|
||||
_F = TypeVar("_F", bound=Callable[..., Any])
|
||||
|
||||
_TRUSTED_ERROR_LABELS = (
|
||||
(RuntimeError, "RuntimeError"),
|
||||
(TimeoutError, "TimeoutError"),
|
||||
(ValueError, "ValueError"),
|
||||
(OSError, "OSError"),
|
||||
(EOFError, "EOFError"),
|
||||
(BrokenPipeError, "BrokenPipeError"),
|
||||
(AssertionError, "AssertionError"),
|
||||
(KeyError, "KeyError"),
|
||||
(IndexError, "IndexError"),
|
||||
(TypeError, "TypeError"),
|
||||
(MemoryError, "MemoryError"),
|
||||
(KeyboardInterrupt, "KeyboardInterrupt"),
|
||||
)
|
||||
|
||||
|
||||
def ple_fail_closed(
|
||||
*,
|
||||
where: str,
|
||||
rank: int,
|
||||
layer: str,
|
||||
sequence: int,
|
||||
error: BaseException,
|
||||
) -> NoReturn:
|
||||
"""Emit one best-effort nonblocking diagnostic and terminate immediately."""
|
||||
try:
|
||||
os.set_blocking(2, False)
|
||||
error_type = type(error)
|
||||
error_name = "Exception"
|
||||
trusted_error = False
|
||||
for trusted_type, trusted_name in _TRUSTED_ERROR_LABELS:
|
||||
if error_type is trusted_type:
|
||||
error_name = trusted_name
|
||||
trusted_error = True
|
||||
break
|
||||
|
||||
safe_where = where[:128] if type(where) is str else "unknown"
|
||||
safe_layer = layer[:256] if type(layer) is str else "unknown"
|
||||
safe_rank = rank if type(rank) is int else -1
|
||||
safe_sequence = sequence if type(sequence) is int else -1
|
||||
detail_suffix = ""
|
||||
if trusted_error:
|
||||
args = BaseException.args.__get__(error, error_type)
|
||||
if (
|
||||
type(args) is tuple
|
||||
and len(args) == 1
|
||||
and type(args[0]) is str
|
||||
):
|
||||
detail = args[0][:512].replace("\n", "\\n")
|
||||
detail_suffix = f":{detail}"
|
||||
message = (
|
||||
f"PLE fatal: where={safe_where} rank={safe_rank} layer={safe_layer} "
|
||||
f"sequence={safe_sequence} error={error_name}{detail_suffix} "
|
||||
f"pid={os.getpid()} "
|
||||
f"monotonic={time.monotonic():.9f} wall={time.time():.9f}\n"
|
||||
)
|
||||
os.write(2, message.encode("utf-8", "backslashreplace")[:2048])
|
||||
except BaseException:
|
||||
pass
|
||||
# THE TRACEBACK, unconditionally, in its own try so it cannot cost us the
|
||||
# line above.
|
||||
#
|
||||
# This path os._exit(1)s, which runs no handlers and flushes nothing, so
|
||||
# whatever is not written here is gone. The line above names the TYPE only
|
||||
# when the type is one of _TRUSTED_ERROR_LABELS and flattens everything
|
||||
# else to the literal string "Exception" -- and the flattened set includes
|
||||
# the two that matter most here: torch.AcceleratorError (a RuntimeError
|
||||
# SUBCLASS, so the `is` comparison misses it) and Triton CompilationError.
|
||||
# An earlier boot died exactly that way: both PP0 ranks
|
||||
# reported "error=Exception" and there was no traceback anywhere in the
|
||||
# log, in any rank, so the cause could not be named at all and the next
|
||||
# step was a re-run rather than a fix.
|
||||
#
|
||||
# A fail-closed path is where evidence matters most, not least: it fires
|
||||
# once, it kills the process, and it is the only witness. Printing the type
|
||||
# of a wrapped exception without the frames that raised it is not a
|
||||
# diagnostic.
|
||||
try:
|
||||
# Three-arg form: it does not depend on being inside an active except
|
||||
# block, and this function is called from one frame further out than
|
||||
# the raise.
|
||||
tb = "".join(
|
||||
traceback.format_exception(type(error), error, error.__traceback__)
|
||||
)
|
||||
raw = tb.encode("utf-8", "backslashreplace")
|
||||
# Truncate from the END, not the start. format_exception puts the
|
||||
# frames first and the exception TYPE AND MESSAGE last, so a head-side
|
||||
# cut throws away the one line that names what went wrong and keeps the
|
||||
# import-time frames nobody needs. A long chained traceback is exactly
|
||||
# the case where that matters.
|
||||
clipped = len(raw) > 8192
|
||||
raw = raw[-8192:]
|
||||
os.write(
|
||||
2,
|
||||
b"PLE fatal traceback (tail) follows:\n" if clipped
|
||||
else b"PLE fatal traceback follows:\n",
|
||||
)
|
||||
os.write(2, raw)
|
||||
if not raw.endswith(b"\n"):
|
||||
os.write(2, b"\n")
|
||||
except BaseException:
|
||||
# Never silent HERE of all places: without this the reader cannot tell
|
||||
# "the wrapper predates the traceback fix" from "the traceback itself
|
||||
# could not be produced", and those call for different next steps.
|
||||
# Still guarded, because os._exit below must run whatever happens: an
|
||||
# escape from this handler would leave the worker alive in the state
|
||||
# the fail-closed path exists to end.
|
||||
try:
|
||||
os.write(2, b"PLE fatal: traceback unavailable\n")
|
||||
except BaseException:
|
||||
pass
|
||||
os._exit(1)
|
||||
|
||||
|
||||
def fail_ple_execute_model(func: _F) -> _F:
|
||||
"""Fail a non-shutdown worker-step escape only on PLE-owning ranks.
|
||||
|
||||
This includes ``sample_tokens``, the second half of the split execute-model
|
||||
step. A rank without a PLE connector rethrows the original exception.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapped(self: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
except SystemExit:
|
||||
# WorkerProc's signal handler uses SystemExit for clean shutdown.
|
||||
raise
|
||||
except BaseException as error:
|
||||
connector = getattr(self, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
model_runner = getattr(self, "model_runner", None)
|
||||
connector = getattr(
|
||||
model_runner, "_ple_offload_connector", None
|
||||
)
|
||||
if connector is None:
|
||||
raise
|
||||
connector._fail_closed(func.__name__, error)
|
||||
|
||||
return wrapped # type: ignore[return-value]
|
||||
@@ -0,0 +1,314 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Independent progress deadline for PLE transactions."""
|
||||
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||
|
||||
_DEFAULT_SERVING_DEADLINE_S = 300.0
|
||||
_DEFAULT_STARTUP_DEADLINE_S = 1800.0
|
||||
_MAX_PROGRESS_DEADLINE_S = 86400.0
|
||||
_POLL_INTERVAL_S = 0.1
|
||||
|
||||
|
||||
def _validate_deadline(value: float, name: str) -> float:
|
||||
value = float(value)
|
||||
if (
|
||||
not math.isfinite(value)
|
||||
or value <= 0
|
||||
or value > _MAX_PROGRESS_DEADLINE_S
|
||||
):
|
||||
raise ValueError(
|
||||
f"{name} must be finite, greater than zero, and no greater than "
|
||||
f"{_MAX_PROGRESS_DEADLINE_S:g} seconds"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _environment_deadline(name: str, default: float) -> float:
|
||||
value = float(os.environ.get(name, str(default)))
|
||||
return _validate_deadline(value, name)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Submission:
|
||||
sequence: int
|
||||
started: float
|
||||
startup: bool
|
||||
num_tokens: int
|
||||
deadline: float
|
||||
depth: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _PendingCompletion:
|
||||
sequence: int
|
||||
started: float
|
||||
startup: bool
|
||||
num_tokens: int
|
||||
deadline: float
|
||||
event: Any
|
||||
|
||||
|
||||
class PleProgressMonitor:
|
||||
"""Bound host submission and the resulting ordered GPU completions."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
rank: int,
|
||||
layer_names: tuple[str, ...],
|
||||
serving_deadline_s: float,
|
||||
startup_deadline_s: float,
|
||||
) -> None:
|
||||
serving_deadline_s = _validate_deadline(
|
||||
serving_deadline_s, "serving_deadline_s"
|
||||
)
|
||||
startup_deadline_s = _validate_deadline(
|
||||
startup_deadline_s, "startup_deadline_s"
|
||||
)
|
||||
self._rank = rank
|
||||
self._layer = ",".join(layer_names)
|
||||
self._serving_deadline_s = serving_deadline_s
|
||||
self._startup_deadline_s = startup_deadline_s
|
||||
self._condition = threading.Condition()
|
||||
self._submission: _Submission | None = None
|
||||
self._pending: deque[_PendingCompletion] = deque()
|
||||
self._sequence = 0
|
||||
self._stopping = False
|
||||
self._thread = threading.Thread(
|
||||
target=self._run,
|
||||
name=f"ple-progress-rank{rank}",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
@classmethod
|
||||
def from_environment(
|
||||
cls, *, rank: int, layer_names: tuple[str, ...]
|
||||
) -> "PleProgressMonitor":
|
||||
return cls(
|
||||
rank=rank,
|
||||
layer_names=layer_names,
|
||||
serving_deadline_s=_environment_deadline(
|
||||
"VLLM_PLE_PROGRESS_DEADLINE_S",
|
||||
_DEFAULT_SERVING_DEADLINE_S,
|
||||
),
|
||||
startup_deadline_s=_environment_deadline(
|
||||
"VLLM_PLE_STARTUP_DEADLINE_S",
|
||||
_DEFAULT_STARTUP_DEADLINE_S,
|
||||
),
|
||||
)
|
||||
|
||||
@property
|
||||
def sequence(self) -> int:
|
||||
with self._condition:
|
||||
return self._sequence
|
||||
|
||||
@property
|
||||
def has_pending_completions(self) -> bool:
|
||||
with self._condition:
|
||||
return bool(self._pending)
|
||||
|
||||
def describe_submission(self, sequence: int) -> str:
|
||||
"""Return the armed transaction descriptor for the active sequence."""
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is None or active.sequence != sequence:
|
||||
raise RuntimeError("PLE fault correlation has no active submission")
|
||||
return (
|
||||
f"sequence:{active.sequence}"
|
||||
f"@begin:{active.started:.9f}"
|
||||
)
|
||||
|
||||
def completion_is_pending(self, sequence: int) -> bool:
|
||||
"""Whether this serving sequence has a retained, incomplete event.
|
||||
|
||||
This method is called only behind the armed fault guard. An event that
|
||||
has completed on the GPU but has not yet been dequeued is not counted.
|
||||
"""
|
||||
with self._condition:
|
||||
pending = next(
|
||||
(
|
||||
item
|
||||
for item in self._pending
|
||||
if item.sequence == sequence and not item.startup
|
||||
),
|
||||
None,
|
||||
)
|
||||
if pending is None:
|
||||
return False
|
||||
return not bool(pending.event.query())
|
||||
|
||||
def begin(self, *, startup: bool, num_tokens: int) -> int:
|
||||
now = time.monotonic()
|
||||
budget = (
|
||||
self._startup_deadline_s if startup else self._serving_deadline_s
|
||||
)
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is not None:
|
||||
if not startup or not active.startup:
|
||||
raise RuntimeError(
|
||||
"Overlapping PLE host submissions violate the serialized "
|
||||
"single-slot protocol"
|
||||
)
|
||||
sequence = active.sequence
|
||||
self._submission = _Submission(
|
||||
sequence=sequence,
|
||||
started=active.started,
|
||||
startup=True,
|
||||
num_tokens=num_tokens,
|
||||
deadline=now + budget,
|
||||
depth=active.depth + 1,
|
||||
)
|
||||
else:
|
||||
self._sequence += 1
|
||||
sequence = self._sequence
|
||||
self._submission = _Submission(
|
||||
sequence=sequence,
|
||||
started=now,
|
||||
startup=startup,
|
||||
num_tokens=num_tokens,
|
||||
deadline=now + budget,
|
||||
depth=1,
|
||||
)
|
||||
self._condition.notify_all()
|
||||
return sequence
|
||||
|
||||
def track_completion(self, event: Any) -> None:
|
||||
"""Retain a release event until the monitor observes GPU completion."""
|
||||
if not callable(getattr(event, "query", None)):
|
||||
raise TypeError("PLE completion event must provide query()")
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is None:
|
||||
raise RuntimeError("PLE completion has no matching submission")
|
||||
self._pending.append(
|
||||
_PendingCompletion(
|
||||
sequence=active.sequence,
|
||||
started=active.started,
|
||||
startup=active.startup,
|
||||
num_tokens=active.num_tokens,
|
||||
deadline=active.deadline,
|
||||
event=event,
|
||||
)
|
||||
)
|
||||
self._condition.notify_all()
|
||||
|
||||
def complete(self) -> None:
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is None or active.depth <= 0:
|
||||
raise RuntimeError("PLE release has no matching transaction")
|
||||
if active.depth == 1:
|
||||
self._submission = None
|
||||
else:
|
||||
self._submission = _Submission(
|
||||
sequence=active.sequence,
|
||||
started=active.started,
|
||||
startup=active.startup,
|
||||
num_tokens=active.num_tokens,
|
||||
deadline=active.deadline,
|
||||
depth=active.depth - 1,
|
||||
)
|
||||
self._condition.notify_all()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._condition:
|
||||
self._stopping = True
|
||||
self._condition.notify_all()
|
||||
self._thread.join(timeout=1)
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
self._monitor()
|
||||
except BaseException as error:
|
||||
ple_fail_closed(
|
||||
where="progress-monitor-thread",
|
||||
rank=self._rank,
|
||||
layer=self._layer,
|
||||
sequence=self._sequence,
|
||||
error=error,
|
||||
)
|
||||
|
||||
def _monitor(self) -> None:
|
||||
while True:
|
||||
with self._condition:
|
||||
if self._stopping:
|
||||
return
|
||||
submission = self._submission
|
||||
pending = self._pending[0] if self._pending else None
|
||||
if submission is None and pending is None:
|
||||
self._condition.wait(timeout=_POLL_INTERVAL_S)
|
||||
continue
|
||||
|
||||
# CUDA completion is observed only here. Never synchronize and never
|
||||
# use the host flag as evidence that the queued reset completed.
|
||||
if pending is not None and pending.event.query():
|
||||
with self._condition:
|
||||
if self._pending and self._pending[0] is pending:
|
||||
self._pending.popleft()
|
||||
self._condition.notify_all()
|
||||
continue
|
||||
|
||||
now = time.monotonic()
|
||||
if pending is not None and now >= pending.deadline:
|
||||
with self._condition:
|
||||
if not self._pending or self._pending[0] is not pending:
|
||||
continue
|
||||
mode = "startup" if pending.startup else "serving"
|
||||
ple_fail_closed(
|
||||
where=(
|
||||
f"progress-deadline:{mode}:gpu-completion:"
|
||||
f"tokens={pending.num_tokens}"
|
||||
),
|
||||
rank=self._rank,
|
||||
layer=self._layer,
|
||||
sequence=pending.sequence,
|
||||
error=TimeoutError("PLE GPU completion made no bounded progress"),
|
||||
)
|
||||
|
||||
if submission is not None and now >= submission.deadline:
|
||||
with self._condition:
|
||||
if self._submission is not submission:
|
||||
continue
|
||||
mode = "startup" if submission.startup else "serving"
|
||||
ple_fail_closed(
|
||||
where=(
|
||||
f"progress-deadline:{mode}:host-submission:"
|
||||
f"tokens={submission.num_tokens}"
|
||||
),
|
||||
rank=self._rank,
|
||||
layer=self._layer,
|
||||
sequence=submission.sequence,
|
||||
error=TimeoutError("PLE host submission made no bounded progress"),
|
||||
)
|
||||
|
||||
deadlines = []
|
||||
if pending is not None:
|
||||
deadlines.append(pending.deadline)
|
||||
if submission is not None:
|
||||
deadlines.append(submission.deadline)
|
||||
timeout = _POLL_INTERVAL_S
|
||||
if deadlines:
|
||||
timeout = min(
|
||||
timeout,
|
||||
max(0.0, min(deadlines) - time.monotonic()),
|
||||
)
|
||||
with self._condition:
|
||||
if self._stopping:
|
||||
return
|
||||
if (
|
||||
self._submission is not submission
|
||||
or (self._pending[0] if self._pending else None) is not pending
|
||||
):
|
||||
continue
|
||||
self._condition.wait(timeout=timeout)
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""IPC message definitions for PLE CPU offload (host-mapped pull design).
|
||||
|
||||
The offload process owns no CUDA context. Every GPU worker owns, per PLE
|
||||
layer, a shared host output buffer and a shared host int32 flag, both
|
||||
page-locked and device-mapped in the worker's context. The offload process
|
||||
waits for ``flag == 0``, writes the layer result into the host buffer and
|
||||
stores ``flag = 1`` with plain CPU stores; the GPU's captured forward waits on
|
||||
the host flag, DMAs host -> GPU on the model stream, and ``release_outputs``
|
||||
resets the flag after the forward.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# IPC message dataclasses
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class PleOffloadRegistration:
|
||||
"""Sent once from each GPU worker during offload setup."""
|
||||
|
||||
worker_id: int
|
||||
tp_rank: int
|
||||
dp_rank: int
|
||||
# Shared CPU tensors per PLE layer, filled by the offload process.
|
||||
host_output_buffers: dict[str, torch.Tensor]
|
||||
# Shared CPU int32 tensors per PLE layer; element 0 is the ready flag.
|
||||
host_flag_tensors: dict[str, torch.Tensor]
|
||||
# CPU tensors are allocated in shared memory and registered once.
|
||||
input_ids_buf: torch.Tensor
|
||||
query_start_loc_buf: torch.Tensor
|
||||
ngram_context_buf: torch.Tensor | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class PleOffloadRequest:
|
||||
"""Sent by each DP rank's TP rank zero at every inference step."""
|
||||
|
||||
dp_rank: int
|
||||
num_tokens: int
|
||||
num_reqs: int
|
||||
|
||||
|
||||
_PLE_OFFLOAD_REQUEST_DECODER = msgspec.msgpack.Decoder(PleOffloadRequest)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,9 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import gc
|
||||
import itertools
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import dataclass, replace
|
||||
from itertools import groupby, product
|
||||
@@ -33,6 +34,9 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.forward_context import BatchDescriptor, set_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.ple_offload_layer import (
|
||||
observe_ple_copy_geometry,
|
||||
)
|
||||
from vllm.model_executor.offloader.base import get_offloader
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -173,6 +177,14 @@ class CudaGraphManager:
|
||||
self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map()
|
||||
|
||||
self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {}
|
||||
# FULL graphs bake every PLE wait/copy into their native graph. Metadata
|
||||
# comes from the actual custom-op implementation during native capture.
|
||||
self._ple_full_copy_geometry: dict[
|
||||
BatchExecutionDescriptor, tuple[tuple[str, int, int], ...]
|
||||
] = {}
|
||||
self._expected_ple_copy_layers: tuple[str, ...] = ()
|
||||
# Only the target-model manager is allowed to capture PLE operations.
|
||||
self._allow_ple_full_graph = False
|
||||
self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None
|
||||
|
||||
self._graphs_captured = False
|
||||
@@ -488,6 +500,7 @@ class CudaGraphManager:
|
||||
torch.accelerator.synchronize()
|
||||
free_before = torch.accelerator.get_memory_info()[0]
|
||||
with (
|
||||
self.capture_ple_copy_geometry(desc),
|
||||
capture_pool(self.pool, self.vllm_config) as pool,
|
||||
torch.cuda.graph(
|
||||
graph, pool, stream=self._capture_stream(desc)
|
||||
@@ -514,6 +527,78 @@ class CudaGraphManager:
|
||||
{desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0}
|
||||
)
|
||||
|
||||
@contextlib.contextmanager
|
||||
def capture_ple_copy_geometry(
|
||||
self, desc: BatchExecutionDescriptor
|
||||
) -> Iterator[None]:
|
||||
"""Production capture callback for actual PLE custom-op geometry."""
|
||||
with observe_ple_copy_geometry() as geometry:
|
||||
yield
|
||||
|
||||
if geometry and not self._allow_ple_full_graph:
|
||||
raise RuntimeError(
|
||||
"PLE wait-copy reached an unsupported FULL graph path; "
|
||||
"speculator/draft FULL graphs may not contain PLE operations"
|
||||
)
|
||||
if geometry:
|
||||
self.record_ple_copy_geometry(desc, geometry)
|
||||
elif self._expected_ple_copy_layers:
|
||||
raise RuntimeError(
|
||||
f"PLE copy geometry was not observed while capturing FULL graph {desc}"
|
||||
)
|
||||
|
||||
def record_ple_copy_geometry(
|
||||
self,
|
||||
desc: BatchExecutionDescriptor,
|
||||
geometry: Iterable[tuple[str, int, int]],
|
||||
) -> None:
|
||||
if desc.cg_mode != CUDAGraphMode.FULL:
|
||||
raise ValueError("PLE capture geometry is recorded only for FULL graphs")
|
||||
observed = tuple(
|
||||
(str(layer), int(rows), int(nbytes))
|
||||
for layer, rows, nbytes in geometry
|
||||
)
|
||||
if not observed:
|
||||
raise RuntimeError(f"PLE FULL graph {desc} observed no wait-copy operations")
|
||||
observed_layers = tuple(layer for layer, _, _ in observed)
|
||||
expected_layers = self._expected_ple_copy_layers
|
||||
if expected_layers and sorted(observed_layers) != sorted(expected_layers):
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} observed layers {observed_layers}, "
|
||||
f"expected {expected_layers}"
|
||||
)
|
||||
inconsistent = [
|
||||
(layer, rows, nbytes)
|
||||
for layer, rows, nbytes in observed
|
||||
if rows != desc.num_tokens or nbytes < 0
|
||||
]
|
||||
if inconsistent:
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} actual PLE copy geometry "
|
||||
f"{inconsistent} does not match descriptor rows={desc.num_tokens}"
|
||||
)
|
||||
previous = self._ple_full_copy_geometry.get(desc)
|
||||
if previous is not None and previous != observed:
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} changed copy geometry from "
|
||||
f"{previous} to {observed}"
|
||||
)
|
||||
self._ple_full_copy_geometry[desc] = observed
|
||||
|
||||
def ple_copy_rows(self, desc: BatchExecutionDescriptor) -> int:
|
||||
try:
|
||||
geometry = self._ple_full_copy_geometry[desc]
|
||||
except KeyError as error:
|
||||
raise RuntimeError(
|
||||
f"PLE copy geometry was not recorded for FULL graph {desc}"
|
||||
) from error
|
||||
rows = {num_rows for _, num_rows, _ in geometry}
|
||||
if len(rows) != 1:
|
||||
raise RuntimeError(
|
||||
f"PLE FULL graph {desc} has inconsistent per-layer rows: {geometry}"
|
||||
)
|
||||
return rows.pop()
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
num_reqs: int,
|
||||
@@ -552,6 +637,11 @@ class CudaGraphManager:
|
||||
f"Expected FULL mode, got {desc.cg_mode}"
|
||||
)
|
||||
assert desc in self.graphs, f"No cudagraph for {desc}"
|
||||
if (
|
||||
desc in self._ple_full_copy_geometry
|
||||
and not self._allow_ple_full_graph
|
||||
):
|
||||
raise RuntimeError("refusing unsupported PLE speculator FULL replay")
|
||||
# Sync offloader before replay - needed when transitioning from
|
||||
# eager/piecewise to full cudagraph (e.g., prefill → decode).
|
||||
# The previous eager iteration's start_prefetch may have queued
|
||||
@@ -597,11 +687,32 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
varlen_decode=varlen_decode,
|
||||
ubatch_runner=ubatch_runner,
|
||||
)
|
||||
self._allow_ple_full_graph = True
|
||||
self.hidden_states: torch.Tensor | None = None
|
||||
self.aux_hidden_states: list[torch.Tensor] = []
|
||||
self.use_aux_hidden_state_outputs = False
|
||||
self.intermediate_tensors: IntermediateTensors | None = None
|
||||
|
||||
def _prepare_ple_capture(
|
||||
self, model: nn.Module, *, initialize_breakable: bool
|
||||
) -> None:
|
||||
expected_ple_copy_layers = tuple(
|
||||
str(getattr(module, "_ple_offload_name", name))
|
||||
for name, module in model.named_modules()
|
||||
if getattr(module, "_is_cpu_offloaded", False)
|
||||
and getattr(module, "_host_output_ptr", None) is not None
|
||||
)
|
||||
if (
|
||||
"<unbound>" in expected_ple_copy_layers
|
||||
or len(set(expected_ple_copy_layers)) != len(expected_ple_copy_layers)
|
||||
):
|
||||
raise RuntimeError(
|
||||
"PLE FULL capture requires distinct, bound connector layer identities"
|
||||
)
|
||||
self._expected_ple_copy_layers = expected_ple_copy_layers
|
||||
if initialize_breakable:
|
||||
self.init_breakable_cg_runner(model)
|
||||
|
||||
def capture(
|
||||
self,
|
||||
model: nn.Module,
|
||||
@@ -619,8 +730,9 @@ class ModelCudaGraphManager(CudaGraphManager):
|
||||
) -> None:
|
||||
"""Capture CUDA graphs for model forward pass."""
|
||||
self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs
|
||||
if self.use_breakable_cg:
|
||||
self.init_breakable_cg_runner(model)
|
||||
self._prepare_ple_capture(
|
||||
model, initialize_breakable=self.use_breakable_cg
|
||||
)
|
||||
|
||||
if self.cudagraph_mode.has_piecewise_cudagraphs() and not (
|
||||
self.use_breakable_cg or has_compiled_submodule(model)
|
||||
@@ -926,7 +1038,9 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int:
|
||||
if manager.use_breakable_cg:
|
||||
# Create the breakable runner before the wrapper pool swap so
|
||||
# its pool is covered as well.
|
||||
manager.init_breakable_cg_runner(runner.model)
|
||||
manager._prepare_ple_capture(
|
||||
runner.model, initialize_breakable=True
|
||||
)
|
||||
all_wrappers = list(CUDAGraphWrapper._all_instances) + list(
|
||||
BreakableCUDAGraphWrapper._all_instances
|
||||
)
|
||||
|
||||
@@ -74,6 +74,9 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.ple_offload.connector import PleOffloadConnector
|
||||
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
|
||||
from vllm.v1.ple_offload.test_fault import _FAULT
|
||||
from vllm.v1.watermarking import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.spec_decode import (
|
||||
@@ -1757,6 +1760,45 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.ec_connector.no_forward(scheduler_output).ec_connector_output,
|
||||
)
|
||||
|
||||
def _prepare_ple_forward(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
batch_desc: BatchExecutionDescriptor,
|
||||
dummy_run: bool,
|
||||
*,
|
||||
serving: bool = False,
|
||||
transaction_id: str | None = None,
|
||||
correlation_id: str | None = None,
|
||||
) -> None:
|
||||
connector = self._ple_offload_connector
|
||||
assert connector is not None
|
||||
request_num_tokens = int(input_batch.num_tokens_after_padding)
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
manager = self.cudagraph_manager
|
||||
if manager is None:
|
||||
raise RuntimeError("PLE FULL replay has no CUDA graph manager")
|
||||
copy_num_tokens = manager.ple_copy_rows(batch_desc)
|
||||
else:
|
||||
# PIECEWISE/eager execute the Python op with this actual padded view.
|
||||
copy_num_tokens = int(input_batch.input_ids.shape[0])
|
||||
|
||||
if request_num_tokens != copy_num_tokens:
|
||||
raise RuntimeError(
|
||||
"PLE request/selected-forward copy-count mismatch: "
|
||||
f"request={request_num_tokens}, copy={copy_num_tokens}, "
|
||||
f"mode={batch_desc.cg_mode}"
|
||||
)
|
||||
connector.prepare_forward(
|
||||
input_batch.num_reqs,
|
||||
request_num_tokens,
|
||||
dummy_run,
|
||||
consumer_num_tokens=copy_num_tokens,
|
||||
serving=serving,
|
||||
transaction_id=transaction_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
|
||||
@fail_ple_execute_model
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
@@ -2038,6 +2080,54 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
self.step_timing.forward_start()
|
||||
|
||||
# prepare_inputs has finalized MRV2's GPU buffers. Stage D2H on this
|
||||
# stream before the PLE placeholder can wait for CPU output.
|
||||
connector = self._ple_offload_connector
|
||||
ple_serving = False
|
||||
ple_sequence = None
|
||||
ple_transaction = None
|
||||
ple_correlation = None
|
||||
if connector is not None:
|
||||
ple_serving = not dummy_run and not is_profile
|
||||
ple_sequence = connector.begin_forward(
|
||||
dummy_run=dummy_run,
|
||||
num_tokens=int(input_batch.num_tokens_after_padding),
|
||||
)
|
||||
if _FAULT is not None and ple_serving:
|
||||
ple_correlation = f"sequence:{ple_sequence}"
|
||||
ple_transaction = connector.fault_transaction(ple_sequence)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_before_prepare",
|
||||
role="gpu",
|
||||
unit="runner-forward",
|
||||
eligible=ple_serving,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_before_prepare")
|
||||
self._prepare_ple_forward(
|
||||
input_batch,
|
||||
batch_desc,
|
||||
dummy_run,
|
||||
serving=ple_serving,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_after_prepare",
|
||||
role="gpu",
|
||||
unit="runner-forward",
|
||||
eligible=ple_serving,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_after_prepare")
|
||||
connector_kwargs = dict(
|
||||
scheduler_output=scheduler_output,
|
||||
request_state_indices=input_batch.idx_mapping,
|
||||
@@ -2095,6 +2185,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
# Eager (NONE): call the raw model directly.
|
||||
model_output = self.model(**model_inputs)
|
||||
|
||||
# The model has consumed every PLE output, so the CPU worker may reuse
|
||||
# the registered buffers for the next request.
|
||||
if connector is not None:
|
||||
connector.release_outputs(
|
||||
serving=ple_serving,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
)
|
||||
if _FAULT is not None and _FAULT.fires(
|
||||
"exc_after_release",
|
||||
role="gpu",
|
||||
unit="runner-forward",
|
||||
eligible=ple_serving,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=ple_correlation,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_after_release")
|
||||
self.kv_connector.finish_forward()
|
||||
|
||||
if self.is_last_pp_rank:
|
||||
@@ -2123,6 +2233,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids=finished_req_ids,
|
||||
ec_connector_output=ec_connector_output,
|
||||
cudagraph_stats=cudagraph_stats,
|
||||
ple_serving=ple_serving,
|
||||
ple_sequence=ple_sequence,
|
||||
ple_transaction=ple_transaction,
|
||||
)
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -2134,6 +2247,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
return None
|
||||
|
||||
@fail_ple_execute_model
|
||||
@torch.inference_mode()
|
||||
@step_eplb_after()
|
||||
def sample_tokens(
|
||||
@@ -2152,6 +2266,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
cudagraph_stats = self.execute_model_state.cudagraph_stats
|
||||
ple_serving = self.execute_model_state.ple_serving
|
||||
ple_sequence = self.execute_model_state.ple_sequence
|
||||
ple_transaction = self.execute_model_state.ple_transaction
|
||||
self.execute_model_state = None
|
||||
|
||||
if not self.is_last_pp_rank:
|
||||
@@ -2161,6 +2278,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
if self.pcp_manager is not None:
|
||||
input_batch = self.pcp_manager.global_batch
|
||||
assert self.pp_handler is not None
|
||||
connector = self._ple_offload_connector
|
||||
if (
|
||||
connector is not None
|
||||
and _FAULT is not None
|
||||
and ple_serving
|
||||
and ple_sequence is not None
|
||||
and connector.completion_is_pending(ple_sequence)
|
||||
and _FAULT.fires(
|
||||
"exc_async_output",
|
||||
role="gpu",
|
||||
unit="runner-output-relay",
|
||||
eligible=True,
|
||||
tp_rank=connector.tp_rank,
|
||||
pp_rank=connector.pp_rank,
|
||||
dp_rank=connector.dp_rank,
|
||||
transaction_id=ple_transaction,
|
||||
correlation_id=f"sequence:{ple_sequence}",
|
||||
)
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_async_output")
|
||||
all_decode_next = self.pp_handler.receive(input_batch)
|
||||
# Optimistically update num_computed_tokens for entire batch here.
|
||||
# Will be adjusted for rejections if necessary in update_requests.
|
||||
@@ -2462,6 +2599,9 @@ class ExecuteModelState(NamedTuple):
|
||||
finished_req_ids: set[str]
|
||||
ec_connector_output: ECConnectorOutput | None
|
||||
cudagraph_stats: CUDAGraphStat | None
|
||||
ple_serving: bool
|
||||
ple_sequence: int | None
|
||||
ple_transaction: str | None
|
||||
|
||||
|
||||
class BatchReqState(NamedTuple):
|
||||
|
||||
@@ -91,6 +91,7 @@ from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
|
||||
from vllm.v1.utils import compute_iteration_details, report_usage_stats
|
||||
from vllm.v1.worker import guard_telemetry
|
||||
from vllm.v1.worker.sentinel.gpu_worker_sentinel import WorkerSentinel
|
||||
@@ -238,6 +239,148 @@ class Worker(WorkerBase):
|
||||
# Resolved lazily on first sleep/wake; persists worker-process state.
|
||||
self._sleep_mode_backend: SleepModeBackend | None = None
|
||||
|
||||
def _has_ple_layers(self) -> bool:
|
||||
"""Return whether this model configuration constructs PLE layers."""
|
||||
if not envs.VLLM_PLE_CPU_OFFLOAD:
|
||||
return False
|
||||
text_config = self.model_config.hf_text_config
|
||||
ple_layer_ids = getattr(text_config, "ple_layer_ids", None)
|
||||
if not ple_layer_ids:
|
||||
return False
|
||||
# PP (vllm#54709, carried from the earlier overlay): only the pipeline rank
|
||||
# that owns a PLE layer takes part in the offload (rank 0 with the
|
||||
# default partition). Other ranks build PPMissingLayer there and must
|
||||
# skip the connector entirely.
|
||||
pp_size = self.parallel_config.pipeline_parallel_size
|
||||
if pp_size > 1:
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
pp_rank = (self.rank // tp_size) % pp_size
|
||||
start, end = get_pp_indices(
|
||||
int(text_config.num_hidden_layers), pp_rank, pp_size
|
||||
)
|
||||
return any(start <= int(i) - 1 < end for i in ple_layer_ids)
|
||||
return True
|
||||
|
||||
def _validate_ple_offload_config(self) -> None:
|
||||
"""Reject unsupported PLE offload execution modes."""
|
||||
parallel_config = self.parallel_config
|
||||
unsupported = []
|
||||
if not self.use_v2_model_runner:
|
||||
unsupported.append("model runner V1")
|
||||
if not current_platform.is_cuda():
|
||||
unsupported.append(f"device={current_platform.device_type}")
|
||||
if parallel_config.nnodes != 1:
|
||||
unsupported.append(f"nnodes={parallel_config.nnodes}")
|
||||
if parallel_config.data_parallel_backend != "mp":
|
||||
unsupported.append(f"DP backend={parallel_config.data_parallel_backend}")
|
||||
if (
|
||||
parallel_config.data_parallel_size_local
|
||||
!= parallel_config.data_parallel_size
|
||||
):
|
||||
unsupported.append(
|
||||
"non-local DP "
|
||||
f"({parallel_config.data_parallel_size_local}/"
|
||||
f"{parallel_config.data_parallel_size} local ranks)"
|
||||
)
|
||||
# PP allowed (vllm#54709): PLE layers must live on pipeline rank 0
|
||||
# (enforced placement-based in models/config.py); num_workers stays
|
||||
# dp_size * tp_size = the ranks of the owning stage.
|
||||
if parallel_config.prefill_context_parallel_size != 1:
|
||||
unsupported.append(f"PCP={parallel_config.prefill_context_parallel_size}")
|
||||
if parallel_config.decode_context_parallel_size != 1:
|
||||
unsupported.append(f"DCP={parallel_config.decode_context_parallel_size}")
|
||||
if parallel_config.use_ubatching:
|
||||
unsupported.append("ubatching/DBO")
|
||||
if self.model_config.architecture not in {
|
||||
"Qwen4ExpForCausalLM",
|
||||
"Qwen4ExpForConditionalGeneration",
|
||||
}:
|
||||
unsupported.append("architecture")
|
||||
if self.vllm_config.weight_transfer_config is not None:
|
||||
unsupported.append("weight transfer")
|
||||
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"VLLM_PLE_CPU_OFFLOAD does not support the requested "
|
||||
"configuration. Unsupported settings: "
|
||||
f"{', '.join(unsupported)}"
|
||||
)
|
||||
|
||||
def spawn_ple_offload(self) -> None:
|
||||
"""Spawn one node-local PLE CPU worker from DP0/TP0."""
|
||||
if (
|
||||
not self._ple_offload_enabled
|
||||
or self.rank != 0
|
||||
or self.parallel_config.data_parallel_rank != 0
|
||||
):
|
||||
return
|
||||
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
ipc_addr = self.parallel_config._ple_offload_ipc_path
|
||||
if not ipc_addr:
|
||||
raise RuntimeError("PLE offload IPC address was not initialized")
|
||||
dp_size = self.parallel_config.data_parallel_size
|
||||
tp_size = self.parallel_config.tensor_parallel_size
|
||||
num_workers = dp_size * tp_size
|
||||
logger.info(
|
||||
"PleOffload: spawning worker "
|
||||
"(rank=%d, local_rank=%d, dp_size=%d, tp_size=%d, "
|
||||
"num_workers=%d, ipc_addr=%s).",
|
||||
self.rank,
|
||||
self.local_rank,
|
||||
dp_size,
|
||||
tp_size,
|
||||
num_workers,
|
||||
ipc_addr,
|
||||
)
|
||||
self._ple_offload_worker_handle = PleOffloadWorker.make_process(
|
||||
self.vllm_config,
|
||||
num_workers,
|
||||
ipc_addr,
|
||||
)
|
||||
self._start_ple_offload_watchdog()
|
||||
|
||||
def _start_ple_offload_watchdog(self) -> None:
|
||||
"""Fail this worker if the PLE offload process exits unexpectedly.
|
||||
|
||||
Host-mapped pull: a dead producer would leave the captured forward
|
||||
waiting on its host flag forever; turn that into a fast, visible
|
||||
engine failure instead of a hang. ``shutdown`` sets the stop marker
|
||||
before closing the handle, so a normal close is not a failure.
|
||||
"""
|
||||
import threading
|
||||
|
||||
handle = self._ple_offload_worker_handle
|
||||
self._ple_offload_stopping = False
|
||||
|
||||
def watch() -> None:
|
||||
returncode = handle.proc.wait()
|
||||
if getattr(self, "_ple_offload_stopping", False):
|
||||
return
|
||||
logger.error(
|
||||
"PLE offload process %s exited unexpectedly (exitcode %s); "
|
||||
"failing this worker so the engine does not hang on a PLE wait",
|
||||
handle.proc.pid,
|
||||
returncode,
|
||||
)
|
||||
os._exit(1)
|
||||
|
||||
threading.Thread(
|
||||
target=watch, name="ple-offload-watchdog", daemon=True
|
||||
).start()
|
||||
|
||||
def wait_ple_offload_ready(self) -> None:
|
||||
"""Wait until weights and every TP registration are ready."""
|
||||
if self._ple_offload_worker_handle is None:
|
||||
return
|
||||
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
PleOffloadWorker.wait_for_ready(self._ple_offload_worker_handle)
|
||||
|
||||
@property
|
||||
def sleep_mode_backend(self) -> "SleepModeBackend":
|
||||
if self._sleep_mode_backend is None:
|
||||
@@ -1263,6 +1406,7 @@ class Worker(WorkerBase):
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput:
|
||||
return self.model_runner.sample_tokens(grammar_output)
|
||||
|
||||
@fail_ple_execute_model
|
||||
@torch.inference_mode()
|
||||
@with_gpu_sync_check
|
||||
def execute_model(
|
||||
@@ -1587,6 +1731,7 @@ class Worker(WorkerBase):
|
||||
self.model_runner.reset_lora_state()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._ple_offload_stopping = True
|
||||
gc.unfreeze()
|
||||
|
||||
# Bounds-guard instrumentation: synchronous final read of the guard
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Minimal PLE child bootstrap; intentionally imports no torch or vLLM."""
|
||||
|
||||
import importlib.abc
|
||||
import importlib.machinery
|
||||
import os
|
||||
import pickle
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
from multiprocessing.connection import Connection
|
||||
from types import ModuleType
|
||||
from typing import Any
|
||||
|
||||
_FAULT = None
|
||||
if os.environ.get("VLLM_PLE_TEST_FAULT") is not None:
|
||||
from vllm_ple_test_fault import (
|
||||
_FAULT as _FAULT,
|
||||
_block_on_harness_gate,
|
||||
)
|
||||
|
||||
|
||||
def _send_partial_ready_frame(fd: int) -> tuple[int, int]:
|
||||
"""Write the complete header and an incomplete body in one syscall."""
|
||||
body = pickle.dumps(
|
||||
{"status": "READY", "layer_names": []},
|
||||
protocol=pickle.HIGHEST_PROTOCOL,
|
||||
)
|
||||
frame = struct.pack("!i", len(body)) + body
|
||||
fragment = frame[: 4 + max(1, len(body) // 2)]
|
||||
try:
|
||||
return os.write(fd, fragment), len(frame)
|
||||
except OSError:
|
||||
return 0, len(frame)
|
||||
|
||||
|
||||
def _send_partial_ready_header(fd: int) -> tuple[int, int]:
|
||||
"""Write fewer than the four required header bytes in one syscall."""
|
||||
header = struct.pack("!i", 1)
|
||||
try:
|
||||
return os.write(fd, header[:2]), len(header)
|
||||
except OSError:
|
||||
return 0, len(header)
|
||||
|
||||
|
||||
def _read_exact(fd: int, size: int) -> bytes:
|
||||
chunks = bytearray()
|
||||
while len(chunks) < size:
|
||||
chunk = os.read(fd, size - len(chunks))
|
||||
if not chunk:
|
||||
raise EOFError("PLE launch payload was truncated")
|
||||
chunks.extend(chunk)
|
||||
return bytes(chunks)
|
||||
|
||||
|
||||
def _recv_launch_payload(fd: int) -> dict:
|
||||
size = struct.unpack("!Q", _read_exact(fd, 8))[0]
|
||||
payload = pickle.loads(_read_exact(fd, size))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("invalid PLE bootstrap payload")
|
||||
return payload
|
||||
|
||||
|
||||
def _fires(
|
||||
cut: str,
|
||||
*,
|
||||
unit: str = "bootstrap-launch",
|
||||
transaction: str = "launch:1",
|
||||
bytes_written: int | None = None,
|
||||
frame_size: int | None = None,
|
||||
) -> bool:
|
||||
return _FAULT is not None and _FAULT.fires(
|
||||
cut,
|
||||
role="bootstrap",
|
||||
unit=unit,
|
||||
eligible=True,
|
||||
transaction_id=transaction,
|
||||
correlation_id="launch:1",
|
||||
bytes_written=bytes_written,
|
||||
frame_size=frame_size,
|
||||
)
|
||||
|
||||
|
||||
def _write_and_ack_partial(
|
||||
fd: int,
|
||||
cut: str,
|
||||
sender: Any,
|
||||
) -> bool:
|
||||
"""A short or failed prefix write cannot acknowledge or enter the park."""
|
||||
if _FAULT is None or _FAULT.cut != cut:
|
||||
# Disarmed or armed for another cut: the control stream is untouched.
|
||||
return False
|
||||
written, frame_size = sender(fd)
|
||||
if written <= 0 or written >= frame_size:
|
||||
return False
|
||||
expected = 2 if cut == "launch_partial_ready_header" else None
|
||||
if expected is not None and written != expected:
|
||||
return False
|
||||
if expected is None and written <= 4:
|
||||
return False
|
||||
return _fires(
|
||||
cut,
|
||||
bytes_written=written,
|
||||
frame_size=frame_size,
|
||||
)
|
||||
|
||||
|
||||
class _FaultImportLoader(importlib.abc.Loader):
|
||||
def __init__(self, delegate: importlib.abc.Loader) -> None:
|
||||
self._delegate = delegate
|
||||
|
||||
def create_module(self, spec):
|
||||
create = getattr(self._delegate, "create_module", None)
|
||||
return None if create is None else create(spec)
|
||||
|
||||
def exec_module(self, module: ModuleType) -> None:
|
||||
if _fires(
|
||||
"launch_block_worker_import",
|
||||
unit="bootstrap-import",
|
||||
transaction="module:vllm.v1.ple_offload.worker",
|
||||
):
|
||||
_block_on_harness_gate()
|
||||
self._delegate.exec_module(module)
|
||||
|
||||
|
||||
class _FaultImportFinder(importlib.abc.MetaPathFinder):
|
||||
_TARGET = "vllm.v1.ple_offload.worker"
|
||||
|
||||
def find_spec(self, fullname: str, path, target=None):
|
||||
if fullname != self._TARGET:
|
||||
return None
|
||||
spec = importlib.machinery.PathFinder.find_spec(fullname, path, target)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ImportError(f"cannot find {fullname}")
|
||||
if not isinstance(spec.loader, importlib.abc.Loader):
|
||||
raise ImportError(f"{fullname} has no executable loader")
|
||||
spec.loader = _FaultImportLoader(spec.loader)
|
||||
return spec
|
||||
|
||||
|
||||
def _install_import_gate() -> None:
|
||||
target = _FaultImportFinder._TARGET
|
||||
if target in sys.modules:
|
||||
raise RuntimeError("PLE worker import gate was a cache hit")
|
||||
sys.meta_path.insert(0, _FaultImportFinder())
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Receive launch state before importing torch or vLLM."""
|
||||
if len(sys.argv) != 3:
|
||||
raise SystemExit("expected inherited PLE control and parent-death fds")
|
||||
control_fd = int(sys.argv[1])
|
||||
death_fd = int(sys.argv[2])
|
||||
if control_fd == death_fd:
|
||||
raise RuntimeError("PLE control and parent-death fds must be distinct")
|
||||
|
||||
os.set_inheritable(control_fd, False)
|
||||
os.set_inheritable(death_fd, False)
|
||||
death_pipe = Connection(death_fd)
|
||||
|
||||
def monitor_parent() -> None:
|
||||
try:
|
||||
death_pipe.recv()
|
||||
except (EOFError, OSError):
|
||||
pass
|
||||
os._exit(1)
|
||||
|
||||
threading.Thread(
|
||||
target=monitor_parent,
|
||||
daemon=True,
|
||||
name="PleOffloadBootstrapDeathMonitor",
|
||||
).start()
|
||||
|
||||
if _fires("launch_park_before_payload_read"):
|
||||
threading.Event().wait()
|
||||
|
||||
# An armed reconstruction payload invokes its reducer inside this loads().
|
||||
payload = _recv_launch_payload(control_fd)
|
||||
|
||||
if _write_and_ack_partial(
|
||||
control_fd,
|
||||
"launch_partial_ready",
|
||||
_send_partial_ready_frame,
|
||||
):
|
||||
threading.Event().wait()
|
||||
|
||||
if _write_and_ack_partial(
|
||||
control_fd,
|
||||
"launch_partial_ready_header",
|
||||
_send_partial_ready_header,
|
||||
):
|
||||
threading.Event().wait()
|
||||
|
||||
if _fires("launch_silent_ready"):
|
||||
threading.Event().wait()
|
||||
|
||||
ready_pipe = Connection(control_fd)
|
||||
vllm_config = pickle.loads(payload["vllm_config_payload"])
|
||||
|
||||
if _FAULT is not None and _FAULT.cut == "launch_block_worker_import":
|
||||
_install_import_gate()
|
||||
from vllm.v1.ple_offload.worker import PleOffloadWorker
|
||||
|
||||
PleOffloadWorker.proc_main(
|
||||
vllm_config=vllm_config,
|
||||
num_workers=int(payload["num_workers"]),
|
||||
ipc_addr=str(payload["ipc_addr"]),
|
||||
ready_pipe=ready_pipe,
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fresh-interpreter entry point for the PLE CPU-offload worker."""
|
||||
|
||||
from vllm_ple_offload_bootstrap import main
|
||||
|
||||
main()
|
||||
Reference in New Issue
Block a user