From 025fd08603245a5880e2c9459f50d6e2a1c67137 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Wed, 23 Sep 2026 00:01:58 -0400 Subject: [PATCH] [PLE] Carry the host-mapped pull transport onto v0.30.0 Restores the fork's PLE offload package (not upstream) and its pull transport: the offload process holds no CUDA context; the captured op waits on a device-mapped host flag and copies host->GPU on the model stream; publication is fenced; releases are synchronous across dummy/capture work; failures close the rank. Includes the test-only fault-injection cuts for the transport. Co-Authored-By: Hinata Signed-off-by: MOVZX --- .../layers/ple_offload_layer.py | 433 ++++++ vllm/v1/executor/multiproc_executor.py | 31 +- vllm/v1/ple_offload/__init__.py | 2 + vllm/v1/ple_offload/connector.py | 855 +++++++++++ vllm/v1/ple_offload/fatal.py | 153 ++ vllm/v1/ple_offload/progress.py | 314 ++++ vllm/v1/ple_offload/protocol.py | 50 + vllm/v1/ple_offload/worker.py | 1298 +++++++++++++++++ vllm/v1/worker/gpu/cudagraph_utils.py | 122 +- vllm/v1/worker/gpu/model_runner.py | 140 ++ vllm/v1/worker/gpu_worker.py | 145 ++ vllm_ple_offload_bootstrap/__init__.py | 210 +++ vllm_ple_offload_bootstrap/__main__.py | 7 + 13 files changed, 3755 insertions(+), 5 deletions(-) create mode 100644 vllm/model_executor/layers/ple_offload_layer.py create mode 100644 vllm/v1/ple_offload/__init__.py create mode 100644 vllm/v1/ple_offload/connector.py create mode 100644 vllm/v1/ple_offload/fatal.py create mode 100644 vllm/v1/ple_offload/progress.py create mode 100644 vllm/v1/ple_offload/protocol.py create mode 100644 vllm/v1/ple_offload/worker.py create mode 100644 vllm_ple_offload_bootstrap/__init__.py create mode 100644 vllm_ple_offload_bootstrap/__main__.py diff --git a/vllm/model_executor/layers/ple_offload_layer.py b/vllm/model_executor/layers/ple_offload_layer.py new file mode 100644 index 0000000000..1b67578d9b --- /dev/null +++ b/vllm/model_executor/layers/ple_offload_layer.py @@ -0,0 +1,433 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Base class for PLE layers that support cross-process CPU offload. + +Synchronization between the offload process and the GPU main stream is +handled by :class:`CpuGpuSemaphore`, which uses CUDA's +``cuStreamWaitValue32`` / ``cuStreamWriteValue32`` stream memory operations. + +Decode-step timeline, host-mapped PULL design +-------------------------------------------------------- +Offload process (CPU, no CUDA) GPU main stream (forward) +------------------------------ ------------------------------ +forward_impl() on CPU ... other GPU ops ... +wait host flag == 0 WaitValue32(host flag == 1) <- in graph +rows -> shared pinned host slot cuMemcpyHtoDAsync(slot -> GPU) <- in graph +publish fence; host flag = 1 return _gpu_output_buffer[:n] + ... model consumes output ... + WriteValue32(host flag = 0) <- release_outputs + +``release_outputs`` performs the sole reset. Safety requires serialized local +forwards on one ordered model stream: a new transaction cannot begin until the +prior release has been enqueued. If any prepared forward is abandoned or its +correspondence becomes uncertain, the worker must terminate; resetting, +retrying, or continuing could reuse stale rows or overwrite live input. +""" + +import contextlib +import functools +import threading +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterator +from typing import Any, cast + +import torch +from cuda.bindings import driver as cuda_driver +from cuda.bindings.driver import CUstreamWaitValue_flags +from torch import nn + +import vllm.envs as envs +from vllm.utils.torch_utils import direct_register_custom_op + +# Module-level flag set to True inside the offload subprocess. +# Because the offload process and GPU worker processes are separate OS +# processes (spawned via multiprocessing), each has its own memory space. +# A plain module-level bool is sufficient -- no thread-local storage needed. +_offload_worker_flag = False + +# Native FULL capture runs on one Python thread. The custom-op implementation +# appends the exact arguments from every wait-copy invocation while that thread +# is inside the graph manager's capture scope. +_ple_copy_capture = threading.local() + + +@contextlib.contextmanager +def observe_ple_copy_geometry() -> Iterator[list[tuple[str, int, int]]]: + """Collect ``(layer, rows, bytes)`` from actual wait-copy executions.""" + if getattr(_ple_copy_capture, "records", None) is not None: + raise RuntimeError("nested PLE copy-geometry observation is unsupported") + records: list[tuple[str, int, int]] = [] + _ple_copy_capture.records = records + try: + yield records + finally: + del _ple_copy_capture.records + + +def _record_ple_copy_geometry( + layer_name: str, + num_tokens: int, + row_bytes: int, +) -> None: + records = getattr(_ple_copy_capture, "records", None) + if records is not None: + rows = int(num_tokens) + records.append((str(layer_name), rows, rows * int(row_bytes))) + + +def is_offload_process() -> bool: + """Return True inside the dedicated PLE CPU-offload subprocess.""" + return _offload_worker_flag + + +def mark_as_offload_worker() -> None: + """Mark the current process as the dedicated PLE offload worker.""" + global _offload_worker_flag + _offload_worker_flag = True + + +def _cuda_check(result: Any, operation: str) -> Any: + """Check the ``(CUresult, ...)`` tuple returned by cuda-python calls.""" + error = result[0] if isinstance(result, tuple) else result + if error.value != 0: + raise RuntimeError(f"{operation} failed: {error}") + return result + + +class CpuGpuSemaphore: + """Cross-process semaphore backed by a one-element int32 CUDA tensor. + + The flag is stored in a regular GPU tensor that is shared through + PyTorch's CUDA IPC mechanism. Both processes obtain device pointers that + map to the same physical GPU memory, so a stream-memory wait in the GPU + worker observes the write issued by the offload process. + """ + + RESET_VALUE = 0 + DONE_VALUE = 1 + + def __init__(self, device: torch.device) -> None: + self._flag_tensor = torch.zeros(1, dtype=torch.int32, device=device) + self._flag_ptr: int | None = None + + @classmethod + def from_ipc_tensor(cls, flag_tensor: torch.Tensor) -> "CpuGpuSemaphore": + """Construct a semaphore from a CUDA tensor received through IPC.""" + semaphore = cls.__new__(cls) + semaphore._flag_tensor = flag_tensor + semaphore._flag_ptr = None + return semaphore + + @classmethod + def from_host_mapped( + cls, host_flag_tensor: torch.Tensor, device_ptr: int + ) -> "CpuGpuSemaphore": + """Semaphore on a page-locked, device-mapped shared host int32. + + ``host_flag_tensor`` keeps the shared storage alive; ``device_ptr`` is + its device address (``cuMemHostGetDevicePointer``). Stream memory ops + on it are executed by the GPU against host memory, so a CPU process + with no CUDA context can signal it with a plain store. + """ + semaphore = cls.__new__(cls) + semaphore._flag_tensor = host_flag_tensor + semaphore._flag_ptr = int(device_ptr) + return semaphore + + @property + def flag_tensor(self) -> torch.Tensor: + """Return the tensor that backs the semaphore flag.""" + return self._flag_tensor + + @property + def flag_ptr(self) -> int: + """Device address used by the stream memory operations.""" + if self._flag_ptr is not None: + return self._flag_ptr + return self._flag_tensor.data_ptr() + + def reset(self, stream: torch.cuda.Stream | None = None) -> None: + """Enqueue ``WriteValue32(flag=0)`` on ``stream``.""" + if stream is None: + stream = torch.cuda.current_stream() + _cuda_check( + cuda_driver.cuStreamWriteValue32( + cuda_driver.CUstream(stream.cuda_stream), + cuda_driver.CUdeviceptr(self.flag_ptr), + self.RESET_VALUE, + 0, + ), + "CpuGpuSemaphore.reset", + ) + + def signal(self, stream: torch.cuda.Stream | None = None) -> None: + """Enqueue ``WriteValue32(flag=1)`` on ``stream``.""" + if stream is None: + stream = torch.cuda.current_stream() + _cuda_check( + cuda_driver.cuStreamWriteValue32( + cuda_driver.CUstream(stream.cuda_stream), + cuda_driver.CUdeviceptr(self.flag_ptr), + self.DONE_VALUE, + 0, + ), + "CpuGpuSemaphore.signal", + ) + + def wait_reset(self, stream: torch.cuda.Stream | None = None) -> None: + """Enqueue ``WaitValue32(flag==0)`` on ``stream``.""" + if stream is None: + stream = torch.cuda.current_stream() + _cuda_check( + cuda_driver.cuStreamWaitValue32( + cuda_driver.CUstream(stream.cuda_stream), + cuda_driver.CUdeviceptr(self.flag_ptr), + self.RESET_VALUE, + CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value, + ), + "CpuGpuSemaphore.wait_reset", + ) + + +# --------------------------------------------------------------------------- +# Custom op: vllm::ple_offload_wait +# --------------------------------------------------------------------------- +# Wraps cuStreamWaitValue32 as a torch custom op so that +# torch.compile(fullgraph=True) treats the CUDA Driver call as an opaque node. +# The hidden_states argument creates a dynamo data-dependency edge that prevents +# the wait from being reordered before preceding GPU work. +# --------------------------------------------------------------------------- + + +def _ple_offload_wait_impl( + sem_flag_tensor: torch.Tensor, + gpu_output_buffer: torch.Tensor, + hidden_states: torch.Tensor, +) -> None: + """Wait for the CPU result without releasing its output buffer.""" + stream = torch.cuda.current_stream() + cuda_stream = cuda_driver.CUstream(stream.cuda_stream) + dev_ptr = cuda_driver.CUdeviceptr(sem_flag_tensor.data_ptr()) + _cuda_check( + cuda_driver.cuStreamWaitValue32( + cuda_stream, + dev_ptr, + CpuGpuSemaphore.DONE_VALUE, + CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value, + ), + "cuStreamWaitValue32(done)", + ) + + +def _ple_offload_wait_fake( + sem_flag_tensor: torch.Tensor, + gpu_output_buffer: torch.Tensor, + hidden_states: torch.Tensor, +) -> None: + """Represent the side-effect-only wait during dynamo tracing.""" + pass + + +direct_register_custom_op( + op_name="ple_offload_wait", + op_func=_ple_offload_wait_impl, + mutates_args=["gpu_output_buffer"], + fake_impl=_ple_offload_wait_fake, +) + + +# --------------------------------------------------------------------------- +# Custom op: vllm::ple_offload_wait_copy (host-mapped pull design) +# --------------------------------------------------------------------------- +# The offload process writes the result into a page-locked shared host buffer +# and stores flag=1 in a page-locked shared host int32; it holds no CUDA +# context. On the CURRENT stream (captured into the graph like the wait above): +# cuStreamWaitValue32(host flag == 1) -> GPU polls host memory +# cuMemcpyAsync(gpu buffer <- host buffer, num_tokens * row_bytes) +# The copy engine reads host memory coherently after the flag was observed, +# and the consumers read device memory as before. The flag is NOT reset here: +# ``release_outputs`` resets it after the forward (unchanged semantics), so +# capture warm-ups and dummy runs behave exactly as with ``ple_offload_wait``. +# Pointers are passed as ints so no CPU tensor enters the compiled graph. +# --------------------------------------------------------------------------- + + +def _ple_offload_wait_copy_impl( + gpu_output_buffer: torch.Tensor, + hidden_states: torch.Tensor, + flag_ptr: int, + host_ptr: int, + row_bytes: int, + num_tokens: int, + layer_name: str, +) -> None: + """Wait for the host-side result, then DMA it into the GPU buffer.""" + if not 0 <= int(num_tokens) <= gpu_output_buffer.shape[0] or int(row_bytes) != ( + gpu_output_buffer.shape[1] * gpu_output_buffer.element_size() + ): + raise RuntimeError( + f"PLE wait_copy out of bounds: {num_tokens} rows x {row_bytes} B vs " + f"GPU buffer {tuple(gpu_output_buffer.shape)} {gpu_output_buffer.dtype}" + ) + nbytes = int(num_tokens) * int(row_bytes) + _record_ple_copy_geometry(layer_name, num_tokens, row_bytes) + stream = cuda_driver.CUstream(torch.cuda.current_stream().cuda_stream) + _cuda_check( + cuda_driver.cuStreamWaitValue32( + stream, + cuda_driver.CUdeviceptr(flag_ptr), + CpuGpuSemaphore.DONE_VALUE, + CUstreamWaitValue_flags.CU_STREAM_WAIT_VALUE_EQ.value, + ), + "cuStreamWaitValue32(host flag done)", + ) + if nbytes > 0: + # host_ptr is this worker's host address of page-locked (registered) + # memory, so the copy is a true async DMA and is capturable. + _cuda_check( + cuda_driver.cuMemcpyHtoDAsync( + cuda_driver.CUdeviceptr(gpu_output_buffer.data_ptr()), + host_ptr, + nbytes, + stream, + ), + "cuMemcpyHtoDAsync(PLE host -> GPU)", + ) + + +def _ple_offload_wait_copy_fake( + gpu_output_buffer: torch.Tensor, + hidden_states: torch.Tensor, + flag_ptr: int, + host_ptr: int, + row_bytes: int, + num_tokens: int, + layer_name: str, +) -> None: + """Represent the side-effect-only wait + copy during dynamo tracing.""" + pass + + +direct_register_custom_op( + op_name="ple_offload_wait_copy", + op_func=_ple_offload_wait_copy_impl, + mutates_args=["gpu_output_buffer"], + fake_impl=_ple_offload_wait_copy_fake, +) + + +class PleOffloadLayer(nn.Module, ABC): + """Base class for embedding-like PLE layers that can run on CPU. + + Subclasses implement :meth:`forward_impl`, which contains the regular + on-device computation. In a GPU worker with PLE offload enabled, subclass + constructors are skipped so their large weights are never allocated. The + CPU process constructs a complete copy and owns those weights instead. + """ + + _is_cpu_offloaded = False + _gpu_output_buffer: torch.Tensor + _sem: CpuGpuSemaphore + # Host-mapped pull design: host address of the shared output buffer and + # the bytes per output row. ``None`` keeps the IPC push behaviour. + _host_output_ptr: int | None = None + _host_row_bytes: int = 0 + _ple_offload_name: str = "" + + def __init_subclass__(cls, **kwargs: object) -> None: + """Skip subclass initialization in cross-process GPU-worker mode.""" + super().__init_subclass__(**kwargs) + original_init_obj = cls.__dict__.get("__init__") + if original_init_obj is None or not callable(original_init_obj): + return + original_init = cast(Callable[..., None], original_init_obj) + + @functools.wraps(original_init) + def guarded_init( + self: "PleOffloadLayer", *args: object, **kwargs: object + ) -> None: + if envs.VLLM_PLE_CPU_OFFLOAD and not is_offload_process(): + nn.Module.__init__(self) + return + original_init(self, *args, **kwargs) + + cls.__init__ = guarded_init # type: ignore[method-assign, assignment] + + @classmethod + def get_target_device(cls) -> torch.device: + """Return CPU for the offload process and the active GPU otherwise.""" + if envs.VLLM_PLE_CPU_OFFLOAD: + return torch.device("cpu") + return torch.device("cuda", torch.accelerator.current_device_index()) + + @abstractmethod + def forward_impl( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + *args: object, + **kwargs: object, + ) -> torch.Tensor: + """Execute the actual embedding computation on the owning device.""" + raise NotImplementedError + + def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype: + """Return the dtype used by cross-process output buffers.""" + return default_dtype + + def get_offload_output_dim(self, default_dim: int) -> int: + """Return the last dimension used by cross-process output buffers.""" + return default_dim + + def initialize_dummy_offload_metadata(self, device: torch.device) -> None: + """Initialize metadata needed by dummy offload execution.""" + + def setup_cross_process_offload( + self, + gpu_output_buffer: torch.Tensor, + semaphore: CpuGpuSemaphore, + host_output_ptr: int | None = None, + host_row_bytes: int = 0, + ) -> None: + """Configure the GPU-worker placeholder with its offload resources.""" + self._is_cpu_offloaded = True + self._gpu_output_buffer = gpu_output_buffer + self._sem = semaphore + self._host_output_ptr = host_output_ptr + self._host_row_bytes = int(host_row_bytes) + + def forward( + self, + hidden_states: torch.Tensor, + input_ids: torch.Tensor, + *args: object, + **kwargs: object, + ) -> torch.Tensor: + """Wait for an offloaded result or delegate to ``forward_impl``.""" + if self._is_cpu_offloaded: + if self._host_output_ptr is not None: + torch.ops.vllm.ple_offload_wait_copy( + self._gpu_output_buffer, + hidden_states, + self._sem.flag_ptr, + self._host_output_ptr, + self._host_row_bytes, + input_ids.shape[0], + self._ple_offload_name, + ) + else: + torch.ops.vllm.ple_offload_wait( + self._sem.flag_tensor, + self._gpu_output_buffer, + hidden_states, + ) + return self._gpu_output_buffer[: input_ids.shape[0]] + return self.forward_impl(hidden_states, input_ids, *args, **kwargs) + + def release_offloaded_output( + self, + stream: torch.cuda.Stream | None = None, + ) -> None: + """Mark the cross-process output buffer reusable on ``stream``.""" + if self._is_cpu_offloaded: + self._sem.reset(stream) diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 07263263a7..bd164656b5 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -352,9 +352,10 @@ class MultiprocExecutor(Executor): proc = next(h.proc for h in workers if h.proc.sentinel == died[0]) logger.error( "Worker proc %s died unexpectedly (exit code: %s), " - "shutting down executor.", + "shutting down executor. executor_shutdown_monotonic=%.9f", proc.name, proc.exitcode, + time.monotonic(), ) _self.shutdown() callback = _self.failure_callback @@ -1039,6 +1040,28 @@ class WorkerProc: SUCCESS = auto() FAILURE = auto() + def _fail_ple_rpc_if_pending( + self, where: str, error: BaseException + ) -> None: + """Fail closed if RPC recovery would abandon queued PLE GPU work.""" + owners = (self.worker, getattr(self.worker, "worker", None)) + for owner in owners: + if owner is None: + continue + runner = getattr(owner, "model_runner", None) + connector = getattr(runner, "_ple_offload_connector", None) + if connector is None: + continue + try: + pending = connector.has_pending_completions + except BaseException as state_error: + connector._fail_closed( + f"{where}:pending-state", state_error + ) + if pending: + connector._fail_closed(where, error) + return + def enqueue_output(self, output: Any): """Prepares output from the worker and enqueues it to the worker_response_mq. If the output is an Exception, it is @@ -1049,6 +1072,9 @@ class WorkerProc: output = output.get_output() except Exception as e: logger.exception("Error getting async model runner output") + self._fail_ple_rpc_if_pending( + "async-output-copy", e + ) output = e if isinstance(output, Exception): @@ -1108,6 +1134,9 @@ class WorkerProc: if output_rank is None or self.rank == output_rank: self.handle_output(output) except Exception as e: + self._fail_ple_rpc_if_pending( + f"worker-rpc:{method}", e + ) # Notes have been introduced in python 3.11 if hasattr(e, "add_note"): e.add_note(traceback.format_exc()) diff --git a/vllm/v1/ple_offload/__init__.py b/vllm/v1/ple_offload/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/ple_offload/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/ple_offload/connector.py b/vllm/v1/ple_offload/connector.py new file mode 100644 index 0000000000..52bc4622e7 --- /dev/null +++ b/vllm/v1/ple_offload/connector.py @@ -0,0 +1,855 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Exchange PLE data between a GPU worker and the CPU-offload process.""" + +import os +import queue +import threading +from dataclasses import dataclass +from multiprocessing.reduction import ForkingPickler +from typing import Any + +import msgspec +import torch +from vllm.v1.ple_offload import rowdump as _rowdump +from vllm.v1.ple_offload.fatal import ple_fail_closed +from vllm.v1.ple_offload.progress import PleProgressMonitor +import torch.nn as nn +import zmq +from cuda.bindings import driver as cuda_driver + +from vllm.config import VllmConfig +from vllm.distributed.parallel_state import get_dp_group, get_pp_group, get_tp_group +from vllm.logger import init_logger +from vllm.model_executor.layers.ple_offload_layer import ( + CpuGpuSemaphore, + PleOffloadLayer, +) +from vllm.v1.ple_offload.protocol import ( + PleOffloadRegistration, + PleOffloadRequest, +) +from vllm.v1.utils import record_function_or_nullcontext +from vllm_ple_test_fault import _FAULT + +logger = init_logger(__name__) + +# CUDA registration alone does not own Python storage. Captured graphs retain +# raw addresses, so every successfully registered host allocation remains +# strongly referenced until process exit. +_PROCESS_LIFETIME_REGISTERED_STORAGE: list[torch.Tensor] = [] + + +def _unlink_filename_shared_storage( + tensor: torch.Tensor, + reserved_filenames: set[bytes], +) -> None: + """Reserve and unlink one already filename-shared CPU storage.""" + try: + storage = tensor.untyped_storage() + if storage.device.type != "cpu" or not storage.is_shared(): + return + try: + storage._get_shared_fd() + except RuntimeError: + # Filename-shared storages do not expose a shared fd. Only now is + # it safe to query the filename: doing so on ordinary or fd-shared + # storage would convert it to a new filename-backed allocation. + pass + else: + return + _, filename, _ = storage._share_filename_cpu_() + if not isinstance(filename, bytes): + filename = os.fsencode(filename) + path = os.path.join("/dev/shm", os.fsdecode(filename).lstrip("/")) + except Exception: + logger.warning( + "Failed to inspect a PLE shared-memory storage during teardown", + exc_info=True, + ) + return + + # RefcountedMapAllocator unlinks the name when its shared counter reaches + # zero. Reserve one count before manual unlink so later storage destruction + # cannot try to unlink the already-removed name. The filename identifies + # the shared counter and also deduplicates tensor aliases and repeated close. + if filename in reserved_filenames: + return + try: + storage._shared_incref() + except Exception: + logger.warning( + "Failed to reserve the PLE shared-memory counter for %s", + path, + exc_info=True, + ) + return + reserved_filenames.add(filename) + + try: + os.unlink(path) + except FileNotFoundError: + # Another teardown participant already removed the name. Retain the + # reservation because this allocator must still avoid a later unlink. + pass + except OSError as error: + # The name remains available, so restore the allocator's original + # counter and permit a later release attempt to retry. + try: + storage._shared_decref() + except Exception: + # The reservation state is uncertain after a failed private call. + # Keep it marked to avoid adding another reservation on retry. + logger.warning( + "Failed to roll back the PLE shared-memory counter for %s", + path, + exc_info=True, + ) + else: + reserved_filenames.discard(filename) + logger.warning("Failed to unlink PLE shared memory %s: %s", path, error) + + +@dataclass(frozen=True) +class _PendingPleOffloadRequest: + """Bind request metadata to its MRV2 D2H completion event.""" + + request: PleOffloadRequest + d2h_done_event: torch.cuda.Event + eligible: bool = False + transaction_id: str | None = None + correlation_id: str | None = None + + +def _cuda_check(result: Any, operation: str) -> Any: + """Check the ``(CUresult, ...)`` tuple returned by cuda-python calls.""" + error = result[0] if isinstance(result, tuple) else result + if error.value != 0: + raise RuntimeError(f"{operation} failed: {error}") + return result + + +class PleOffloadConnector: + """Connect a model runner V2 GPU worker to the PLE CPU worker.""" + + def __init__( + self, + vllm_config: VllmConfig, + model: nn.Module, + device: torch.device, + ipc_addr: str, + *, + input_ids_source: torch.Tensor, + query_start_loc_source: torch.Tensor, + ngram_context_source: torch.Tensor | None, + ) -> None: + self.device = device + self.rank = int(vllm_config.parallel_config.rank) + self.dp_rank = get_dp_group().rank_in_group + self.tp_rank = get_tp_group().rank_in_group + self.pp_rank = get_pp_group().rank_in_group + # Host-mapped pull design: per layer a shared host output buffer and a + # shared host int32 flag (element 0), filled/stored by the offload + # process with plain CPU writes. Page-locked + device-mapped here only + # after registration, because serialising for registration may move + # the shared storage to a new address. + self._host_output_bufs: dict[str, torch.Tensor] = {} + self._host_flags: dict[str, torch.Tensor] = {} + self._reserved_host_storage_filenames: set[bytes] = set() + self._gpu_output_bufs: dict[str, torch.Tensor] = {} + self._layers = self._setup_layers(vllm_config, model) + self._progress = PleProgressMonitor.from_environment( + rank=self.rank, + layer_names=tuple(self._layers), + ) + + # TP0 registers shared buffers with CUDA for asynchronous D2H copies. + scheduler_config = vllm_config.scheduler_config + self._input_ids_buf = torch.empty( + scheduler_config.max_num_batched_tokens, + dtype=torch.int32, + device="cpu", + ).share_memory_() + self._query_start_loc_buf = torch.empty( + scheduler_config.max_num_seqs + 1, + dtype=torch.int32, + device="cpu", + ).share_memory_() + self._ngram_context_buf = None + config = vllm_config.model_config.hf_text_config + ngram_context_len = int(config.ngram_size) - 1 + if ngram_context_len > 0: + self._ngram_context_buf = torch.empty( + scheduler_config.max_num_seqs, + ngram_context_len, + dtype=torch.int32, + device="cpu", + ).share_memory_() + + # Runner input allocations are address-stable, so bind them once and + # pass only batch sizes through the per-forward request queue. + self._input_ids_source = input_ids_source + self._query_start_loc_source = query_start_loc_source + self._ngram_context_source = ngram_context_source + self._validate_input_sources() + + self._pinned_input_buffers: list[torch.Tensor] = [] + # Device-mapped host outputs/flags referenced by raw address from + # captured graphs: never unregistered while the process lives + # (unregistering under a pending graph wait unmaps it). + self._mapped_host_buffers: list[torch.Tensor] = [] + # True while dummy/capture work (seeds, dummy forwards, capture + # cleanup resets) has run since the last real forward. Releases in + # that state complete synchronously, and the next real forward first + # passes a TP barrier, so no rank can publish a real request while + # another rank still has a dummy/cleanup reset pending. + # In this tree real-path kernel warm-ups run BEFORE + # capture and capture adds seeds/resets after them, so this must hold + # at every dummy->real transition, not only at the first real one. + self._after_dummy = True + self._request_queue: queue.Queue[_PendingPleOffloadRequest | None] = ( + queue.Queue(maxsize=vllm_config.max_concurrent_batches) + ) + self._dropped_events: list[torch.cuda.Event] = [] + self._request_thread: threading.Thread | None = None + self._request_thread_ready = threading.Event() + self._zmq_ctx: zmq.Context | None = None + self._registration_socket: zmq.Socket | None = None + self._d2h_event_pool: queue.Queue[torch.cuda.Event] | None = None + + try: + self._zmq_ctx = zmq.Context() + self._registration_socket = self._zmq_ctx.socket(zmq.PUSH) + self._registration_socket.connect(ipc_addr) + payload = self._serialize_registration(vllm_config) + + # Serialising moved the shared storages to their final addresses. + # Page-lock/map exactly those, THEN publish, so a received + # registration always means "storage usable". + with torch.accelerator.device_index(self.device.index): + self._map_host_outputs(vllm_config) + if self.tp_rank == 0: + self._pin_input_buffers() + self._send_registration(payload, ipc_addr) + + if self.tp_rank == 0: + with torch.accelerator.device_index(self.device.index): + self._d2h_event_pool = queue.Queue( + maxsize=vllm_config.max_concurrent_batches + ) + for _ in range(vllm_config.max_concurrent_batches): + self._d2h_event_pool.put_nowait(torch.cuda.Event()) + self._start_request_thread(ipc_addr) + except Exception: + self.close() + raise + + def _setup_layers( + self, + vllm_config: VllmConfig, + model: nn.Module, + ) -> dict[str, PleOffloadLayer]: + """Attach output buffers and semaphores to GPU PLE placeholders.""" + layers = { + name: module + for name, module in model.named_modules() + if isinstance(module, PleOffloadLayer) + } + if not layers: + raise RuntimeError( + "VLLM_PLE_CPU_OFFLOAD is enabled, but the model has no PleOffloadLayer" + ) + + config = vllm_config.model_config.hf_text_config + max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens + for name, layer in layers.items(): + if vllm_config.load_config.load_format == "dummy": + layer.initialize_dummy_offload_metadata(self.device) + out_dim = layer.get_offload_output_dim(int(config.ple_embed_dim)) + out_dtype = layer.get_offload_output_dtype(vllm_config.model_config.dtype) + # The captured forward DMAs the host result into this device + # buffer after the host flag is set; consumers read it as before. + self._gpu_output_bufs[name] = torch.empty( + max_num_tokens, out_dim, dtype=out_dtype, device=self.device + ) + # Zero-initialised so dummy/profile forwards copy finite values. + self._host_output_bufs[name] = torch.zeros( + max_num_tokens, out_dim, dtype=out_dtype, device="cpu" + ).share_memory_() + # 16 x int32 = one 64-byte line; element 0 is the flag. + self._host_flags[name] = torch.zeros( + 16, dtype=torch.int32, device="cpu" + ).share_memory_() + return layers + + def _map_host_outputs(self, vllm_config: VllmConfig) -> None: + """Page-lock + device-map host outputs/flags and arm the placeholders.""" + for name, layer in self._layers.items(): + layer._ple_offload_name = name + host_out = self._host_output_bufs[name] + host_flag = self._host_flags[name] + for buffer in (host_out, host_flag): + self._register_host_buffer(buffer, device_map=True) + flag_dev_ptr = _cuda_check( + cuda_driver.cuMemHostGetDevicePointer(host_flag.data_ptr(), 0), + "cuMemHostGetDevicePointer(PLE host flag)", + )[1] + out_dev_ptr = _cuda_check( + cuda_driver.cuMemHostGetDevicePointer(host_out.data_ptr(), 0), + "cuMemHostGetDevicePointer(PLE host output)", + )[1] + row_bytes = host_out.shape[1] * host_out.element_size() + gpu_buf = self._gpu_output_bufs[name] + if ( + gpu_buf.shape[0] != host_out.shape[0] + or gpu_buf.shape[1] * gpu_buf.element_size() != row_bytes + ): + raise RuntimeError(f"PLE host/GPU buffer geometry mismatch for {name}") + # Stream memops use the flag's DEVICE pointer; the H2D copy uses + # this worker's HOST pointer (no raw addresses cross + # processes; the offload writes through its own mapping). + layer.setup_cross_process_offload( + gpu_buf, + CpuGpuSemaphore.from_host_mapped(host_flag, int(flag_dev_ptr)), + host_output_ptr=host_out.data_ptr(), + host_row_bytes=row_bytes, + ) + logger.info( + "PleOffload: host-mapped pull for %s (rows %d x %d B, flag dev " + "0x%x, out dev 0x%x, host==dev %s)", + name, + host_out.shape[0], + row_bytes, + int(flag_dev_ptr), + int(out_dev_ptr), + int(out_dev_ptr) == host_out.data_ptr(), + ) + + def _pin_input_buffers(self) -> None: + """Page-lock shared input allocations without replacing their storage.""" + buffers = [self._input_ids_buf, self._query_start_loc_buf] + if self._ngram_context_buf is not None: + buffers.append(self._ngram_context_buf) + for buffer in buffers: + self._register_host_buffer(buffer, device_map=False) + + def _register_host_buffer(self, buffer: torch.Tensor, device_map: bool) -> None: + """Page-lock (and optionally device-map) a shared CPU allocation.""" + if buffer.device.type != "cpu" or not buffer.is_shared(): + raise RuntimeError("PLE host buffers must be shared CPU tensors") + if not buffer.is_contiguous(): + raise RuntimeError("PLE host buffers must be contiguous") + flags = cuda_driver.CU_MEMHOSTREGISTER_PORTABLE + if device_map: + flags |= cuda_driver.CU_MEMHOSTREGISTER_DEVICEMAP + _cuda_check( + cuda_driver.cuMemHostRegister( + buffer.data_ptr(), + buffer.numel() * buffer.element_size(), + flags, + ), + "cuMemHostRegister(PLE host buffer)", + ) + _PROCESS_LIFETIME_REGISTERED_STORAGE.append(buffer) + if device_map: + self._mapped_host_buffers.append(buffer) + else: + self._pinned_input_buffers.append(buffer) + if not buffer.is_pinned(): + raise RuntimeError("CUDA did not page-lock a PLE host buffer") + + def _unpin_input_buffers(self) -> None: + """Release CUDA registrations after the request thread has stopped.""" + for buffer in reversed(self._pinned_input_buffers): + try: + _cuda_check( + cuda_driver.cuMemHostUnregister(buffer.data_ptr()), + "cuMemHostUnregister(PLE input buffer)", + ) + except RuntimeError: + logger.exception("Failed to unregister a PLE input buffer") + self._pinned_input_buffers.clear() + + def _serialize_registration(self, vllm_config: VllmConfig) -> Any: + """Build and serialise the registration (may move shared storages).""" + # Each GPU worker owns distinct host output buffers and flags, while + # TP0's shared inputs become the request source for its DP rank. + registration = PleOffloadRegistration( + worker_id=( + self.dp_rank * vllm_config.parallel_config.world_size + + vllm_config.parallel_config.rank + ), + tp_rank=self.tp_rank, + dp_rank=self.dp_rank, + host_output_buffers=dict(self._host_output_bufs), + host_flag_tensors=dict(self._host_flags), + input_ids_buf=self._input_ids_buf, + query_start_loc_buf=self._query_start_loc_buf, + ngram_context_buf=self._ngram_context_buf, + ) + + # ForkingPickler transmits tensors through shared-memory and CUDA IPC. + import torch.multiprocessing as torch_mp + + original_strategy = torch_mp.get_sharing_strategy() + torch_mp.set_sharing_strategy("file_system") + try: + payload = ForkingPickler.dumps(registration) + finally: + torch_mp.set_sharing_strategy(original_strategy) + return payload + + def _send_registration(self, payload: Any, ipc_addr: str) -> None: + """Publish the serialised registration (storages already mapped).""" + assert self._registration_socket is not None + self._registration_socket.send(payload) + + logger.info( + "PleOffload: registered %d PleOffloadLayer(s) " + "(dp_rank=%d, tp_rank=%d, ipc_addr=%s): %s", + len(self._layers), + self.dp_rank, + self.tp_rank, + ipc_addr, + sorted(self._layers), + ) + + def _start_request_thread(self, ipc_addr: str) -> None: + """Start the thread that publishes batches after inputs are ready.""" + self._request_thread = threading.Thread( + target=self._request_loop, + args=(ipc_addr,), + name=f"ple-offload-dp{self.dp_rank}", + daemon=True, + ) + self._request_thread.start() + if not self._request_thread_ready.wait(timeout=10): + raise RuntimeError("Timed out starting the PLE request thread") + + def _request_loop(self, ipc_addr: str) -> None: + """Wait for staged inputs, then notify the CPU worker.""" + socket: zmq.Socket | None = None + try: + if self._zmq_ctx is None: + raise RuntimeError("PLE ZMQ context closed before thread startup") + socket = self._zmq_ctx.socket(zmq.PUSH) + socket.connect(ipc_addr) + self._request_thread_ready.set() + while True: + request = self._request_queue.get() + if request is None: + return + self._process_request(request, socket) + except Exception: + logger.exception("PLE request thread failed") + os._exit(1) + finally: + self._request_thread_ready.set() + if socket is not None: + socket.close(linger=0) + + def _process_request( + self, pending: _PendingPleOffloadRequest, socket: zmq.Socket + ) -> None: + """Wait for one staged batch and publish its request.""" + request = pending.request + event_pool = self._d2h_event_pool + event = pending.d2h_done_event + assert event_pool is not None, "PLE D2H event pool is not initialized" + if _FAULT is not None and _FAULT.fires( + "block_input_completion", + role="gpu", + unit="connector-request", + eligible=pending.eligible, + tp_rank=self.tp_rank, + pp_rank=self.pp_rank, + dp_rank=self.dp_rank, + transaction_id=pending.transaction_id, + correlation_id=pending.correlation_id, + ): + threading.Event().wait() + + with torch.accelerator.device_index(self.device.index): + event.synchronize() + + payload = msgspec.msgpack.encode(request) + if _FAULT is None: + socket.send(payload) + else: + # Armed runs only: correlation metadata rides as a second frame. + metadata = msgspec.msgpack.encode( + { + "serving": pending.eligible, + "transaction": pending.transaction_id, + "correlation": pending.correlation_id, + } + ) + socket.send_multipart([payload, metadata]) + event_pool.put_nowait(event) + + def _validate_input_sources(self) -> None: + """Validate fixed runner sources against shared input buffers.""" + sources = [ + ("input_ids", self._input_ids_source, self._input_ids_buf), + ( + "query_start_loc", + self._query_start_loc_source, + self._query_start_loc_buf, + ), + ] + if (self._ngram_context_source is None) != (self._ngram_context_buf is None): + raise ValueError("PLE ngram_context source and buffer must match") + if self._ngram_context_source is not None: + assert self._ngram_context_buf is not None + sources.append( + ( + "ngram_context", + self._ngram_context_source, + self._ngram_context_buf, + ) + ) + + for name, source, buffer in sources: + if ( + source.device != self.device + or source.dtype != buffer.dtype + or source.ndim != buffer.ndim + or source.shape[0] < buffer.shape[0] + or source.shape[1:] != buffer.shape[1:] + ): + raise ValueError(f"PLE {name} source is incompatible") + + def _enqueue_cuda_inputs( + self, + request: PleOffloadRequest, + d2h_done_event: torch.cuda.Event, + ) -> None: + """Stage inputs on the model stream and record D2H completion.""" + with ( + record_function_or_nullcontext("ple_offload: enqueue_cuda_inputs"), + torch.accelerator.device_index(self.device.index), + ): + stream = torch.cuda.current_stream(self.device) + self._input_ids_buf[: request.num_tokens].copy_( + self._input_ids_source[: request.num_tokens], + non_blocking=True, + ) + self._query_start_loc_buf[: request.num_reqs + 1].copy_( + self._query_start_loc_source[: request.num_reqs + 1], + non_blocking=True, + ) + if self._ngram_context_buf is not None: + assert self._ngram_context_source is not None + self._ngram_context_buf[: request.num_reqs].copy_( + self._ngram_context_source[: request.num_reqs], + non_blocking=True, + ) + d2h_done_event.record(stream) + + def _launch( + self, + num_reqs: int, + num_tokens: int, + *, + serving: bool = False, + transaction_id: str | None = None, + correlation_id: str | None = None, + ) -> None: + """Stage or queue one batch for request publication.""" + # Inputs are replicated across TP ranks. One request per DP rank drives + # the CPU result fan-out to every registered TP output buffer. + if self.tp_rank != 0: + return + + request = PleOffloadRequest( + dp_rank=self.dp_rank, + num_tokens=num_tokens, + num_reqs=num_reqs, + ) + assert self._d2h_event_pool is not None, "PLE D2H event pool is not initialized" + try: + d2h_done_event = self._d2h_event_pool.get_nowait() + except queue.Empty as exc: + raise RuntimeError( + "PLE has more requests than configured concurrent batches" + ) from exc + self._enqueue_cuda_inputs(request, d2h_done_event) + if _FAULT is not None and _FAULT.fires( + "drop_request", + role="gpu", + unit="connector-request", + eligible=serving, + tp_rank=self.tp_rank, + pp_rank=self.pp_rank, + dp_rank=self.dp_rank, + transaction_id=transaction_id, + correlation_id=correlation_id, + ): + # Deliberately retain the event: the fault is loss after D2H, not + # premature event-pool reuse while that D2H may still be in flight. + self._dropped_events.append(d2h_done_event) + return + self._request_queue.put_nowait( + _PendingPleOffloadRequest( + request, + d2h_done_event, + eligible=serving, + transaction_id=transaction_id, + correlation_id=correlation_id, + ) + ) + + def _fail_closed(self, where: str, error: BaseException) -> None: + ple_fail_closed( + where=where, + rank=self.rank, + layer=",".join(self._layers), + sequence=self._progress.sequence, + error=error, + ) + + @property + def has_pending_completions(self) -> bool: + return self._progress.has_pending_completions + + def begin_forward(self, *, dummy_run: bool, num_tokens: int) -> int: + """Begin the bounded transaction before any runner fault boundary.""" + try: + return self._progress.begin( + startup=dummy_run, + num_tokens=num_tokens, + ) + except Exception as error: + self._fail_closed("prepare_forward", error) + + def fault_transaction(self, sequence: int) -> str: + """Read diagnostic state only when the caller has established arming.""" + return self._progress.describe_submission(sequence) + + def completion_is_pending(self, sequence: int) -> bool: + """Return exact-sequence incomplete completion state for cut 6.""" + return self._progress.completion_is_pending(sequence) + + def prepare_forward( + self, + num_reqs: int, + num_tokens: int, + dummy_run: bool, + *, + consumer_num_tokens: int | None = None, + serving: bool = False, + transaction_id: str | None = None, + correlation_id: str | None = None, + ) -> None: + """Prepare one PLE transaction; any ordinary exception is fatal.""" + try: + self._prepare_forward_impl( + num_reqs, + num_tokens, + dummy_run, + consumer_num_tokens=consumer_num_tokens, + serving=serving, + transaction_id=transaction_id, + correlation_id=correlation_id, + ) + except Exception as error: + self._fail_closed("prepare_forward", error) + + def _prepare_forward_impl( + self, + num_reqs: int, + num_tokens: int, + dummy_run: bool, + *, + consumer_num_tokens: int | None = None, + serving: bool = False, + transaction_id: str | None = None, + correlation_id: str | None = None, + ) -> None: + """Submit real inputs or satisfy the PLE wait for a dummy forward.""" + if consumer_num_tokens is None: + consumer_num_tokens = num_tokens + if int(num_tokens) != int(consumer_num_tokens): + raise RuntimeError( + "PLE request/consumer token-count mismatch: " + f"request={num_tokens}, consumer={consumer_num_tokens}" + ) + if not 0 <= int(num_tokens) <= self._input_ids_buf.shape[0]: + raise RuntimeError( + f"PLE token count {num_tokens} exceeds connector capacity " + f"{self._input_ids_buf.shape[0]}" + ) + if not 0 <= int(num_reqs) < self._query_start_loc_buf.shape[0]: + raise RuntimeError( + f"PLE request count {num_reqs} exceeds connector capacity " + f"{self._query_start_loc_buf.shape[0] - 1}" + ) + self._rowdump_n = None if dummy_run else num_tokens + if dummy_run: + self._signal_dummy_outputs_impl(num_tokens) + return + if self._after_dummy: + # Every dummy/cleanup reset on THIS rank already completed + # (synchronous releases); the barrier extends that to every + # stage-0 TP rank before TP0 can publish (CPU group, no GPU work; + # all TP ranks run the same forwards in the same order). + get_tp_group().barrier() + self._after_dummy = False + logger.info( + "PleOffload: dummy->real transition (dp_rank=%d, tp_rank=%d): " + "releases were synchronous, TP barrier passed", + self.dp_rank, + self.tp_rank, + ) + self._launch( + num_reqs, + num_tokens, + serving=serving, + transaction_id=transaction_id, + correlation_id=correlation_id, + ) + + def signal_dummy_outputs(self, num_tokens: int) -> None: + """Locally satisfy PLE waits for dummy and capture forwards.""" + self._progress.begin(startup=True, num_tokens=num_tokens) + self._signal_dummy_outputs_impl(num_tokens) + + def _signal_dummy_outputs_impl(self, num_tokens: int) -> None: + """Signal dummy outputs after the progress transaction is armed.""" + # Dummy and capture forwards do not send CPU requests, but every PLE + # placeholder still waits for a completed output semaphore. + self._after_dummy = True + stream = torch.cuda.current_stream(self.device) + for layer in self._layers.values(): + layer._gpu_output_buffer[:num_tokens].zero_() + layer._sem.signal(stream) + + def release_outputs( + self, + *, + serving: bool = False, + transaction_id: str | None = None, + correlation_id: str | None = None, + ) -> None: + """Release one PLE transaction; any ordinary exception is fatal.""" + try: + self._release_outputs_impl( + serving=serving, + transaction_id=transaction_id, + correlation_id=correlation_id, + ) + self._progress.complete() + except Exception as error: + self._fail_closed("release_outputs", error) + + def _release_outputs_impl( + self, + *, + serving: bool = False, + transaction_id: str | None = None, + correlation_id: str | None = None, + ) -> None: + """Mark GPU output buffers reusable after the model consumes them.""" + if _FAULT is not None and _FAULT.fires( + "exc_in_release", + role="gpu", + unit="connector-release", + eligible=serving, + tp_rank=self.tp_rank, + pp_rank=self.pp_rank, + dp_rank=self.dp_rank, + transaction_id=transaction_id, + correlation_id=correlation_id, + ): + raise RuntimeError("PLE test fault: exc_in_release") + n = getattr(self, "_rowdump_n", None) + self._rowdump_n = None + if _rowdump.enabled() and n is not None and not torch.cuda.is_current_stream_capturing(): + torch.cuda.current_stream(self.device).synchronize() + seq = getattr(self, "_rowdump_seq", 0) + self._rowdump_seq = seq + 1 + for layer in self._layers.values(): + _rowdump.write(f"gpu-tp{self.tp_rank}", 0, self.tp_rank, seq, + self._input_ids_source[:n], layer._gpu_output_buffer[:n]) + # Reset only after the consumer forward so the CPU worker cannot + # overwrite an output that a GPU PLE placeholder may still read. + stream = torch.cuda.current_stream(self.device) + for layer in self._layers.values(): + layer.release_offloaded_output(stream) + # Retain the serving deadline until the model stream has actually + # executed every reset above. The monitor thread is the sole query site. + completion_event = torch.cuda.Event() + completion_event.record(stream) + self._progress.track_completion(completion_event) + # After dummy/capture work (profile, warm-up sizes, capture seed, + # adaptive-verification dummy replays, capture cleanup): complete the + # reset before returning, so it cannot still be pending when a real + # request lets the offload publish. Real-forward releases stay async. + # Never inside a stream capture. + if self._after_dummy and not torch.cuda.is_current_stream_capturing(): + stream.synchronize() + + def release_host_storage(self) -> None: + """Unlink host-storage names while retaining mappings and tensors.""" + buffers = [ + *self._host_output_bufs.values(), + *self._host_flags.values(), + self._input_ids_buf, + self._query_start_loc_buf, + ] + if self._ngram_context_buf is not None: + buffers.append(self._ngram_context_buf) + for buffer in buffers: + _unlink_filename_shared_storage( + buffer, + self._reserved_host_storage_filenames, + ) + + def close(self) -> None: + """Stop transport and unlink names, retaining host mappings.""" + request_thread = self._request_thread + if request_thread is not None and request_thread.is_alive(): + try: + self._request_queue.put(None, timeout=5) + except queue.Full: + logger.error("Timed out stopping the PLE request thread") + request_thread.join(timeout=5) + if request_thread is not None and request_thread.is_alive(): + # The thread may still access the registered buffers or ZMQ context. + # It may also still race another process opening a shared name, so + # this deferred-cleanup branch must not unlink host storage. + logger.error("PLE request thread did not stop; deferring resource cleanup") + self._progress.close() + return + self._request_thread = None + + # Inputs, outputs, and flags remain registered and strongly referenced. + # A captured graph or delayed request can still contain their addresses; + # the CUDA driver releases registrations when the process exits. + registered = len(self._pinned_input_buffers) + len(self._mapped_host_buffers) + if registered: + logger.debug( + "PleOffload: %d registered host buffers retained until exit", + registered, + ) + self._d2h_event_pool = None + if self._registration_socket is not None: + self._registration_socket.close(linger=0) + self._registration_socket = None + if self._zmq_ctx is not None: + self._zmq_ctx.term() + self._zmq_ctx = None + self._progress.close() + + # __init__'s failure handler can reach here before the child has rebuilt + # a registration. Startup is already failing in that case; if a payload + # was sent, a late rebuild may also observe the removed name. Successful + # startup accepts every registration before an external close begins. + # + # NOTE: Graceful shutdown unlinks the PLE shared-memory names and the + # PLE ipc socket. The remaining 17 /tmp UUID sockets per boot are + # upstream vLLM VLLM_RPC_BASE_PATH behaviour; a tmpfiles/launch-profile + # sweep is recommended. + # Keep this last: unlinking preserves every existing CPU/CUDA mapping, + # but no transport user may still need to open a name at this point. + self.release_host_storage() diff --git a/vllm/v1/ple_offload/fatal.py b/vllm/v1/ple_offload/fatal.py new file mode 100644 index 0000000000..a80cad4b8d --- /dev/null +++ b/vllm/v1/ple_offload/fatal.py @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fail-closed helpers for the PLE single-slot transport.""" + +import functools +import os +import time +import traceback +from collections.abc import Callable +from typing import Any, NoReturn, TypeVar + +_F = TypeVar("_F", bound=Callable[..., Any]) + +_TRUSTED_ERROR_LABELS = ( + (RuntimeError, "RuntimeError"), + (TimeoutError, "TimeoutError"), + (ValueError, "ValueError"), + (OSError, "OSError"), + (EOFError, "EOFError"), + (BrokenPipeError, "BrokenPipeError"), + (AssertionError, "AssertionError"), + (KeyError, "KeyError"), + (IndexError, "IndexError"), + (TypeError, "TypeError"), + (MemoryError, "MemoryError"), + (KeyboardInterrupt, "KeyboardInterrupt"), +) + + +def ple_fail_closed( + *, + where: str, + rank: int, + layer: str, + sequence: int, + error: BaseException, +) -> NoReturn: + """Emit one best-effort nonblocking diagnostic and terminate immediately.""" + try: + os.set_blocking(2, False) + error_type = type(error) + error_name = "Exception" + trusted_error = False + for trusted_type, trusted_name in _TRUSTED_ERROR_LABELS: + if error_type is trusted_type: + error_name = trusted_name + trusted_error = True + break + + safe_where = where[:128] if type(where) is str else "unknown" + safe_layer = layer[:256] if type(layer) is str else "unknown" + safe_rank = rank if type(rank) is int else -1 + safe_sequence = sequence if type(sequence) is int else -1 + detail_suffix = "" + if trusted_error: + args = BaseException.args.__get__(error, error_type) + if ( + type(args) is tuple + and len(args) == 1 + and type(args[0]) is str + ): + detail = args[0][:512].replace("\n", "\\n") + detail_suffix = f":{detail}" + message = ( + f"PLE fatal: where={safe_where} rank={safe_rank} layer={safe_layer} " + f"sequence={safe_sequence} error={error_name}{detail_suffix} " + f"pid={os.getpid()} " + f"monotonic={time.monotonic():.9f} wall={time.time():.9f}\n" + ) + os.write(2, message.encode("utf-8", "backslashreplace")[:2048]) + except BaseException: + pass + # THE TRACEBACK, unconditionally, in its own try so it cannot cost us the + # line above. + # + # This path os._exit(1)s, which runs no handlers and flushes nothing, so + # whatever is not written here is gone. The line above names the TYPE only + # when the type is one of _TRUSTED_ERROR_LABELS and flattens everything + # else to the literal string "Exception" -- and the flattened set includes + # the two that matter most here: torch.AcceleratorError (a RuntimeError + # SUBCLASS, so the `is` comparison misses it) and Triton CompilationError. + # An earlier boot died exactly that way: both PP0 ranks + # reported "error=Exception" and there was no traceback anywhere in the + # log, in any rank, so the cause could not be named at all and the next + # step was a re-run rather than a fix. + # + # A fail-closed path is where evidence matters most, not least: it fires + # once, it kills the process, and it is the only witness. Printing the type + # of a wrapped exception without the frames that raised it is not a + # diagnostic. + try: + # Three-arg form: it does not depend on being inside an active except + # block, and this function is called from one frame further out than + # the raise. + tb = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + raw = tb.encode("utf-8", "backslashreplace") + # Truncate from the END, not the start. format_exception puts the + # frames first and the exception TYPE AND MESSAGE last, so a head-side + # cut throws away the one line that names what went wrong and keeps the + # import-time frames nobody needs. A long chained traceback is exactly + # the case where that matters. + clipped = len(raw) > 8192 + raw = raw[-8192:] + os.write( + 2, + b"PLE fatal traceback (tail) follows:\n" if clipped + else b"PLE fatal traceback follows:\n", + ) + os.write(2, raw) + if not raw.endswith(b"\n"): + os.write(2, b"\n") + except BaseException: + # Never silent HERE of all places: without this the reader cannot tell + # "the wrapper predates the traceback fix" from "the traceback itself + # could not be produced", and those call for different next steps. + # Still guarded, because os._exit below must run whatever happens: an + # escape from this handler would leave the worker alive in the state + # the fail-closed path exists to end. + try: + os.write(2, b"PLE fatal: traceback unavailable\n") + except BaseException: + pass + os._exit(1) + + +def fail_ple_execute_model(func: _F) -> _F: + """Fail a non-shutdown worker-step escape only on PLE-owning ranks. + + This includes ``sample_tokens``, the second half of the split execute-model + step. A rank without a PLE connector rethrows the original exception. + """ + + @functools.wraps(func) + def wrapped(self: Any, *args: Any, **kwargs: Any) -> Any: + try: + return func(self, *args, **kwargs) + except SystemExit: + # WorkerProc's signal handler uses SystemExit for clean shutdown. + raise + except BaseException as error: + connector = getattr(self, "_ple_offload_connector", None) + if connector is None: + model_runner = getattr(self, "model_runner", None) + connector = getattr( + model_runner, "_ple_offload_connector", None + ) + if connector is None: + raise + connector._fail_closed(func.__name__, error) + + return wrapped # type: ignore[return-value] diff --git a/vllm/v1/ple_offload/progress.py b/vllm/v1/ple_offload/progress.py new file mode 100644 index 0000000000..bf88291e8e --- /dev/null +++ b/vllm/v1/ple_offload/progress.py @@ -0,0 +1,314 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Independent progress deadline for PLE transactions.""" + +import math +import os +import threading +import time +from collections import deque +from dataclasses import dataclass +from typing import Any + +from vllm.v1.ple_offload.fatal import ple_fail_closed + +_DEFAULT_SERVING_DEADLINE_S = 300.0 +_DEFAULT_STARTUP_DEADLINE_S = 1800.0 +_MAX_PROGRESS_DEADLINE_S = 86400.0 +_POLL_INTERVAL_S = 0.1 + + +def _validate_deadline(value: float, name: str) -> float: + value = float(value) + if ( + not math.isfinite(value) + or value <= 0 + or value > _MAX_PROGRESS_DEADLINE_S + ): + raise ValueError( + f"{name} must be finite, greater than zero, and no greater than " + f"{_MAX_PROGRESS_DEADLINE_S:g} seconds" + ) + return value + + +def _environment_deadline(name: str, default: float) -> float: + value = float(os.environ.get(name, str(default))) + return _validate_deadline(value, name) + + +@dataclass(frozen=True) +class _Submission: + sequence: int + started: float + startup: bool + num_tokens: int + deadline: float + depth: int + + +@dataclass(frozen=True) +class _PendingCompletion: + sequence: int + started: float + startup: bool + num_tokens: int + deadline: float + event: Any + + +class PleProgressMonitor: + """Bound host submission and the resulting ordered GPU completions.""" + + def __init__( + self, + *, + rank: int, + layer_names: tuple[str, ...], + serving_deadline_s: float, + startup_deadline_s: float, + ) -> None: + serving_deadline_s = _validate_deadline( + serving_deadline_s, "serving_deadline_s" + ) + startup_deadline_s = _validate_deadline( + startup_deadline_s, "startup_deadline_s" + ) + self._rank = rank + self._layer = ",".join(layer_names) + self._serving_deadline_s = serving_deadline_s + self._startup_deadline_s = startup_deadline_s + self._condition = threading.Condition() + self._submission: _Submission | None = None + self._pending: deque[_PendingCompletion] = deque() + self._sequence = 0 + self._stopping = False + self._thread = threading.Thread( + target=self._run, + name=f"ple-progress-rank{rank}", + daemon=True, + ) + self._thread.start() + + @classmethod + def from_environment( + cls, *, rank: int, layer_names: tuple[str, ...] + ) -> "PleProgressMonitor": + return cls( + rank=rank, + layer_names=layer_names, + serving_deadline_s=_environment_deadline( + "VLLM_PLE_PROGRESS_DEADLINE_S", + _DEFAULT_SERVING_DEADLINE_S, + ), + startup_deadline_s=_environment_deadline( + "VLLM_PLE_STARTUP_DEADLINE_S", + _DEFAULT_STARTUP_DEADLINE_S, + ), + ) + + @property + def sequence(self) -> int: + with self._condition: + return self._sequence + + @property + def has_pending_completions(self) -> bool: + with self._condition: + return bool(self._pending) + + def describe_submission(self, sequence: int) -> str: + """Return the armed transaction descriptor for the active sequence.""" + with self._condition: + active = self._submission + if active is None or active.sequence != sequence: + raise RuntimeError("PLE fault correlation has no active submission") + return ( + f"sequence:{active.sequence}" + f"@begin:{active.started:.9f}" + ) + + def completion_is_pending(self, sequence: int) -> bool: + """Whether this serving sequence has a retained, incomplete event. + + This method is called only behind the armed fault guard. An event that + has completed on the GPU but has not yet been dequeued is not counted. + """ + with self._condition: + pending = next( + ( + item + for item in self._pending + if item.sequence == sequence and not item.startup + ), + None, + ) + if pending is None: + return False + return not bool(pending.event.query()) + + def begin(self, *, startup: bool, num_tokens: int) -> int: + now = time.monotonic() + budget = ( + self._startup_deadline_s if startup else self._serving_deadline_s + ) + with self._condition: + active = self._submission + if active is not None: + if not startup or not active.startup: + raise RuntimeError( + "Overlapping PLE host submissions violate the serialized " + "single-slot protocol" + ) + sequence = active.sequence + self._submission = _Submission( + sequence=sequence, + started=active.started, + startup=True, + num_tokens=num_tokens, + deadline=now + budget, + depth=active.depth + 1, + ) + else: + self._sequence += 1 + sequence = self._sequence + self._submission = _Submission( + sequence=sequence, + started=now, + startup=startup, + num_tokens=num_tokens, + deadline=now + budget, + depth=1, + ) + self._condition.notify_all() + return sequence + + def track_completion(self, event: Any) -> None: + """Retain a release event until the monitor observes GPU completion.""" + if not callable(getattr(event, "query", None)): + raise TypeError("PLE completion event must provide query()") + with self._condition: + active = self._submission + if active is None: + raise RuntimeError("PLE completion has no matching submission") + self._pending.append( + _PendingCompletion( + sequence=active.sequence, + started=active.started, + startup=active.startup, + num_tokens=active.num_tokens, + deadline=active.deadline, + event=event, + ) + ) + self._condition.notify_all() + + def complete(self) -> None: + with self._condition: + active = self._submission + if active is None or active.depth <= 0: + raise RuntimeError("PLE release has no matching transaction") + if active.depth == 1: + self._submission = None + else: + self._submission = _Submission( + sequence=active.sequence, + started=active.started, + startup=active.startup, + num_tokens=active.num_tokens, + deadline=active.deadline, + depth=active.depth - 1, + ) + self._condition.notify_all() + + def close(self) -> None: + with self._condition: + self._stopping = True + self._condition.notify_all() + self._thread.join(timeout=1) + + def _run(self) -> None: + try: + self._monitor() + except BaseException as error: + ple_fail_closed( + where="progress-monitor-thread", + rank=self._rank, + layer=self._layer, + sequence=self._sequence, + error=error, + ) + + def _monitor(self) -> None: + while True: + with self._condition: + if self._stopping: + return + submission = self._submission + pending = self._pending[0] if self._pending else None + if submission is None and pending is None: + self._condition.wait(timeout=_POLL_INTERVAL_S) + continue + + # CUDA completion is observed only here. Never synchronize and never + # use the host flag as evidence that the queued reset completed. + if pending is not None and pending.event.query(): + with self._condition: + if self._pending and self._pending[0] is pending: + self._pending.popleft() + self._condition.notify_all() + continue + + now = time.monotonic() + if pending is not None and now >= pending.deadline: + with self._condition: + if not self._pending or self._pending[0] is not pending: + continue + mode = "startup" if pending.startup else "serving" + ple_fail_closed( + where=( + f"progress-deadline:{mode}:gpu-completion:" + f"tokens={pending.num_tokens}" + ), + rank=self._rank, + layer=self._layer, + sequence=pending.sequence, + error=TimeoutError("PLE GPU completion made no bounded progress"), + ) + + if submission is not None and now >= submission.deadline: + with self._condition: + if self._submission is not submission: + continue + mode = "startup" if submission.startup else "serving" + ple_fail_closed( + where=( + f"progress-deadline:{mode}:host-submission:" + f"tokens={submission.num_tokens}" + ), + rank=self._rank, + layer=self._layer, + sequence=submission.sequence, + error=TimeoutError("PLE host submission made no bounded progress"), + ) + + deadlines = [] + if pending is not None: + deadlines.append(pending.deadline) + if submission is not None: + deadlines.append(submission.deadline) + timeout = _POLL_INTERVAL_S + if deadlines: + timeout = min( + timeout, + max(0.0, min(deadlines) - time.monotonic()), + ) + with self._condition: + if self._stopping: + return + if ( + self._submission is not submission + or (self._pending[0] if self._pending else None) is not pending + ): + continue + self._condition.wait(timeout=timeout) diff --git a/vllm/v1/ple_offload/protocol.py b/vllm/v1/ple_offload/protocol.py new file mode 100644 index 0000000000..87eba1319b --- /dev/null +++ b/vllm/v1/ple_offload/protocol.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""IPC message definitions for PLE CPU offload (host-mapped pull design). + +The offload process owns no CUDA context. Every GPU worker owns, per PLE +layer, a shared host output buffer and a shared host int32 flag, both +page-locked and device-mapped in the worker's context. The offload process +waits for ``flag == 0``, writes the layer result into the host buffer and +stores ``flag = 1`` with plain CPU stores; the GPU's captured forward waits on +the host flag, DMAs host -> GPU on the model stream, and ``release_outputs`` +resets the flag after the forward. +""" + +from dataclasses import dataclass + +import msgspec +import torch + +# --------------------------------------------------------------------------- +# IPC message dataclasses +# --------------------------------------------------------------------------- + + +@dataclass +class PleOffloadRegistration: + """Sent once from each GPU worker during offload setup.""" + + worker_id: int + tp_rank: int + dp_rank: int + # Shared CPU tensors per PLE layer, filled by the offload process. + host_output_buffers: dict[str, torch.Tensor] + # Shared CPU int32 tensors per PLE layer; element 0 is the ready flag. + host_flag_tensors: dict[str, torch.Tensor] + # CPU tensors are allocated in shared memory and registered once. + input_ids_buf: torch.Tensor + query_start_loc_buf: torch.Tensor + ngram_context_buf: torch.Tensor | None + + +@dataclass +class PleOffloadRequest: + """Sent by each DP rank's TP rank zero at every inference step.""" + + dp_rank: int + num_tokens: int + num_reqs: int + + +_PLE_OFFLOAD_REQUEST_DECODER = msgspec.msgpack.Decoder(PleOffloadRequest) diff --git a/vllm/v1/ple_offload/worker.py b/vllm/v1/ple_offload/worker.py new file mode 100644 index 0000000000..43caf80962 --- /dev/null +++ b/vllm/v1/ple_offload/worker.py @@ -0,0 +1,1298 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Dedicated CPU-offload process for PLE embedding layers. + +This module implements a standalone process that: +1. Loads only the :class:`PleOffloadLayer` weights into CPU memory. +2. Accepts per-step computation requests from GPU worker processes. +3. Runs ``forward_impl()`` on CPU, writes the rows into every TP worker's + shared host output buffer for the requesting DP rank and sets that + worker's shared host flag (host-mapped pull design: this process holds no + CUDA context; the GPU waits on the host flag and DMAs host -> GPU inside + its captured forward). + +The TP workers within one DP rank receive identical inputs, so the CPU result +is computed once per DP rank and fanned out to all of its TP ranks. + +Class structure mirrors the GPU worker pattern in multiproc_executor.py: + + PleOffloadWorkerHandle -- handle held by the spawning GPU worker + PleOffloadWorker -- process lifecycle and READY handshake + PleOffloadRunner -- owns weights and serves inference requests +""" + +import contextlib +import os +import pickle +import select +import signal +import socket +import struct +import subprocess +import sys +import tempfile +import threading +import time +from collections.abc import Iterable +from dataclasses import dataclass +from multiprocessing.connection import Connection +from typing import Any, cast + +import msgspec +import torch +import torch.distributed as dist +import zmq + +import vllm.envs as envs +from vllm.config import VllmConfig, set_current_vllm_config +from vllm.distributed.parallel_state import ( + ensure_model_parallel_initialized, + init_distributed_environment, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.ple_offload_layer import ( + PleOffloadLayer, + mark_as_offload_worker, +) +from vllm.model_executor.model_loader import get_model_loader +from vllm.model_executor.model_loader.default_loader import DefaultModelLoader +from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader +from vllm.model_executor.model_loader.utils import ( + initialize_model, + process_weights_after_loading, +) +from vllm.model_executor.model_loader.weight_utils import initialize_dummy_weights +from vllm.utils.system_utils import decorate_logs +from vllm.utils.torch_utils import set_default_torch_dtype +from vllm.v1.ple_offload.protocol import ( + _PLE_OFFLOAD_REQUEST_DECODER, + PleOffloadRegistration, + PleOffloadRequest, +) +from vllm.v1.utils import record_function_or_nullcontext +from vllm_ple_test_fault import ( + _FAULT, + _FaultReconstructionPayload, +) + +logger = init_logger(__name__) + +# Publication fence for the host-mapped pull: a locked read-modify-write (the +# uncontended lock acquire) is a full barrier on x86 and also drains +# write-combining buffers, so every row store of a result is globally visible +# before the flag store that tells the GPU to DMA it (do not rely +# on "memcpy uses no non-temporal stores at this size"). +_PUBLISH_FENCE = threading.Lock() + + +def _unlink_ipc_socket(ipc_addr: str) -> None: + """Best-effort unlink of the exact filesystem IPC endpoint PLE owns.""" + if not ipc_addr.startswith("ipc://"): + return + path = ipc_addr.removeprefix("ipc://") + # PLE receives an absolute UUID path from get_open_zmq_ipc_path(). Avoid + # treating abstract or relative ZMQ addresses as filesystem ownership. + if not os.path.isabs(path): + return + try: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as error: + logger.warning("Failed to unlink PLE IPC socket %s: %s", path, error) + + +def _publish_fence() -> None: + with _PUBLISH_FENCE: + pass + + +def _offload_child_environment() -> dict[str, str]: + """Build the offload environment without mutating the parent.""" + child_env = dict(os.environ) + child_env["VLLM_DISABLE_SHARED_EXPERTS_STREAM"] = "1" + child_env.setdefault("CUTLASS_CUDA_DEVICE_ID", "0") + child_env.pop("VLLM_PP_LAYER_PARTITION", None) + + home_device = child_env.get("VLLM_PLE_OFFLOAD_HOME_DEVICE", "").strip() + if home_device: + parent_visible = child_env.get("CUDA_VISIBLE_DEVICES", "").strip() + try: + physical = ( + parent_visible.split(",")[int(home_device)].strip() + if parent_visible + else home_device + ) + except (IndexError, ValueError) as error: + raise ValueError("Invalid VLLM_PLE_OFFLOAD_HOME_DEVICE") from error + child_env["CUDA_VISIBLE_DEVICES"] = physical + return child_env + + +def _prepare_cuda_free_process() -> None: + """Keep the PLE offload process free of any CUDA context. + + The host-mapped pull design issues no CUDA work here, yet model + construction reached CUDA incidentally (found by the offload-startup + replays): + 1. ``VLLM_DISABLE_SHARED_EXPERTS_STREAM=1``: each MoE layer's shared-experts + runner calls ``aux_stream()`` -> ``torch.cuda.Stream()``, whose C++ + constructor creates the primary context without calling + ``torch.cuda._lazy_init`` (``torch.cuda.is_initialized()`` stays False). + This is the context the replay caught (stack dumped inside + ``Stream.__new__``). This process never runs an MoE layer, so it never + needs the stream. vLLM reads the variable when the layer is built; this + process does not enable vLLM's env cache. + 2. ``CUTLASS_CUDA_DEVICE_ID``: CUTLASS python (``cutlass_cppgen``) runs + ``initialize_cuda_context()`` -> ``cudart.cudaFree(0)`` at import unless + it is set. Not observed on this import path; kept as a cheap defence. + 3. The torch guard (``_install_cuda_guard``). + """ + os.environ["VLLM_DISABLE_SHARED_EXPERTS_STREAM"] = "1" + os.environ.setdefault("CUTLASS_CUDA_DEVICE_ID", "0") + _install_cuda_guard() + + +def _install_cuda_guard() -> None: + """Make torch-level CUDA initialization raise in the PLE offload process. + + An offload-startup check found the host-mapped pull still got a 322 MiB context + on its home card: importing the Qwen4Exp model imports the vendored FLA + ops, whose utils call triton's ``get_current_target()`` at import time, + i.e. ``torch.cuda.current_device()`` -> lazy init. FLA catches + ``RuntimeError`` and falls back to "cpu" (and then skips its Hopper + probe), so raising here keeps this process context-free. Any other call + site fails loudly at offload startup with its stack logged, instead of + silently taking VRAM on a stage-0 card. The GPU workers are untouched. + + torch also has C++ entry points that create the context without calling + ``torch.cuda._lazy_init``: stream construction (``torch.cuda.Stream``) and + the pinned host allocator (``Tensor.pin_memory``). Both raise here too; + this process needs neither (all of its buffers are plain CPU memory). + ``torch.cuda.is_initialized()`` only reflects the Python-level path, so + the per-PID GPU sampler, not that flag, is the authority on contexts. + """ + import traceback + + if torch.cuda.is_initialized(): + # Fail closed: a context that already exists would stay on a + # stage-0 card for the whole run. + raise RuntimeError( + "CUDA is already initialized in the PLE offload process before its " + "guard; the host-mapped pull offload must stay CUDA-free" + ) + + def _refuse(what: str) -> None: + stack = "".join(traceback.format_stack(limit=12)[:-2]) + logger.info( + "PLE offload CUDA guard: blocked a CUDA initialization attempt " + "(%s; the caller must fall back to CPU or fail); call site:\n%s", + what, + stack, + ) + raise RuntimeError( + f"CUDA is disabled in the PLE offload process (host-mapped pull): {what}" + ) + + def _blocked_lazy_init() -> None: + _refuse("torch.cuda lazy init") + + def _blocked_stream(cls, *args, **kwargs): + _refuse("torch.cuda.Stream construction") + + def _blocked_pin_memory(self, *args, **kwargs): + _refuse("Tensor.pin_memory") + + torch.cuda._lazy_init = _blocked_lazy_init # type: ignore[assignment] + torch.cuda.Stream.__new__ = staticmethod(_blocked_stream) # type: ignore[assignment] + torch.Tensor.pin_memory = _blocked_pin_memory # type: ignore[assignment] + logger.info( + "PLE offload CUDA guard installed (torch CUDA lazy init, stream " + "construction and pin_memory raise)" + ) + + +@contextlib.contextmanager +def _platform_device_is_meta(): + """During the offload's meta-model construction, report the meta device. + + Some layer constructors pass an explicit ``current_platform.current_device()`` + (e.g. the GDN output norm in qwen_gdn_linear_attn.py), which overrides the + ``torch.device("meta")`` context and would allocate on CUDA. In the offload + process nothing but the CPU PLE layers is ever materialized, so those + tensors belong on meta too. The patch is an instance attribute removed on + exit, restoring the platform's own method. + """ + from vllm.platforms import current_platform + + current_platform.current_device = lambda *args, **kwargs: torch.device("meta") + try: + yield + finally: + with contextlib.suppress(AttributeError): + del current_platform.current_device + + +# Bounded wait for the GPU to release a result (seconds). A timed-out ownership +# transfer is fatal: the offload process raises and exits, never overwrites. +_PLE_WAIT_TIMEOUT_S = float(os.environ.get("VLLM_PLE_OFFLOAD_WAIT_TIMEOUT", "120")) + + +@dataclass +class PleOffloadOutputTarget: + """Host output destination and ready flag for one TP worker. + + Both are shared CPU tensors owned by the GPU worker, page-locked and + device-mapped in that worker's context. This process only reads/writes + them with plain CPU loads/stores (no CUDA): wait for ``flag == 0`` (the GPU + released the previous result), write the rows, then store ``flag = 1``. + """ + + tp_rank: int + host_output_buffer: torch.Tensor + host_flag: torch.Tensor # int32, element 0 is the flag + + +@dataclass +class PleOffloadInputBuffers: + """Shared-memory input buffers registered for one DP rank.""" + + input_ids_buf: torch.Tensor # int32 (max_num_tokens,) + query_start_loc_buf: torch.Tensor # int32 (max_num_reqs + 1,) + ngram_context_buf: torch.Tensor | None # int32 (max_num_reqs, ngram_context_len) + + +@dataclass(frozen=True) +class _ReceivedPleOffloadRequest: + request: PleOffloadRequest + serving: bool + transaction_id: str | None + correlation_id: str | None + + +def _decode_offload_request( + frames: list[bytes], +) -> _ReceivedPleOffloadRequest: + if len(frames) != 2: + raise RuntimeError("PLE offload request must contain two frames") + request = _PLE_OFFLOAD_REQUEST_DECODER.decode(frames[0]) + metadata = msgspec.msgpack.decode(frames[1]) + if type(metadata) is not dict or set(metadata) != { + "serving", + "transaction", + "correlation", + }: + raise RuntimeError("PLE offload request metadata is invalid") + serving = metadata["serving"] + transaction = metadata["transaction"] + correlation = metadata["correlation"] + if type(serving) is not bool: + raise RuntimeError("PLE offload serving metadata is invalid") + if transaction is not None and type(transaction) is not str: + raise RuntimeError("PLE offload transaction metadata is invalid") + if correlation is not None and type(correlation) is not str: + raise RuntimeError("PLE offload correlation metadata is invalid") + return _ReceivedPleOffloadRequest( + request=request, + serving=serving, + transaction_id=transaction, + correlation_id=correlation, + ) + + +def _receive_offload_request( + socket: zmq.Socket, flags: int = 0 +) -> _ReceivedPleOffloadRequest | PleOffloadRequest: + """Disarmed: the unchanged single-frame request. Armed: request + metadata.""" + if _FAULT is None: + return _PLE_OFFLOAD_REQUEST_DECODER.decode(socket.recv(flags)) + return _decode_offload_request(socket.recv_multipart(flags)) + + +def _send_framed_payload(fd: int, payload: bytes, deadline: float) -> None: + """Send one launch payload without blocking beyond an absolute deadline.""" + frame = memoryview(struct.pack("!Q", len(payload)) + payload) + offset = 0 + os.set_blocking(fd, False) + try: + while offset < len(frame): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("PLE launch payload transmission timed out") + _, writable, _ = select.select([], [fd], [], remaining) + if not writable: + raise TimeoutError("PLE launch payload transmission timed out") + try: + sent = os.write(fd, frame[offset:]) + except BlockingIOError: + continue + if sent <= 0: + raise BrokenPipeError("PLE launch control socket closed") + offset += sent + if ( + _FAULT is not None + and _FAULT.cut == "launch_park_before_payload_read" + ): + _FAULT.acknowledge_launch_send( + bytes_sent=offset, + frame_size=len(frame), + ) + finally: + with contextlib.suppress(OSError): + os.set_blocking(fd, True) + + +_PLE_READY_MAX_PAYLOAD_BYTES = 1 << 20 + +class _BoundedStartupResponseWriter: + """Preflight startup responses against the parent receiver's frame cap.""" + + def __init__(self, writer: Connection) -> None: + self._writer = writer + + def send(self, message: Any) -> None: + payload = pickle.dumps(message, protocol=pickle.HIGHEST_PROTOCOL) + if len(payload) > _PLE_READY_MAX_PAYLOAD_BYTES: + payload = pickle.dumps( + { + "status": "FAILURE", + "error": "PLE startup response exceeded size limit", + }, + protocol=pickle.HIGHEST_PROTOCOL, + ) + if len(payload) > _PLE_READY_MAX_PAYLOAD_BYTES: + raise RuntimeError( + "Bounded PLE startup failure response exceeds its frame cap" + ) + self._writer.send_bytes(payload) + + def close(self) -> None: + self._writer.close() + + +def _read_exact_before_deadline( + fd: int, + size: int, + deadline: float, +) -> bytes: + """Read exactly ``size`` bytes without crossing an absolute deadline.""" + chunks = bytearray() + while len(chunks) < size: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError( + "PLE offload worker did not become ready before its " + "absolute launch deadline" + ) + readable, _, _ = select.select([fd], [], [], remaining) + if not readable: + raise TimeoutError( + "PLE offload worker did not become ready before its " + "absolute launch deadline" + ) + try: + chunk = os.read(fd, size - len(chunks)) + except BlockingIOError: + continue + if not chunk: + raise EOFError("PLE startup response was truncated") + chunks.extend(chunk) + return bytes(chunks) + + +def _recv_bounded_connection_message( + reader: Connection, + deadline: float, +) -> Any: + """Receive one bounded multiprocessing Connection frame by ``deadline``.""" + fd = reader.fileno() + os.set_blocking(fd, False) + try: + header = _read_exact_before_deadline(fd, 4, deadline) + size = struct.unpack("!i", header)[0] + if size == -1: + size = struct.unpack( + "!Q", _read_exact_before_deadline(fd, 8, deadline) + )[0] + elif size < 0: + raise RuntimeError("PLE startup response has an invalid frame size") + if size > _PLE_READY_MAX_PAYLOAD_BYTES: + raise RuntimeError( + f"PLE startup response exceeds " + f"{_PLE_READY_MAX_PAYLOAD_BYTES} bytes" + ) + frame = _read_exact_before_deadline(fd, size, deadline) + finally: + with contextlib.suppress(OSError): + os.set_blocking(fd, True) + try: + return pickle.loads(frame) + except BaseException as error: + raise RuntimeError( + "PLE startup response is not a valid serialized message" + ) from error + + +@dataclass +class PleOffloadWorkerHandle: + """Resources owned by the GPU worker that spawned the offload process.""" + + proc: subprocess.Popen + death_writer: Connection | None + ready_pipe_reader: Connection | None + launch_deadline: float + ipc_addr: str | None = None + + def close(self, *, graceful: bool = True) -> None: + """Release all process resources. Safe to call more than once.""" + if self.ready_pipe_reader is not None: + self.ready_pipe_reader.close() + self.ready_pipe_reader = None + if graceful and self.proc.poll() is None: + # Keep parent-death supervision connected while SIGTERM lets + # proc_main run its shutdown event and finally cleanup. + self.proc.terminate() + try: + self.proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + if self.death_writer is not None: + self.death_writer.close() + self.death_writer = None + if not graceful and self.proc.poll() is None: + # Expired launch: retain the immediate EOF -> TERM path. + self.proc.terminate() + try: + self.proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + if self.proc.poll() is None: + self.proc.kill() + try: + self.proc.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + # The child normally removes the endpoint in proc_main's finally. + # Once it has exited, this exact PLE-owned address is safe to remove as + # a fallback for SIGKILL or an interrupted child cleanup. + if self.proc.poll() is not None and self.ipc_addr is not None: + _unlink_ipc_socket(self.ipc_addr) + + +def _init_offload_distributed() -> None: + """Initialize the single-rank Gloo world required by TP-aware layers.""" + if dist.is_initialized(): + return + + # VocabParallelEmbedding reads the TP process group during construction. + # The offload process owns the full embedding table, so it uses an isolated + # TP1/PP1 Gloo world and never joins the GPU workers' NCCL groups. + store_dir = tempfile.mkdtemp(prefix="vllm_ple_offload_") + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method=f"file://{store_dir}/store", + local_rank=0, + backend="gloo", + ) + # initialize_model_parallel reads the active VllmConfig in the current + # vLLM version. Explicitly configure DP1/TP1/PP1 to match the isolated + # world, regardless of any DP environment variables inherited from the GPU + # worker. The real DP/TP configuration is used later for model construction, + # registration, and request routing. + offload_config = VllmConfig() + offload_parallel_config = offload_config.parallel_config + offload_parallel_config.data_parallel_size = 1 + offload_parallel_config.data_parallel_size_local = 1 + offload_parallel_config.data_parallel_rank = 0 + offload_parallel_config.data_parallel_rank_local = 0 + offload_parallel_config.data_parallel_index = 0 + offload_parallel_config.tensor_parallel_size = 1 + offload_parallel_config.pipeline_parallel_size = 1 + offload_parallel_config.prefill_context_parallel_size = 1 + offload_parallel_config.decode_context_parallel_size = 1 + offload_parallel_config.world_size = 1 + offload_parallel_config.nnodes = 1 + offload_parallel_config.node_rank = 0 + with set_current_vllm_config(offload_config): + ensure_model_parallel_initialized( + tensor_model_parallel_size=1, + pipeline_model_parallel_size=1, + backend="gloo", + ) + logger.info( + "Distributed environment initialized (backend=gloo, rank=0, world_size=1)." + ) + + +class PleOffloadWorker: + """Manage process creation, READY handshake, and the child entry point.""" + + READY_STR = "READY" + + @staticmethod + def make_process( + vllm_config: VllmConfig, + num_workers: int, + ipc_addr: str, + ) -> PleOffloadWorkerHandle: + """Start one CPU offload process in a fresh interpreter.""" + launch_deadline = ( + time.monotonic() + float(envs.VLLM_PLE_OFFLOAD_READY_TIMEOUT) + ) + config_payload = pickle.dumps( + vllm_config, protocol=pickle.HIGHEST_PROTOCOL + ) + child_env = _offload_child_environment() + + # READY and parent-death supervision must never compete for reads. + parent_control, child_control = socket.socketpair() + parent_death, child_death = socket.socketpair() + ready_reader = Connection(parent_control.detach()) + death_writer = Connection(parent_death.detach()) + try: + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "vllm_ple_offload_bootstrap", + str(child_control.fileno()), + str(child_death.fileno()), + ], + env=child_env, + close_fds=True, + pass_fds=(child_control.fileno(), child_death.fileno()), + ) + except BaseException: + ready_reader.close() + death_writer.close() + raise + finally: + child_control.close() + child_death.close() + + handle = PleOffloadWorkerHandle( + proc=proc, + death_writer=death_writer, + ready_pipe_reader=ready_reader, + launch_deadline=launch_deadline, + ipc_addr=ipc_addr, + ) + try: + config_for_launch: bytes | _FaultReconstructionPayload = config_payload + if ( + _FAULT is not None + and _FAULT.cut == "launch_block_reconstruction" + ): + config_for_launch = _FaultReconstructionPayload(config_payload) + launch_state = { + "vllm_config_payload": config_for_launch, + "num_workers": num_workers, + "ipc_addr": ipc_addr, + } + if ( + _FAULT is not None + and _FAULT.cut == "launch_park_before_payload_read" + ): + # Larger than ordinary AF_UNIX socket buffers: an acknowledged + # non-reading child leaves the parent inside framed transmission. + launch_state["_test_padding"] = b"\0" * (8 << 20) + launch_payload = pickle.dumps( + launch_state, + protocol=pickle.HIGHEST_PROTOCOL, + ) + _send_framed_payload( + ready_reader.fileno(), launch_payload, launch_deadline + ) + except BaseException: + handle.close(graceful=False) + raise + + return handle + + @staticmethod + def wait_for_ready(handle: PleOffloadWorkerHandle) -> None: + """Wait until weights and all GPU registrations are ready to serve.""" + reader = handle.ready_pipe_reader + if reader is None: + return + try: + message = _recv_bounded_connection_message( + reader, handle.launch_deadline + ) + if type(message) is not dict: + raise RuntimeError("PLE offload worker sent an invalid response") + status = dict.get(message, "status") + if type(status) is not str: + raise RuntimeError( + "PLE offload worker sent an invalid response status" + ) + if status == PleOffloadWorker.READY_STR: + pass + elif status == "FAILURE": + error_detail = dict.get(message, "error") + if type(error_detail) is not str: + error_detail = "unknown error" + raise RuntimeError( + "PLE offload worker failed during startup: " + f"{error_detail}" + ) + else: + raise RuntimeError("PLE offload worker sent unsupported status") + layer_names = dict.get(message, "layer_names") + if ( + type(layer_names) is not list + or any( + type(name) is not str for name in layer_names + ) + ): + raise RuntimeError( + "PLE offload worker sent invalid READY layer_names" + ) + logger.info( + "Worker ready - %d PleOffloadLayer(s): %s", + len(layer_names), + layer_names, + ) + except EOFError as error: + handle.close(graceful=False) + raise RuntimeError("PLE offload worker exited during startup") from error + except BaseException: + handle.close(graceful=False) + raise + finally: + if handle.ready_pipe_reader is not None: + handle.ready_pipe_reader.close() + handle.ready_pipe_reader = None + + @staticmethod + def proc_main( + vllm_config: VllmConfig, + num_workers: int, + ipc_addr: str, + ready_pipe: Connection, + ) -> None: + """Load PLE weights, accept registrations, and run the request loop.""" + decorate_logs("PleOffloadWorker") + ready_writer = _BoundedStartupResponseWriter(ready_pipe) + shutdown_event = threading.Event() + + def handle_signal(_signum: int, _frame: object) -> None: + shutdown_event.set() + + signal.signal(signal.SIGTERM, handle_signal) + signal.signal(signal.SIGINT, handle_signal) + + zmq_context: zmq.Context | None = None + pull_socket: zmq.Socket | None = None + owns_ipc_socket = False + try: + # The flag lets PleOffloadLayer subclasses execute their complete + # constructors instead of becoming empty GPU-worker placeholders. + mark_as_offload_worker() + + # The bootstrap established child-only visibility before importing + # torch or vLLM. Retain the Python-level guards for known CUDA + # initialization paths and fail if initialization already occurred. + _prepare_cuda_free_process() + home_device = os.environ.get("VLLM_PLE_OFFLOAD_HOME_DEVICE", "").strip() + if home_device: + logger.info( + "PLE offload bootstrap visibility is CUDA_VISIBLE_DEVICES=%s " + "(host-mapped pull: no CUDA context expected)", + os.environ.get("CUDA_VISIBLE_DEVICES", ""), + ) + + # Initialize Gloo before installing the real VllmConfig. This keeps + # the CPU process in an isolated rank-zero, world-size-one group. + # The offload worker is a PP=1 world; a VLLM_PP_LAYER_PARTITION set + # for the GPU workers (e.g. "25,23") would make make_layers raise + # "len(partitions) does not match pp_size=1" while the meta model + # is built here. Drop it in this process only. + os.environ.pop("VLLM_PP_LAYER_PARTITION", None) + _init_offload_distributed() + + # Model components read the active VllmConfig while the meta model + # is constructed, so keep the context around runner initialization. + with set_current_vllm_config(vllm_config): + runner = PleOffloadRunner(vllm_config) + + zmq_context = zmq.Context() + pull_socket = zmq_context.socket(zmq.PULL) + pull_socket.bind(ipc_addr) + owns_ipc_socket = True + logger.info( + "Bound IPC address %s; waiting for %d GPU worker registration(s).", + ipc_addr, + num_workers, + ) + + # READY means that the process can immediately serve requests. Wait + # for every DP/TP worker to register before notifying the parent. + runner.accept_registrations(pull_socket, num_workers) + # Checked: the host-mapped pull design must not create a context. + logger.info( + "PLE offload host-mapped pull: CUDA initialized in this process = %s", + torch.cuda.is_initialized(), + ) + ready_writer.send( + { + "status": PleOffloadWorker.READY_STR, + "layer_names": sorted(runner.layer_names), + } + ) + ready_writer.close() + ready_writer = None # type: ignore[assignment] + + runner.busy_loop(pull_socket, shutdown_event) + except Exception as error: + logger.exception("Unexpected failure in PLE offload worker.") + if ready_writer is not None: + with contextlib.suppress(Exception): + ready_writer.send( + {"status": "FAILURE", "error": repr(error)[:4096]} + ) + raise + finally: + if pull_socket is not None: + pull_socket.close(linger=0) + if zmq_context is not None: + zmq_context.term() + if owns_ipc_socket: + _unlink_ipc_socket(ipc_addr) + if ready_writer is not None: + ready_writer.close() + + +class PleOffloadRunner: + """Own all discovered PLE tables and serve every local DP rank.""" + + def __init__(self, vllm_config: VllmConfig) -> None: + self.vllm_config = vllm_config + self._clamp_input_ids = ( + getattr(vllm_config, "speculative_config", None) is not None + ) + # name -> PleOffloadLayer (CPU) + self._layers: dict[str, PleOffloadLayer] = {} + # dp_rank -> layer_name -> one destination per TP rank + self._worker_targets: dict[int, dict[str, list[PleOffloadOutputTarget]]] = {} + # Each (dp_rank, layer_name) pair owns a plain CPU scratch buffer that + # forward_impl writes; the rows are then copied into every TP worker's + # shared host buffer. Nothing in this process is pinned or on a GPU. + self._scratch_bufs: dict[int, dict[str, torch.Tensor]] = {} + self._shutdown_event: threading.Event | None = None + self._step = 0 # number of handled request batches (diagnostics) + self._request_index = 0 + # Shared-memory inputs are registered once per DP rank by TP rank zero. + self._input_bufs: dict[int, PleOffloadInputBuffers] = {} + self._load_weights() + + @property + def layer_names(self) -> list[str]: + """Return PleOffloadLayer names in model traversal order.""" + return list(self._layers) + + def _load_weights(self) -> None: + """Load only :class:`PleOffloadLayer` subtrees into CPU memory. + + Strategy: + 1. Build the entire model on ``meta`` so non-offloaded parameters use no + physical memory. PleOffloadLayer constructors explicitly target CPU. + 2. Discover all PleOffloadLayer modules from the complete model. + 3. Stream the checkpoint through a prefix filter so only matching PLE + tensors are materialized and passed to ``model.load_weights``. + 4. Run post-load processing only on the CPU-owned PLE subtrees. + """ + model_config = self.vllm_config.model_config + load_config = self.vllm_config.load_config + + # Step 1: build complete structure, while only PLE subtrees allocate CPU + # memory. All transformer, MoE, and vision parameters remain on meta. + logger.info("Initializing model structure for PLE weight discovery ...") + model_dtype = cast(torch.dtype, model_config.dtype) + with ( + set_default_torch_dtype(model_dtype), + torch.device("meta"), + _platform_device_is_meta(), + ): + model = initialize_model( + vllm_config=self.vllm_config, + model_config=model_config, + ) + + # Step 2: preserve named_modules DFS order so CPU execution follows the + # same layer order as the GPU model forward. + offload_layers = { + name: module + for name, module in model.named_modules() + if isinstance(module, PleOffloadLayer) + } + if not offload_layers: + raise RuntimeError( + "VLLM_PLE_CPU_OFFLOAD is enabled, but no PleOffloadLayer " + "was found in the initialized model" + ) + logger.info( + "Found %d PleOffloadLayer(s): %s", + len(offload_layers), + sorted(offload_layers), + ) + offload_prefixes = tuple(f"{name}." for name in offload_layers) + + # Step 3: filter checkpoint tensors before model.load_weights(). The + # conditional-generation checkpoint uses HF names such as + # ``model.language_model.*`` while named_modules exposes mapped vLLM + # names such as ``language_model.model.*``. Apply the model mapper only + # for matching, then yield the original pair so load_weights performs + # its normal single mapping pass. + mapper = getattr(model, "hf_to_vllm_mapper", None) + matched_checkpoint_tensors = 0 + + def offload_only_iter( + weights: Iterable[tuple[str, torch.Tensor]], + ) -> Iterable[tuple[str, torch.Tensor]]: + nonlocal matched_checkpoint_tensors + for weight_name, tensor in weights: + mapped_name: str | None = weight_name + if mapper is not None: + mapped_names = mapper.apply_list([weight_name]) + mapped_name = mapped_names[0] if mapped_names else None + if mapped_name is not None and mapped_name.startswith(offload_prefixes): + matched_checkpoint_tensors += 1 + yield weight_name, tensor + + loader = get_model_loader(load_config) + if isinstance(loader, DummyModelLoader): + logger.info( + "Initializing dummy weights for %d PleOffloadLayer(s) ...", + len(offload_layers), + ) + for layer in offload_layers.values(): + initialize_dummy_weights(layer, model_config) + elif isinstance(loader, DefaultModelLoader): + all_weights = loader.get_all_weights(model_config, model) + loaded_params = model.load_weights(offload_only_iter(all_weights)) + if matched_checkpoint_tensors == 0: + raise RuntimeError( + "PLE offload checkpoint filter matched no weights for " + f"layers: {sorted(offload_layers)}" + ) + + expected_offload_params = { + f"{layer_name}.{param_name}" + for layer_name, layer in offload_layers.items() + for param_name, _ in layer.named_parameters() + } + loaded_offload_entries = { + name for name in loaded_params if name.startswith(offload_prefixes) + } + loaded_expected_params = expected_offload_params.intersection(loaded_params) + missing_offload_params = sorted( + expected_offload_params.difference(loaded_expected_params) + ) + if missing_offload_params: + raise RuntimeError( + "PLE offload checkpoint did not load all materialized " + f"parameters: {missing_offload_params}" + ) + logger.info( + "PLE offload matched %d checkpoint tensor(s), loaded %d " + "offload entries, and verified %d/%d materialized " + "parameter(s) for layers: %s", + matched_checkpoint_tensors, + len(loaded_offload_entries), + len(loaded_expected_params), + len(expected_offload_params), + sorted(offload_layers), + ) + else: + raise NotImplementedError( + "PLE offload requires the default or dummy model loader, got " + f"{type(loader).__name__}" + ) + + # Step 4: post-load processing is restricted to CPU-owned PLE modules; + # the remainder of the model is still on meta and must not be visited. + for layer in offload_layers.values(): + process_weights_after_loading(layer, model_config, torch.device("cpu")) + + self._layers.update(offload_layers) + del model + logger.info("PLE weight loading complete.") + + def accept_registrations( + self, + pull_socket: zmq.Socket, + num_workers: int, + ) -> None: + """Receive every local DP/TP worker's IPC and shared-memory buffers.""" + logger.info("Waiting for %d GPU worker registration(s) ...", num_workers) + registrations: list[PleOffloadRegistration] = [] + for index in range(num_workers): + item = pickle.loads(pull_socket.recv()) + if not isinstance(item, PleOffloadRegistration): + raise RuntimeError( + "Expected PleOffloadRegistration during setup, got " + f"{type(item).__name__} ({index + 1}/{num_workers})" + ) + registrations.append(item) + logger.info( + "GPU worker %d registered (dp_rank=%d, tp_rank=%d, layers=%s).", + item.worker_id, + item.dp_rank, + item.tp_rank, + sorted(item.host_output_buffers), + ) + + dp_size = self.vllm_config.parallel_config.data_parallel_size + tp_size = self.vllm_config.parallel_config.tensor_parallel_size + if num_workers != dp_size * tp_size: + raise RuntimeError( + f"Expected {dp_size * tp_size} registrations for DP={dp_size}, " + f"TP={tp_size}, got {num_workers}" + ) + + registrations_by_dp: dict[int, list[PleOffloadRegistration]] = {} + for registration in registrations: + registrations_by_dp.setdefault(registration.dp_rank, []).append( + registration + ) + if set(registrations_by_dp) != set(range(dp_size)): + raise RuntimeError( + f"Expected DP ranks {set(range(dp_size))}, " + f"got {set(registrations_by_dp)}" + ) + for dp_rank, dp_registrations in registrations_by_dp.items(): + tp_ranks = {registration.tp_rank for registration in dp_registrations} + if tp_ranks != set(range(tp_size)): + raise RuntimeError( + f"DP rank {dp_rank} expected TP ranks {set(range(tp_size))}, " + f"got {tp_ranks}" + ) + + for registration in registrations: + if set(registration.host_output_buffers) != set(self.layer_names) or set( + registration.host_flag_tensors + ) != set(self.layer_names): + raise RuntimeError( + "Registered PLE layers do not match CPU layers: " + f"registered={sorted(registration.host_output_buffers)} / " + f"{sorted(registration.host_flag_tensors)}, " + f"cpu={sorted(self.layer_names)}" + ) + targets_for_dp = self._worker_targets.setdefault(registration.dp_rank, {}) + for layer_name, host_buffer in registration.host_output_buffers.items(): + host_flag = registration.host_flag_tensors[layer_name] + for tensor in (host_buffer, host_flag): + if tensor.device.type != "cpu" or not tensor.is_shared(): + raise RuntimeError( + f"PLE host buffer for {layer_name} (dp=" + f"{registration.dp_rank}, tp={registration.tp_rank}) " + "is not a shared CPU tensor" + ) + if host_flag.dtype != torch.int32 or host_flag.numel() < 1: + raise RuntimeError(f"PLE host flag for {layer_name} must be int32") + target = PleOffloadOutputTarget( + tp_rank=registration.tp_rank, + host_output_buffer=host_buffer, + host_flag=host_flag, + ) + targets_for_dp.setdefault(layer_name, []).append(target) + # All TP ranks in one DP group receive the same input, so buffers + # registered by TP rank zero are sufficient for that DP rank. + if registration.tp_rank == 0: + self._input_bufs[registration.dp_rank] = PleOffloadInputBuffers( + input_ids_buf=registration.input_ids_buf, + query_start_loc_buf=registration.query_start_loc_buf, + ngram_context_buf=registration.ngram_context_buf, + ) + + if set(self._input_bufs) != set(range(dp_size)): + raise RuntimeError( + "TP rank zero did not register PLE input buffers for every DP " + f"rank: expected={set(range(dp_size))}, got={set(self._input_bufs)}" + ) + + config = self.vllm_config.model_config.hf_text_config + max_tokens = self.vllm_config.scheduler_config.max_num_batched_tokens + for dp_rank, layer_targets in self._worker_targets.items(): + self._scratch_bufs[dp_rank] = {} + for layer_name, targets in layer_targets.items(): + if len(targets) != tp_size: + raise RuntimeError( + f"PLE layer {layer_name} for DP rank {dp_rank} received " + f"{len(targets)} targets, expected {tp_size}" + ) + targets.sort(key=lambda target: target.tp_rank) + out_dim = self._layers[layer_name].get_offload_output_dim( + int(config.ple_embed_dim) + ) + out_dtype = self._layers[layer_name].get_offload_output_dtype( + self.vllm_config.model_config.dtype + ) + for target in targets: + shape = tuple(target.host_output_buffer.shape) + if shape != (max_tokens, out_dim) or ( + target.host_output_buffer.dtype != out_dtype + ): + raise RuntimeError( + f"PLE host buffer for {layer_name} (dp={dp_rank}, " + f"tp={target.tp_rank}) is {shape} " + f"{target.host_output_buffer.dtype}, expected " + f"{(max_tokens, out_dim)} {out_dtype}" + ) + self._scratch_bufs[dp_rank][layer_name] = torch.empty( + max_tokens, out_dim, dtype=out_dtype, device="cpu" + ) + logger.info( + "Registrations complete (dp_size=%d, tp_size=%d, layers=%s).", + dp_size, + tp_size, + sorted(self.layer_names), + ) + + @torch.inference_mode() + def busy_loop( + self, + pull_socket: zmq.Socket, + shutdown_event: threading.Event, + ) -> None: + """Decode and batch available requests by DP rank until shutdown.""" + logger.info("Busy-loop started.") + self._shutdown_event = shutdown_event + poller = zmq.Poller() + poller.register(pull_socket, zmq.POLLIN) + while not shutdown_event.is_set(): + if pull_socket not in dict(poller.poll(timeout=100)): + continue + + requests = [] + try: + requests.append(_receive_offload_request(pull_socket)) + while True: + requests.append( + _receive_offload_request(pull_socket, zmq.NOBLOCK) + ) + except zmq.Again: + pass + except (msgspec.DecodeError, RuntimeError) as error: + raise RuntimeError("Unexpected PLE offload request") from error + + self._handle_requests(requests) + + def _handle_requests( + self, requests: list[_ReceivedPleOffloadRequest] + ) -> None: + """Run requests layer-first so each DP rank can resume promptly.""" + with record_function_or_nullcontext("ple_offload: handle_requests"): + self._handle_requests_impl(requests) + + def _validate_requests_before_publication( + self, requests: list[PleOffloadRequest] + ) -> None: + """Validate a drained request batch before computation or publication.""" + for request in requests: + dp_rank = request.dp_rank + if dp_rank not in self._worker_targets or dp_rank not in self._input_bufs: + raise RuntimeError( + f"No PLE buffers or output targets for dp_rank={dp_rank}; " + "refusing request" + ) + + input_bufs = self._input_bufs[dp_rank] + if not 0 < request.num_tokens <= input_bufs.input_ids_buf.numel(): + raise RuntimeError( + f"PLE num_tokens={request.num_tokens} is outside capacity " + f"{input_bufs.input_ids_buf.numel()} for dp_rank={dp_rank}" + ) + max_reqs = input_bufs.query_start_loc_buf.numel() - 1 + if not 0 < request.num_reqs <= max_reqs: + raise RuntimeError( + f"PLE num_reqs={request.num_reqs} is outside capacity " + f"{max_reqs} for dp_rank={dp_rank}" + ) + if ( + input_bufs.query_start_loc_buf.dtype != torch.int32 + or input_bufs.query_start_loc_buf.dim() != 1 + ): + raise RuntimeError( + f"PLE query_start_loc for dp_rank={dp_rank} must be 1-D int32" + ) + if ( + input_bufs.ngram_context_buf is not None + and request.num_reqs > input_bufs.ngram_context_buf.shape[0] + ): + raise RuntimeError( + f"PLE num_reqs={request.num_reqs} exceeds ngram context " + f"capacity for dp_rank={dp_rank}" + ) + + query_start_loc = input_bufs.query_start_loc_buf[ + : request.num_reqs + 1 + ].tolist() + if ( + query_start_loc[0] != 0 + or query_start_loc[-1] > request.num_tokens + or any(a > b for a, b in zip(query_start_loc, query_start_loc[1:])) + ): + raise RuntimeError( + f"Invalid PLE query_start_loc for dp_rank={dp_rank}: " + f"{query_start_loc}, num_tokens={request.num_tokens}" + ) + + for layer_name, targets in self._worker_targets[dp_rank].items(): + scratch = self._scratch_bufs[dp_rank][layer_name] + if request.num_tokens > scratch.shape[0] or any( + request.num_tokens > target.host_output_buffer.shape[0] + for target in targets + ): + raise RuntimeError( + f"PLE num_tokens={request.num_tokens} exceeds output " + f"capacity for {layer_name}, dp_rank={dp_rank}" + ) + + def _handle_requests_impl( + self, + requests: list[_ReceivedPleOffloadRequest | PleOffloadRequest], + ) -> None: + """Process one batch of PLE requests.""" + received = [ + item + if isinstance(item, _ReceivedPleOffloadRequest) + else _ReceivedPleOffloadRequest(item, False, None, None) + for item in requests + ] + plain_requests = [item.request for item in received] + self._validate_requests_before_publication(plain_requests) + received_by_dp: dict[int, _ReceivedPleOffloadRequest] = {} + for item in received: + request = item.request + if request.dp_rank not in self._worker_targets: + raise RuntimeError( + f"No PLE output targets for dp_rank={request.dp_rank}; " + "refusing request" + ) + if request.dp_rank in received_by_dp: + # One input slot and one output slot per DP rank: a second + # request in the same batch would alias them. Fatal, never + # silently dropped. + raise RuntimeError( + f"Two PLE requests for dp_rank={request.dp_rank} in one batch " + f"(step {self._step}); the single-slot protocol is violated" + ) + received_by_dp[request.dp_rank] = item + self._step += 1 + if _FAULT is not None: + for item in received_by_dp.values(): + if not item.serving: + continue + self._request_index += 1 + transaction = f"request:{self._request_index}" + if item.transaction_id is not None: + transaction = f"{transaction}@{item.transaction_id}" + for cut in ("park_before_publish", "park_before_publish_ack"): + if _FAULT is not None and _FAULT.fires( + cut, + role="offload", + unit="offload-request", + eligible=True, + dp_rank=item.request.dp_rank, + transaction_id=transaction, + correlation_id=item.correlation_id, + ): + # No multiprocessing or fault-state lock is held here. + threading.Event().wait() + + requests_by_dp = { + dp_rank: item.request + for dp_rank, item in received_by_dp.items() + } + + # Speculative placeholders are not vocabulary IDs. Normalize each DP + # input once before all PLE layers consume the shared buffer. + if self._clamp_input_ids: + for dp_rank, request in requests_by_dp.items(): + self._input_bufs[dp_rank].input_ids_buf[ + : request.num_tokens + ].clamp_min_(0) + + for layer_name, layer in self._layers.items(): + for dp_rank, request in requests_by_dp.items(): + targets = self._worker_targets[dp_rank][layer_name] + + input_bufs = self._input_bufs[dp_rank] + ngram_context = ( + input_bufs.ngram_context_buf[: request.num_reqs] + if input_bufs.ngram_context_buf is not None + else None + ) + result = layer.forward_impl( + input_bufs.input_ids_buf[: request.num_tokens], + input_bufs.input_ids_buf[: request.num_tokens], + input_bufs.query_start_loc_buf[: request.num_reqs + 1], + ngram_context, + output_buffer=self._scratch_bufs[dp_rank][layer_name], + ) + if _rowdump.enabled(): + _seq = getattr(self, "_rowdump_seq", 0) + self._rowdump_seq = _seq + 1 + _rowdump.write(f"offload-dp{dp_rank}", 1, dp_rank, _seq, + input_bufs.input_ids_buf[: request.num_tokens], result) + + # The result is identical on every TP rank in this DP group. + # A worker's host buffer may be overwritten only after its GPU + # released the previous result (``release_outputs`` stores + # flag=0 on the model stream after the forward, which is also + # after the in-graph DMA read it). Then write rows, then flag=1. + scratch = self._scratch_bufs[dp_rank][layer_name] + if result.dim() != 2 or result.shape[0] != request.num_tokens or ( + result.shape[1] != scratch.shape[1] + ) or result.dtype != scratch.dtype or result.device.type != "cpu": + raise RuntimeError( + f"PLE result {tuple(result.shape)} {result.dtype} " + f"{result.device} for {layer_name} does not " + f"match the request ({request.num_tokens} rows x " + f"{scratch.shape[1]} {scratch.dtype} CPU); refusing to " + f"publish (step {self._step})" + ) + slices = tuple(slice(0, size) for size in result.shape) + for target in targets: + self._wait_flag_released(target, layer_name, dp_rank) + target.host_output_buffer[slices].copy_(result[slices]) + _publish_fence() + target.host_flag[0] = 1 + + def _wait_flag_released( + self, target: PleOffloadOutputTarget, layer_name: str, dp_rank: int + ) -> None: + """Spin until the GPU reset this target's flag (previous result used).""" + flag = target.host_flag + if int(flag[0]) == 0: + return + spins = 0 + warned = False + start = time.monotonic() + while int(flag[0]) != 0: + spins += 1 + if self._shutdown_event is not None and self._shutdown_event.is_set(): + raise RuntimeError("PLE offload shutting down while waiting for GPU") + if spins > 200: + time.sleep(0.0001) + waited = time.monotonic() - start + if not warned and waited > 5.0: + warned = True + logger.warning( + "PLE offload waited >5 s for the GPU to release %s " + "(dp=%d, tp=%d, step %d); flag=%d", + layer_name, + dp_rank, + target.tp_rank, + self._step, + int(flag[0]), + ) + if waited > _PLE_WAIT_TIMEOUT_S: + raise RuntimeError( + f"PLE offload: GPU did not release {layer_name} (dp={dp_rank}, " + f"tp={target.tp_rank}, step {self._step}) within " + f"{_PLE_WAIT_TIMEOUT_S:.0f} s; refusing to overwrite" + ) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 6778fea729..55462283db 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextlib import gc import itertools from collections import defaultdict -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Iterator from contextlib import ExitStack from dataclasses import dataclass, replace from itertools import groupby, product @@ -33,6 +34,9 @@ from vllm.distributed.parallel_state import ( ) from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger +from vllm.model_executor.layers.ple_offload_layer import ( + observe_ple_copy_geometry, +) from vllm.model_executor.offloader.base import get_offloader from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors @@ -173,6 +177,14 @@ class CudaGraphManager: self._lora_dispatch_map, self._max_lora_case = self._build_lora_dispatch_map() self.graphs: dict[BatchExecutionDescriptor, torch.cuda.CUDAGraph] = {} + # FULL graphs bake every PLE wait/copy into their native graph. Metadata + # comes from the actual custom-op implementation during native capture. + self._ple_full_copy_geometry: dict[ + BatchExecutionDescriptor, tuple[tuple[str, int, int], ...] + ] = {} + self._expected_ple_copy_layers: tuple[str, ...] = () + # Only the target-model manager is allowed to capture PLE operations. + self._allow_ple_full_graph = False self.pool = current_platform.get_global_graph_pool() if cudagraph_mode else None self._graphs_captured = False @@ -488,6 +500,7 @@ class CudaGraphManager: torch.accelerator.synchronize() free_before = torch.accelerator.get_memory_info()[0] with ( + self.capture_ple_copy_geometry(desc), capture_pool(self.pool, self.vllm_config) as pool, torch.cuda.graph( graph, pool, stream=self._capture_stream(desc) @@ -514,6 +527,78 @@ class CudaGraphManager: {desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0} ) + @contextlib.contextmanager + def capture_ple_copy_geometry( + self, desc: BatchExecutionDescriptor + ) -> Iterator[None]: + """Production capture callback for actual PLE custom-op geometry.""" + with observe_ple_copy_geometry() as geometry: + yield + + if geometry and not self._allow_ple_full_graph: + raise RuntimeError( + "PLE wait-copy reached an unsupported FULL graph path; " + "speculator/draft FULL graphs may not contain PLE operations" + ) + if geometry: + self.record_ple_copy_geometry(desc, geometry) + elif self._expected_ple_copy_layers: + raise RuntimeError( + f"PLE copy geometry was not observed while capturing FULL graph {desc}" + ) + + def record_ple_copy_geometry( + self, + desc: BatchExecutionDescriptor, + geometry: Iterable[tuple[str, int, int]], + ) -> None: + if desc.cg_mode != CUDAGraphMode.FULL: + raise ValueError("PLE capture geometry is recorded only for FULL graphs") + observed = tuple( + (str(layer), int(rows), int(nbytes)) + for layer, rows, nbytes in geometry + ) + if not observed: + raise RuntimeError(f"PLE FULL graph {desc} observed no wait-copy operations") + observed_layers = tuple(layer for layer, _, _ in observed) + expected_layers = self._expected_ple_copy_layers + if expected_layers and sorted(observed_layers) != sorted(expected_layers): + raise RuntimeError( + f"PLE FULL graph {desc} observed layers {observed_layers}, " + f"expected {expected_layers}" + ) + inconsistent = [ + (layer, rows, nbytes) + for layer, rows, nbytes in observed + if rows != desc.num_tokens or nbytes < 0 + ] + if inconsistent: + raise RuntimeError( + f"PLE FULL graph {desc} actual PLE copy geometry " + f"{inconsistent} does not match descriptor rows={desc.num_tokens}" + ) + previous = self._ple_full_copy_geometry.get(desc) + if previous is not None and previous != observed: + raise RuntimeError( + f"PLE FULL graph {desc} changed copy geometry from " + f"{previous} to {observed}" + ) + self._ple_full_copy_geometry[desc] = observed + + def ple_copy_rows(self, desc: BatchExecutionDescriptor) -> int: + try: + geometry = self._ple_full_copy_geometry[desc] + except KeyError as error: + raise RuntimeError( + f"PLE copy geometry was not recorded for FULL graph {desc}" + ) from error + rows = {num_rows for _, num_rows, _ in geometry} + if len(rows) != 1: + raise RuntimeError( + f"PLE FULL graph {desc} has inconsistent per-layer rows: {geometry}" + ) + return rows.pop() + def dispatch( self, num_reqs: int, @@ -552,6 +637,11 @@ class CudaGraphManager: f"Expected FULL mode, got {desc.cg_mode}" ) assert desc in self.graphs, f"No cudagraph for {desc}" + if ( + desc in self._ple_full_copy_geometry + and not self._allow_ple_full_graph + ): + raise RuntimeError("refusing unsupported PLE speculator FULL replay") # Sync offloader before replay - needed when transitioning from # eager/piecewise to full cudagraph (e.g., prefill → decode). # The previous eager iteration's start_prefetch may have queued @@ -597,11 +687,32 @@ class ModelCudaGraphManager(CudaGraphManager): varlen_decode=varlen_decode, ubatch_runner=ubatch_runner, ) + self._allow_ple_full_graph = True self.hidden_states: torch.Tensor | None = None self.aux_hidden_states: list[torch.Tensor] = [] self.use_aux_hidden_state_outputs = False self.intermediate_tensors: IntermediateTensors | None = None + def _prepare_ple_capture( + self, model: nn.Module, *, initialize_breakable: bool + ) -> None: + expected_ple_copy_layers = tuple( + str(getattr(module, "_ple_offload_name", name)) + for name, module in model.named_modules() + if getattr(module, "_is_cpu_offloaded", False) + and getattr(module, "_host_output_ptr", None) is not None + ) + if ( + "" in expected_ple_copy_layers + or len(set(expected_ple_copy_layers)) != len(expected_ple_copy_layers) + ): + raise RuntimeError( + "PLE FULL capture requires distinct, bound connector layer identities" + ) + self._expected_ple_copy_layers = expected_ple_copy_layers + if initialize_breakable: + self.init_breakable_cg_runner(model) + def capture( self, model: nn.Module, @@ -619,8 +730,9 @@ class ModelCudaGraphManager(CudaGraphManager): ) -> None: """Capture CUDA graphs for model forward pass.""" self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs - if self.use_breakable_cg: - self.init_breakable_cg_runner(model) + self._prepare_ple_capture( + model, initialize_breakable=self.use_breakable_cg + ) if self.cudagraph_mode.has_piecewise_cudagraphs() and not ( self.use_breakable_cg or has_compiled_submodule(model) @@ -926,7 +1038,9 @@ def profile_cudagraph_memory(runner: "GPUModelRunner") -> int: if manager.use_breakable_cg: # Create the breakable runner before the wrapper pool swap so # its pool is covered as well. - manager.init_breakable_cg_runner(runner.model) + manager._prepare_ple_capture( + runner.model, initialize_breakable=True + ) all_wrappers = list(CUDAGraphWrapper._all_instances) + list( BreakableCUDAGraphWrapper._all_instances ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 8988f551d1..9b96775453 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -74,6 +74,9 @@ from vllm.v1.outputs import ( ECConnectorOutput, ModelRunnerOutput, ) +from vllm.v1.ple_offload.connector import PleOffloadConnector +from vllm.v1.ple_offload.fatal import fail_ple_execute_model +from vllm.v1.ple_offload.test_fault import _FAULT from vllm.v1.watermarking import create_watermarker from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler from vllm.v1.watermarking.spec_decode import ( @@ -1757,6 +1760,45 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.ec_connector.no_forward(scheduler_output).ec_connector_output, ) + def _prepare_ple_forward( + self, + input_batch: InputBatch, + batch_desc: BatchExecutionDescriptor, + dummy_run: bool, + *, + serving: bool = False, + transaction_id: str | None = None, + correlation_id: str | None = None, + ) -> None: + connector = self._ple_offload_connector + assert connector is not None + request_num_tokens = int(input_batch.num_tokens_after_padding) + if batch_desc.cg_mode == CUDAGraphMode.FULL: + manager = self.cudagraph_manager + if manager is None: + raise RuntimeError("PLE FULL replay has no CUDA graph manager") + copy_num_tokens = manager.ple_copy_rows(batch_desc) + else: + # PIECEWISE/eager execute the Python op with this actual padded view. + copy_num_tokens = int(input_batch.input_ids.shape[0]) + + if request_num_tokens != copy_num_tokens: + raise RuntimeError( + "PLE request/selected-forward copy-count mismatch: " + f"request={request_num_tokens}, copy={copy_num_tokens}, " + f"mode={batch_desc.cg_mode}" + ) + connector.prepare_forward( + input_batch.num_reqs, + request_num_tokens, + dummy_run, + consumer_num_tokens=copy_num_tokens, + serving=serving, + transaction_id=transaction_id, + correlation_id=correlation_id, + ) + + @fail_ple_execute_model @torch.inference_mode() def execute_model( self, @@ -2038,6 +2080,54 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) self.step_timing.forward_start() + # prepare_inputs has finalized MRV2's GPU buffers. Stage D2H on this + # stream before the PLE placeholder can wait for CPU output. + connector = self._ple_offload_connector + ple_serving = False + ple_sequence = None + ple_transaction = None + ple_correlation = None + if connector is not None: + ple_serving = not dummy_run and not is_profile + ple_sequence = connector.begin_forward( + dummy_run=dummy_run, + num_tokens=int(input_batch.num_tokens_after_padding), + ) + if _FAULT is not None and ple_serving: + ple_correlation = f"sequence:{ple_sequence}" + ple_transaction = connector.fault_transaction(ple_sequence) + if _FAULT is not None and _FAULT.fires( + "exc_before_prepare", + role="gpu", + unit="runner-forward", + eligible=ple_serving, + tp_rank=connector.tp_rank, + pp_rank=connector.pp_rank, + dp_rank=connector.dp_rank, + transaction_id=ple_transaction, + correlation_id=ple_correlation, + ): + raise RuntimeError("PLE test fault: exc_before_prepare") + self._prepare_ple_forward( + input_batch, + batch_desc, + dummy_run, + serving=ple_serving, + transaction_id=ple_transaction, + correlation_id=ple_correlation, + ) + if _FAULT is not None and _FAULT.fires( + "exc_after_prepare", + role="gpu", + unit="runner-forward", + eligible=ple_serving, + tp_rank=connector.tp_rank, + pp_rank=connector.pp_rank, + dp_rank=connector.dp_rank, + transaction_id=ple_transaction, + correlation_id=ple_correlation, + ): + raise RuntimeError("PLE test fault: exc_after_prepare") connector_kwargs = dict( scheduler_output=scheduler_output, request_state_indices=input_batch.idx_mapping, @@ -2095,6 +2185,26 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Eager (NONE): call the raw model directly. model_output = self.model(**model_inputs) + # The model has consumed every PLE output, so the CPU worker may reuse + # the registered buffers for the next request. + if connector is not None: + connector.release_outputs( + serving=ple_serving, + transaction_id=ple_transaction, + correlation_id=ple_correlation, + ) + if _FAULT is not None and _FAULT.fires( + "exc_after_release", + role="gpu", + unit="runner-forward", + eligible=ple_serving, + tp_rank=connector.tp_rank, + pp_rank=connector.pp_rank, + dp_rank=connector.dp_rank, + transaction_id=ple_transaction, + correlation_id=ple_correlation, + ): + raise RuntimeError("PLE test fault: exc_after_release") self.kv_connector.finish_forward() if self.is_last_pp_rank: @@ -2123,6 +2233,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): finished_req_ids=finished_req_ids, ec_connector_output=ec_connector_output, cudagraph_stats=cudagraph_stats, + ple_serving=ple_serving, + ple_sequence=ple_sequence, + ple_transaction=ple_transaction, ) if not self.is_last_pp_rank: @@ -2134,6 +2247,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) return None + @fail_ple_execute_model @torch.inference_mode() @step_eplb_after() def sample_tokens( @@ -2152,6 +2266,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): finished_req_ids = self.execute_model_state.finished_req_ids ec_connector_output = self.execute_model_state.ec_connector_output cudagraph_stats = self.execute_model_state.cudagraph_stats + ple_serving = self.execute_model_state.ple_serving + ple_sequence = self.execute_model_state.ple_sequence + ple_transaction = self.execute_model_state.ple_transaction self.execute_model_state = None if not self.is_last_pp_rank: @@ -2161,6 +2278,26 @@ class GPUModelRunner(LoRAModelRunnerMixin): if self.pcp_manager is not None: input_batch = self.pcp_manager.global_batch assert self.pp_handler is not None + connector = self._ple_offload_connector + if ( + connector is not None + and _FAULT is not None + and ple_serving + and ple_sequence is not None + and connector.completion_is_pending(ple_sequence) + and _FAULT.fires( + "exc_async_output", + role="gpu", + unit="runner-output-relay", + eligible=True, + tp_rank=connector.tp_rank, + pp_rank=connector.pp_rank, + dp_rank=connector.dp_rank, + transaction_id=ple_transaction, + correlation_id=f"sequence:{ple_sequence}", + ) + ): + raise RuntimeError("PLE test fault: exc_async_output") all_decode_next = self.pp_handler.receive(input_batch) # Optimistically update num_computed_tokens for entire batch here. # Will be adjusted for rejections if necessary in update_requests. @@ -2462,6 +2599,9 @@ class ExecuteModelState(NamedTuple): finished_req_ids: set[str] ec_connector_output: ECConnectorOutput | None cudagraph_stats: CUDAGraphStat | None + ple_serving: bool + ple_sequence: int | None + ple_transaction: str | None class BatchReqState(NamedTuple): diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 61374835a1..00d5806278 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -91,6 +91,7 @@ from vllm.v1.outputs import ( DraftTokenIds, ModelRunnerOutput, ) +from vllm.v1.ple_offload.fatal import fail_ple_execute_model from vllm.v1.utils import compute_iteration_details, report_usage_stats from vllm.v1.worker import guard_telemetry from vllm.v1.worker.sentinel.gpu_worker_sentinel import WorkerSentinel @@ -238,6 +239,148 @@ class Worker(WorkerBase): # Resolved lazily on first sleep/wake; persists worker-process state. self._sleep_mode_backend: SleepModeBackend | None = None + def _has_ple_layers(self) -> bool: + """Return whether this model configuration constructs PLE layers.""" + if not envs.VLLM_PLE_CPU_OFFLOAD: + return False + text_config = self.model_config.hf_text_config + ple_layer_ids = getattr(text_config, "ple_layer_ids", None) + if not ple_layer_ids: + return False + # PP (vllm#54709, carried from the earlier overlay): only the pipeline rank + # that owns a PLE layer takes part in the offload (rank 0 with the + # default partition). Other ranks build PPMissingLayer there and must + # skip the connector entirely. + pp_size = self.parallel_config.pipeline_parallel_size + if pp_size > 1: + from vllm.distributed.utils import get_pp_indices + + tp_size = self.parallel_config.tensor_parallel_size + pp_rank = (self.rank // tp_size) % pp_size + start, end = get_pp_indices( + int(text_config.num_hidden_layers), pp_rank, pp_size + ) + return any(start <= int(i) - 1 < end for i in ple_layer_ids) + return True + + def _validate_ple_offload_config(self) -> None: + """Reject unsupported PLE offload execution modes.""" + parallel_config = self.parallel_config + unsupported = [] + if not self.use_v2_model_runner: + unsupported.append("model runner V1") + if not current_platform.is_cuda(): + unsupported.append(f"device={current_platform.device_type}") + if parallel_config.nnodes != 1: + unsupported.append(f"nnodes={parallel_config.nnodes}") + if parallel_config.data_parallel_backend != "mp": + unsupported.append(f"DP backend={parallel_config.data_parallel_backend}") + if ( + parallel_config.data_parallel_size_local + != parallel_config.data_parallel_size + ): + unsupported.append( + "non-local DP " + f"({parallel_config.data_parallel_size_local}/" + f"{parallel_config.data_parallel_size} local ranks)" + ) + # PP allowed (vllm#54709): PLE layers must live on pipeline rank 0 + # (enforced placement-based in models/config.py); num_workers stays + # dp_size * tp_size = the ranks of the owning stage. + if parallel_config.prefill_context_parallel_size != 1: + unsupported.append(f"PCP={parallel_config.prefill_context_parallel_size}") + if parallel_config.decode_context_parallel_size != 1: + unsupported.append(f"DCP={parallel_config.decode_context_parallel_size}") + if parallel_config.use_ubatching: + unsupported.append("ubatching/DBO") + if self.model_config.architecture not in { + "Qwen4ExpForCausalLM", + "Qwen4ExpForConditionalGeneration", + }: + unsupported.append("architecture") + if self.vllm_config.weight_transfer_config is not None: + unsupported.append("weight transfer") + + if unsupported: + raise ValueError( + "VLLM_PLE_CPU_OFFLOAD does not support the requested " + "configuration. Unsupported settings: " + f"{', '.join(unsupported)}" + ) + + def spawn_ple_offload(self) -> None: + """Spawn one node-local PLE CPU worker from DP0/TP0.""" + if ( + not self._ple_offload_enabled + or self.rank != 0 + or self.parallel_config.data_parallel_rank != 0 + ): + return + + from vllm.v1.ple_offload.worker import PleOffloadWorker + + ipc_addr = self.parallel_config._ple_offload_ipc_path + if not ipc_addr: + raise RuntimeError("PLE offload IPC address was not initialized") + dp_size = self.parallel_config.data_parallel_size + tp_size = self.parallel_config.tensor_parallel_size + num_workers = dp_size * tp_size + logger.info( + "PleOffload: spawning worker " + "(rank=%d, local_rank=%d, dp_size=%d, tp_size=%d, " + "num_workers=%d, ipc_addr=%s).", + self.rank, + self.local_rank, + dp_size, + tp_size, + num_workers, + ipc_addr, + ) + self._ple_offload_worker_handle = PleOffloadWorker.make_process( + self.vllm_config, + num_workers, + ipc_addr, + ) + self._start_ple_offload_watchdog() + + def _start_ple_offload_watchdog(self) -> None: + """Fail this worker if the PLE offload process exits unexpectedly. + + Host-mapped pull: a dead producer would leave the captured forward + waiting on its host flag forever; turn that into a fast, visible + engine failure instead of a hang. ``shutdown`` sets the stop marker + before closing the handle, so a normal close is not a failure. + """ + import threading + + handle = self._ple_offload_worker_handle + self._ple_offload_stopping = False + + def watch() -> None: + returncode = handle.proc.wait() + if getattr(self, "_ple_offload_stopping", False): + return + logger.error( + "PLE offload process %s exited unexpectedly (exitcode %s); " + "failing this worker so the engine does not hang on a PLE wait", + handle.proc.pid, + returncode, + ) + os._exit(1) + + threading.Thread( + target=watch, name="ple-offload-watchdog", daemon=True + ).start() + + def wait_ple_offload_ready(self) -> None: + """Wait until weights and every TP registration are ready.""" + if self._ple_offload_worker_handle is None: + return + + from vllm.v1.ple_offload.worker import PleOffloadWorker + + PleOffloadWorker.wait_for_ready(self._ple_offload_worker_handle) + @property def sleep_mode_backend(self) -> "SleepModeBackend": if self._sleep_mode_backend is None: @@ -1263,6 +1406,7 @@ class Worker(WorkerBase): ) -> ModelRunnerOutput | AsyncModelRunnerOutput: return self.model_runner.sample_tokens(grammar_output) + @fail_ple_execute_model @torch.inference_mode() @with_gpu_sync_check def execute_model( @@ -1587,6 +1731,7 @@ class Worker(WorkerBase): self.model_runner.reset_lora_state() def shutdown(self) -> None: + self._ple_offload_stopping = True gc.unfreeze() # Bounds-guard instrumentation: synchronous final read of the guard diff --git a/vllm_ple_offload_bootstrap/__init__.py b/vllm_ple_offload_bootstrap/__init__.py new file mode 100644 index 0000000000..60ddda472d --- /dev/null +++ b/vllm_ple_offload_bootstrap/__init__.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Minimal PLE child bootstrap; intentionally imports no torch or vLLM.""" + +import importlib.abc +import importlib.machinery +import os +import pickle +import struct +import sys +import threading +from multiprocessing.connection import Connection +from types import ModuleType +from typing import Any + +_FAULT = None +if os.environ.get("VLLM_PLE_TEST_FAULT") is not None: + from vllm_ple_test_fault import ( + _FAULT as _FAULT, + _block_on_harness_gate, + ) + + +def _send_partial_ready_frame(fd: int) -> tuple[int, int]: + """Write the complete header and an incomplete body in one syscall.""" + body = pickle.dumps( + {"status": "READY", "layer_names": []}, + protocol=pickle.HIGHEST_PROTOCOL, + ) + frame = struct.pack("!i", len(body)) + body + fragment = frame[: 4 + max(1, len(body) // 2)] + try: + return os.write(fd, fragment), len(frame) + except OSError: + return 0, len(frame) + + +def _send_partial_ready_header(fd: int) -> tuple[int, int]: + """Write fewer than the four required header bytes in one syscall.""" + header = struct.pack("!i", 1) + try: + return os.write(fd, header[:2]), len(header) + except OSError: + return 0, len(header) + + +def _read_exact(fd: int, size: int) -> bytes: + chunks = bytearray() + while len(chunks) < size: + chunk = os.read(fd, size - len(chunks)) + if not chunk: + raise EOFError("PLE launch payload was truncated") + chunks.extend(chunk) + return bytes(chunks) + + +def _recv_launch_payload(fd: int) -> dict: + size = struct.unpack("!Q", _read_exact(fd, 8))[0] + payload = pickle.loads(_read_exact(fd, size)) + if not isinstance(payload, dict): + raise RuntimeError("invalid PLE bootstrap payload") + return payload + + +def _fires( + cut: str, + *, + unit: str = "bootstrap-launch", + transaction: str = "launch:1", + bytes_written: int | None = None, + frame_size: int | None = None, +) -> bool: + return _FAULT is not None and _FAULT.fires( + cut, + role="bootstrap", + unit=unit, + eligible=True, + transaction_id=transaction, + correlation_id="launch:1", + bytes_written=bytes_written, + frame_size=frame_size, + ) + + +def _write_and_ack_partial( + fd: int, + cut: str, + sender: Any, +) -> bool: + """A short or failed prefix write cannot acknowledge or enter the park.""" + if _FAULT is None or _FAULT.cut != cut: + # Disarmed or armed for another cut: the control stream is untouched. + return False + written, frame_size = sender(fd) + if written <= 0 or written >= frame_size: + return False + expected = 2 if cut == "launch_partial_ready_header" else None + if expected is not None and written != expected: + return False + if expected is None and written <= 4: + return False + return _fires( + cut, + bytes_written=written, + frame_size=frame_size, + ) + + +class _FaultImportLoader(importlib.abc.Loader): + def __init__(self, delegate: importlib.abc.Loader) -> None: + self._delegate = delegate + + def create_module(self, spec): + create = getattr(self._delegate, "create_module", None) + return None if create is None else create(spec) + + def exec_module(self, module: ModuleType) -> None: + if _fires( + "launch_block_worker_import", + unit="bootstrap-import", + transaction="module:vllm.v1.ple_offload.worker", + ): + _block_on_harness_gate() + self._delegate.exec_module(module) + + +class _FaultImportFinder(importlib.abc.MetaPathFinder): + _TARGET = "vllm.v1.ple_offload.worker" + + def find_spec(self, fullname: str, path, target=None): + if fullname != self._TARGET: + return None + spec = importlib.machinery.PathFinder.find_spec(fullname, path, target) + if spec is None or spec.loader is None: + raise ImportError(f"cannot find {fullname}") + if not isinstance(spec.loader, importlib.abc.Loader): + raise ImportError(f"{fullname} has no executable loader") + spec.loader = _FaultImportLoader(spec.loader) + return spec + + +def _install_import_gate() -> None: + target = _FaultImportFinder._TARGET + if target in sys.modules: + raise RuntimeError("PLE worker import gate was a cache hit") + sys.meta_path.insert(0, _FaultImportFinder()) + + +def main() -> None: + """Receive launch state before importing torch or vLLM.""" + if len(sys.argv) != 3: + raise SystemExit("expected inherited PLE control and parent-death fds") + control_fd = int(sys.argv[1]) + death_fd = int(sys.argv[2]) + if control_fd == death_fd: + raise RuntimeError("PLE control and parent-death fds must be distinct") + + os.set_inheritable(control_fd, False) + os.set_inheritable(death_fd, False) + death_pipe = Connection(death_fd) + + def monitor_parent() -> None: + try: + death_pipe.recv() + except (EOFError, OSError): + pass + os._exit(1) + + threading.Thread( + target=monitor_parent, + daemon=True, + name="PleOffloadBootstrapDeathMonitor", + ).start() + + if _fires("launch_park_before_payload_read"): + threading.Event().wait() + + # An armed reconstruction payload invokes its reducer inside this loads(). + payload = _recv_launch_payload(control_fd) + + if _write_and_ack_partial( + control_fd, + "launch_partial_ready", + _send_partial_ready_frame, + ): + threading.Event().wait() + + if _write_and_ack_partial( + control_fd, + "launch_partial_ready_header", + _send_partial_ready_header, + ): + threading.Event().wait() + + if _fires("launch_silent_ready"): + threading.Event().wait() + + ready_pipe = Connection(control_fd) + vllm_config = pickle.loads(payload["vllm_config_payload"]) + + if _FAULT is not None and _FAULT.cut == "launch_block_worker_import": + _install_import_gate() + from vllm.v1.ple_offload.worker import PleOffloadWorker + + PleOffloadWorker.proc_main( + vllm_config=vllm_config, + num_workers=int(payload["num_workers"]), + ipc_addr=str(payload["ipc_addr"]), + ready_pipe=ready_pipe, + ) diff --git a/vllm_ple_offload_bootstrap/__main__.py b/vllm_ple_offload_bootstrap/__main__.py new file mode 100644 index 0000000000..5b3ca57787 --- /dev/null +++ b/vllm_ple_offload_bootstrap/__main__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fresh-interpreter entry point for the PLE CPU-offload worker.""" + +from vllm_ple_offload_bootstrap import main + +main()