[PLE] Integrate the pull transport on v0.30.0 with upstream UVA refused

- VLLM_E2_PLE_PULL_TRANSPORT replaces the fork's old switch name, which
  v0.30.0 reuses for its UVA backend with the opposite default; strict parser;
  compile/capture factor.
- The pull transport is reachable on v0.30.0 through a pull embedding class
  that inherits upstream's without editing it; the upstream UVA backend is
  provably off while the pull switch is on.
- resolve_ple_mode()/assert_ple_mode() log and assert the effective PLE mode
  for target and drafter, and refuse a mixed configuration.
- Producer log lines for the PLE backend and weight dtype.
- CPU tests for mode resolution and the mixed-config refusal.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:52:54 +07:00
co-authored by Hinata
parent 3b5fc7c7be
commit 23349e43b2
19 changed files with 711 additions and 24 deletions
+19 -3
View File
@@ -3,9 +3,10 @@
from typing import TYPE_CHECKING
from pydantic import model_validator
from pydantic import Field, model_validator
from typing_extensions import Self
import vllm.envs as envs
from vllm.config.utils import config, get_hash_factors, hash_factors
if TYPE_CHECKING:
@@ -21,6 +22,17 @@ _NGRAM_LAYER_FIELDS = {
}
def _default_cpu_offload() -> bool:
# Fork: with VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a
# VLLM_PLE_CPU_OFFLOAD value that would enable UVA fails the boot. With the
# pull transport off, the upstream default below is returned unchanged.
from vllm.v1.e2_guards import pull_transport_cpu_offload_default
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
return envs.VLLM_PLE_CPU_OFFLOAD
return pull_transport_cpu_offload_default(False)
def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
"""Whether the model carries n-gram embedding layers."""
if model_config is None:
@@ -35,8 +47,12 @@ def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
class EngramConfig:
"""Configuration for Engram embedding storage and sharding."""
cpu_offload: bool = True
"""Store embedding weights in pinned CPU memory for UVA lookup."""
cpu_offload: bool = Field(default_factory=_default_cpu_offload)
"""Store embedding weights in pinned CPU memory for UVA lookup.
Defaults to VLLM_PLE_CPU_OFFLOAD, which is enabled by default. An explicit
value takes precedence over the environment variable. With
VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a value that would
enable UVA fails the boot (vllm/v1/e2_guards.py)."""
host_file_gather: bool = False
"""Serve Qwen4Exp PLE rows by gathering them on the CPU from the loader's
+9 -1
View File
@@ -18,7 +18,7 @@ from vllm.config.fault_tolerance import FaultToleranceConfig
from vllm.config.utils import config
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_ports_list
from vllm.utils.network_utils import get_open_ports_list, get_open_zmq_ipc_path
if TYPE_CHECKING:
from ray.runtime_env import RuntimeEnv
@@ -369,6 +369,10 @@ class ParallelConfig:
"""Port of the coordination TCPStore. Can be set by the API server; workers
connect as clients to exchange self-picked group ports at runtime."""
_ple_offload_ipc_path: str = ""
"""Node-local ZMQ IPC address for the fork's PLE pull-transport offload
process. Set only when VLLM_E2_PLE_PULL_TRANSPORT=1."""
decode_context_parallel_size: int = Field(default=1, ge=1)
"""Number of ranks that shard the decode KV cache. DCP does not expand
the process world size. Without PCP, DCP reuses TP ranks. With PCP, DCP
@@ -517,6 +521,9 @@ class ParallelConfig:
"The FT system assumes one AsyncMPClient manages all engines."
)
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not self._ple_offload_ipc_path:
self._ple_offload_ipc_path = get_open_zmq_ipc_path()
if self.all2all_backend in ["pplx", "naive"]:
logger.warning(
"The '%s' all2all backend has been removed. "
@@ -910,6 +917,7 @@ class ParallelConfig:
"worker_extension_cls",
"_api_process_count",
"_api_process_rank",
"_ple_offload_ipc_path",
# NUMA binding is per-rank host-side memory locality; it does
# not affect collective-communication semantics. When numa_bind
# is enabled with auto-detection, each DP rank stores its own
+5
View File
@@ -1249,6 +1249,11 @@ class VllmConfig:
# PyTorch's expandable_segments allocator uses CUDA VMM, which can
# remap a virtual address range to different physical pages over the
# engine's lifetime. KV connectors that pin KV cache memory (e.g.
# Fork: an explicit cpu_offload=True beside the pull transport is
# a double offload; refused here, before any worker starts.
from vllm.v1.e2_guards import verify_engram_against_pull
verify_engram_against_pull(self.engram_config)
# NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their
# registrations pointing at stale physical pages after any remap,
# producing RDMA failures like IBV_WC_REM_ACCESS_ERR /
+31
View File
@@ -382,6 +382,18 @@ if TYPE_CHECKING:
# to decide behaviour. Registered with the medium-path exact restart
# so the cache-factor contract is complete.
MERLIN_TOPK_EXACT_MEDIUM: str | None = None
# The fork's PLE pull transport -- the host-mapped,
# fail-closed offload process in vllm/v1/ple_offload/. Renamed from the E2
# base's VLLM_PLE_CPU_OFFLOAD, which vLLM v0.30.0 reuses for a DIFFERENT
# subsystem with the OPPOSITE default (upstream: pinned-host UVA lookup,
# default ON). The two subsystems no longer share a switch.
# Strict: unset or "0" = off, "1" = on, anything else is a boot error.
# On: the PLE table lives in the offload process, the GPU placeholder waits
# on a host flag and DMAs rows inside the captured forward, and upstream's
# UVA backend is refused (vllm/v1/e2_guards.py). Off: the tree is upstream.
# AOT: yes (different module class and forward). CAPTURE: yes (the captured
# graph contains the wait/copy ops instead of a table lookup). Hashed.
VLLM_E2_PLE_PULL_TRANSPORT: bool = False
#
# RUNTIME-ONLY (registered for validate_environ + a single cached read):
# Static K/V scale sidecar: fill_()ed into existing fp32 device buffers
@@ -780,6 +792,21 @@ def _resolve_rust_cli_path() -> str | None:
return raw
def _e2_strict_switch(name: str) -> bool:
"""Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``.
Unset means off. Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
``"01"`` -- is a boot error: a switch that selects a different transport
must not be satisfiable by a spelling nobody meant.
"""
raw = os.environ.get(name)
if raw is None or raw == "0":
return False
if raw == "1":
return True
raise ValueError(f'{name} must be exactly "0" or "1" (or unset), got {raw!r}')
def _e2_diagnostic_mode(name: str) -> str:
"""Normalize an E2 diagnostic gate to off|warn|raise.
@@ -2328,6 +2355,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_QSA_KV_COLLECT": lambda: os.getenv("VLLM_QSA_KV_COLLECT", "").strip(),
# Capture-configuration factor, read in C++; raw passthrough on purpose.
"MERLIN_TOPK_EXACT_MEDIUM": lambda: os.getenv("MERLIN_TOPK_EXACT_MEDIUM", None),
# PLE pull transport switch; strict "0"/"1" (see the TYPE_CHECKING block).
"VLLM_E2_PLE_PULL_TRANSPORT": lambda: _e2_strict_switch(
"VLLM_E2_PLE_PULL_TRANSPORT"
),
# Runtime only; all of the following are in compile_factors.ignored_factors.
"VLLM_E2_ALLOW_BREAKABLE": lambda: os.getenv("VLLM_E2_ALLOW_BREAKABLE", "") == "1",
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE": lambda: os.getenv(
@@ -346,7 +346,7 @@ class PleOffloadLayer(nn.Module, ABC):
def guarded_init(
self: "PleOffloadLayer", *args: object, **kwargs: object
) -> None:
if envs.VLLM_PLE_CPU_OFFLOAD and not is_offload_process():
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not is_offload_process():
nn.Module.__init__(self)
return
original_init(self, *args, **kwargs)
@@ -356,7 +356,7 @@ class PleOffloadLayer(nn.Module, ABC):
@classmethod
def get_target_device(cls) -> torch.device:
"""Return CPU for the offload process and the active GPU otherwise."""
if envs.VLLM_PLE_CPU_OFFLOAD:
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
return torch.device("cpu")
return torch.device("cuda", torch.accelerator.current_device_index())
@@ -15,6 +15,7 @@ import torch
import torch.nn.functional as F
from torch import nn
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
from vllm.forward_context import DPMetadata, get_forward_context
@@ -422,6 +423,13 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
max_total_tokens: int = 0,
data_parallel_rank: int = 0,
) -> None:
# Fork: never build the UVA table beside the E2 pull transport --
# that would be a second offload of the same PLE table.
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
raise RuntimeError(
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
)
if not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
super().__init__(
+2 -1
View File
@@ -78,7 +78,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.e2_guards import assert_graph_profile_is_executable
from vllm.v1.e2_guards import assert_graph_profile_is_executable, assert_ple_mode
from vllm.v1.kv_cache_interface import MambaSpec
from ..common.mtp import make_mtp_hidden_buffer
@@ -767,6 +767,7 @@ class Qwen4ExpForCausalLM(
# identity nobody chose. Qwen4ExpForConditionalGeneration constructs
# this class; Qwen4ExpMTP does not, and guards itself.
assert_graph_profile_is_executable(vllm_config, role="target")
assert_ple_mode(vllm_config, role="target")
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
if vllm_config.lora_config is not None:
# LoRA does not support the merged QKV/indexer projection, so its
+2 -1
View File
@@ -51,7 +51,7 @@ from vllm.model_executor.models.utils import (
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.e2_guards import assert_graph_profile_is_executable
from vllm.v1.e2_guards import assert_graph_profile_is_executable, assert_ple_mode
from .hyperconnection import GatedResidual, HyperConnectionConfig
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
@@ -400,6 +400,7 @@ class Qwen4ExpMTP(
# this class builds Qwen4ExpMultiTokenPredictor directly, from a config
# rebuilt at mtp.py:159 -- so its graph profile is checked here.
assert_graph_profile_is_executable(vllm_config, role="draft")
assert_ple_mode(vllm_config, role="draft")
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
self.vllm_config = vllm_config
self.quant_config = vllm_config.quant_config
@@ -10,6 +10,7 @@ import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
import vllm.envs as envs
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.base_config import (
@@ -357,11 +358,18 @@ class Qwen4ExpNGramEmbedding(nn.Module):
if params_dtype is None:
params_dtype = torch.get_default_dtype()
engram_config = get_current_vllm_config().engram_config
use_uva = engram_config is not None and engram_config.cpu_offload
if use_uva and envs.VLLM_E2_PLE_PULL_TRANSPORT:
# Fork: no silent fallback between transports.
raise RuntimeError(
"PLE backend selection: cpu_offload=True (upstream UVA) with "
"VLLM_E2_PLE_PULL_TRANSPORT=1 -- refusing a double offload"
)
embedding_cls = (
Qwen4ExpPLEFileGatherEmbedding
if engram_config is not None and engram_config.host_file_gather
else Qwen4ExpPLEPinnedHostEmbedding
if engram_config is not None and engram_config.cpu_offload
if use_uva
else Qwen4ExpPLEDeviceEmbedding
)
self.ngram_embedding = embedding_cls(
@@ -382,14 +390,21 @@ class Qwen4ExpNGramEmbedding(nn.Module):
max_total_tokens, self.ngram_heads, dtype=torch.long
)
weight = self.ngram_embedding.weight
# Fork: backend= names the class (or, inside the pull transport's
# offload process, the table it serves), so a smoke need not derive
# the backend from pinned=. Fields before it are upstream's, unchanged.
backend = type(self.ngram_embedding).__name__
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
backend = f"E2PullTransport.offload_table:{backend}"
logger.info(
"Initialized PLE embedding %s: quantization_method=%s, "
"weight_dtype=%s, weight_device=%s, pinned=%s",
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
embedding_prefix,
type(embedding_quant_method).__name__,
weight.dtype,
weight.device,
weight.is_pinned(),
backend,
)
@staticmethod
@@ -402,6 +417,13 @@ class Qwen4ExpNGramEmbedding(nn.Module):
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
# Fork: never build the UVA table beside the E2 pull transport --
# that would be a second offload of the same PLE table.
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
raise RuntimeError(
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
)
previous_eos = torch.cat(
[
eos_positions.new_full((batch_size, 1), -1),
+31 -10
View File
@@ -8,6 +8,7 @@ import torch
from torch import nn
from transformers import Qwen4ExpTextConfig
import vllm.envs as envs
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
from vllm.forward_context import get_forward_context
@@ -91,16 +92,32 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
self.num_spec_tokens = vllm_config.num_speculative_tokens
self.activation = "silu"
self.ple_embedding = Qwen4ExpNGramEmbedding(
config,
int(config.ple_embed_dim),
self.ple_dense_layer_id,
vllm_config.scheduler_config.max_num_batched_tokens,
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
prefix=f"{prefix}.ple_embedding",
quant_config=quant_config,
params_dtype=model_config.dtype,
)
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
# Fork: the E2 pull transport owns the table in a CPU-only
# offload process; upstream's device/UVA backends are never built.
from .ple_pull import Qwen4ExpNGramEmbeddingPull
self.ple_embedding = Qwen4ExpNGramEmbeddingPull.build(
config,
int(config.ple_embed_dim),
self.ple_dense_layer_id,
vllm_config.scheduler_config.max_num_batched_tokens,
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
prefix=f"{prefix}.ple_embedding",
quant_config=quant_config,
params_dtype=model_config.dtype,
)
else:
self.ple_embedding = Qwen4ExpNGramEmbedding(
config,
int(config.ple_embed_dim),
self.ple_dense_layer_id,
vllm_config.scheduler_config.max_num_batched_tokens,
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
prefix=f"{prefix}.ple_embedding",
quant_config=quant_config,
params_dtype=model_config.dtype,
)
# The PLE cache is TP-replicated, so this merged projection is too.
self.kv_proj = MergedColumnParallelLinear(
int(config.ple_embed_dim),
@@ -145,6 +162,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
) -> torch.Tensor:
"""Dequantize PLE lookup output."""
return self.ple_embedding.ngram_embedding.dequantize(
dequantize_pulled = getattr(self.ple_embedding, "dequantize_pulled", None)
if dequantize_pulled is not None:
# Fork pull transport: no local table; FP8 scale kept here.
return dequantize_pulled(embeddings, output_dtype)
embeddings,
output_dtype,
)
+342
View File
@@ -0,0 +1,342 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The fork's PLE pull transport, re-seated on vLLM v0.30.0.
v0.30.0 split the E2 base's offload-aware ``Qwen4ExpNGramEmbedding`` into an
upstream-only module (``ngram_embedding.py``, device or pinned-host UVA table).
This module puts the E2 pull transport back WITHOUT editing that upstream
class: ``Qwen4ExpNGramEmbeddingPull`` inherits it and ``PleOffloadLayer``.
* In a GPU worker, ``PleOffloadLayer.__init_subclass__`` skips the constructor:
the module is an empty placeholder with no table. ``PleOffloadConnector``
binds it to a host-mapped flag/buffer; its forward is the captured
``ple_offload_wait_copy`` op. Only the FP8 global scale is kept on the GPU.
* In the CPU-only offload process the constructor runs in full, on CPU. The
table is upstream's ``Qwen4ExpPLEDeviceEmbedding`` allocated on CPU;
``forward_impl`` computes n-gram IDs and gathers raw (FP8) rows.
Selected only when ``VLLM_E2_PLE_PULL_TRANSPORT=1``; with the switch off this
module is never imported by the model and the tree is upstream.
Supported table formats: FP8 (global scale) and unquantized -- the two
``Qwen4ExpPLEEmbeddingMethod`` families v0.30.0 itself supports. The E2 NVFP4
path is not carried (v0.30.0 has no NVFP4 PLE method to select).
"""
from collections.abc import Iterable
import torch
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.ple_offload_layer import (
PleOffloadLayer,
is_offload_process,
)
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.transformers_utils.configs.qwen4_exp import Qwen4ExpTextConfig
from .ngram_embedding import (
Qwen4ExpNGramEmbedding,
Qwen4ExpPLEEmbeddingMethod,
Qwen4ExpPLEFp8EmbeddingMethod,
Qwen4ExpPLEUnquantizedEmbeddingMethod,
)
logger = init_logger(__name__)
#: The backend name this module reports in the PLE producer log line.
PULL_BACKEND_NAME = "E2PullTransport"
class Qwen4ExpNGramEmbeddingPull(PleOffloadLayer, Qwen4ExpNGramEmbedding):
"""Upstream n-gram embedding whose table lives in the offload process."""
#: GPU placeholder: the method the offload process will use for the table,
#: resolved with upstream's own selector (set by ``build``).
_offload_quant_method: Qwen4ExpPLEEmbeddingMethod | None = None
def __init__(
self,
config: Qwen4ExpTextConfig,
embedding_dim: int,
ple_dense_layer_id: int,
max_total_tokens: int,
*,
data_parallel_rank: int,
prefix: str,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
) -> None:
# Runs only in the offload process (guarded by PleOffloadLayer).
engram_config = get_current_vllm_config().engram_config
if engram_config is not None and engram_config.cpu_offload:
raise RuntimeError(
"PLE pull transport: the offload process resolved "
"cpu_offload=True; refusing to build upstream's UVA table here."
)
Qwen4ExpNGramEmbedding.__init__(
self,
config,
embedding_dim,
ple_dense_layer_id,
max_total_tokens,
data_parallel_rank=data_parallel_rank,
prefix=prefix,
quant_config=quant_config,
params_dtype=params_dtype,
)
@classmethod
def build(
cls,
config: Qwen4ExpTextConfig,
embedding_dim: int,
ple_dense_layer_id: int,
max_total_tokens: int,
*,
data_parallel_rank: int,
prefix: str,
quant_config: QuantizationConfig | None,
params_dtype: torch.dtype | None,
) -> "Qwen4ExpNGramEmbeddingPull":
"""Construct the placeholder (GPU) or the full CPU module (offload)."""
# The offload process builds the surrounding model on meta; this
# subtree must own real CPU storage. In a GPU worker the constructor is
# skipped, so the device context allocates nothing.
with torch.device(cls.get_target_device()):
module = cls(
config,
embedding_dim,
ple_dense_layer_id,
max_total_tokens,
data_parallel_rank=data_parallel_rank,
prefix=prefix,
quant_config=quant_config,
params_dtype=params_dtype,
)
embedding_prefix = f"{prefix}.ngram_embedding"
method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
quant_config,
embedding_prefix,
getattr(config, "ple_embedding_dtype", None),
)
if not isinstance(
method,
(Qwen4ExpPLEFp8EmbeddingMethod, Qwen4ExpPLEUnquantizedEmbeddingMethod),
):
raise NotImplementedError(
f"PLE pull transport does not support {type(method).__name__}"
)
module._offload_quant_method = method
module._pull_params_dtype = params_dtype or torch.get_default_dtype()
module._pull_embedding_prefix = embedding_prefix
if not is_offload_process():
# The GPU-rank producer line, in upstream's format plus backend=
# (the offload process logs its own table line from upstream's
# constructor). The table lives in the offload process's CPU memory
# and is not pinned; rows cross via host-mapped buffers.
logger.info(
"Initialized PLE embedding %s: quantization_method=%s, "
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
embedding_prefix,
type(method).__name__,
module.get_offload_output_dtype(module._pull_params_dtype),
"cpu",
False,
PULL_BACKEND_NAME,
)
return module
# ------------------------------------------------------------- formats
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
"""Rows cross the transport in their table storage dtype."""
embedding = getattr(self, "ngram_embedding", None)
weight = getattr(embedding, "weight", None)
if weight is not None:
return weight.dtype
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
return torch.float8_e4m3fn
return getattr(self, "_pull_params_dtype", default_dtype)
def get_offload_output_dim(self, default_dim: int) -> int:
return default_dim
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
"""Dummy loads skip checkpoint scales; give FP8 a unit scale."""
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
self.register_buffer(
"_offload_weight_scale",
torch.ones((), dtype=torch.float32, device=device),
persistent=False,
)
def dequantize_pulled(
self, embeddings: torch.Tensor, output_dtype: torch.dtype
) -> torch.Tensor:
"""GPU-side dequantization of rows DMA'd from the offload process.
Same arithmetic as upstream ``Qwen4ExpPLEFp8EmbeddingMethod.dequantize``
(and the E2 ``_dequantize_embeddings`` FP8 branch).
"""
method = self._offload_quant_method
if isinstance(method, Qwen4ExpPLEUnquantizedEmbeddingMethod):
return embeddings
weight_scale = getattr(self, "_offload_weight_scale", None)
if weight_scale is None:
raise RuntimeError(
"FP8 PLE pull transport is missing its global scale on the GPU"
)
if weight_scale.device != embeddings.device:
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
# ------------------------------------------------------------- execution
def start_prefetch(
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> None:
"""No prefetch: rows are published by the offload process."""
return None
def _compute_ngram_ids_offload(
self,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
) -> torch.Tensor:
"""CPU n-gram IDs, carried unchanged from the E2 offload path.
Differs from upstream's CPU branch in two E2 fixes: the packed layout
is (num_reqs, max_seq_len) rather than (num_reqs, num_tokens), and
CUDA-graph padding rows beyond ``query_start_loc[-1]`` are not written,
so they cannot overwrite a request's final real token.
"""
input_ids = input_ids.reshape(-1).long()
query_start_loc = query_start_loc.long()
num_reqs = query_start_loc.numel() - 1
num_tokens = input_ids.shape[0]
if num_reqs <= 0:
raise ValueError("PLE requires at least one request")
max_seq_len = max(
1,
int((query_start_loc[1:] - query_start_loc[:-1]).max().item()),
)
num_valid_tokens = min(int(query_start_loc[-1].item()), num_tokens)
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
packed = torch.full(
(num_reqs, max_seq_len),
self.eos_token_id,
device=input_ids.device,
dtype=torch.int64,
)
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
request_indices.clamp_(max=num_reqs - 1)
columns = (positions - query_start_loc[request_indices]).clamp(
0, packed.shape[1] - 1
)
packed[request_indices[:num_valid_tokens], columns[:num_valid_tokens]] = (
input_ids[:num_valid_tokens]
)
ngram_context = ngram_context[:num_reqs].to(
device=input_ids.device, dtype=torch.long
)
context = torch.cat([ngram_context, packed], dim=-1)
positions_2d, position_in_segment = self._shift_precompute(
context, self.eos_token_id
)
shifted = [context]
for shift in range(1, self.ngram_size):
shifted.append(
self._shift_apply(
context,
positions_2d,
position_in_segment,
shift,
self.eos_token_id,
)
)
adjusted_columns = columns + self.ngram_size - 1
id_blocks = []
for ngram in range(2, self.ngram_size + 1):
start = (ngram - 2) * self.heads_per_ngram
end = start + self.heads_per_ngram
mixed = shifted[0] * self.layer_multipliers[0]
for index in range(1, ngram):
mixed = torch.bitwise_xor(
mixed, shifted[index] * self.layer_multipliers[index]
)
sizes = self.ngram_heads_vocab_sizes[start:end]
offsets = self.ngram_heads_offsets[start:end]
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
id_blocks.append(ids[request_indices, adjusted_columns])
return torch.cat(id_blocks, dim=-1)
def forward_impl( # type: ignore[override]
self,
hidden_states: torch.Tensor,
input_ids: torch.Tensor,
query_start_loc: torch.Tensor,
ngram_context: torch.Tensor,
output_buffer: torch.Tensor | None = None,
) -> torch.Tensor:
"""Offload process only: gather raw table rows for these tokens."""
del hidden_states
if not is_offload_process():
raise RuntimeError(
"PLE pull transport: forward_impl ran outside the offload "
"process -- the GPU placeholder was never bound by the connector"
)
ngram_ids = self._compute_ngram_ids_offload(
input_ids, query_start_loc, ngram_context
)
num_tokens = input_ids.reshape(-1).shape[0]
weight = self.ngram_embedding.weight
if output_buffer is not None:
output = output_buffer[:num_tokens, : self.embedding_dim]
torch.index_select(
weight,
0,
ngram_ids.reshape(-1),
out=output.reshape(-1, self.head_dim),
)
return output
return torch.index_select(weight, 0, ngram_ids.reshape(-1)).reshape(
num_tokens, self.embedding_dim
)
# ------------------------------------------------------------- loading
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Offload process: upstream loading. GPU worker: keep the FP8 scale.
The GPU placeholder owns no table; every other checkpoint tensor for
this subtree is consumed and dropped here (the offload process loads
the table itself). Prefix grouping may call this more than once.
"""
if is_offload_process():
return Qwen4ExpNGramEmbedding.load_weights(self, weights)
retained: set[str] = set()
fp8 = isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod)
for name, loaded_weight in weights:
if fp8 and name == "ngram_embedding.weight_scale":
self.register_buffer(
"_offload_weight_scale",
loaded_weight.reshape(()).to(
device=torch.accelerator.current_accelerator(),
dtype=torch.float32,
),
persistent=False,
)
retained.add(name)
return retained
__all__ = ["PULL_BACKEND_NAME", "Qwen4ExpNGramEmbeddingPull"]
+146
View File
@@ -214,5 +214,151 @@ def assert_graph_profile_is_executable(vllm_config, *, role: str = "target") ->
)
# --------------------------------------------------------------------------
# PLE transport: carry the pull transport, UVA disabled.
# --------------------------------------------------------------------------
#: The fork's pull-transport switch (strict "0"/"1"; registered and hashed in
#: vllm/envs.py). Renamed from the E2 base's VLLM_PLE_CPU_OFFLOAD.
PULL_TRANSPORT_ENV = "VLLM_E2_PLE_PULL_TRANSPORT"
#: Upstream v0.30.0's switch for the pinned-host UVA PLE backend. Same name as
#: the E2 base's pull switch, opposite default (upstream: "1").
UPSTREAM_UVA_ENV = "VLLM_PLE_CPU_OFFLOAD"
def pull_transport_enabled() -> bool:
"""Whether the fork's pull transport is on. Strict; raises on bad values.
One source: the registered accessor ``envs.VLLM_E2_PLE_PULL_TRANSPORT``,
whose parser (``envs._e2_strict_switch``) accepts exactly unset/"0"/"1".
"""
import vllm.envs as envs
return bool(envs.VLLM_E2_PLE_PULL_TRANSPORT)
def check_upstream_uva_env_against_pull() -> None:
"""Fail loud if the old name would turn upstream's UVA path on.
With the pull transport on, ``VLLM_PLE_CPU_OFFLOAD`` may be unset (the
fork then supplies ``cpu_offload=False`` as the Engram default) or exactly
``"0"``. ``"1"`` -- the E2-era spelling of "enable our offload" -- would
enable upstream's pinned-host UVA backend beside ours: a double offload.
Any other value is rejected rather than interpreted. With the pull
transport off this function does nothing: the name is upstream's.
"""
if not pull_transport_enabled():
return
raw = os.environ.get(UPSTREAM_UVA_ENV)
if raw is None or raw == "0":
return
if raw == "1":
raise RuntimeError(
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}=1. On "
f"vLLM v0.30.0 {UPSTREAM_UVA_ENV} is UPSTREAM's switch for the "
"pinned-host UVA PLE backend (default on), not the E2 pull "
"transport, so this launch asks for both transports at once. Unset "
f"{UPSTREAM_UVA_ENV} or export {UPSTREAM_UVA_ENV}=0 "
"(the pull transport replaces that switch)."
)
raise ValueError(
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}={raw!r}. "
f'With the pull transport on, {UPSTREAM_UVA_ENV} must be unset or '
'exactly "0"; any other value is refused rather than guessed.'
)
def pull_transport_cpu_offload_default(upstream_default: bool) -> bool:
"""The Engram ``cpu_offload`` default, as this fork resolves it.
Pull off: ``upstream_default`` unchanged (the tree behaves as upstream).
Pull on: the old-name check above, then ``False`` -- upstream's UVA
backend is never the default beside the pull transport.
"""
if not pull_transport_enabled():
return upstream_default
check_upstream_uva_env_against_pull()
return False
def verify_engram_against_pull(engram_config) -> None:
"""Refuse an explicit ``cpu_offload=True`` while the pull transport is on.
``--engram-config '{"cpu_offload": true}'`` takes precedence over every
default, so it is checked on the resolved object, not the environment.
"""
if not pull_transport_enabled():
return
check_upstream_uva_env_against_pull()
if engram_config is not None and bool(engram_config.cpu_offload):
raise RuntimeError(
f"e2-guard: {PULL_TRANSPORT_ENV}=1 but the resolved EngramConfig "
"has cpu_offload=True (upstream's pinned-host UVA PLE backend). "
"Both transports would own the PLE table. Drop cpu_offload from "
"--engram-config (or set it false), or turn the pull transport off."
)
#: Effective PLE modes, as the guard names them.
PLE_MODE_PULL = "pull" # E2 pull transport (VLLM_E2_PLE_PULL_TRANSPORT=1)
PLE_MODE_UVA = "uva" # upstream Qwen4ExpPLEPinnedHostEmbedding
PLE_MODE_DEVICE = "device" # upstream Qwen4ExpPLEDeviceEmbedding
PLE_MODE_NONE = "none" # no Engram config resolved (no PLE layers / not CUDA)
def resolve_ple_mode(vllm_config) -> str:
"""The PLE transport this process will build, or an error if mixed.
Reads the two inputs that decide it -- the pull switch (strict) and the
RESOLVED ``engram_config.cpu_offload`` -- and nothing else. Mixed states
raise; there is no fallback from one transport to another.
"""
pull = pull_transport_enabled()
check_upstream_uva_env_against_pull()
engram_config = getattr(vllm_config, "engram_config", None)
cpu_offload = (
None if engram_config is None else bool(engram_config.cpu_offload)
)
if pull:
if cpu_offload:
raise RuntimeError(
f"e2-guard: {PULL_TRANSPORT_ENV}=1 with a resolved Engram "
"cpu_offload=True: the pull transport and upstream's UVA "
"backend would both own the PLE table."
)
return PLE_MODE_PULL
if cpu_offload is None:
return PLE_MODE_NONE
return PLE_MODE_UVA if cpu_offload else PLE_MODE_DEVICE
def assert_ple_mode(vllm_config, *, role: str = "target") -> str:
"""Startup guard: resolve, assert and log the effective PLE mode.
Runs beside ``assert_graph_profile_is_executable`` at model construction
for ``role="target"`` and ``role="draft"`` (the MTP drafter carries the
target's EngramConfig and the same process environment, so a mixed
configuration fails there too). NOT disabled by VLLM_E2_ALLOW_BREAKABLE:
that opt-out is about graph identity, not about owning the PLE table twice.
"""
mode = resolve_ple_mode(vllm_config)
model_config = getattr(vllm_config, "model_config", None)
text_config = getattr(model_config, "hf_text_config", None)
ple_layer_ids = list(getattr(text_config, "ple_layer_ids", None) or [])
engram_config = getattr(vllm_config, "engram_config", None)
logger.info(
"e2-guard[%s]: ple_mode=%s pull_transport=%d engram_cpu_offload=%s "
"%s=%s ple_layer_ids=%s.",
role,
mode,
1 if mode == PLE_MODE_PULL else 0,
None if engram_config is None else bool(engram_config.cpu_offload),
UPSTREAM_UVA_ENV,
os.environ.get(UPSTREAM_UVA_ENV, "<unset>"),
ple_layer_ids,
)
return mode
#: Back-compatible name for the earlier call sites.
assert_breakable_cudagraph_pinned = assert_graph_profile_is_executable
+6
View File
@@ -716,6 +716,10 @@ class WorkerProc:
# Load model
self.worker.init_device()
# PLE pull transport: DP0/TP0 starts the CPU-only PLE process before
# load_model, so its weight load overlaps ours; READY is awaited after.
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
self.worker.spawn_ple_offload()
# Update process title now that parallel groups are initialized
self.setup_proc_title_and_log_prefix(
enable_ep=vllm_config.parallel_config.enable_expert_parallel
@@ -724,6 +728,8 @@ class WorkerProc:
self.worker.elastic_ep_execute("load_model")
else:
self.worker.load_model()
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
self.worker.wait_ple_offload_ready()
scheduler_config = vllm_config.scheduler_config
self.use_async_scheduling = scheduler_config.async_scheduling
+4
View File
@@ -67,11 +67,15 @@ class UniProcExecutor(Executor):
self.driver_worker.init_worker(all_kwargs=[kwargs])
self.driver_worker.init_device()
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
self.driver_worker.spawn_ple_offload()
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
self.driver_worker.elastic_ep_execute("load_model")
else:
self.driver_worker.load_model()
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
self.driver_worker.wait_ple_offload_ready()
current_platform.update_block_size_for_backend(self.vllm_config)
def _distributed_args(self) -> tuple[str, int, int]:
+1 -1
View File
@@ -257,7 +257,7 @@ class PleOffloadConnector:
}
if not layers:
raise RuntimeError(
"VLLM_PLE_CPU_OFFLOAD is enabled, but the model has no PleOffloadLayer"
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no PleOffloadLayer"
)
config = vllm_config.model_config.hf_text_config
+13
View File
@@ -0,0 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compatibility exports for the top-level test-only fault singleton."""
from vllm_ple_test_fault import (
_FAULT as _FAULT,
)
from vllm_ple_test_fault import ( # noqa: F401
FaultSpec as FaultSpec,
)
from vllm_ple_test_fault import (
_parse_fault_spec as _parse_fault_spec,
)
+1 -1
View File
@@ -830,7 +830,7 @@ class PleOffloadRunner:
}
if not offload_layers:
raise RuntimeError(
"VLLM_PLE_CPU_OFFLOAD is enabled, but no PleOffloadLayer "
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but no PleOffloadLayer "
"was found in the initialized model"
)
logger.info(
+33
View File
@@ -385,6 +385,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# For transferring state from execute_model to subsequent sample_tokens call.
self.execute_model_state: ExecuteModelState | None = None
# PLE pull transport: set by _setup_ple_offload() from Worker.load_model
# when VLLM_E2_PLE_PULL_TRANSPORT=1 and this rank owns a PLE layer.
# None otherwise, and every hook below is then a no-op (upstream path).
self._ple_offload_connector: PleOffloadConnector | None = None
# Expert parallelism load balancer.
self.eplb = EPLBController(self.parallel_config, self.device)
@@ -408,6 +412,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
tasks.extend(PoolingRunner.get_supported_tasks(self.model))
return tuple(tasks)
def _setup_ple_offload(self, ipc_addr: str) -> None:
"""Initialize PLE offload after the model state is available."""
# PLE placeholders are created by model loading, so CUDA output
# registration cannot happen in the runner constructor.
query_start_loc_source = getattr(self.model_state, "ple_query_start_loc", None)
ngram_context_source = getattr(self.model_state, "ngram_context", None)
if not isinstance(query_start_loc_source, torch.Tensor):
raise RuntimeError("PLE offload requires a query_start_loc source")
if not isinstance(ngram_context_source, torch.Tensor):
raise RuntimeError("PLE offload requires an ngram_context source")
self._ple_offload_connector = PleOffloadConnector(
self.vllm_config,
self.model,
self.device,
ipc_addr,
input_ids_source=self.input_buffers.input_ids,
query_start_loc_source=query_start_loc_source,
ngram_context_source=ngram_context_source,
)
def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:
time_before_load = time.perf_counter()
if load_dummy_weights:
@@ -1102,6 +1126,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
torch.accelerator.empty_cache()
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
# Decoder capture calls the model outside execute_model. Keep dummy
# PLE outputs signaled until all graphs capture their semaphore wait.
if self._ple_offload_connector is not None and capture_decoder:
self._ple_offload_connector.signal_dummy_outputs(self.max_num_tokens)
with self.maybe_setup_dummy_loras(self.lora_config):
if capture_encoder:
self.model_state.encoder_runner.capture()
@@ -1137,6 +1165,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.adaptive_verification.set_initial_cost_curves(timings)
self.kv_connector.reset_capture_state()
if self._ple_offload_connector is not None and capture_decoder:
self._ple_offload_connector.release_outputs()
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
if not profile_only:
@@ -2479,6 +2509,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return None
input_batch = self.execute_model_state.input_batch
if self._ple_offload_connector is not None:
self._ple_offload_connector.close()
self._ple_offload_connector = None
hidden_states = self.execute_model_state.hidden_states
finished_req_ids = self.execute_model_state.finished_req_ids
ec_connector_output = self.execute_model_state.ec_connector_output
+32 -2
View File
@@ -233,6 +233,22 @@ class Worker(WorkerBase):
validate_worker_profiler_config(self.profiler_config)
self.use_v2_model_runner = vllm_config.use_v2_model_runner
# PLE pull transport (VLLM_E2_PLE_PULL_TRANSPORT). Off: both stay
# None/False and every PLE hook in this class is a no-op.
self._ple_offload_worker_handle: Any | None = None
self._ple_offload_stopping = False
self._ple_offload_enabled = self._has_ple_layers()
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
if self._ple_offload_enabled:
self._validate_ple_offload_config()
elif self.rank == 0 and self.parallel_config.data_parallel_rank == 0:
text_config = self.model_config.hf_text_config
logger.warning(
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but this rank owns "
"no PLE layer (ple_layer_ids=%s); no PLE offload connector "
"is created here.",
getattr(text_config, "ple_layer_ids", None),
)
# Device handles of the previous step's PP intermediate-tensor send.
self._pp_send_work: list[Handle] = []
@@ -242,7 +258,7 @@ class Worker(WorkerBase):
def _has_ple_layers(self) -> bool:
"""Return whether this model configuration constructs PLE layers."""
if not envs.VLLM_PLE_CPU_OFFLOAD:
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
return False
text_config = self.model_config.hf_text_config
ple_layer_ids = getattr(text_config, "ple_layer_ids", None)
@@ -304,7 +320,7 @@ class Worker(WorkerBase):
if unsupported:
raise ValueError(
"VLLM_PLE_CPU_OFFLOAD does not support the requested "
"VLLM_E2_PLE_PULL_TRANSPORT does not support the requested "
"configuration. Unsupported settings: "
f"{', '.join(unsupported)}"
)
@@ -718,6 +734,12 @@ class Worker(WorkerBase):
):
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
# Model loading constructs the PLE placeholders whose buffers are
# registered with the offload process by the model runner V2 connector.
if self._ple_offload_enabled:
model_runner = cast("GPUModelRunnerV2", self.model_runner)
model_runner._setup_ple_offload(self.parallel_config._ple_offload_ipc_path)
if has_ec_transfer():
get_ec_transfer().start_worker_services()
@@ -738,6 +760,10 @@ class Worker(WorkerBase):
self.model_runner.update_config(overrides)
def reload_weights(self, *args, **kwargs) -> None:
if self._ple_offload_enabled:
raise NotImplementedError(
"Weight reload is not supported with the PLE pull transport"
)
with set_current_vllm_config(self.vllm_config):
self.model_runner.reload_weights(*args, **kwargs)
@@ -1759,6 +1785,10 @@ class Worker(WorkerBase):
self.elastic_ep_executor.shutdown()
if self._ple_offload_worker_handle is not None:
self._ple_offload_worker_handle.close()
self._ple_offload_worker_handle = None
# Release GPU resources held by the model runner so that memory
# can be reclaimed when running in-process
if model_runner := getattr(self, "model_runner", None):