[PLE] Integrate the pull transport on v0.30.0 with upstream UVA refused
- VLLM_E2_PLE_PULL_TRANSPORT replaces the fork's old switch name, which v0.30.0 reuses for its UVA backend with the opposite default; strict parser; compile/capture factor. - The pull transport is reachable on v0.30.0 through a pull embedding class that inherits upstream's without editing it; the upstream UVA backend is provably off while the pull switch is on. - resolve_ple_mode()/assert_ple_mode() log and assert the effective PLE mode for target and drafter, and refuse a mixed configuration. - Producer log lines for the PLE backend and weight dtype. - CPU tests for mode resolution and the mixed-config refusal. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+19
-3
@@ -3,9 +3,10 @@
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic import Field, model_validator
|
||||
from typing_extensions import Self
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import config, get_hash_factors, hash_factors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -21,6 +22,17 @@ _NGRAM_LAYER_FIELDS = {
|
||||
}
|
||||
|
||||
|
||||
def _default_cpu_offload() -> bool:
|
||||
# Fork: with VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a
|
||||
# VLLM_PLE_CPU_OFFLOAD value that would enable UVA fails the boot. With the
|
||||
# pull transport off, the upstream default below is returned unchanged.
|
||||
from vllm.v1.e2_guards import pull_transport_cpu_offload_default
|
||||
|
||||
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
return envs.VLLM_PLE_CPU_OFFLOAD
|
||||
return pull_transport_cpu_offload_default(False)
|
||||
|
||||
|
||||
def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
||||
"""Whether the model carries n-gram embedding layers."""
|
||||
if model_config is None:
|
||||
@@ -35,8 +47,12 @@ def model_has_engram_layers(model_config: "ModelConfig | None") -> bool:
|
||||
class EngramConfig:
|
||||
"""Configuration for Engram embedding storage and sharding."""
|
||||
|
||||
cpu_offload: bool = True
|
||||
"""Store embedding weights in pinned CPU memory for UVA lookup."""
|
||||
cpu_offload: bool = Field(default_factory=_default_cpu_offload)
|
||||
"""Store embedding weights in pinned CPU memory for UVA lookup.
|
||||
Defaults to VLLM_PLE_CPU_OFFLOAD, which is enabled by default. An explicit
|
||||
value takes precedence over the environment variable. With
|
||||
VLLM_E2_PLE_PULL_TRANSPORT=1 the default is False and a value that would
|
||||
enable UVA fails the boot (vllm/v1/e2_guards.py)."""
|
||||
|
||||
host_file_gather: bool = False
|
||||
"""Serve Qwen4Exp PLE rows by gathering them on the CPU from the loader's
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.config.fault_tolerance import FaultToleranceConfig
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_ports_list
|
||||
from vllm.utils.network_utils import get_open_ports_list, get_open_zmq_ipc_path
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ray.runtime_env import RuntimeEnv
|
||||
@@ -369,6 +369,10 @@ class ParallelConfig:
|
||||
"""Port of the coordination TCPStore. Can be set by the API server; workers
|
||||
connect as clients to exchange self-picked group ports at runtime."""
|
||||
|
||||
_ple_offload_ipc_path: str = ""
|
||||
"""Node-local ZMQ IPC address for the fork's PLE pull-transport offload
|
||||
process. Set only when VLLM_E2_PLE_PULL_TRANSPORT=1."""
|
||||
|
||||
decode_context_parallel_size: int = Field(default=1, ge=1)
|
||||
"""Number of ranks that shard the decode KV cache. DCP does not expand
|
||||
the process world size. Without PCP, DCP reuses TP ranks. With PCP, DCP
|
||||
@@ -517,6 +521,9 @@ class ParallelConfig:
|
||||
"The FT system assumes one AsyncMPClient manages all engines."
|
||||
)
|
||||
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not self._ple_offload_ipc_path:
|
||||
self._ple_offload_ipc_path = get_open_zmq_ipc_path()
|
||||
|
||||
if self.all2all_backend in ["pplx", "naive"]:
|
||||
logger.warning(
|
||||
"The '%s' all2all backend has been removed. "
|
||||
@@ -910,6 +917,7 @@ class ParallelConfig:
|
||||
"worker_extension_cls",
|
||||
"_api_process_count",
|
||||
"_api_process_rank",
|
||||
"_ple_offload_ipc_path",
|
||||
# NUMA binding is per-rank host-side memory locality; it does
|
||||
# not affect collective-communication semantics. When numa_bind
|
||||
# is enabled with auto-detection, each DP rank stores its own
|
||||
|
||||
@@ -1249,6 +1249,11 @@ class VllmConfig:
|
||||
# PyTorch's expandable_segments allocator uses CUDA VMM, which can
|
||||
# remap a virtual address range to different physical pages over the
|
||||
# engine's lifetime. KV connectors that pin KV cache memory (e.g.
|
||||
# Fork: an explicit cpu_offload=True beside the pull transport is
|
||||
# a double offload; refused here, before any worker starts.
|
||||
from vllm.v1.e2_guards import verify_engram_against_pull
|
||||
|
||||
verify_engram_against_pull(self.engram_config)
|
||||
# NixlConnector via ibv_reg_mr, MooncakeConnector) end up with their
|
||||
# registrations pointing at stale physical pages after any remap,
|
||||
# producing RDMA failures like IBV_WC_REM_ACCESS_ERR /
|
||||
|
||||
@@ -382,6 +382,18 @@ if TYPE_CHECKING:
|
||||
# to decide behaviour. Registered with the medium-path exact restart
|
||||
# so the cache-factor contract is complete.
|
||||
MERLIN_TOPK_EXACT_MEDIUM: str | None = None
|
||||
# The fork's PLE pull transport -- the host-mapped,
|
||||
# fail-closed offload process in vllm/v1/ple_offload/. Renamed from the E2
|
||||
# base's VLLM_PLE_CPU_OFFLOAD, which vLLM v0.30.0 reuses for a DIFFERENT
|
||||
# subsystem with the OPPOSITE default (upstream: pinned-host UVA lookup,
|
||||
# default ON). The two subsystems no longer share a switch.
|
||||
# Strict: unset or "0" = off, "1" = on, anything else is a boot error.
|
||||
# On: the PLE table lives in the offload process, the GPU placeholder waits
|
||||
# on a host flag and DMAs rows inside the captured forward, and upstream's
|
||||
# UVA backend is refused (vllm/v1/e2_guards.py). Off: the tree is upstream.
|
||||
# AOT: yes (different module class and forward). CAPTURE: yes (the captured
|
||||
# graph contains the wait/copy ops instead of a table lookup). Hashed.
|
||||
VLLM_E2_PLE_PULL_TRANSPORT: bool = False
|
||||
#
|
||||
# RUNTIME-ONLY (registered for validate_environ + a single cached read):
|
||||
# Static K/V scale sidecar: fill_()ed into existing fp32 device buffers
|
||||
@@ -780,6 +792,21 @@ def _resolve_rust_cli_path() -> str | None:
|
||||
return raw
|
||||
|
||||
|
||||
def _e2_strict_switch(name: str) -> bool:
|
||||
"""Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``.
|
||||
|
||||
Unset means off. Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
|
||||
``"01"`` -- is a boot error: a switch that selects a different transport
|
||||
must not be satisfiable by a spelling nobody meant.
|
||||
"""
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw == "0":
|
||||
return False
|
||||
if raw == "1":
|
||||
return True
|
||||
raise ValueError(f'{name} must be exactly "0" or "1" (or unset), got {raw!r}')
|
||||
|
||||
|
||||
def _e2_diagnostic_mode(name: str) -> str:
|
||||
"""Normalize an E2 diagnostic gate to off|warn|raise.
|
||||
|
||||
@@ -2328,6 +2355,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_QSA_KV_COLLECT": lambda: os.getenv("VLLM_QSA_KV_COLLECT", "").strip(),
|
||||
# Capture-configuration factor, read in C++; raw passthrough on purpose.
|
||||
"MERLIN_TOPK_EXACT_MEDIUM": lambda: os.getenv("MERLIN_TOPK_EXACT_MEDIUM", None),
|
||||
# PLE pull transport switch; strict "0"/"1" (see the TYPE_CHECKING block).
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT"
|
||||
),
|
||||
# Runtime only; all of the following are in compile_factors.ignored_factors.
|
||||
"VLLM_E2_ALLOW_BREAKABLE": lambda: os.getenv("VLLM_E2_ALLOW_BREAKABLE", "") == "1",
|
||||
"VLLM_E2_EXPECTED_CUDAGRAPH_MODE": lambda: os.getenv(
|
||||
|
||||
@@ -346,7 +346,7 @@ class PleOffloadLayer(nn.Module, ABC):
|
||||
def guarded_init(
|
||||
self: "PleOffloadLayer", *args: object, **kwargs: object
|
||||
) -> None:
|
||||
if envs.VLLM_PLE_CPU_OFFLOAD and not is_offload_process():
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT and not is_offload_process():
|
||||
nn.Module.__init__(self)
|
||||
return
|
||||
original_init(self, *args, **kwargs)
|
||||
@@ -356,7 +356,7 @@ class PleOffloadLayer(nn.Module, ABC):
|
||||
@classmethod
|
||||
def get_target_device(cls) -> torch.device:
|
||||
"""Return CPU for the offload process and the active GPU otherwise."""
|
||||
if envs.VLLM_PLE_CPU_OFFLOAD:
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
return torch.device("cpu")
|
||||
return torch.device("cuda", torch.accelerator.current_device_index())
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.distributed import get_dp_group, get_etp_group, get_tp_group
|
||||
from vllm.forward_context import DPMetadata, get_forward_context
|
||||
@@ -422,6 +423,13 @@ class Qwen4ExpPLEPinnedHostEmbedding(Qwen4ExpPLEEmbedding):
|
||||
max_total_tokens: int = 0,
|
||||
data_parallel_rank: int = 0,
|
||||
) -> None:
|
||||
# Fork: never build the UVA table beside the E2 pull transport --
|
||||
# that would be a second offload of the same PLE table.
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
raise RuntimeError(
|
||||
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
|
||||
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
|
||||
)
|
||||
if not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
super().__init__(
|
||||
|
||||
@@ -78,7 +78,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.e2_guards import assert_graph_profile_is_executable
|
||||
from vllm.v1.e2_guards import assert_graph_profile_is_executable, assert_ple_mode
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
from ..common.mtp import make_mtp_hidden_buffer
|
||||
@@ -767,6 +767,7 @@ class Qwen4ExpForCausalLM(
|
||||
# identity nobody chose. Qwen4ExpForConditionalGeneration constructs
|
||||
# this class; Qwen4ExpMTP does not, and guards itself.
|
||||
assert_graph_profile_is_executable(vllm_config, role="target")
|
||||
assert_ple_mode(vllm_config, role="target")
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
if vllm_config.lora_config is not None:
|
||||
# LoRA does not support the merged QKV/indexer projection, so its
|
||||
|
||||
@@ -51,7 +51,7 @@ from vllm.model_executor.models.utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.e2_guards import assert_graph_profile_is_executable
|
||||
from vllm.v1.e2_guards import assert_graph_profile_is_executable, assert_ple_mode
|
||||
|
||||
from .hyperconnection import GatedResidual, HyperConnectionConfig
|
||||
from .low_latency_gemm import enable_qwen4_exp_low_latency_gemm
|
||||
@@ -400,6 +400,7 @@ class Qwen4ExpMTP(
|
||||
# this class builds Qwen4ExpMultiTokenPredictor directly, from a config
|
||||
# rebuilt at mtp.py:159 -- so its graph profile is checked here.
|
||||
assert_graph_profile_is_executable(vllm_config, role="draft")
|
||||
assert_ple_mode(vllm_config, role="draft")
|
||||
config: Qwen4ExpTextConfig = vllm_config.model_config.hf_text_config
|
||||
self.vllm_config = vllm_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
@@ -10,6 +10,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
@@ -357,11 +358,18 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
use_uva = engram_config is not None and engram_config.cpu_offload
|
||||
if use_uva and envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
# Fork: no silent fallback between transports.
|
||||
raise RuntimeError(
|
||||
"PLE backend selection: cpu_offload=True (upstream UVA) with "
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT=1 -- refusing a double offload"
|
||||
)
|
||||
embedding_cls = (
|
||||
Qwen4ExpPLEFileGatherEmbedding
|
||||
if engram_config is not None and engram_config.host_file_gather
|
||||
else Qwen4ExpPLEPinnedHostEmbedding
|
||||
if engram_config is not None and engram_config.cpu_offload
|
||||
if use_uva
|
||||
else Qwen4ExpPLEDeviceEmbedding
|
||||
)
|
||||
self.ngram_embedding = embedding_cls(
|
||||
@@ -382,14 +390,21 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
max_total_tokens, self.ngram_heads, dtype=torch.long
|
||||
)
|
||||
weight = self.ngram_embedding.weight
|
||||
# Fork: backend= names the class (or, inside the pull transport's
|
||||
# offload process, the table it serves), so a smoke need not derive
|
||||
# the backend from pinned=. Fields before it are upstream's, unchanged.
|
||||
backend = type(self.ngram_embedding).__name__
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
backend = f"E2PullTransport.offload_table:{backend}"
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s",
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
|
||||
embedding_prefix,
|
||||
type(embedding_quant_method).__name__,
|
||||
weight.dtype,
|
||||
weight.device,
|
||||
weight.is_pinned(),
|
||||
backend,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -402,6 +417,13 @@ class Qwen4ExpNGramEmbedding(nn.Module):
|
||||
positions = torch.arange(seq_len, device=tokens.device, dtype=torch.int64)
|
||||
eos_positions = torch.where(tokens == eos_token_id, positions, -1)
|
||||
previous_eos_inclusive = torch.cummax(eos_positions, dim=1).values
|
||||
# Fork: never build the UVA table beside the E2 pull transport --
|
||||
# that would be a second offload of the same PLE table.
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
raise RuntimeError(
|
||||
"Qwen4ExpPLEPinnedHostEmbedding (upstream UVA PLE backend) "
|
||||
"refused: VLLM_E2_PLE_PULL_TRANSPORT=1 owns the PLE table"
|
||||
)
|
||||
previous_eos = torch.cat(
|
||||
[
|
||||
eos_positions.new_full((batch_size, 1), -1),
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen4ExpTextConfig
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context
|
||||
@@ -91,16 +92,32 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
self.conv_state_len = (self.conv_kernel_size - 1) * self.short_conv_dilation
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
self.activation = "silu"
|
||||
self.ple_embedding = Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
# Fork: the E2 pull transport owns the table in a CPU-only
|
||||
# offload process; upstream's device/UVA backends are never built.
|
||||
from .ple_pull import Qwen4ExpNGramEmbeddingPull
|
||||
|
||||
self.ple_embedding = Qwen4ExpNGramEmbeddingPull.build(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
else:
|
||||
self.ple_embedding = Qwen4ExpNGramEmbedding(
|
||||
config,
|
||||
int(config.ple_embed_dim),
|
||||
self.ple_dense_layer_id,
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
data_parallel_rank=vllm_config.parallel_config.data_parallel_rank,
|
||||
prefix=f"{prefix}.ple_embedding",
|
||||
quant_config=quant_config,
|
||||
params_dtype=model_config.dtype,
|
||||
)
|
||||
# The PLE cache is TP-replicated, so this merged projection is too.
|
||||
self.kv_proj = MergedColumnParallelLinear(
|
||||
int(config.ple_embed_dim),
|
||||
@@ -145,6 +162,10 @@ class Qwen4ExpPLELayer(nn.Module, MambaBase):
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize PLE lookup output."""
|
||||
return self.ple_embedding.ngram_embedding.dequantize(
|
||||
dequantize_pulled = getattr(self.ple_embedding, "dequantize_pulled", None)
|
||||
if dequantize_pulled is not None:
|
||||
# Fork pull transport: no local table; FP8 scale kept here.
|
||||
return dequantize_pulled(embeddings, output_dtype)
|
||||
embeddings,
|
||||
output_dtype,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The fork's PLE pull transport, re-seated on vLLM v0.30.0.
|
||||
|
||||
v0.30.0 split the E2 base's offload-aware ``Qwen4ExpNGramEmbedding`` into an
|
||||
upstream-only module (``ngram_embedding.py``, device or pinned-host UVA table).
|
||||
This module puts the E2 pull transport back WITHOUT editing that upstream
|
||||
class: ``Qwen4ExpNGramEmbeddingPull`` inherits it and ``PleOffloadLayer``.
|
||||
|
||||
* In a GPU worker, ``PleOffloadLayer.__init_subclass__`` skips the constructor:
|
||||
the module is an empty placeholder with no table. ``PleOffloadConnector``
|
||||
binds it to a host-mapped flag/buffer; its forward is the captured
|
||||
``ple_offload_wait_copy`` op. Only the FP8 global scale is kept on the GPU.
|
||||
* In the CPU-only offload process the constructor runs in full, on CPU. The
|
||||
table is upstream's ``Qwen4ExpPLEDeviceEmbedding`` allocated on CPU;
|
||||
``forward_impl`` computes n-gram IDs and gathers raw (FP8) rows.
|
||||
|
||||
Selected only when ``VLLM_E2_PLE_PULL_TRANSPORT=1``; with the switch off this
|
||||
module is never imported by the model and the tree is upstream.
|
||||
|
||||
Supported table formats: FP8 (global scale) and unquantized -- the two
|
||||
``Qwen4ExpPLEEmbeddingMethod`` families v0.30.0 itself supports. The E2 NVFP4
|
||||
path is not carried (v0.30.0 has no NVFP4 PLE method to select).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.ple_offload_layer import (
|
||||
PleOffloadLayer,
|
||||
is_offload_process,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
|
||||
from vllm.transformers_utils.configs.qwen4_exp import Qwen4ExpTextConfig
|
||||
|
||||
from .ngram_embedding import (
|
||||
Qwen4ExpNGramEmbedding,
|
||||
Qwen4ExpPLEEmbeddingMethod,
|
||||
Qwen4ExpPLEFp8EmbeddingMethod,
|
||||
Qwen4ExpPLEUnquantizedEmbeddingMethod,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
#: The backend name this module reports in the PLE producer log line.
|
||||
PULL_BACKEND_NAME = "E2PullTransport"
|
||||
|
||||
|
||||
class Qwen4ExpNGramEmbeddingPull(PleOffloadLayer, Qwen4ExpNGramEmbedding):
|
||||
"""Upstream n-gram embedding whose table lives in the offload process."""
|
||||
|
||||
#: GPU placeholder: the method the offload process will use for the table,
|
||||
#: resolved with upstream's own selector (set by ``build``).
|
||||
_offload_quant_method: Qwen4ExpPLEEmbeddingMethod | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
*,
|
||||
data_parallel_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
# Runs only in the offload process (guarded by PleOffloadLayer).
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
if engram_config is not None and engram_config.cpu_offload:
|
||||
raise RuntimeError(
|
||||
"PLE pull transport: the offload process resolved "
|
||||
"cpu_offload=True; refusing to build upstream's UVA table here."
|
||||
)
|
||||
Qwen4ExpNGramEmbedding.__init__(
|
||||
self,
|
||||
config,
|
||||
embedding_dim,
|
||||
ple_dense_layer_id,
|
||||
max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prefix=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=params_dtype,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def build(
|
||||
cls,
|
||||
config: Qwen4ExpTextConfig,
|
||||
embedding_dim: int,
|
||||
ple_dense_layer_id: int,
|
||||
max_total_tokens: int,
|
||||
*,
|
||||
data_parallel_rank: int,
|
||||
prefix: str,
|
||||
quant_config: QuantizationConfig | None,
|
||||
params_dtype: torch.dtype | None,
|
||||
) -> "Qwen4ExpNGramEmbeddingPull":
|
||||
"""Construct the placeholder (GPU) or the full CPU module (offload)."""
|
||||
# The offload process builds the surrounding model on meta; this
|
||||
# subtree must own real CPU storage. In a GPU worker the constructor is
|
||||
# skipped, so the device context allocates nothing.
|
||||
with torch.device(cls.get_target_device()):
|
||||
module = cls(
|
||||
config,
|
||||
embedding_dim,
|
||||
ple_dense_layer_id,
|
||||
max_total_tokens,
|
||||
data_parallel_rank=data_parallel_rank,
|
||||
prefix=prefix,
|
||||
quant_config=quant_config,
|
||||
params_dtype=params_dtype,
|
||||
)
|
||||
embedding_prefix = f"{prefix}.ngram_embedding"
|
||||
method = Qwen4ExpPLEEmbeddingMethod.from_quant_config(
|
||||
quant_config,
|
||||
embedding_prefix,
|
||||
getattr(config, "ple_embedding_dtype", None),
|
||||
)
|
||||
if not isinstance(
|
||||
method,
|
||||
(Qwen4ExpPLEFp8EmbeddingMethod, Qwen4ExpPLEUnquantizedEmbeddingMethod),
|
||||
):
|
||||
raise NotImplementedError(
|
||||
f"PLE pull transport does not support {type(method).__name__}"
|
||||
)
|
||||
module._offload_quant_method = method
|
||||
module._pull_params_dtype = params_dtype or torch.get_default_dtype()
|
||||
module._pull_embedding_prefix = embedding_prefix
|
||||
if not is_offload_process():
|
||||
# The GPU-rank producer line, in upstream's format plus backend=
|
||||
# (the offload process logs its own table line from upstream's
|
||||
# constructor). The table lives in the offload process's CPU memory
|
||||
# and is not pinned; rows cross via host-mapped buffers.
|
||||
logger.info(
|
||||
"Initialized PLE embedding %s: quantization_method=%s, "
|
||||
"weight_dtype=%s, weight_device=%s, pinned=%s, backend=%s",
|
||||
embedding_prefix,
|
||||
type(method).__name__,
|
||||
module.get_offload_output_dtype(module._pull_params_dtype),
|
||||
"cpu",
|
||||
False,
|
||||
PULL_BACKEND_NAME,
|
||||
)
|
||||
return module
|
||||
|
||||
# ------------------------------------------------------------- formats
|
||||
|
||||
def get_offload_output_dtype(self, default_dtype: torch.dtype) -> torch.dtype:
|
||||
"""Rows cross the transport in their table storage dtype."""
|
||||
embedding = getattr(self, "ngram_embedding", None)
|
||||
weight = getattr(embedding, "weight", None)
|
||||
if weight is not None:
|
||||
return weight.dtype
|
||||
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
|
||||
return torch.float8_e4m3fn
|
||||
return getattr(self, "_pull_params_dtype", default_dtype)
|
||||
|
||||
def get_offload_output_dim(self, default_dim: int) -> int:
|
||||
return default_dim
|
||||
|
||||
def initialize_dummy_offload_metadata(self, device: torch.device) -> None:
|
||||
"""Dummy loads skip checkpoint scales; give FP8 a unit scale."""
|
||||
if isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod):
|
||||
self.register_buffer(
|
||||
"_offload_weight_scale",
|
||||
torch.ones((), dtype=torch.float32, device=device),
|
||||
persistent=False,
|
||||
)
|
||||
|
||||
def dequantize_pulled(
|
||||
self, embeddings: torch.Tensor, output_dtype: torch.dtype
|
||||
) -> torch.Tensor:
|
||||
"""GPU-side dequantization of rows DMA'd from the offload process.
|
||||
|
||||
Same arithmetic as upstream ``Qwen4ExpPLEFp8EmbeddingMethod.dequantize``
|
||||
(and the E2 ``_dequantize_embeddings`` FP8 branch).
|
||||
"""
|
||||
method = self._offload_quant_method
|
||||
if isinstance(method, Qwen4ExpPLEUnquantizedEmbeddingMethod):
|
||||
return embeddings
|
||||
weight_scale = getattr(self, "_offload_weight_scale", None)
|
||||
if weight_scale is None:
|
||||
raise RuntimeError(
|
||||
"FP8 PLE pull transport is missing its global scale on the GPU"
|
||||
)
|
||||
if weight_scale.device != embeddings.device:
|
||||
raise RuntimeError("FP8 PLE embedding scale must be on the output device")
|
||||
return embeddings.to(output_dtype) * weight_scale.to(output_dtype)
|
||||
|
||||
# ------------------------------------------------------------- execution
|
||||
|
||||
def start_prefetch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> None:
|
||||
"""No prefetch: rows are published by the offload process."""
|
||||
return None
|
||||
|
||||
def _compute_ngram_ids_offload(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""CPU n-gram IDs, carried unchanged from the E2 offload path.
|
||||
|
||||
Differs from upstream's CPU branch in two E2 fixes: the packed layout
|
||||
is (num_reqs, max_seq_len) rather than (num_reqs, num_tokens), and
|
||||
CUDA-graph padding rows beyond ``query_start_loc[-1]`` are not written,
|
||||
so they cannot overwrite a request's final real token.
|
||||
"""
|
||||
input_ids = input_ids.reshape(-1).long()
|
||||
query_start_loc = query_start_loc.long()
|
||||
num_reqs = query_start_loc.numel() - 1
|
||||
num_tokens = input_ids.shape[0]
|
||||
if num_reqs <= 0:
|
||||
raise ValueError("PLE requires at least one request")
|
||||
max_seq_len = max(
|
||||
1,
|
||||
int((query_start_loc[1:] - query_start_loc[:-1]).max().item()),
|
||||
)
|
||||
num_valid_tokens = min(int(query_start_loc[-1].item()), num_tokens)
|
||||
|
||||
positions = torch.arange(num_tokens, device=input_ids.device, dtype=torch.int64)
|
||||
packed = torch.full(
|
||||
(num_reqs, max_seq_len),
|
||||
self.eos_token_id,
|
||||
device=input_ids.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
request_indices = torch.searchsorted(query_start_loc, positions, right=True) - 1
|
||||
request_indices.clamp_(max=num_reqs - 1)
|
||||
columns = (positions - query_start_loc[request_indices]).clamp(
|
||||
0, packed.shape[1] - 1
|
||||
)
|
||||
packed[request_indices[:num_valid_tokens], columns[:num_valid_tokens]] = (
|
||||
input_ids[:num_valid_tokens]
|
||||
)
|
||||
ngram_context = ngram_context[:num_reqs].to(
|
||||
device=input_ids.device, dtype=torch.long
|
||||
)
|
||||
|
||||
context = torch.cat([ngram_context, packed], dim=-1)
|
||||
positions_2d, position_in_segment = self._shift_precompute(
|
||||
context, self.eos_token_id
|
||||
)
|
||||
shifted = [context]
|
||||
for shift in range(1, self.ngram_size):
|
||||
shifted.append(
|
||||
self._shift_apply(
|
||||
context,
|
||||
positions_2d,
|
||||
position_in_segment,
|
||||
shift,
|
||||
self.eos_token_id,
|
||||
)
|
||||
)
|
||||
adjusted_columns = columns + self.ngram_size - 1
|
||||
id_blocks = []
|
||||
for ngram in range(2, self.ngram_size + 1):
|
||||
start = (ngram - 2) * self.heads_per_ngram
|
||||
end = start + self.heads_per_ngram
|
||||
mixed = shifted[0] * self.layer_multipliers[0]
|
||||
for index in range(1, ngram):
|
||||
mixed = torch.bitwise_xor(
|
||||
mixed, shifted[index] * self.layer_multipliers[index]
|
||||
)
|
||||
sizes = self.ngram_heads_vocab_sizes[start:end]
|
||||
offsets = self.ngram_heads_offsets[start:end]
|
||||
ids = torch.remainder(mixed.unsqueeze(-1), sizes) + offsets
|
||||
id_blocks.append(ids[request_indices, adjusted_columns])
|
||||
return torch.cat(id_blocks, dim=-1)
|
||||
|
||||
def forward_impl( # type: ignore[override]
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
input_ids: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
ngram_context: torch.Tensor,
|
||||
output_buffer: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Offload process only: gather raw table rows for these tokens."""
|
||||
del hidden_states
|
||||
if not is_offload_process():
|
||||
raise RuntimeError(
|
||||
"PLE pull transport: forward_impl ran outside the offload "
|
||||
"process -- the GPU placeholder was never bound by the connector"
|
||||
)
|
||||
ngram_ids = self._compute_ngram_ids_offload(
|
||||
input_ids, query_start_loc, ngram_context
|
||||
)
|
||||
num_tokens = input_ids.reshape(-1).shape[0]
|
||||
weight = self.ngram_embedding.weight
|
||||
if output_buffer is not None:
|
||||
output = output_buffer[:num_tokens, : self.embedding_dim]
|
||||
torch.index_select(
|
||||
weight,
|
||||
0,
|
||||
ngram_ids.reshape(-1),
|
||||
out=output.reshape(-1, self.head_dim),
|
||||
)
|
||||
return output
|
||||
return torch.index_select(weight, 0, ngram_ids.reshape(-1)).reshape(
|
||||
num_tokens, self.embedding_dim
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------- loading
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Offload process: upstream loading. GPU worker: keep the FP8 scale.
|
||||
|
||||
The GPU placeholder owns no table; every other checkpoint tensor for
|
||||
this subtree is consumed and dropped here (the offload process loads
|
||||
the table itself). Prefix grouping may call this more than once.
|
||||
"""
|
||||
if is_offload_process():
|
||||
return Qwen4ExpNGramEmbedding.load_weights(self, weights)
|
||||
retained: set[str] = set()
|
||||
fp8 = isinstance(self._offload_quant_method, Qwen4ExpPLEFp8EmbeddingMethod)
|
||||
for name, loaded_weight in weights:
|
||||
if fp8 and name == "ngram_embedding.weight_scale":
|
||||
self.register_buffer(
|
||||
"_offload_weight_scale",
|
||||
loaded_weight.reshape(()).to(
|
||||
device=torch.accelerator.current_accelerator(),
|
||||
dtype=torch.float32,
|
||||
),
|
||||
persistent=False,
|
||||
)
|
||||
retained.add(name)
|
||||
return retained
|
||||
|
||||
|
||||
__all__ = ["PULL_BACKEND_NAME", "Qwen4ExpNGramEmbeddingPull"]
|
||||
@@ -214,5 +214,151 @@ def assert_graph_profile_is_executable(vllm_config, *, role: str = "target") ->
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# PLE transport: carry the pull transport, UVA disabled.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: The fork's pull-transport switch (strict "0"/"1"; registered and hashed in
|
||||
#: vllm/envs.py). Renamed from the E2 base's VLLM_PLE_CPU_OFFLOAD.
|
||||
PULL_TRANSPORT_ENV = "VLLM_E2_PLE_PULL_TRANSPORT"
|
||||
#: Upstream v0.30.0's switch for the pinned-host UVA PLE backend. Same name as
|
||||
#: the E2 base's pull switch, opposite default (upstream: "1").
|
||||
UPSTREAM_UVA_ENV = "VLLM_PLE_CPU_OFFLOAD"
|
||||
|
||||
|
||||
def pull_transport_enabled() -> bool:
|
||||
"""Whether the fork's pull transport is on. Strict; raises on bad values.
|
||||
|
||||
One source: the registered accessor ``envs.VLLM_E2_PLE_PULL_TRANSPORT``,
|
||||
whose parser (``envs._e2_strict_switch``) accepts exactly unset/"0"/"1".
|
||||
"""
|
||||
import vllm.envs as envs
|
||||
|
||||
return bool(envs.VLLM_E2_PLE_PULL_TRANSPORT)
|
||||
|
||||
|
||||
def check_upstream_uva_env_against_pull() -> None:
|
||||
"""Fail loud if the old name would turn upstream's UVA path on.
|
||||
|
||||
With the pull transport on, ``VLLM_PLE_CPU_OFFLOAD`` may be unset (the
|
||||
fork then supplies ``cpu_offload=False`` as the Engram default) or exactly
|
||||
``"0"``. ``"1"`` -- the E2-era spelling of "enable our offload" -- would
|
||||
enable upstream's pinned-host UVA backend beside ours: a double offload.
|
||||
Any other value is rejected rather than interpreted. With the pull
|
||||
transport off this function does nothing: the name is upstream's.
|
||||
"""
|
||||
if not pull_transport_enabled():
|
||||
return
|
||||
raw = os.environ.get(UPSTREAM_UVA_ENV)
|
||||
if raw is None or raw == "0":
|
||||
return
|
||||
if raw == "1":
|
||||
raise RuntimeError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}=1. On "
|
||||
f"vLLM v0.30.0 {UPSTREAM_UVA_ENV} is UPSTREAM's switch for the "
|
||||
"pinned-host UVA PLE backend (default on), not the E2 pull "
|
||||
"transport, so this launch asks for both transports at once. Unset "
|
||||
f"{UPSTREAM_UVA_ENV} or export {UPSTREAM_UVA_ENV}=0 "
|
||||
"(the pull transport replaces that switch)."
|
||||
)
|
||||
raise ValueError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 and {UPSTREAM_UVA_ENV}={raw!r}. "
|
||||
f'With the pull transport on, {UPSTREAM_UVA_ENV} must be unset or '
|
||||
'exactly "0"; any other value is refused rather than guessed.'
|
||||
)
|
||||
|
||||
|
||||
def pull_transport_cpu_offload_default(upstream_default: bool) -> bool:
|
||||
"""The Engram ``cpu_offload`` default, as this fork resolves it.
|
||||
|
||||
Pull off: ``upstream_default`` unchanged (the tree behaves as upstream).
|
||||
Pull on: the old-name check above, then ``False`` -- upstream's UVA
|
||||
backend is never the default beside the pull transport.
|
||||
"""
|
||||
if not pull_transport_enabled():
|
||||
return upstream_default
|
||||
check_upstream_uva_env_against_pull()
|
||||
return False
|
||||
|
||||
|
||||
def verify_engram_against_pull(engram_config) -> None:
|
||||
"""Refuse an explicit ``cpu_offload=True`` while the pull transport is on.
|
||||
|
||||
``--engram-config '{"cpu_offload": true}'`` takes precedence over every
|
||||
default, so it is checked on the resolved object, not the environment.
|
||||
"""
|
||||
if not pull_transport_enabled():
|
||||
return
|
||||
check_upstream_uva_env_against_pull()
|
||||
if engram_config is not None and bool(engram_config.cpu_offload):
|
||||
raise RuntimeError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 but the resolved EngramConfig "
|
||||
"has cpu_offload=True (upstream's pinned-host UVA PLE backend). "
|
||||
"Both transports would own the PLE table. Drop cpu_offload from "
|
||||
"--engram-config (or set it false), or turn the pull transport off."
|
||||
)
|
||||
|
||||
|
||||
#: Effective PLE modes, as the guard names them.
|
||||
PLE_MODE_PULL = "pull" # E2 pull transport (VLLM_E2_PLE_PULL_TRANSPORT=1)
|
||||
PLE_MODE_UVA = "uva" # upstream Qwen4ExpPLEPinnedHostEmbedding
|
||||
PLE_MODE_DEVICE = "device" # upstream Qwen4ExpPLEDeviceEmbedding
|
||||
PLE_MODE_NONE = "none" # no Engram config resolved (no PLE layers / not CUDA)
|
||||
|
||||
|
||||
def resolve_ple_mode(vllm_config) -> str:
|
||||
"""The PLE transport this process will build, or an error if mixed.
|
||||
|
||||
Reads the two inputs that decide it -- the pull switch (strict) and the
|
||||
RESOLVED ``engram_config.cpu_offload`` -- and nothing else. Mixed states
|
||||
raise; there is no fallback from one transport to another.
|
||||
"""
|
||||
pull = pull_transport_enabled()
|
||||
check_upstream_uva_env_against_pull()
|
||||
engram_config = getattr(vllm_config, "engram_config", None)
|
||||
cpu_offload = (
|
||||
None if engram_config is None else bool(engram_config.cpu_offload)
|
||||
)
|
||||
if pull:
|
||||
if cpu_offload:
|
||||
raise RuntimeError(
|
||||
f"e2-guard: {PULL_TRANSPORT_ENV}=1 with a resolved Engram "
|
||||
"cpu_offload=True: the pull transport and upstream's UVA "
|
||||
"backend would both own the PLE table."
|
||||
)
|
||||
return PLE_MODE_PULL
|
||||
if cpu_offload is None:
|
||||
return PLE_MODE_NONE
|
||||
return PLE_MODE_UVA if cpu_offload else PLE_MODE_DEVICE
|
||||
|
||||
|
||||
def assert_ple_mode(vllm_config, *, role: str = "target") -> str:
|
||||
"""Startup guard: resolve, assert and log the effective PLE mode.
|
||||
|
||||
Runs beside ``assert_graph_profile_is_executable`` at model construction
|
||||
for ``role="target"`` and ``role="draft"`` (the MTP drafter carries the
|
||||
target's EngramConfig and the same process environment, so a mixed
|
||||
configuration fails there too). NOT disabled by VLLM_E2_ALLOW_BREAKABLE:
|
||||
that opt-out is about graph identity, not about owning the PLE table twice.
|
||||
"""
|
||||
mode = resolve_ple_mode(vllm_config)
|
||||
model_config = getattr(vllm_config, "model_config", None)
|
||||
text_config = getattr(model_config, "hf_text_config", None)
|
||||
ple_layer_ids = list(getattr(text_config, "ple_layer_ids", None) or [])
|
||||
engram_config = getattr(vllm_config, "engram_config", None)
|
||||
logger.info(
|
||||
"e2-guard[%s]: ple_mode=%s pull_transport=%d engram_cpu_offload=%s "
|
||||
"%s=%s ple_layer_ids=%s.",
|
||||
role,
|
||||
mode,
|
||||
1 if mode == PLE_MODE_PULL else 0,
|
||||
None if engram_config is None else bool(engram_config.cpu_offload),
|
||||
UPSTREAM_UVA_ENV,
|
||||
os.environ.get(UPSTREAM_UVA_ENV, "<unset>"),
|
||||
ple_layer_ids,
|
||||
)
|
||||
return mode
|
||||
|
||||
|
||||
#: Back-compatible name for the earlier call sites.
|
||||
assert_breakable_cudagraph_pinned = assert_graph_profile_is_executable
|
||||
|
||||
@@ -716,6 +716,10 @@ class WorkerProc:
|
||||
|
||||
# Load model
|
||||
self.worker.init_device()
|
||||
# PLE pull transport: DP0/TP0 starts the CPU-only PLE process before
|
||||
# load_model, so its weight load overlaps ours; READY is awaited after.
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.worker.spawn_ple_offload()
|
||||
# Update process title now that parallel groups are initialized
|
||||
self.setup_proc_title_and_log_prefix(
|
||||
enable_ep=vllm_config.parallel_config.enable_expert_parallel
|
||||
@@ -724,6 +728,8 @@ class WorkerProc:
|
||||
self.worker.elastic_ep_execute("load_model")
|
||||
else:
|
||||
self.worker.load_model()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.worker.wait_ple_offload_ready()
|
||||
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
self.use_async_scheduling = scheduler_config.async_scheduling
|
||||
|
||||
@@ -67,11 +67,15 @@ class UniProcExecutor(Executor):
|
||||
|
||||
self.driver_worker.init_worker(all_kwargs=[kwargs])
|
||||
self.driver_worker.init_device()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.driver_worker.spawn_ple_offload()
|
||||
|
||||
if envs.VLLM_ELASTIC_EP_SCALE_UP_LAUNCH:
|
||||
self.driver_worker.elastic_ep_execute("load_model")
|
||||
else:
|
||||
self.driver_worker.load_model()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
self.driver_worker.wait_ple_offload_ready()
|
||||
current_platform.update_block_size_for_backend(self.vllm_config)
|
||||
|
||||
def _distributed_args(self) -> tuple[str, int, int]:
|
||||
|
||||
@@ -257,7 +257,7 @@ class PleOffloadConnector:
|
||||
}
|
||||
if not layers:
|
||||
raise RuntimeError(
|
||||
"VLLM_PLE_CPU_OFFLOAD is enabled, but the model has no PleOffloadLayer"
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no PleOffloadLayer"
|
||||
)
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compatibility exports for the top-level test-only fault singleton."""
|
||||
|
||||
from vllm_ple_test_fault import (
|
||||
_FAULT as _FAULT,
|
||||
)
|
||||
from vllm_ple_test_fault import ( # noqa: F401
|
||||
FaultSpec as FaultSpec,
|
||||
)
|
||||
from vllm_ple_test_fault import (
|
||||
_parse_fault_spec as _parse_fault_spec,
|
||||
)
|
||||
@@ -830,7 +830,7 @@ class PleOffloadRunner:
|
||||
}
|
||||
if not offload_layers:
|
||||
raise RuntimeError(
|
||||
"VLLM_PLE_CPU_OFFLOAD is enabled, but no PleOffloadLayer "
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but no PleOffloadLayer "
|
||||
"was found in the initialized model"
|
||||
)
|
||||
logger.info(
|
||||
|
||||
@@ -385,6 +385,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
|
||||
# For transferring state from execute_model to subsequent sample_tokens call.
|
||||
self.execute_model_state: ExecuteModelState | None = None
|
||||
# PLE pull transport: set by _setup_ple_offload() from Worker.load_model
|
||||
# when VLLM_E2_PLE_PULL_TRANSPORT=1 and this rank owns a PLE layer.
|
||||
# None otherwise, and every hook below is then a no-op (upstream path).
|
||||
self._ple_offload_connector: PleOffloadConnector | None = None
|
||||
|
||||
# Expert parallelism load balancer.
|
||||
self.eplb = EPLBController(self.parallel_config, self.device)
|
||||
@@ -408,6 +412,26 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
tasks.extend(PoolingRunner.get_supported_tasks(self.model))
|
||||
return tuple(tasks)
|
||||
|
||||
def _setup_ple_offload(self, ipc_addr: str) -> None:
|
||||
"""Initialize PLE offload after the model state is available."""
|
||||
# PLE placeholders are created by model loading, so CUDA output
|
||||
# registration cannot happen in the runner constructor.
|
||||
query_start_loc_source = getattr(self.model_state, "ple_query_start_loc", None)
|
||||
ngram_context_source = getattr(self.model_state, "ngram_context", None)
|
||||
if not isinstance(query_start_loc_source, torch.Tensor):
|
||||
raise RuntimeError("PLE offload requires a query_start_loc source")
|
||||
if not isinstance(ngram_context_source, torch.Tensor):
|
||||
raise RuntimeError("PLE offload requires an ngram_context source")
|
||||
self._ple_offload_connector = PleOffloadConnector(
|
||||
self.vllm_config,
|
||||
self.model,
|
||||
self.device,
|
||||
ipc_addr,
|
||||
input_ids_source=self.input_buffers.input_ids,
|
||||
query_start_loc_source=query_start_loc_source,
|
||||
ngram_context_source=ngram_context_source,
|
||||
)
|
||||
|
||||
def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None:
|
||||
time_before_load = time.perf_counter()
|
||||
if load_dummy_weights:
|
||||
@@ -1102,6 +1126,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
torch.accelerator.empty_cache()
|
||||
start_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
# Decoder capture calls the model outside execute_model. Keep dummy
|
||||
# PLE outputs signaled until all graphs capture their semaphore wait.
|
||||
if self._ple_offload_connector is not None and capture_decoder:
|
||||
self._ple_offload_connector.signal_dummy_outputs(self.max_num_tokens)
|
||||
with self.maybe_setup_dummy_loras(self.lora_config):
|
||||
if capture_encoder:
|
||||
self.model_state.encoder_runner.capture()
|
||||
@@ -1137,6 +1165,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.adaptive_verification.set_initial_cost_curves(timings)
|
||||
self.kv_connector.reset_capture_state()
|
||||
|
||||
if self._ple_offload_connector is not None and capture_decoder:
|
||||
self._ple_offload_connector.release_outputs()
|
||||
end_free_gpu_memory = torch.accelerator.get_memory_info()[0]
|
||||
|
||||
if not profile_only:
|
||||
@@ -2479,6 +2509,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return None
|
||||
|
||||
input_batch = self.execute_model_state.input_batch
|
||||
if self._ple_offload_connector is not None:
|
||||
self._ple_offload_connector.close()
|
||||
self._ple_offload_connector = None
|
||||
hidden_states = self.execute_model_state.hidden_states
|
||||
finished_req_ids = self.execute_model_state.finished_req_ids
|
||||
ec_connector_output = self.execute_model_state.ec_connector_output
|
||||
|
||||
@@ -233,6 +233,22 @@ class Worker(WorkerBase):
|
||||
validate_worker_profiler_config(self.profiler_config)
|
||||
|
||||
self.use_v2_model_runner = vllm_config.use_v2_model_runner
|
||||
# PLE pull transport (VLLM_E2_PLE_PULL_TRANSPORT). Off: both stay
|
||||
# None/False and every PLE hook in this class is a no-op.
|
||||
self._ple_offload_worker_handle: Any | None = None
|
||||
self._ple_offload_stopping = False
|
||||
self._ple_offload_enabled = self._has_ple_layers()
|
||||
if envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
if self._ple_offload_enabled:
|
||||
self._validate_ple_offload_config()
|
||||
elif self.rank == 0 and self.parallel_config.data_parallel_rank == 0:
|
||||
text_config = self.model_config.hf_text_config
|
||||
logger.warning(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but this rank owns "
|
||||
"no PLE layer (ple_layer_ids=%s); no PLE offload connector "
|
||||
"is created here.",
|
||||
getattr(text_config, "ple_layer_ids", None),
|
||||
)
|
||||
|
||||
# Device handles of the previous step's PP intermediate-tensor send.
|
||||
self._pp_send_work: list[Handle] = []
|
||||
@@ -242,7 +258,7 @@ class Worker(WorkerBase):
|
||||
|
||||
def _has_ple_layers(self) -> bool:
|
||||
"""Return whether this model configuration constructs PLE layers."""
|
||||
if not envs.VLLM_PLE_CPU_OFFLOAD:
|
||||
if not envs.VLLM_E2_PLE_PULL_TRANSPORT:
|
||||
return False
|
||||
text_config = self.model_config.hf_text_config
|
||||
ple_layer_ids = getattr(text_config, "ple_layer_ids", None)
|
||||
@@ -304,7 +320,7 @@ class Worker(WorkerBase):
|
||||
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
"VLLM_PLE_CPU_OFFLOAD does not support the requested "
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT does not support the requested "
|
||||
"configuration. Unsupported settings: "
|
||||
f"{', '.join(unsupported)}"
|
||||
)
|
||||
@@ -718,6 +734,12 @@ class Worker(WorkerBase):
|
||||
):
|
||||
self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
|
||||
# Model loading constructs the PLE placeholders whose buffers are
|
||||
# registered with the offload process by the model runner V2 connector.
|
||||
if self._ple_offload_enabled:
|
||||
model_runner = cast("GPUModelRunnerV2", self.model_runner)
|
||||
model_runner._setup_ple_offload(self.parallel_config._ple_offload_ipc_path)
|
||||
|
||||
if has_ec_transfer():
|
||||
get_ec_transfer().start_worker_services()
|
||||
|
||||
@@ -738,6 +760,10 @@ class Worker(WorkerBase):
|
||||
self.model_runner.update_config(overrides)
|
||||
|
||||
def reload_weights(self, *args, **kwargs) -> None:
|
||||
if self._ple_offload_enabled:
|
||||
raise NotImplementedError(
|
||||
"Weight reload is not supported with the PLE pull transport"
|
||||
)
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
self.model_runner.reload_weights(*args, **kwargs)
|
||||
|
||||
@@ -1759,6 +1785,10 @@ class Worker(WorkerBase):
|
||||
|
||||
self.elastic_ep_executor.shutdown()
|
||||
|
||||
if self._ple_offload_worker_handle is not None:
|
||||
self._ple_offload_worker_handle.close()
|
||||
self._ple_offload_worker_handle = None
|
||||
|
||||
# Release GPU resources held by the model runner so that memory
|
||||
# can be reclaimed when running in-process
|
||||
if model_runner := getattr(self, "model_runner", None):
|
||||
|
||||
Reference in New Issue
Block a user