[Qwen4Exp] Host-resident token embedding tables (VLLM_HOST_EMBED_TABLE)

Keeps the token embedding table in pinned host memory behind a UVA view for the
stage-0 target and the last-stage MTP drafter, freeing GPU memory for KV. TP
sharding, masking and the all-reduce are retained. The knob is registered as a
compile-cache factor.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:43:53 +07:00
co-authored by Hinata
parent 66b3d8c108
commit 41f6691c26
4 changed files with 231 additions and 3 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from collections.abc import Sequence
from dataclasses import dataclass
@@ -31,8 +32,26 @@ from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
from vllm.model_executor.parameter import BasevLLMParameter
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
PIN_MEMORY,
get_accelerator_view_from_cpu_tensor,
)
DEFAULT_VOCAB_PADDING_SIZE = 64
HOST_EMBED_TABLE_ENV = "VLLM_HOST_EMBED_TABLE"
def host_embed_table_enabled() -> bool:
"""Return whether the exact host-embedding opt-in is set."""
return os.environ.get(HOST_EMBED_TABLE_ENV, "") == "1"
def should_use_host_embed_table() -> bool:
"""Keep the PLE offload worker's model construction on the meta device."""
return (
host_embed_table_enabled()
and torch.empty(0).device.type != "meta"
)
class UnquantizedEmbeddingMethod(QuantizeMethodBase):
@@ -91,6 +110,46 @@ class UnquantizedEmbeddingMethod(QuantizeMethodBase):
return layer
class _PinnedHostUnquantizedEmbeddingMethod(UnquantizedEmbeddingMethod):
"""Allocate an unquantized embedding shard without a CUDA transient."""
requires_device_loading = False
def __init__(self, *, device: torch.device, pin_memory: bool) -> None:
self.device = device
self.pin_memory = pin_memory
def create_weights(
self,
layer: torch.nn.Module,
input_size_per_partition: int,
output_partition_sizes: list[int],
input_size: int,
output_size: int,
params_dtype: torch.dtype,
**extra_weight_attrs,
) -> None:
del input_size, output_size
weight = Parameter(
torch.empty(
sum(output_partition_sizes),
input_size_per_partition,
dtype=params_dtype,
device=self.device,
pin_memory=self.pin_memory,
),
requires_grad=False,
)
set_weight_attrs(weight, {"input_dim": 1, "output_dim": 0})
layer.register_parameter("weight", weight)
set_weight_attrs(weight, extra_weight_attrs)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# The serving layer must retain the same pinned allocation whose mapped
# address was captured. CPU-only reference tests need no GEMM dispatch.
del layer
def pad_vocab_size(vocab_size: int, pad_to: int = DEFAULT_VOCAB_PADDING_SIZE) -> int:
"""Pad the vocab size to the given value."""
return ((vocab_size + pad_to - 1) // pad_to) * pad_to
@@ -572,6 +631,124 @@ class VocabParallelEmbedding(PluggableLayer):
return s
class PinnedHostVocabParallelEmbedding(VocabParallelEmbedding):
"""TP-sharded embedding read on CUDA through mapped pinned host memory.
Host provisioning must use measured pinned allocation, not tensor payload.
"""
def __init__(
self,
num_embeddings: int,
embedding_dim: int,
params_dtype: torch.dtype | None = None,
org_num_embeddings: int | None = None,
padding_size: int = DEFAULT_VOCAB_PADDING_SIZE,
prefix: str = "",
*,
disable_tp: bool = False,
create_accelerator_view: bool = True,
) -> None:
construction_device = torch.empty(0).device
is_meta = construction_device.type == "meta"
if (
create_accelerator_view
and not is_meta
and current_platform.is_cuda()
and not PIN_MEMORY
):
raise RuntimeError(
"VLLM_HOST_EMBED_TABLE requires available pinned host memory"
)
storage_device = construction_device if is_meta else torch.device("cpu")
pin_memory = (
not is_meta
and current_platform.is_cuda()
and PIN_MEMORY
)
quant_method = _PinnedHostUnquantizedEmbeddingMethod(
device=storage_device,
pin_memory=pin_memory,
)
super().__init__(
num_embeddings,
embedding_dim,
params_dtype=params_dtype,
org_num_embeddings=org_num_embeddings,
padding_size=padding_size,
prefix=prefix,
disable_tp=disable_tp,
quant_method=quant_method,
)
# The custom fused kernel is specified for ordinary device-resident
# shards, not device-mapped host allocations. Use the regular masked
# embedding path whose CUDA tensor argument is the mapped view.
self.use_fused_embedding = False
self._uva_weight: torch.Tensor | None = None
if (
create_accelerator_view
and self.weight.device.type == "cpu"
and current_platform.is_cuda()
):
if not self.weight.is_pinned():
raise RuntimeError(
"VLLM_HOST_EMBED_TABLE weight was not allocated pinned"
)
self._uva_weight = get_accelerator_view_from_cpu_tensor(self.weight)
@property
def accelerator_weight_view(self) -> torch.Tensor | None:
return self._uva_weight
def _apply(self, fn, recurse: bool = True):
# Probe the transformation without applying it to the pinned shard.
# Device-only parent moves remain no-ops, but silently retaining this
# layer's dtype during a parent dtype conversion would create a
# mixed-dtype model.
del recurse
probe = torch.empty(0, dtype=self.weight.dtype, device=self.weight.device)
if fn(probe).dtype != self.weight.dtype:
raise RuntimeError("Pinned host embedding has a fixed dtype")
return self
def _lookup_weight(self, input_: torch.Tensor) -> torch.Tensor:
if input_.device.type == "cpu":
return self.weight
if self._uva_weight is None:
raise RuntimeError(
"Host embedding received accelerator IDs without a UVA view"
)
if self._uva_weight.device != input_.device:
raise RuntimeError(
"Host embedding UVA view and input IDs are on different devices"
)
return self._uva_weight
def forward(self, input_: torch.Tensor) -> torch.Tensor:
if self.tp_size == 1:
return F.embedding(
input_.long(),
self._lookup_weight(input_),
)
masked_input, input_mask = get_masked_input_and_mask(
input_,
self.shard_indices.org_vocab_start_index,
self.shard_indices.org_vocab_end_index,
self.shard_indices.num_org_vocab_padding,
self.shard_indices.added_vocab_start_index,
self.shard_indices.added_vocab_end_index,
)
output_parallel = F.embedding(
masked_input.long(),
self._lookup_weight(input_),
)
output_parallel.masked_fill_(input_mask.unsqueeze(-1), 0)
return tensor_model_parallel_all_reduce(output_parallel)
# --8<-- [start:parallel_lm_head]
@PluggableLayer.register("parallel_lm_head")
class ParallelLMHead(VocabParallelEmbedding):