[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: ywang96 <>
Co-authored-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Juntian Liu
2026-09-13 02:44:23 -07:00
committed by GitHub
co-authored by Roger Wang ywang96 <> Woosuk Kwon Codex
parent e7a3963339
commit d2d649e674
12 changed files with 1821 additions and 195 deletions
+112 -93
View File
@@ -51,8 +51,6 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.utils import set_weight_attrs
from vllm.triton_utils import tl, triton
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
logger = init_logger(__name__)
@@ -236,6 +234,7 @@ def _write_hash_cache_kernel(
tl.store(cache + slot, value, valid)
# Keep request shapes out of the cache key so one warmed variant covers runtime batches.
@triton.jit(
do_not_specialize=[
"num_tokens",
@@ -462,6 +461,21 @@ class NgramHashState(nn.Module):
self._kv_cache_ref = weakref.ref(kv_cache)
return True
def dummy_hashes(
self, input_ids: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
"""Participate in DP lookups without valid rows or hash-cache updates."""
num_tokens = input_ids.shape[0]
num_layers, max_ngram = self.multipliers.shape
num_heads = self.primes.shape[-1]
hashes = input_ids.new_full(
(num_tokens, num_layers, (max_ngram - 1) * num_heads),
DEAD_ID,
dtype=torch.int32,
)
keep = torch.zeros(num_tokens, dtype=torch.bool, device=input_ids.device)
return hashes, keep
def forward(
self,
input_ids: torch.Tensor,
@@ -561,7 +575,16 @@ def _engram_head_shard_weight_loader(
param.data.copy_(shard)
@triton.jit
@triton.jit(
do_not_specialize=[
"vocab_start",
"vocab_end",
"num_rows",
"ids_stride_t",
"ids_stride_h",
"GRID",
]
)
def _engram_lookup_kernel(
weight,
scales,
@@ -578,7 +601,7 @@ def _engram_lookup_kernel(
DIM: tl.constexpr,
QUANT_BLOCK: tl.constexpr,
BLOCK_R: tl.constexpr,
GRID: tl.constexpr,
GRID,
):
"""Gather fp8 rows, apply their ue8m0 block scales, write bf16.
@@ -620,12 +643,9 @@ def _engram_lookup_kernel(
class ParallelEngramEmbedding(nn.Module):
"""The n-gram hash table, sharded by complete hash heads over TP ranks.
Rows stay fp8 and are dequantized with ue8m0 per-32 scales on lookup.
"""TP-sharded hash heads with FP8 rows and per-block E8M0 scales."""
With `cpu_offload` the shard lives in pinned host memory and is read over
UVA instead of HBM; the TP sharding is unchanged either way.
"""
_weight_loader = staticmethod(_engram_head_shard_weight_loader)
def __init__(
self,
@@ -633,89 +653,55 @@ class ParallelEngramEmbedding(nn.Module):
dim: int,
head_sizes: tuple[int, ...],
block_size: int = 32,
cpu_offload: bool = False,
):
) -> None:
super().__init__()
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
assert head_sizes and all(size > 0 for size in head_sizes)
assert sum(head_sizes) <= num_embeddings
if cpu_offload and not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
self.num_embeddings = num_embeddings
self.dim = dim
self.block_size = block_size
self.n_hash_cols = len(head_sizes)
self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, tp_size)
self.head_start = tp_rank * self.part_n_hash_cols
self.tp_size = get_tensor_model_parallel_world_size()
num_shards, head_rank = self._get_shard_info()
self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, num_shards)
# TODO: Support row-wise sharding when there are too few hash heads.
assert (num_shards - 1) * self.part_n_hash_cols < self.n_hash_cols, (
f"Engram sharding leaves ranks without hash heads: "
f"{self.n_hash_cols} heads over {num_shards} shards"
)
self.head_start = head_rank * self.part_n_hash_cols
head_end = self.head_start + self.part_n_hash_cols
self.vocab_start_idx = sum(head_sizes[: self.head_start])
self.vocab_end_idx = sum(head_sizes[:head_end])
self.part_num_embeddings = self.vocab_end_idx - self.vocab_start_idx
self.tp_size = tp_size
self.cpu_offload = cpu_offload
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
self._view_src: tuple[int, int] | None = None
self._num_sms = torch.cuda.get_device_properties(
torch.accelerator.current_device_index()
).multi_processor_count
# Explicit device: model init runs under a `torch.device("cuda")`
# context, which would otherwise put the shard in HBM.
kwargs = {"device": "cpu", "pin_memory": True} if cpu_offload else {}
self.weight = nn.Parameter(
torch.empty(
self.part_num_embeddings, dim, dtype=torch.float8_e4m3fn, **kwargs
),
requires_grad=False,
)
self.weight_scale_inv = nn.Parameter(
torch.empty(
self.part_num_embeddings,
dim // block_size,
dtype=torch.uint8,
**kwargs,
),
requires_grad=False,
)
weight, scales = self._allocate_weights()
self.weight = nn.Parameter(weight, requires_grad=False)
self.weight_scale_inv = nn.Parameter(scales, requires_grad=False)
for param in (self.weight, self.weight_scale_inv):
set_weight_attrs(
param,
{
"weight_loader": _engram_head_shard_weight_loader,
"weight_loader": self._weight_loader,
"engram_vocab_start": self.vocab_start_idx,
},
)
if cpu_offload:
# Constant dummy values avoid randomizing huge CPU lookup tables.
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
logger.info(
"Engram table offloaded to pinned host memory: %d rows x %d, "
"%.2f GiB per rank",
self.part_num_embeddings,
dim,
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
)
def _get_shard_info(self) -> tuple[int, int]:
return self.tp_size, get_tensor_model_parallel_rank()
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
return (
torch.empty(self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn),
torch.empty(
self.part_num_embeddings, self.dim // self.block_size, dtype=torch.uint8
),
)
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
"""Parameters when resident, else cached UVA views of the pinned shard.
Rebuilt if anything swaps `.data`, so a stale device pointer cannot
survive silently.
"""
if not self.cpu_offload:
return self.weight.data, self.weight_scale_inv.data
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
if self._view_src != src:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight.data),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
)
self._view_src = src
assert self._views is not None
return self._views
return self.weight.data, self.weight_scale_inv.data
def lookup(
self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
@@ -753,7 +739,7 @@ class ParallelEngramEmbedding(nn.Module):
def forward(self, indices: torch.Tensor) -> torch.Tensor:
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
bf16, gathered from all TP shards."""
bf16, gathered from all shards for this replica's tokens."""
out = torch.empty(
(indices.shape[0], self.part_n_hash_cols, self.dim),
dtype=torch.bfloat16,
@@ -762,8 +748,7 @@ class ParallelEngramEmbedding(nn.Module):
self.lookup(indices, out)
if self.tp_size > 1:
out = tensor_model_parallel_all_gather(out, dim=1)
out = out[:, : self.n_hash_cols]
return out
return out[:, : self.n_hash_cols]
@triton.jit(do_not_specialize=["num_kv_tokens"])
@@ -859,7 +844,7 @@ def _fused_engram_post_wkv_kernel(
@triton.jit(do_not_specialize=["num_tokens", "token_start", "num_elements"])
def _engram_sp_rows_kernel(
def _engram_select_rows_kernel(
gathered,
output,
num_tokens,
@@ -880,6 +865,32 @@ def _engram_sp_rows_kernel(
tl.store(output + offsets, values, offsets < num_elements)
def _engram_select_rows(
gathered: torch.Tensor,
output: torch.Tensor,
source_tokens: int,
token_start: int,
local_width: int,
) -> None:
"""Copy one token window out of a rank-major gathered buffer.
Both gathers land rank-major ([rank][token][local width]); this walks the
window the rank keeps and lays its ranks out side by side as width.
"""
if output.numel() == 0:
return
_engram_select_rows_kernel[(triton.cdiv(output.numel(), 1024),)](
gathered,
output,
source_tokens,
token_start,
output.numel(),
local_width,
output.shape[1] * output.shape[2],
BLOCK_SIZE=1024,
)
class Engram(nn.Module):
"""Writes an n-gram lookup into the residual stream, gated by how well it
matches that stream.
@@ -910,13 +921,7 @@ class Engram(nn.Module):
# Named ``embed_tokens`` so the checkpoint's ``engram.embed.weight``
# survives the mapper's ``embed.weight`` -> ``embed_tokens.weight``
# suffix rule.
engram_config = get_current_vllm_config().engram_config
self.embed_tokens = ParallelEngramEmbedding(
layout.num_embeddings[layer_hash_index],
layout.head_dim,
tuple(size for order in layout.primes[layer_hash_index] for size in order),
cpu_offload=engram_config.cpu_offload if engram_config else True,
)
self.embed_tokens = self._create_embedding(layout, layer_hash_index)
n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads
self.wkv = ReplicatedLinear(
n_hash_cols * layout.head_dim,
@@ -936,40 +941,54 @@ class Engram(nn.Module):
)
max_tokens = get_current_vllm_config().scheduler_config.max_num_batched_tokens
# Keep lookup results alive across breakable graph segments.
self._init_staging(max_tokens, layout.head_dim)
def _create_embedding(
self, layout: EngramLayout, layer_hash_index: int
) -> ParallelEngramEmbedding:
return ParallelEngramEmbedding(
layout.num_embeddings[layer_hash_index],
layout.head_dim,
tuple(size for order in layout.primes[layer_hash_index] for size in order),
)
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
# Persistent storage keeps lookup addresses stable across graph replays.
self.staged_rows = torch.empty(
max_tokens,
self.embed_tokens.part_n_hash_cols,
layout.head_dim,
head_dim,
dtype=torch.bfloat16,
)
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
"""Gather this layer's rows on the main stream before decoder layers."""
self.embed_tokens.lookup(hash_ids, self.staged_rows[: hash_ids.shape[0]])
"""Look up rows before the decoder layers consume them."""
rows = self.staged_rows[: hash_ids.shape[0]]
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
self.embed_tokens.lookup(hash_ids, rows)
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
return self.staged_rows[:num_tokens]
def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
"""Gather heads, returning only local tokens when SP is enabled."""
rows = self.staged_rows[: hash_ids.shape[0]]
rows = self._ready_rows(hash_ids.shape[0])
if self.embed_tokens.tp_size == 1:
return rows
return rows[:, : self.embed_tokens.n_hash_cols]
if self.use_sequence_parallel:
tp_size = self.embed_tokens.tp_size
num_tokens, local_heads, dim = rows.shape
gathered = tensor_model_parallel_all_gather(rows, dim=0)
chunk = (num_tokens + tp_size - 1) // tp_size
rows = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
_engram_sp_rows_kernel[(triton.cdiv(rows.numel(), 1024),)](
out = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
_engram_select_rows(
gathered,
rows,
out,
num_tokens,
get_tensor_model_parallel_rank() * chunk,
rows.numel(),
local_heads * dim,
self.embed_tokens.n_hash_cols * dim,
BLOCK_SIZE=1024,
)
return rows
return out
rows = tensor_model_parallel_all_gather(rows, dim=1)
return rows[:, : self.embed_tokens.n_hash_cols]
+403
View File
@@ -0,0 +1,403 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
import mmap
import tempfile
import weakref
from contextlib import ExitStack
import numpy as np
import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_dp_group,
get_engram_dp_group,
get_engram_dp_size,
get_tensor_model_parallel_rank,
tensor_model_parallel_all_gather,
)
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.deepseek_v4_1.common.engram import (
DEAD_ID,
EngramLayout,
_engram_head_shard_weight_loader,
_engram_select_rows,
)
from vllm.models.deepseek_v4_1.common.engram import (
Engram as BaseEngram,
)
from vllm.models.deepseek_v4_1.common.engram import (
ParallelEngramEmbedding as BaseParallelEngramEmbedding,
)
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
logger = init_logger(__name__)
def engram_head_shard_rank() -> int:
"""This rank's slot among the hash-head shards of one engram table.
TP-major, so the shards a DP gather brings in are contiguous heads and
the following TP gather completes the head order.
"""
dp_group = get_engram_dp_group()
dp_size = dp_group.world_size if dp_group is not None else 1
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
return get_tensor_model_parallel_rank() * dp_size + dp_rank
def engram_gathered_num_tokens() -> int:
"""Per-replica token slot for the node-local Engram DP group."""
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is None:
raise RuntimeError("a DP-shared engram table needs DP token metadata")
group = get_engram_dp_group()
assert group is not None
# Engram groups are contiguous slices of the full DP group.
start = get_dp_group().rank_in_group - group.rank_in_group
return int(
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
)
def gather_engram_hashes(
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
) -> torch.Tensor:
"""Collect the n-gram ids of every DP replica sharing one table.
Replicas are padded to a common token slot, so the gathered shape is
static under CUDA graph capture (where DP already pads alike).
"""
dp_group = get_engram_dp_group()
if dp_group is None or dp_shared_memory:
return hash_ids
slot = engram_gathered_num_tokens()
if hash_ids.shape[0] > slot:
raise ValueError("Engram token count exceeds the DP token slot")
if hash_ids.shape[0] < slot:
pad = hash_ids.new_full(
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
)
hash_ids = torch.cat((hash_ids, pad))
return dp_group.all_gather(hash_ids, dim=0)
class DPSharedEngramStorage:
"""Registered host weights shared by a node-local DP group with one writer."""
def __init__(
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
) -> None:
self.group = group
weight_bytes = num_rows * dim
storage = self._allocate(weight_bytes + weight_bytes // block_size)
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
def _allocate(self, num_bytes: int) -> torch.Tensor:
"""Map and register one physical allocation across a node-local DP group."""
from vllm.distributed.device_communicators.shm_broadcast import (
check_shm_free_space,
)
group = self.group
with ExitStack() as stack:
path, error = None, None
if group.rank_in_group == 0:
try:
check_shm_free_space(num_bytes)
backing_file = stack.enter_context(
tempfile.NamedTemporaryFile(
prefix="vllm_engram_", dir="/dev/shm"
)
)
backing_file.truncate(num_bytes)
path = backing_file.name
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
path, error = group.broadcast_object((path, error))
if error is not None:
raise RuntimeError(
"Engram shared-memory creation failed on EDP rank 0: " + error
)
mapping = owner = tensor = finalizer = None
stage = "open"
try:
try:
with open(path, "r+b") as file:
stage = "mmap"
mapping = mmap.mmap(
file.fileno(), num_bytes, flags=mmap.MAP_SHARED
)
stage = "cudaHostRegister"
owner = np.frombuffer(mapping, dtype=np.uint8)
pointer = owner.ctypes.data
tensor = torch.from_numpy(owner)
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
if result.value != 0:
raise RuntimeError(f"cudaHostRegister failed: {result}")
finalizer = weakref.finalize(
owner, self._unregister, mapping, pointer
)
finalizer.atexit = False # type: ignore[misc]
# The UVA helper otherwise allocates a private pinned copy.
if not tensor.is_pinned():
raise RuntimeError(
"CUDA did not recognize the shared Engram registration"
)
except Exception as exc:
error = f"{stage}: {type(exc).__name__}: {exc}"
errors: list[str | None] = [None] * group.world_size
# Also fences peer mappings before the leader unlinks the file.
torch.distributed.all_gather_object(
errors, error, group=group.cpu_group
)
failures = "; ".join(
f"EDP rank {rank}: {error}"
for rank, error in enumerate(errors)
if error is not None
)
if failures:
raise RuntimeError(
"Engram shared-memory initialization failed: " + failures
)
assert tensor is not None
return tensor
except Exception:
if finalizer is not None:
finalizer()
tensor = owner = None
if mapping is not None:
mapping.close()
raise
@staticmethod
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
# Torch storage retains the numpy owner, including through cached UVA views.
# Keep its mmap alive until CUDA has released the registration.
result = torch.cuda.cudart().cudaHostUnregister(pointer)
if result.value != 0:
logger.warning("Engram cudaHostUnregister failed: %s", result)
def load_weight(
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
) -> None:
if self.group.rank_in_group == 0:
_engram_head_shard_weight_loader(param, loaded_weight)
# Read order may differ across ranks. Equal load counts ensure all shared
# weights are ready after the last weight-loader call returns.
torch.distributed.barrier(group=self.group.cpu_group)
def get_views(
self, weight: torch.Tensor, scales: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
if (weight.data_ptr(), scales.data_ptr()) != (
self.weight.data_ptr(),
self.weight_scale_inv.data_ptr(),
):
raise RuntimeError("Shared Engram parameter storage must not be replaced")
if self._views is None:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
)
return self._views
class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
"""Extend TP lookup with DP head sharding or shared, CPU-offloaded TP slices."""
_shared_memory: DPSharedEngramStorage | None = None
def __init__(
self,
num_embeddings: int,
dim: int,
head_sizes: tuple[int, ...],
block_size: int = 32,
cpu_offload: bool = False,
dp_shared_memory: bool = False,
) -> None:
self.cpu_offload = cpu_offload
self.dp_shared_memory = dp_shared_memory
self.dp_size = get_engram_dp_size()
if dp_shared_memory:
if not cpu_offload:
raise ValueError("dp_shared_memory requires cpu_offload=True")
if self.dp_size <= 1:
raise ValueError(
"dp_shared_memory requires a node-local Engram DP "
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
"Check that the node layout and rank placement allow complete "
"DP replicas to be co-located."
)
self.dp_size = 1
if cpu_offload and not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
self._view_src: tuple[int, int] | None = None
super().__init__(num_embeddings, dim, head_sizes, block_size)
if cpu_offload:
# Constant dummy values avoid randomizing huge CPU lookup tables.
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
logger.info(
"Engram table offloaded to pinned host memory: %d rows x %d, "
"%.2f GiB %s",
self.part_num_embeddings,
dim,
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
"shared across DP replicas" if dp_shared_memory else "per rank",
)
def _get_shard_info(self) -> tuple[int, int]:
if self.dp_size == 1:
return super()._get_shard_info()
return self.tp_size * self.dp_size, engram_head_shard_rank()
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
if self.dp_shared_memory:
group = get_engram_dp_group()
assert group is not None
storage = DPSharedEngramStorage(
self.part_num_embeddings, self.dim, self.block_size, group
)
self._shared_memory = storage
self._weight_loader = storage.load_weight
return storage.weight, storage.weight_scale_inv
if not self.cpu_offload:
return super()._allocate_weights()
# Model initialization may be inside a CUDA device context.
return (
torch.empty(
self.part_num_embeddings,
self.dim,
dtype=torch.float8_e4m3fn,
device="cpu",
pin_memory=True,
),
torch.empty(
self.part_num_embeddings,
self.dim // self.block_size,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
),
)
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
if self._shared_memory is not None:
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
if not self.cpu_offload:
return super()._storage()
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
if self._view_src != src:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight.data),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
)
self._view_src = src
assert self._views is not None
return self._views
def forward(self, indices: torch.Tensor) -> torch.Tensor:
if self.dp_size == 1:
return super().forward(indices)
num_tokens = indices.shape[0]
indices = gather_engram_hashes(indices)
out = torch.empty(
(indices.shape[0], self.part_n_hash_cols, self.dim),
dtype=torch.bfloat16,
device=indices.device,
)
self.lookup(indices, out)
out = _gather_engram_rows(out, num_tokens)
if self.tp_size > 1:
out = tensor_model_parallel_all_gather(out, dim=1)
return out[:, : self.n_hash_cols]
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
dp_group = get_engram_dp_group()
assert dp_group is not None
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
assert remainder == 0 and 0 <= num_tokens <= slot
gathered = dp_group.all_gather(staged, dim=0)
local_heads, dim = staged.shape[1:]
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
_engram_select_rows(
gathered,
rows,
staged.shape[0],
dp_group.rank_in_group * slot,
local_heads * dim,
)
return rows
class Engram(BaseEngram):
"""NVIDIA Engram with asynchronous offload and node-local DP lookup."""
_prefetch_stream: torch.cuda.Stream | None = None
def _create_embedding(
self, layout: EngramLayout, layer_hash_index: int
) -> ParallelEngramEmbedding:
engram_config = get_current_vllm_config().engram_config
assert engram_config is not None
return ParallelEngramEmbedding(
layout.num_embeddings[layer_hash_index],
layout.head_dim,
tuple(size for order in layout.primes[layer_hash_index] for size in order),
cpu_offload=engram_config.cpu_offload,
dp_shared_memory=engram_config.dp_shared_memory,
)
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim)
if self.embed_tokens.cpu_offload:
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
"""Prefetch local shared rows or the DP group's gathered hash IDs."""
if self._prefetch_stream is None:
return super().prepare_embeddings(hash_ids)
rows = self.staged_rows[: hash_ids.shape[0]]
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
self._start_prefetch(hash_ids, rows, self._prefetch_stream)
@eager_break_during_capture
def _start_prefetch(
self, hash_ids: torch.Tensor, rows: torch.Tensor, stream: torch.cuda.Stream
) -> None:
# Eager boundaries let the lookup span piecewise graph segments.
stream.wait_stream(torch.cuda.current_stream())
# Keep temporary hash storage alive until lookup finishes reading it.
hash_ids.record_stream(stream)
with torch.cuda.stream(stream):
self.embed_tokens.lookup(hash_ids, rows, background=True)
@eager_break_during_capture
def _finish_prefetch(self, stream: torch.cuda.Stream) -> None:
torch.cuda.current_stream().wait_stream(stream)
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
if self._prefetch_stream is not None:
self._finish_prefetch(self._prefetch_stream)
if self.embed_tokens.dp_size > 1:
slot = engram_gathered_num_tokens()
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
return _gather_engram_rows(staged, num_tokens)
return super()._ready_rows(num_tokens)
+16 -2
View File
@@ -12,6 +12,7 @@ import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.config.kernel import MEGA_MOE_BACKENDS
from vllm.distributed import (
get_engram_dp_size,
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -74,8 +75,9 @@ from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import Engram, EngramLayout, NgramHashState
from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, gather_engram_hashes
if typing.TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
@@ -466,6 +468,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
# layer's sliding-window KV cache. Only PP ranks owning an engram
# layer need it.
self.engram_hash: NgramHashState | None = None
self.engram_dp_shared_memory = bool(
vllm_config.engram_config and vllm_config.engram_config.dp_shared_memory
)
self.engram_swa_prefix: str | None = None
if self.engram_layout is not None:
local_engram = any(
@@ -594,12 +599,21 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
swa_metadata.slot_mapping,
swa_metadata.block_table,
)
elif not self.engram_dp_shared_memory and get_engram_dp_size() > 1:
# DP-sharded lookups are collective, so a replica skipping the
# hash still has to reach them.
engram_hashes, engram_mask = self.engram_hash.dummy_hashes(input_ids)
if engram_hashes is not None:
# Gather all Engram rows before entering the decoder layers.
# One gather feeds every layer sharing the DP-split table.
gathered_hashes = gather_engram_hashes(
engram_hashes, dp_shared_memory=self.engram_dp_shared_memory
)
for layer in islice(self.layers, self.start_layer, self.end_layer):
engram = getattr(layer, "engram", None)
if engram is not None:
engram.prepare_embeddings(
engram_hashes[:, engram.layer_hash_index]
gathered_hashes[:, engram.layer_hash_index]
)
full_num_tokens = positions.shape[0]