[Perf][DSv4.1] Faster Engram host lookups: sorted rows, inline big lookups (#59327)

Signed-off-by: Juntian Liu <Juntianl777@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Juntian Liu
2026-10-01 11:54:44 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 4ed438b79c
commit 4c2d277643
6 changed files with 547 additions and 556 deletions
+24 -5
View File
@@ -39,10 +39,10 @@ from vllm.distributed.parallel_state import (
initialize_model_parallel,
)
from vllm.forward_context import set_forward_context
from vllm.models.deepseek_v41.common.engram import EngramLayout
from vllm.models.deepseek_v41.nvidia import engram as engram_ops
from vllm.models.deepseek_v41.nvidia.engram import (
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram,
EngramLayout,
ParallelEngramEmbedding,
engram_head_shard_rank,
gather_engram_hashes,
@@ -686,11 +686,30 @@ def test_engram_dp_shared_memory_runtime_requirements(
)
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch):
"""A /dev/shm smaller than the tables must fall back instead of failing startup."""
@pytest.mark.parametrize("query_fails", [False, True])
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
"""A /dev/shm that is too small or cannot be queried must fall back, and the
leader must still broadcast that decision so its peers do not hang."""
broadcasts = []
def broadcast(obj):
broadcasts.append(obj)
return obj
def query(*args, **kwargs):
raise OSError("statvfs failed")
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
monkeypatch.setattr(
engram_ops,
"get_engram_dp_group",
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
)
if query_fails:
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
assert not engram_ops.can_share_engram_tables(layout)
assert len(broadcasts) == 1 and broadcasts[0] is not None
@pytest.mark.parametrize(
+25 -25
View File
@@ -9,13 +9,10 @@ import torch
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram as CommonEngram,
)
from vllm.models.deepseek_v41.common.engram import (
Engram,
NgramHashState,
ParallelEngramEmbedding,
)
from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops
from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding
from vllm.platforms import current_platform
@@ -111,7 +108,7 @@ def test_fused_engram_post_wkv_matches_reference(
)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
module = CommonEngram.__new__(CommonEngram)
module = Engram.__new__(Engram)
torch.nn.Module.__init__(module)
module.dim = dim
module.hc_mult = hc_mult
@@ -121,6 +118,7 @@ def test_fused_engram_post_wkv_matches_reference(
module.embed_tokens = torch.nn.Identity()
# `forward` reads rows staged by `prepare_embeddings`, so inject kv there.
module.embed_tokens.tp_size = module.embed_tokens.n_hash_cols = 1
module.embed_tokens.dp_size = 1
module.staged_rows = kv.unsqueeze(1)
if use_sequence_parallel:
padded = torch.nn.functional.pad(kv, (0, 0, 0, (-num_kv_tokens) % tp_size))
@@ -592,9 +590,8 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
monkeypatch.setattr(
engram_ops, "get_tensor_model_parallel_world_size", lambda: tp_size
)
monkeypatch.setattr(nvidia_engram_ops, "get_engram_dp_size", lambda: dp_size)
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(nvidia_engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
with pytest.raises(AssertionError, match="ranks without hash heads"):
ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads)
@@ -602,10 +599,19 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="CUDA or ROCm required"
)
@pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize(
"cpu_offload,sort_rows", [(False, False), (True, False), (True, True)]
)
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch):
"""Keep complete buckets and reconstruct head order, including TP padding."""
def test_engram_head_shards_reconstruct_checkpoint(
cpu_offload, sort_rows, tp_size, monkeypatch
):
"""Keep complete buckets and reconstruct head order, including TP padding,
whether or not the host lookup sorts its rows by table offset."""
if sort_rows:
monkeypatch.setattr(
engram_ops, "_engram_lookup_thresholds", lambda _: (0, None)
)
head_sizes = (17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73)
num_rows, dim = sum(head_sizes), 64
torch.manual_seed(0)
@@ -796,10 +802,8 @@ def test_engram_shared_prefetch_stream_waits_per_layer():
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize(
"backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)]
)
def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
@pytest.mark.parametrize("cpu_offload", [None, False])
def test_engram_constructor_honors_offload(monkeypatch, cpu_offload):
"""Default offload uses pinned storage and the given stream; False uses HBM."""
from vllm.config import EngramConfig
@@ -817,23 +821,18 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
else EngramConfig(cpu_offload=cpu_offload),
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
)
offloaded = backend == "nvidia" and vllm_config.engram_config.cpu_offload
if backend == "common":
vllm_config.engram_config = None
offloaded = vllm_config.engram_config.cpu_offload
monkeypatch.setattr(engram_ops, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(
nvidia_engram_ops, "get_current_vllm_config", lambda: vllm_config
)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity()
)
stream = torch.cuda.Stream()
kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {}
with torch.device("cuda"):
cls = Engram if backend == "nvidia" else CommonEngram
module = cls(config, None, layout, 0, False, "engram", **kwargs)
module = Engram(
config, None, layout, 0, False, "engram", prefetch_stream=stream
)
layer = module.embed_tokens
if offloaded:
assert layer.weight.device.type == "cpu" and layer.weight.is_pinned()
@@ -897,7 +896,8 @@ def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_depende
"The allocator reuse negative control requires the native allocator"
)
def start(self, hash_ids, rows, stream):
def start(self, hash_ids, rows):
stream = self._prefetch_stream
if missing_dependency != "producer_wait":
stream.wait_stream(torch.cuda.current_stream())
if missing_dependency != "record_stream":
+1 -5
View File
@@ -65,13 +65,9 @@ from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import EngramLayout, NgramHashState
from ..common.engram import Engram, EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
# Engram host offload and its prefetch stream are neither ROCm- nor
# NVIDIA-specific, so they are imported rather than duplicated.
from ..nvidia.engram import Engram
if typing.TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
+490 -22
View File
@@ -34,23 +34,41 @@ chunk-by-chunk while an n-gram at position ``p`` needs the token ids at
cache for the rest.
"""
import ctypes
import mmap
import os
import tempfile
import weakref
from contextlib import ExitStack
import numpy as np
import torch
from torch import nn
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed import (
get_dp_group,
get_engram_dp_group,
get_engram_dp_size,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
)
from vllm.distributed.device_communicators.shm_broadcast import (
SHM_PATH,
check_shm_free_space,
)
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
logger = init_logger(__name__)
@@ -58,6 +76,18 @@ logger = init_logger(__name__)
DEAD_ID = -1
def _engram_lookup_thresholds(device: torch.device) -> tuple[int | None, int | None]:
"""Host lookup thresholds on `device`: the rows from which to sort, and the
tokens (max over DP ranks) from which to run inline. None disables that path."""
if current_platform.is_cuda() and current_platform.is_device_capability_family(
100, device.index or 0
):
# Measured on GB200.
return 36864, 4096
# TODO: Add hand-tuned thresholds for other GPU families.
return None, None
def _is_prime(n: int) -> bool:
"""Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import)."""
if n < 2:
@@ -583,12 +613,14 @@ def _engram_head_shard_weight_loader(
"ids_stride_t",
"ids_stride_h",
"GRID",
"SORTED",
]
)
def _engram_lookup_kernel(
weight,
scales,
ids,
sorted_dst,
out,
vocab_start,
vocab_end,
@@ -602,27 +634,37 @@ def _engram_lookup_kernel(
QUANT_BLOCK: tl.constexpr,
BLOCK_R: tl.constexpr,
GRID,
SORTED,
):
"""Gather fp8 rows, apply their ue8m0 block scales, write bf16.
Only this rank's heads are read; padded heads write zeros for all-gather.
`weight`/`scales` may address pinned host memory through UVA.
`weight`/`scales` may address pinned host memory through UVA. If SORTED,
`ids` holds rows relative to vocab_start in table order and `sorted_dst`
their output rows.
"""
cols = tl.arange(0, DIM)
scale_cols = cols // QUANT_BLOCK
for base in tl.range(tl.program_id(0) * BLOCK_R, num_rows, GRID * BLOCK_R):
rows = base + tl.arange(0, BLOCK_R)
valid = rows < num_rows
head = HEAD_START + rows % LOCAL_HEADS
token = (rows // LOCAL_HEADS).to(tl.int64)
index = tl.load(
ids + token * ids_stride_t + head * ids_stride_h,
mask=valid & (head < TOTAL_HEADS),
other=-1,
).to(tl.int64)
owned = valid & (head < TOTAL_HEADS)
owned &= (index >= vocab_start) & (index < vocab_end)
local = tl.where(owned, index - vocab_start, 0)
if SORTED:
local = tl.load(ids + rows, mask=valid, other=-1).to(tl.int64)
dst = tl.load(sorted_dst + rows, mask=valid, other=0).to(tl.int32)
owned = (local >= 0) & (local < vocab_end - vocab_start)
else:
head = HEAD_START + rows % LOCAL_HEADS
token = (rows // LOCAL_HEADS).to(tl.int64)
index = tl.load(
ids + token * ids_stride_t + head * ids_stride_h,
mask=valid & (head < TOTAL_HEADS),
other=-1,
).to(tl.int64)
dst = rows
owned = valid & (head < TOTAL_HEADS)
owned &= (index >= vocab_start) & (index < vocab_end)
local = index - vocab_start
local = tl.where(owned, local, 0)
values = tl.load(
weight + local[:, None] * DIM + cols[None, :],
mask=owned[:, None],
@@ -636,16 +678,240 @@ def _engram_lookup_kernel(
# ue8m0 is a power of two, so its byte *is* the fp32 exponent field.
scale = (scale.to(tl.int32) << 23).to(tl.float32, bitcast=True)
tl.store(
out + rows[:, None] * DIM + cols[None, :],
out + dst[:, None] * DIM + cols[None, :],
(values.to(tl.float32) * scale).to(tl.bfloat16),
mask=valid[:, None],
)
def engram_head_shard_rank() -> int:
"""This rank's slot among the hash-head shards of one engram table.
TP-major, so the shards a DP gather brings in are contiguous heads and
the following TP gather completes the head order.
"""
dp_group = get_engram_dp_group()
dp_size = dp_group.world_size if dp_group is not None else 1
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
return get_tensor_model_parallel_rank() * dp_size + dp_rank
def engram_gathered_num_tokens() -> int:
"""Per-replica token slot for the node-local Engram DP group."""
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is None:
raise RuntimeError("a DP-shared engram table needs DP token metadata")
group = get_engram_dp_group()
assert group is not None
# Engram groups are contiguous slices of the full DP group.
start = get_dp_group().rank_in_group - group.rank_in_group
return int(
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
)
def gather_engram_hashes(
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
) -> torch.Tensor:
"""Collect the n-gram ids of every DP replica sharing one table.
Replicas are padded to a common token slot, so the gathered shape is
static under CUDA graph capture (where DP already pads alike).
"""
dp_group = get_engram_dp_group()
if dp_group is None or dp_shared_memory:
return hash_ids
slot = engram_gathered_num_tokens()
if hash_ids.shape[0] > slot:
raise ValueError("Engram token count exceeds the DP token slot")
if hash_ids.shape[0] < slot:
pad = hash_ids.new_full(
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
)
hash_ids = torch.cat((hash_ids, pad))
return dp_group.all_gather(hash_ids, dim=0)
class DPSharedEngramStorage:
"""Registered host weights shared by a node-local DP group with one writer."""
def __init__(
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
) -> None:
self.group = group
weight_bytes = num_rows * dim
storage = self._allocate(weight_bytes + weight_bytes // block_size)
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
def _allocate(self, num_bytes: int) -> torch.Tensor:
"""Map and register one physical allocation across a node-local DP group."""
group = self.group
with ExitStack() as stack:
path = error = None
if group.rank_in_group == 0:
try:
check_shm_free_space(num_bytes)
backing_file = stack.enter_context(
tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
)
backing_file.truncate(num_bytes)
path = backing_file.name
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
path, error = group.broadcast_object((path, error))
if error is not None:
raise RuntimeError(
"Engram shared-memory creation failed on EDP rank 0: " + error
)
mapping = owner = tensor = None
try:
with open(path, "r+b") as file:
mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
owner = np.frombuffer(mapping, dtype=np.uint8)
pointer = owner.ctypes.data
tensor = torch.from_numpy(owner)
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
if result.value != 0:
raise RuntimeError(f"cudaHostRegister failed: {result}")
finalizer = weakref.finalize(owner, self._unregister, mapping, pointer)
finalizer.atexit = False # type: ignore[misc]
# The UVA helper otherwise allocates a private pinned copy.
if not tensor.is_pinned():
raise RuntimeError(
"cudaHostRegister did not pin the shared Engram mapping"
)
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
# Also fences peer mappings before the leader unlinks the file.
errors: list[str | None] = [None] * group.world_size
torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
failures = "; ".join(
f"EDP rank {rank}: {error}"
for rank, error in enumerate(errors)
if error is not None
)
if failures:
# Dropping the owner runs the finalizer; the mapping then unmaps.
del tensor, owner, mapping
raise RuntimeError(
"Engram shared-memory initialization failed: " + failures
)
assert tensor is not None
return tensor
@staticmethod
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
# Torch storage retains the numpy owner, including through cached UVA views.
# Keep its mmap alive until CUDA has released the registration.
result = torch.cuda.cudart().cudaHostUnregister(pointer)
if result.value != 0:
logger.warning("Engram cudaHostUnregister failed: %s", result)
def load_weight(
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
) -> None:
if self.group.rank_in_group == 0:
_engram_head_shard_weight_loader(param, loaded_weight)
# Read order may differ across ranks. Equal load counts ensure all shared
# weights are ready after the last weight-loader call returns.
torch.distributed.barrier(group=self.group.cpu_group)
def get_views(
self, weight: torch.Tensor, scales: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
if (weight.data_ptr(), scales.data_ptr()) != (
self.weight.data_ptr(),
self.weight_scale_inv.data_ptr(),
):
raise RuntimeError("Shared Engram parameter storage must not be replaced")
if self._views is None:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
)
return self._views
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
if get_engram_dp_size() == 1:
logger.warning_once(
"Engram DP replicas are not co-located on one node; "
"storing the offloaded tables per rank instead of sharing them."
)
return False
num_bytes = sum(layout.num_embeddings) * (
layout.head_dim + layout.head_dim // block_size
)
group = get_engram_dp_group()
assert group is not None
# Only the leader allocates, so every replica follows its check.
error = None
if group.rank_in_group == 0:
if not os.path.isdir(SHM_PATH):
error = f"{SHM_PATH} is not mounted"
else:
try:
check_shm_free_space(num_bytes, allocation_name="Engram tables")
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
error = group.broadcast_object(error)
if error is not None:
logger.warning_once(
"Sharding the offloaded Engram tables across DP replicas: %s", error
)
return error is None
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
"""Register prefaulted huge pages, or return None for pinned-memory fallback."""
try:
mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
mapping.madvise(mmap.MADV_HUGEPAGE)
except OSError as exc:
logger.warning(
"Engram huge-page allocation failed (%s); using pinned memory.", exc
)
return None
owner = np.frombuffer(mapping, dtype=np.uint8)
# Fault the pages in before CUDA pins them, so they can be huge.
owner[:: mmap.PAGESIZE] = 0
tensor = torch.from_numpy(owner)
pointer = tensor.data_ptr()
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
if result.value != 0:
logger.warning(
"Engram cudaHostRegister failed (%s); using pinned memory.", result
)
return None
# Tensor views retain owner; its finalizer retains mapping until unregister.
finalizer = weakref.finalize(
owner, DPSharedEngramStorage._unregister, mapping, pointer
)
finalizer.atexit = False # type: ignore[misc]
if not tensor.is_pinned():
logger.warning(
"CUDA did not recognize Engram registration; using pinned memory."
)
return None
return tensor
class ParallelEngramEmbedding(nn.Module):
"""TP-sharded hash heads with FP8 rows and per-block E8M0 scales."""
"""Hash heads with FP8 rows and per-block E8M0 scales.
Heads are TP-sharded, and additionally DP-sharded across a node-local
group unless the table lives in (DP-shared) pinned host memory.
"""
_weight_loader = staticmethod(_engram_head_shard_weight_loader)
_shared_memory: DPSharedEngramStorage | None = None
_packed: torch.Tensor | None = None
def __init__(
self,
@@ -653,10 +919,32 @@ class ParallelEngramEmbedding(nn.Module):
dim: int,
head_sizes: tuple[int, ...],
block_size: int = 32,
cpu_offload: bool = False,
dp_shared_memory: bool = False,
use_thp: bool = False,
) -> None:
super().__init__()
assert head_sizes and all(size > 0 for size in head_sizes)
assert sum(head_sizes) <= num_embeddings
self.cpu_offload = cpu_offload
self.dp_shared_memory = dp_shared_memory
self.use_thp = use_thp
self.dp_size = get_engram_dp_size()
if dp_shared_memory:
if not cpu_offload:
raise ValueError("dp_shared_memory requires cpu_offload=True")
if self.dp_size <= 1:
raise ValueError(
"dp_shared_memory requires a node-local Engram DP "
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
"Check that the node layout and rank placement allow complete "
"DP replicas to be co-located."
)
self.dp_size = 1
if cpu_offload and not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
self._view_src: tuple[int, int] | None = None
self.num_embeddings = num_embeddings
self.dim = dim
self.block_size = block_size
@@ -688,20 +976,99 @@ class ParallelEngramEmbedding(nn.Module):
"engram_vocab_start": self.vocab_start_idx,
},
)
if cpu_offload:
# Constant dummy values avoid randomizing huge CPU lookup tables.
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
logger.info(
"Engram table offloaded to pinned host memory: %d rows x %d, "
"%.2f GiB %s",
self.part_num_embeddings,
dim,
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
"shared across DP replicas" if dp_shared_memory else "per rank",
)
def _get_shard_info(self) -> tuple[int, int]:
return self.tp_size, get_tensor_model_parallel_rank()
if self.dp_size == 1:
return self.tp_size, get_tensor_model_parallel_rank()
return self.tp_size * self.dp_size, engram_head_shard_rank()
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
if self.dp_shared_memory:
group = get_engram_dp_group()
assert group is not None
storage = DPSharedEngramStorage(
self.part_num_embeddings, self.dim, self.block_size, group
)
self._shared_memory = storage
self._weight_loader = storage.load_weight
return storage.weight, storage.weight_scale_inv
scale_dim = self.dim // self.block_size
if not self.cpu_offload:
return (
torch.empty(
self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn
),
torch.empty(self.part_num_embeddings, scale_dim, dtype=torch.uint8),
)
if self.use_thp:
weight_bytes = self.part_num_embeddings * self.dim
packed = _allocate_huge_page_storage(
weight_bytes + weight_bytes // self.block_size
)
if packed is not None:
self._packed = packed
return (
packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim),
packed[weight_bytes:].view(-1, scale_dim),
)
# Model initialization may be inside a CUDA device context.
return (
torch.empty(self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn),
torch.empty(
self.part_num_embeddings, self.dim // self.block_size, dtype=torch.uint8
self.part_num_embeddings,
self.dim,
dtype=torch.float8_e4m3fn,
device="cpu",
pin_memory=True,
),
torch.empty(
self.part_num_embeddings,
scale_dim,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
),
)
def collapse_huge_pages(self) -> None:
"""Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
if self._packed is None:
return
addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
libc = ctypes.CDLL(None, use_errno=True)
libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
if libc.madvise(addr, num_bytes, 25) != 0:
logger.warning(
"Engram MADV_COLLAPSE failed; keeping existing pages: %s",
os.strerror(ctypes.get_errno()),
)
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
return self.weight.data, self.weight_scale_inv.data
if self._shared_memory is not None:
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
if not self.cpu_offload:
return self.weight.data, self.weight_scale_inv.data
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
if self._view_src != src:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight.data),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
)
self._view_src = src
assert self._views is not None
return self._views
def lookup(
self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
@@ -714,6 +1081,24 @@ class ParallelEngramEmbedding(nn.Module):
if not rows:
return
weight, scales = self._storage()
ids_stride_t, ids_stride_h = indices.stride()
sort_min_rows, _ = _engram_lookup_thresholds(indices.device)
sort = (
self.cpu_offload
and self._packed is None
and sort_min_rows is not None
and rows >= sort_min_rows
)
dst = indices
if sort:
# Small-page host tables miss the TLB per row; in table order, rows on
# one page share it.
head_end = self.head_start + self.part_n_hash_cols
local = indices[:, self.head_start : head_end] - self.vocab_start_idx
if (pad := self.part_n_hash_cols - local.shape[1]) > 0:
local = nn.functional.pad(local, (0, pad), value=-1)
indices, dst = local.flatten().sort()
dst = dst.int()
# The table dwarfs TLB reach, so a persistent grid near the SM count
# beats one program per row; halve it to leave SMs for the main stream.
tiles = triton.cdiv(rows, 16)
@@ -722,12 +1107,13 @@ class ParallelEngramEmbedding(nn.Module):
weight,
scales,
indices,
dst,
out,
self.vocab_start_idx,
self.vocab_end_idx,
rows,
indices.stride(0),
indices.stride(1),
ids_stride_t,
ids_stride_h,
HEAD_START=self.head_start,
LOCAL_HEADS=self.part_n_hash_cols,
TOTAL_HEADS=self.n_hash_cols,
@@ -735,17 +1121,23 @@ class ParallelEngramEmbedding(nn.Module):
QUANT_BLOCK=self.block_size,
BLOCK_R=16,
GRID=grid,
SORTED=sort,
)
def forward(self, indices: torch.Tensor) -> torch.Tensor:
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
bf16, gathered from all shards for this replica's tokens."""
num_tokens = indices.shape[0]
if self.dp_size > 1:
indices = gather_engram_hashes(indices)
out = torch.empty(
(indices.shape[0], self.part_n_hash_cols, self.dim),
dtype=torch.bfloat16,
device=indices.device,
)
self.lookup(indices, out)
if self.dp_size > 1:
out = _gather_engram_rows(out, num_tokens)
if self.tp_size > 1:
out = tensor_model_parallel_all_gather(out, dim=1)
return out[:, : self.n_hash_cols]
@@ -891,6 +1283,25 @@ def _engram_select_rows(
)
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
dp_group = get_engram_dp_group()
assert dp_group is not None
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
assert remainder == 0 and 0 <= num_tokens <= slot
gathered = dp_group.all_gather(staged, dim=0)
local_heads, dim = staged.shape[1:]
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
_engram_select_rows(
gathered,
rows,
staged.shape[0],
dp_group.rank_in_group * slot,
local_heads * dim,
)
return rows
class Engram(nn.Module):
"""Writes an n-gram lookup into the residual stream, gated by how well it
matches that stream.
@@ -901,6 +1312,9 @@ class Engram(nn.Module):
training kernel).
"""
_prefetch_stream: torch.cuda.Stream | None = None
_prefetch_done: torch.cuda.Event | None = None
def __init__(
self,
config,
@@ -909,8 +1323,12 @@ class Engram(nn.Module):
layer_hash_index: int,
use_sequence_parallel: bool,
prefix: str,
prefetch_stream: torch.cuda.Stream | None = None,
) -> None:
super().__init__()
# Layers sharing one stream serialize their offloaded lookups, so they
# take turns instead of jointly starving decoder compute of SMs.
self._prefetch_stream = prefetch_stream
self.layer_hash_index = layer_hash_index
self.dim = config.hidden_size
self.hc_mult = config.hc_mult
@@ -950,28 +1368,78 @@ class Engram(nn.Module):
def _create_embedding(
self, layout: EngramLayout, layer_hash_index: int
) -> ParallelEngramEmbedding:
engram_config = get_current_vllm_config().engram_config
assert engram_config is not None
return ParallelEngramEmbedding(
layout.num_embeddings[layer_hash_index],
layout.head_dim,
tuple(size for order in layout.primes[layer_hash_index] for size in order),
cpu_offload=engram_config.cpu_offload,
dp_shared_memory=bool(engram_config.dp_shared_memory),
use_thp=engram_config.use_thp,
)
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
# Persistent storage keeps lookup addresses stable across graph replays.
self.staged_rows = torch.empty(
max_tokens,
max_tokens * self.embed_tokens.dp_size,
self.embed_tokens.part_n_hash_cols,
head_dim,
dtype=torch.bfloat16,
)
if not self.embed_tokens.cpu_offload:
self._prefetch_stream = None
return
if self._prefetch_stream is None:
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
self._prefetch_done = torch.cuda.Event()
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
"""Look up rows before the decoder layers consume them."""
"""Look up, or prefetch from host memory, rows before the decoder layers
consume them; DP-sharded tables take the group's gathered hash IDs."""
rows = self.staged_rows[: hash_ids.shape[0]]
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
self.embed_tokens.lookup(hash_ids, rows)
if self._prefetch_stream is None:
self.embed_tokens.lookup(hash_ids, rows)
else:
self._start_prefetch(hash_ids, rows)
@eager_break_during_capture
def _start_prefetch(self, hash_ids: torch.Tensor, rows: torch.Tensor) -> None:
# Eager boundaries let the lookup span piecewise graph segments, and
# decide per replay (on the replay stream) whether it runs inline.
num_tokens = hash_ids.shape[0]
# All DP ranks must agree, or EP collectives wait on the slowest one.
if is_forward_context_available() and (dp := get_forward_context().dp_metadata):
num_tokens = int(dp.num_tokens_across_dp_cpu.max())
current = stream = torch.cuda.current_stream()
# Big lookups stall persistent main-stream kernels anyway: run them inline.
_, inline_min_tokens = _engram_lookup_thresholds(hash_ids.device)
if inline_min_tokens is not None and num_tokens >= inline_min_tokens:
self.embed_tokens.lookup(hash_ids, rows)
else:
stream = self._prefetch_stream
assert stream is not None
stream.wait_stream(current)
# Keep temporary hash storage alive until lookup finishes reading it.
hash_ids.record_stream(stream)
with torch.cuda.stream(stream):
self.embed_tokens.lookup(hash_ids, rows, background=True)
assert self._prefetch_done is not None
self._prefetch_done.record(stream)
@eager_break_during_capture
def _finish_prefetch(self, event: torch.cuda.Event) -> None:
# Wait for this layer's lookup only, not for later layers on the stream.
torch.cuda.current_stream().wait_event(event)
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
if self._prefetch_done is not None:
self._finish_prefetch(self._prefetch_done)
if self.embed_tokens.dp_size > 1:
slot = engram_gathered_num_tokens()
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
return _gather_engram_rows(staged, num_tokens)
return self.staged_rows[:num_tokens]
def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
-497
View File
@@ -1,497 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
import ctypes
import mmap
import os
import shutil
import tempfile
import weakref
from contextlib import ExitStack
import numpy as np
import torch
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_dp_group,
get_engram_dp_group,
get_engram_dp_size,
get_tensor_model_parallel_rank,
tensor_model_parallel_all_gather,
)
from vllm.distributed.device_communicators.shm_broadcast import (
SHM_PATH,
check_shm_free_space,
)
from vllm.distributed.parallel_state import GroupCoordinator
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.deepseek_v41.common.engram import (
DEAD_ID,
EngramLayout,
_engram_head_shard_weight_loader,
_engram_select_rows,
)
from vllm.models.deepseek_v41.common.engram import (
Engram as BaseEngram,
)
from vllm.models.deepseek_v41.common.engram import (
ParallelEngramEmbedding as BaseParallelEngramEmbedding,
)
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
logger = init_logger(__name__)
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
"""Register prefaulted huge pages, or return None for pinned-memory fallback."""
try:
mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
mapping.madvise(mmap.MADV_HUGEPAGE)
except OSError as exc:
logger.warning(
"Engram huge-page allocation failed (%s); using pinned memory.", exc
)
return None
owner = np.frombuffer(mapping, dtype=np.uint8)
# Fault the pages in before CUDA pins them, so they can be huge.
owner[:: mmap.PAGESIZE] = 0
tensor = torch.from_numpy(owner)
pointer = tensor.data_ptr()
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
if result.value != 0:
logger.warning(
"Engram cudaHostRegister failed (%s); using pinned memory.", result
)
return None
# Tensor views retain owner; its finalizer retains mapping until unregister.
finalizer = weakref.finalize(
owner, DPSharedEngramStorage._unregister, mapping, pointer
)
finalizer.atexit = False # type: ignore[misc]
if not tensor.is_pinned():
logger.warning(
"CUDA did not recognize Engram registration; using pinned memory."
)
return None
return tensor
def engram_head_shard_rank() -> int:
"""This rank's slot among the hash-head shards of one engram table.
TP-major, so the shards a DP gather brings in are contiguous heads and
the following TP gather completes the head order.
"""
dp_group = get_engram_dp_group()
dp_size = dp_group.world_size if dp_group is not None else 1
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
return get_tensor_model_parallel_rank() * dp_size + dp_rank
def engram_gathered_num_tokens() -> int:
"""Per-replica token slot for the node-local Engram DP group."""
dp_metadata = get_forward_context().dp_metadata
if dp_metadata is None:
raise RuntimeError("a DP-shared engram table needs DP token metadata")
group = get_engram_dp_group()
assert group is not None
# Engram groups are contiguous slices of the full DP group.
start = get_dp_group().rank_in_group - group.rank_in_group
return int(
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
)
def gather_engram_hashes(
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
) -> torch.Tensor:
"""Collect the n-gram ids of every DP replica sharing one table.
Replicas are padded to a common token slot, so the gathered shape is
static under CUDA graph capture (where DP already pads alike).
"""
dp_group = get_engram_dp_group()
if dp_group is None or dp_shared_memory:
return hash_ids
slot = engram_gathered_num_tokens()
if hash_ids.shape[0] > slot:
raise ValueError("Engram token count exceeds the DP token slot")
if hash_ids.shape[0] < slot:
pad = hash_ids.new_full(
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
)
hash_ids = torch.cat((hash_ids, pad))
return dp_group.all_gather(hash_ids, dim=0)
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
if not os.path.isdir(SHM_PATH):
logger.warning_once(
"%s is not mounted; storing the offloaded Engram tables per rank "
"instead of sharing them across DP replicas.",
SHM_PATH,
)
return False
if get_engram_dp_size() == 1:
logger.warning_once(
"Engram DP replicas are not co-located on one node; "
"storing the offloaded tables per rank instead of sharing them."
)
return False
num_bytes = sum(layout.num_embeddings) * (
layout.head_dim + layout.head_dim // block_size
)
if shutil.disk_usage(SHM_PATH).total < num_bytes:
logger.warning_once(
"Sharing Engram tables across DP replicas needs %.1f GiB of %s "
"(--shm-size or --ipc=host); sharding them across replicas instead.",
num_bytes / 1024**3,
SHM_PATH,
)
return False
return True
class DPSharedEngramStorage:
"""Registered host weights shared by a node-local DP group with one writer."""
def __init__(
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
) -> None:
self.group = group
weight_bytes = num_rows * dim
storage = self._allocate(weight_bytes + weight_bytes // block_size)
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
def _allocate(self, num_bytes: int) -> torch.Tensor:
"""Map and register one physical allocation across a node-local DP group."""
group = self.group
with ExitStack() as stack:
path = error = None
if group.rank_in_group == 0:
try:
check_shm_free_space(num_bytes)
backing_file = stack.enter_context(
tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
)
backing_file.truncate(num_bytes)
path = backing_file.name
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
path, error = group.broadcast_object((path, error))
if error is not None:
raise RuntimeError(
"Engram shared-memory creation failed on EDP rank 0: " + error
)
mapping = owner = tensor = None
try:
with open(path, "r+b") as file:
mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
owner = np.frombuffer(mapping, dtype=np.uint8)
pointer = owner.ctypes.data
tensor = torch.from_numpy(owner)
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
if result.value != 0:
raise RuntimeError(f"cudaHostRegister failed: {result}")
finalizer = weakref.finalize(owner, self._unregister, mapping, pointer)
finalizer.atexit = False # type: ignore[misc]
# The UVA helper otherwise allocates a private pinned copy.
if not tensor.is_pinned():
raise RuntimeError(
"cudaHostRegister did not pin the shared Engram mapping"
)
except Exception as exc:
error = f"{type(exc).__name__}: {exc}"
# Also fences peer mappings before the leader unlinks the file.
errors: list[str | None] = [None] * group.world_size
torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
failures = "; ".join(
f"EDP rank {rank}: {error}"
for rank, error in enumerate(errors)
if error is not None
)
if failures:
# Dropping the owner runs the finalizer; the mapping then unmaps.
del tensor, owner, mapping
raise RuntimeError(
"Engram shared-memory initialization failed: " + failures
)
assert tensor is not None
return tensor
@staticmethod
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
# Torch storage retains the numpy owner, including through cached UVA views.
# Keep its mmap alive until CUDA has released the registration.
result = torch.cuda.cudart().cudaHostUnregister(pointer)
if result.value != 0:
logger.warning("Engram cudaHostUnregister failed: %s", result)
def load_weight(
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
) -> None:
if self.group.rank_in_group == 0:
_engram_head_shard_weight_loader(param, loaded_weight)
# Read order may differ across ranks. Equal load counts ensure all shared
# weights are ready after the last weight-loader call returns.
torch.distributed.barrier(group=self.group.cpu_group)
def get_views(
self, weight: torch.Tensor, scales: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
if (weight.data_ptr(), scales.data_ptr()) != (
self.weight.data_ptr(),
self.weight_scale_inv.data_ptr(),
):
raise RuntimeError("Shared Engram parameter storage must not be replaced")
if self._views is None:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
)
return self._views
class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
"""Extend TP lookup with DP head sharding or shared, CPU-offloaded TP slices."""
_shared_memory: DPSharedEngramStorage | None = None
def __init__(
self,
num_embeddings: int,
dim: int,
head_sizes: tuple[int, ...],
block_size: int = 32,
cpu_offload: bool = False,
dp_shared_memory: bool = False,
use_thp: bool = False,
) -> None:
self.cpu_offload = cpu_offload
self.dp_shared_memory = dp_shared_memory
self.use_thp = use_thp
self._packed: torch.Tensor | None = None
self.dp_size = get_engram_dp_size()
if dp_shared_memory:
if not cpu_offload:
raise ValueError("dp_shared_memory requires cpu_offload=True")
if self.dp_size <= 1:
raise ValueError(
"dp_shared_memory requires a node-local Engram DP "
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
"Check that the node layout and rank placement allow complete "
"DP replicas to be co-located."
)
self.dp_size = 1
if cpu_offload and not is_uva_available():
raise RuntimeError("Engram CPU offload requires UVA support")
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
self._view_src: tuple[int, int] | None = None
super().__init__(num_embeddings, dim, head_sizes, block_size)
if cpu_offload:
# Constant dummy values avoid randomizing huge CPU lookup tables.
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
logger.info(
"Engram table offloaded to pinned host memory: %d rows x %d, "
"%.2f GiB %s",
self.part_num_embeddings,
dim,
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
"shared across DP replicas" if dp_shared_memory else "per rank",
)
def _get_shard_info(self) -> tuple[int, int]:
if self.dp_size == 1:
return super()._get_shard_info()
return self.tp_size * self.dp_size, engram_head_shard_rank()
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
if self.dp_shared_memory:
group = get_engram_dp_group()
assert group is not None
storage = DPSharedEngramStorage(
self.part_num_embeddings, self.dim, self.block_size, group
)
self._shared_memory = storage
self._weight_loader = storage.load_weight
return storage.weight, storage.weight_scale_inv
if not self.cpu_offload:
return super()._allocate_weights()
if self.use_thp:
weight_bytes = self.part_num_embeddings * self.dim
packed = _allocate_huge_page_storage(
weight_bytes + weight_bytes // self.block_size
)
if packed is not None:
self._packed = packed
return (
packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim),
packed[weight_bytes:].view(-1, self.dim // self.block_size),
)
# Model initialization may be inside a CUDA device context.
return (
torch.empty(
self.part_num_embeddings,
self.dim,
dtype=torch.float8_e4m3fn,
device="cpu",
pin_memory=True,
),
torch.empty(
self.part_num_embeddings,
self.dim // self.block_size,
dtype=torch.uint8,
device="cpu",
pin_memory=True,
),
)
def collapse_huge_pages(self) -> None:
"""Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
if self._packed is None:
return
addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
libc = ctypes.CDLL(None, use_errno=True)
libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
if libc.madvise(addr, num_bytes, 25) != 0:
logger.warning(
"Engram MADV_COLLAPSE failed; keeping existing pages: %s",
os.strerror(ctypes.get_errno()),
)
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
if self._shared_memory is not None:
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
if not self.cpu_offload:
return super()._storage()
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
if self._view_src != src:
self._views = (
get_accelerator_view_from_cpu_tensor(self.weight.data),
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
)
self._view_src = src
assert self._views is not None
return self._views
def forward(self, indices: torch.Tensor) -> torch.Tensor:
if self.dp_size == 1:
return super().forward(indices)
num_tokens = indices.shape[0]
indices = gather_engram_hashes(indices)
out = torch.empty(
(indices.shape[0], self.part_n_hash_cols, self.dim),
dtype=torch.bfloat16,
device=indices.device,
)
self.lookup(indices, out)
out = _gather_engram_rows(out, num_tokens)
if self.tp_size > 1:
out = tensor_model_parallel_all_gather(out, dim=1)
return out[:, : self.n_hash_cols]
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
dp_group = get_engram_dp_group()
assert dp_group is not None
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
assert remainder == 0 and 0 <= num_tokens <= slot
gathered = dp_group.all_gather(staged, dim=0)
local_heads, dim = staged.shape[1:]
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
_engram_select_rows(
gathered,
rows,
staged.shape[0],
dp_group.rank_in_group * slot,
local_heads * dim,
)
return rows
class Engram(BaseEngram):
"""NVIDIA Engram with asynchronous offload and node-local DP lookup."""
_prefetch_stream: torch.cuda.Stream | None = None
_prefetch_done: torch.cuda.Event | None = None
def __init__(
self, *args, prefetch_stream: torch.cuda.Stream | None = None, **kwargs
) -> None:
# Layers sharing one stream serialize their offloaded lookups, so they
# take turns instead of jointly starving decoder compute of SMs.
self._prefetch_stream = prefetch_stream
super().__init__(*args, **kwargs)
def _create_embedding(
self, layout: EngramLayout, layer_hash_index: int
) -> ParallelEngramEmbedding:
engram_config = get_current_vllm_config().engram_config
assert engram_config is not None
return ParallelEngramEmbedding(
layout.num_embeddings[layer_hash_index],
layout.head_dim,
tuple(size for order in layout.primes[layer_hash_index] for size in order),
cpu_offload=engram_config.cpu_offload,
dp_shared_memory=bool(engram_config.dp_shared_memory),
use_thp=engram_config.use_thp,
)
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim)
if not self.embed_tokens.cpu_offload:
self._prefetch_stream = None
return
if self._prefetch_stream is None:
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
self._prefetch_done = torch.cuda.Event()
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
"""Prefetch local shared rows or the DP group's gathered hash IDs."""
if self._prefetch_stream is None:
return super().prepare_embeddings(hash_ids)
rows = self.staged_rows[: hash_ids.shape[0]]
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
self._start_prefetch(hash_ids, rows, self._prefetch_stream)
@eager_break_during_capture
def _start_prefetch(
self, hash_ids: torch.Tensor, rows: torch.Tensor, stream: torch.cuda.Stream
) -> None:
# Eager boundaries let the lookup span piecewise graph segments.
stream.wait_stream(torch.cuda.current_stream())
# Keep temporary hash storage alive until lookup finishes reading it.
hash_ids.record_stream(stream)
with torch.cuda.stream(stream):
self.embed_tokens.lookup(hash_ids, rows, background=True)
assert self._prefetch_done is not None
self._prefetch_done.record(stream)
@eager_break_during_capture
def _finish_prefetch(self, event: torch.cuda.Event) -> None:
# Wait for this layer's lookup only, not for later layers on the stream.
torch.cuda.current_stream().wait_event(event)
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
if self._prefetch_done is not None:
self._finish_prefetch(self._prefetch_done)
if self.embed_tokens.dp_size > 1:
slot = engram_gathered_num_tokens()
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
return _gather_engram_rows(staged, num_tokens)
return super()._ready_rows(num_tokens)
+7 -2
View File
@@ -98,9 +98,14 @@ from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import EngramLayout, NgramHashState
from ..common.engram import (
Engram,
EngramLayout,
NgramHashState,
can_share_engram_tables,
gather_engram_hashes,
)
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, can_share_engram_tables, gather_engram_hashes
from .ops.mhc import (
MHC_OVERLAP_MAX_TOKENS,
init_mhc_all_reduce,