[DS V4.1][Engram] Support async prefetch for offloaded engram lookups and engram DP sharding (#56512)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: ywang96 <> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Roger Wang
ywang96 <>
Woosuk Kwon
Codex
parent
e7a3963339
commit
d2d649e674
@@ -51,8 +51,6 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -236,6 +234,7 @@ def _write_hash_cache_kernel(
|
||||
tl.store(cache + slot, value, valid)
|
||||
|
||||
|
||||
# Keep request shapes out of the cache key so one warmed variant covers runtime batches.
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"num_tokens",
|
||||
@@ -462,6 +461,21 @@ class NgramHashState(nn.Module):
|
||||
self._kv_cache_ref = weakref.ref(kv_cache)
|
||||
return True
|
||||
|
||||
def dummy_hashes(
|
||||
self, input_ids: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Participate in DP lookups without valid rows or hash-cache updates."""
|
||||
num_tokens = input_ids.shape[0]
|
||||
num_layers, max_ngram = self.multipliers.shape
|
||||
num_heads = self.primes.shape[-1]
|
||||
hashes = input_ids.new_full(
|
||||
(num_tokens, num_layers, (max_ngram - 1) * num_heads),
|
||||
DEAD_ID,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
keep = torch.zeros(num_tokens, dtype=torch.bool, device=input_ids.device)
|
||||
return hashes, keep
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -561,7 +575,16 @@ def _engram_head_shard_weight_loader(
|
||||
param.data.copy_(shard)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(
|
||||
do_not_specialize=[
|
||||
"vocab_start",
|
||||
"vocab_end",
|
||||
"num_rows",
|
||||
"ids_stride_t",
|
||||
"ids_stride_h",
|
||||
"GRID",
|
||||
]
|
||||
)
|
||||
def _engram_lookup_kernel(
|
||||
weight,
|
||||
scales,
|
||||
@@ -578,7 +601,7 @@ def _engram_lookup_kernel(
|
||||
DIM: tl.constexpr,
|
||||
QUANT_BLOCK: tl.constexpr,
|
||||
BLOCK_R: tl.constexpr,
|
||||
GRID: tl.constexpr,
|
||||
GRID,
|
||||
):
|
||||
"""Gather fp8 rows, apply their ue8m0 block scales, write bf16.
|
||||
|
||||
@@ -620,12 +643,9 @@ def _engram_lookup_kernel(
|
||||
|
||||
|
||||
class ParallelEngramEmbedding(nn.Module):
|
||||
"""The n-gram hash table, sharded by complete hash heads over TP ranks.
|
||||
Rows stay fp8 and are dequantized with ue8m0 per-32 scales on lookup.
|
||||
"""TP-sharded hash heads with FP8 rows and per-block E8M0 scales."""
|
||||
|
||||
With `cpu_offload` the shard lives in pinned host memory and is read over
|
||||
UVA instead of HBM; the TP sharding is unchanged either way.
|
||||
"""
|
||||
_weight_loader = staticmethod(_engram_head_shard_weight_loader)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -633,89 +653,55 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
dim: int,
|
||||
head_sizes: tuple[int, ...],
|
||||
block_size: int = 32,
|
||||
cpu_offload: bool = False,
|
||||
):
|
||||
) -> None:
|
||||
super().__init__()
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
assert head_sizes and all(size > 0 for size in head_sizes)
|
||||
assert sum(head_sizes) <= num_embeddings
|
||||
if cpu_offload and not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
self.num_embeddings = num_embeddings
|
||||
self.dim = dim
|
||||
self.block_size = block_size
|
||||
self.n_hash_cols = len(head_sizes)
|
||||
self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, tp_size)
|
||||
self.head_start = tp_rank * self.part_n_hash_cols
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
num_shards, head_rank = self._get_shard_info()
|
||||
self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, num_shards)
|
||||
# TODO: Support row-wise sharding when there are too few hash heads.
|
||||
assert (num_shards - 1) * self.part_n_hash_cols < self.n_hash_cols, (
|
||||
f"Engram sharding leaves ranks without hash heads: "
|
||||
f"{self.n_hash_cols} heads over {num_shards} shards"
|
||||
)
|
||||
self.head_start = head_rank * self.part_n_hash_cols
|
||||
head_end = self.head_start + self.part_n_hash_cols
|
||||
self.vocab_start_idx = sum(head_sizes[: self.head_start])
|
||||
self.vocab_end_idx = sum(head_sizes[:head_end])
|
||||
self.part_num_embeddings = self.vocab_end_idx - self.vocab_start_idx
|
||||
self.tp_size = tp_size
|
||||
self.cpu_offload = cpu_offload
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._view_src: tuple[int, int] | None = None
|
||||
self._num_sms = torch.cuda.get_device_properties(
|
||||
torch.accelerator.current_device_index()
|
||||
).multi_processor_count
|
||||
|
||||
# Explicit device: model init runs under a `torch.device("cuda")`
|
||||
# context, which would otherwise put the shard in HBM.
|
||||
kwargs = {"device": "cpu", "pin_memory": True} if cpu_offload else {}
|
||||
self.weight = nn.Parameter(
|
||||
torch.empty(
|
||||
self.part_num_embeddings, dim, dtype=torch.float8_e4m3fn, **kwargs
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
self.weight_scale_inv = nn.Parameter(
|
||||
torch.empty(
|
||||
self.part_num_embeddings,
|
||||
dim // block_size,
|
||||
dtype=torch.uint8,
|
||||
**kwargs,
|
||||
),
|
||||
requires_grad=False,
|
||||
)
|
||||
weight, scales = self._allocate_weights()
|
||||
self.weight = nn.Parameter(weight, requires_grad=False)
|
||||
self.weight_scale_inv = nn.Parameter(scales, requires_grad=False)
|
||||
for param in (self.weight, self.weight_scale_inv):
|
||||
set_weight_attrs(
|
||||
param,
|
||||
{
|
||||
"weight_loader": _engram_head_shard_weight_loader,
|
||||
"weight_loader": self._weight_loader,
|
||||
"engram_vocab_start": self.vocab_start_idx,
|
||||
},
|
||||
)
|
||||
if cpu_offload:
|
||||
# Constant dummy values avoid randomizing huge CPU lookup tables.
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
|
||||
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
|
||||
logger.info(
|
||||
"Engram table offloaded to pinned host memory: %d rows x %d, "
|
||||
"%.2f GiB per rank",
|
||||
self.part_num_embeddings,
|
||||
dim,
|
||||
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
|
||||
)
|
||||
|
||||
def _get_shard_info(self) -> tuple[int, int]:
|
||||
return self.tp_size, get_tensor_model_parallel_rank()
|
||||
|
||||
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return (
|
||||
torch.empty(self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn),
|
||||
torch.empty(
|
||||
self.part_num_embeddings, self.dim // self.block_size, dtype=torch.uint8
|
||||
),
|
||||
)
|
||||
|
||||
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Parameters when resident, else cached UVA views of the pinned shard.
|
||||
|
||||
Rebuilt if anything swaps `.data`, so a stale device pointer cannot
|
||||
survive silently.
|
||||
"""
|
||||
if not self.cpu_offload:
|
||||
return self.weight.data, self.weight_scale_inv.data
|
||||
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
|
||||
if self._view_src != src:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight.data),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
|
||||
)
|
||||
self._view_src = src
|
||||
assert self._views is not None
|
||||
return self._views
|
||||
return self.weight.data, self.weight_scale_inv.data
|
||||
|
||||
def lookup(
|
||||
self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
|
||||
@@ -753,7 +739,7 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
|
||||
bf16, gathered from all TP shards."""
|
||||
bf16, gathered from all shards for this replica's tokens."""
|
||||
out = torch.empty(
|
||||
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
||||
dtype=torch.bfloat16,
|
||||
@@ -762,8 +748,7 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
self.lookup(indices, out)
|
||||
if self.tp_size > 1:
|
||||
out = tensor_model_parallel_all_gather(out, dim=1)
|
||||
out = out[:, : self.n_hash_cols]
|
||||
return out
|
||||
return out[:, : self.n_hash_cols]
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_kv_tokens"])
|
||||
@@ -859,7 +844,7 @@ def _fused_engram_post_wkv_kernel(
|
||||
|
||||
|
||||
@triton.jit(do_not_specialize=["num_tokens", "token_start", "num_elements"])
|
||||
def _engram_sp_rows_kernel(
|
||||
def _engram_select_rows_kernel(
|
||||
gathered,
|
||||
output,
|
||||
num_tokens,
|
||||
@@ -880,6 +865,32 @@ def _engram_sp_rows_kernel(
|
||||
tl.store(output + offsets, values, offsets < num_elements)
|
||||
|
||||
|
||||
def _engram_select_rows(
|
||||
gathered: torch.Tensor,
|
||||
output: torch.Tensor,
|
||||
source_tokens: int,
|
||||
token_start: int,
|
||||
local_width: int,
|
||||
) -> None:
|
||||
"""Copy one token window out of a rank-major gathered buffer.
|
||||
|
||||
Both gathers land rank-major ([rank][token][local width]); this walks the
|
||||
window the rank keeps and lays its ranks out side by side as width.
|
||||
"""
|
||||
if output.numel() == 0:
|
||||
return
|
||||
_engram_select_rows_kernel[(triton.cdiv(output.numel(), 1024),)](
|
||||
gathered,
|
||||
output,
|
||||
source_tokens,
|
||||
token_start,
|
||||
output.numel(),
|
||||
local_width,
|
||||
output.shape[1] * output.shape[2],
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
|
||||
class Engram(nn.Module):
|
||||
"""Writes an n-gram lookup into the residual stream, gated by how well it
|
||||
matches that stream.
|
||||
@@ -910,13 +921,7 @@ class Engram(nn.Module):
|
||||
# Named ``embed_tokens`` so the checkpoint's ``engram.embed.weight``
|
||||
# survives the mapper's ``embed.weight`` -> ``embed_tokens.weight``
|
||||
# suffix rule.
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
self.embed_tokens = ParallelEngramEmbedding(
|
||||
layout.num_embeddings[layer_hash_index],
|
||||
layout.head_dim,
|
||||
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
||||
cpu_offload=engram_config.cpu_offload if engram_config else True,
|
||||
)
|
||||
self.embed_tokens = self._create_embedding(layout, layer_hash_index)
|
||||
n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads
|
||||
self.wkv = ReplicatedLinear(
|
||||
n_hash_cols * layout.head_dim,
|
||||
@@ -936,40 +941,54 @@ class Engram(nn.Module):
|
||||
)
|
||||
|
||||
max_tokens = get_current_vllm_config().scheduler_config.max_num_batched_tokens
|
||||
# Keep lookup results alive across breakable graph segments.
|
||||
self._init_staging(max_tokens, layout.head_dim)
|
||||
|
||||
def _create_embedding(
|
||||
self, layout: EngramLayout, layer_hash_index: int
|
||||
) -> ParallelEngramEmbedding:
|
||||
return ParallelEngramEmbedding(
|
||||
layout.num_embeddings[layer_hash_index],
|
||||
layout.head_dim,
|
||||
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
||||
)
|
||||
|
||||
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
|
||||
# Persistent storage keeps lookup addresses stable across graph replays.
|
||||
self.staged_rows = torch.empty(
|
||||
max_tokens,
|
||||
self.embed_tokens.part_n_hash_cols,
|
||||
layout.head_dim,
|
||||
head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
|
||||
"""Gather this layer's rows on the main stream before decoder layers."""
|
||||
self.embed_tokens.lookup(hash_ids, self.staged_rows[: hash_ids.shape[0]])
|
||||
"""Look up rows before the decoder layers consume them."""
|
||||
rows = self.staged_rows[: hash_ids.shape[0]]
|
||||
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
|
||||
self.embed_tokens.lookup(hash_ids, rows)
|
||||
|
||||
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
|
||||
return self.staged_rows[:num_tokens]
|
||||
|
||||
def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Gather heads, returning only local tokens when SP is enabled."""
|
||||
rows = self.staged_rows[: hash_ids.shape[0]]
|
||||
rows = self._ready_rows(hash_ids.shape[0])
|
||||
if self.embed_tokens.tp_size == 1:
|
||||
return rows
|
||||
return rows[:, : self.embed_tokens.n_hash_cols]
|
||||
if self.use_sequence_parallel:
|
||||
tp_size = self.embed_tokens.tp_size
|
||||
num_tokens, local_heads, dim = rows.shape
|
||||
gathered = tensor_model_parallel_all_gather(rows, dim=0)
|
||||
chunk = (num_tokens + tp_size - 1) // tp_size
|
||||
rows = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
|
||||
_engram_sp_rows_kernel[(triton.cdiv(rows.numel(), 1024),)](
|
||||
out = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
|
||||
_engram_select_rows(
|
||||
gathered,
|
||||
rows,
|
||||
out,
|
||||
num_tokens,
|
||||
get_tensor_model_parallel_rank() * chunk,
|
||||
rows.numel(),
|
||||
local_heads * dim,
|
||||
self.embed_tokens.n_hash_cols * dim,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
return rows
|
||||
return out
|
||||
rows = tensor_model_parallel_all_gather(rows, dim=1)
|
||||
return rows[:, : self.embed_tokens.n_hash_cols]
|
||||
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
|
||||
|
||||
import mmap
|
||||
import tempfile
|
||||
import weakref
|
||||
from contextlib import ExitStack
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_engram_dp_group,
|
||||
get_engram_dp_size,
|
||||
get_tensor_model_parallel_rank,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v4_1.common.engram import (
|
||||
DEAD_ID,
|
||||
EngramLayout,
|
||||
_engram_head_shard_weight_loader,
|
||||
_engram_select_rows,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.common.engram import (
|
||||
Engram as BaseEngram,
|
||||
)
|
||||
from vllm.models.deepseek_v4_1.common.engram import (
|
||||
ParallelEngramEmbedding as BaseParallelEngramEmbedding,
|
||||
)
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engram_head_shard_rank() -> int:
|
||||
"""This rank's slot among the hash-head shards of one engram table.
|
||||
|
||||
TP-major, so the shards a DP gather brings in are contiguous heads and
|
||||
the following TP gather completes the head order.
|
||||
"""
|
||||
dp_group = get_engram_dp_group()
|
||||
dp_size = dp_group.world_size if dp_group is not None else 1
|
||||
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
|
||||
return get_tensor_model_parallel_rank() * dp_size + dp_rank
|
||||
|
||||
|
||||
def engram_gathered_num_tokens() -> int:
|
||||
"""Per-replica token slot for the node-local Engram DP group."""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
if dp_metadata is None:
|
||||
raise RuntimeError("a DP-shared engram table needs DP token metadata")
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
# Engram groups are contiguous slices of the full DP group.
|
||||
start = get_dp_group().rank_in_group - group.rank_in_group
|
||||
return int(
|
||||
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
|
||||
)
|
||||
|
||||
|
||||
def gather_engram_hashes(
|
||||
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""Collect the n-gram ids of every DP replica sharing one table.
|
||||
|
||||
Replicas are padded to a common token slot, so the gathered shape is
|
||||
static under CUDA graph capture (where DP already pads alike).
|
||||
"""
|
||||
dp_group = get_engram_dp_group()
|
||||
if dp_group is None or dp_shared_memory:
|
||||
return hash_ids
|
||||
slot = engram_gathered_num_tokens()
|
||||
if hash_ids.shape[0] > slot:
|
||||
raise ValueError("Engram token count exceeds the DP token slot")
|
||||
if hash_ids.shape[0] < slot:
|
||||
pad = hash_ids.new_full(
|
||||
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
|
||||
)
|
||||
hash_ids = torch.cat((hash_ids, pad))
|
||||
return dp_group.all_gather(hash_ids, dim=0)
|
||||
|
||||
|
||||
class DPSharedEngramStorage:
|
||||
"""Registered host weights shared by a node-local DP group with one writer."""
|
||||
|
||||
def __init__(
|
||||
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
|
||||
) -> None:
|
||||
self.group = group
|
||||
weight_bytes = num_rows * dim
|
||||
storage = self._allocate(weight_bytes + weight_bytes // block_size)
|
||||
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
|
||||
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
|
||||
def _allocate(self, num_bytes: int) -> torch.Tensor:
|
||||
"""Map and register one physical allocation across a node-local DP group."""
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
check_shm_free_space,
|
||||
)
|
||||
|
||||
group = self.group
|
||||
with ExitStack() as stack:
|
||||
path, error = None, None
|
||||
if group.rank_in_group == 0:
|
||||
try:
|
||||
check_shm_free_space(num_bytes)
|
||||
backing_file = stack.enter_context(
|
||||
tempfile.NamedTemporaryFile(
|
||||
prefix="vllm_engram_", dir="/dev/shm"
|
||||
)
|
||||
)
|
||||
backing_file.truncate(num_bytes)
|
||||
path = backing_file.name
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
path, error = group.broadcast_object((path, error))
|
||||
if error is not None:
|
||||
raise RuntimeError(
|
||||
"Engram shared-memory creation failed on EDP rank 0: " + error
|
||||
)
|
||||
|
||||
mapping = owner = tensor = finalizer = None
|
||||
stage = "open"
|
||||
try:
|
||||
try:
|
||||
with open(path, "r+b") as file:
|
||||
stage = "mmap"
|
||||
mapping = mmap.mmap(
|
||||
file.fileno(), num_bytes, flags=mmap.MAP_SHARED
|
||||
)
|
||||
stage = "cudaHostRegister"
|
||||
owner = np.frombuffer(mapping, dtype=np.uint8)
|
||||
pointer = owner.ctypes.data
|
||||
tensor = torch.from_numpy(owner)
|
||||
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
||||
if result.value != 0:
|
||||
raise RuntimeError(f"cudaHostRegister failed: {result}")
|
||||
finalizer = weakref.finalize(
|
||||
owner, self._unregister, mapping, pointer
|
||||
)
|
||||
finalizer.atexit = False # type: ignore[misc]
|
||||
# The UVA helper otherwise allocates a private pinned copy.
|
||||
if not tensor.is_pinned():
|
||||
raise RuntimeError(
|
||||
"CUDA did not recognize the shared Engram registration"
|
||||
)
|
||||
except Exception as exc:
|
||||
error = f"{stage}: {type(exc).__name__}: {exc}"
|
||||
|
||||
errors: list[str | None] = [None] * group.world_size
|
||||
# Also fences peer mappings before the leader unlinks the file.
|
||||
torch.distributed.all_gather_object(
|
||||
errors, error, group=group.cpu_group
|
||||
)
|
||||
failures = "; ".join(
|
||||
f"EDP rank {rank}: {error}"
|
||||
for rank, error in enumerate(errors)
|
||||
if error is not None
|
||||
)
|
||||
if failures:
|
||||
raise RuntimeError(
|
||||
"Engram shared-memory initialization failed: " + failures
|
||||
)
|
||||
assert tensor is not None
|
||||
return tensor
|
||||
except Exception:
|
||||
if finalizer is not None:
|
||||
finalizer()
|
||||
tensor = owner = None
|
||||
if mapping is not None:
|
||||
mapping.close()
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
|
||||
# Torch storage retains the numpy owner, including through cached UVA views.
|
||||
# Keep its mmap alive until CUDA has released the registration.
|
||||
result = torch.cuda.cudart().cudaHostUnregister(pointer)
|
||||
if result.value != 0:
|
||||
logger.warning("Engram cudaHostUnregister failed: %s", result)
|
||||
|
||||
def load_weight(
|
||||
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
if self.group.rank_in_group == 0:
|
||||
_engram_head_shard_weight_loader(param, loaded_weight)
|
||||
# Read order may differ across ranks. Equal load counts ensure all shared
|
||||
# weights are ready after the last weight-loader call returns.
|
||||
torch.distributed.barrier(group=self.group.cpu_group)
|
||||
|
||||
def get_views(
|
||||
self, weight: torch.Tensor, scales: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if (weight.data_ptr(), scales.data_ptr()) != (
|
||||
self.weight.data_ptr(),
|
||||
self.weight_scale_inv.data_ptr(),
|
||||
):
|
||||
raise RuntimeError("Shared Engram parameter storage must not be replaced")
|
||||
if self._views is None:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
|
||||
)
|
||||
return self._views
|
||||
|
||||
|
||||
class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
|
||||
"""Extend TP lookup with DP head sharding or shared, CPU-offloaded TP slices."""
|
||||
|
||||
_shared_memory: DPSharedEngramStorage | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
dim: int,
|
||||
head_sizes: tuple[int, ...],
|
||||
block_size: int = 32,
|
||||
cpu_offload: bool = False,
|
||||
dp_shared_memory: bool = False,
|
||||
) -> None:
|
||||
self.cpu_offload = cpu_offload
|
||||
self.dp_shared_memory = dp_shared_memory
|
||||
self.dp_size = get_engram_dp_size()
|
||||
if dp_shared_memory:
|
||||
if not cpu_offload:
|
||||
raise ValueError("dp_shared_memory requires cpu_offload=True")
|
||||
if self.dp_size <= 1:
|
||||
raise ValueError(
|
||||
"dp_shared_memory requires a node-local Engram DP "
|
||||
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
|
||||
"Check that the node layout and rank placement allow complete "
|
||||
"DP replicas to be co-located."
|
||||
)
|
||||
self.dp_size = 1
|
||||
if cpu_offload and not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._view_src: tuple[int, int] | None = None
|
||||
super().__init__(num_embeddings, dim, head_sizes, block_size)
|
||||
if cpu_offload:
|
||||
# Constant dummy values avoid randomizing huge CPU lookup tables.
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
|
||||
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
|
||||
logger.info(
|
||||
"Engram table offloaded to pinned host memory: %d rows x %d, "
|
||||
"%.2f GiB %s",
|
||||
self.part_num_embeddings,
|
||||
dim,
|
||||
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
|
||||
"shared across DP replicas" if dp_shared_memory else "per rank",
|
||||
)
|
||||
|
||||
def _get_shard_info(self) -> tuple[int, int]:
|
||||
if self.dp_size == 1:
|
||||
return super()._get_shard_info()
|
||||
return self.tp_size * self.dp_size, engram_head_shard_rank()
|
||||
|
||||
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.dp_shared_memory:
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
storage = DPSharedEngramStorage(
|
||||
self.part_num_embeddings, self.dim, self.block_size, group
|
||||
)
|
||||
self._shared_memory = storage
|
||||
self._weight_loader = storage.load_weight
|
||||
return storage.weight, storage.weight_scale_inv
|
||||
if not self.cpu_offload:
|
||||
return super()._allocate_weights()
|
||||
# Model initialization may be inside a CUDA device context.
|
||||
return (
|
||||
torch.empty(
|
||||
self.part_num_embeddings,
|
||||
self.dim,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
),
|
||||
torch.empty(
|
||||
self.part_num_embeddings,
|
||||
self.dim // self.block_size,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
),
|
||||
)
|
||||
|
||||
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self._shared_memory is not None:
|
||||
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
|
||||
if not self.cpu_offload:
|
||||
return super()._storage()
|
||||
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
|
||||
if self._view_src != src:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight.data),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
|
||||
)
|
||||
self._view_src = src
|
||||
assert self._views is not None
|
||||
return self._views
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
if self.dp_size == 1:
|
||||
return super().forward(indices)
|
||||
num_tokens = indices.shape[0]
|
||||
indices = gather_engram_hashes(indices)
|
||||
out = torch.empty(
|
||||
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=indices.device,
|
||||
)
|
||||
self.lookup(indices, out)
|
||||
out = _gather_engram_rows(out, num_tokens)
|
||||
if self.tp_size > 1:
|
||||
out = tensor_model_parallel_all_gather(out, dim=1)
|
||||
return out[:, : self.n_hash_cols]
|
||||
|
||||
|
||||
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
|
||||
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
|
||||
dp_group = get_engram_dp_group()
|
||||
assert dp_group is not None
|
||||
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
|
||||
assert remainder == 0 and 0 <= num_tokens <= slot
|
||||
gathered = dp_group.all_gather(staged, dim=0)
|
||||
local_heads, dim = staged.shape[1:]
|
||||
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
|
||||
_engram_select_rows(
|
||||
gathered,
|
||||
rows,
|
||||
staged.shape[0],
|
||||
dp_group.rank_in_group * slot,
|
||||
local_heads * dim,
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
class Engram(BaseEngram):
|
||||
"""NVIDIA Engram with asynchronous offload and node-local DP lookup."""
|
||||
|
||||
_prefetch_stream: torch.cuda.Stream | None = None
|
||||
|
||||
def _create_embedding(
|
||||
self, layout: EngramLayout, layer_hash_index: int
|
||||
) -> ParallelEngramEmbedding:
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
assert engram_config is not None
|
||||
return ParallelEngramEmbedding(
|
||||
layout.num_embeddings[layer_hash_index],
|
||||
layout.head_dim,
|
||||
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
||||
cpu_offload=engram_config.cpu_offload,
|
||||
dp_shared_memory=engram_config.dp_shared_memory,
|
||||
)
|
||||
|
||||
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
|
||||
super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim)
|
||||
if self.embed_tokens.cpu_offload:
|
||||
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
|
||||
|
||||
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
|
||||
"""Prefetch local shared rows or the DP group's gathered hash IDs."""
|
||||
if self._prefetch_stream is None:
|
||||
return super().prepare_embeddings(hash_ids)
|
||||
rows = self.staged_rows[: hash_ids.shape[0]]
|
||||
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
|
||||
self._start_prefetch(hash_ids, rows, self._prefetch_stream)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _start_prefetch(
|
||||
self, hash_ids: torch.Tensor, rows: torch.Tensor, stream: torch.cuda.Stream
|
||||
) -> None:
|
||||
# Eager boundaries let the lookup span piecewise graph segments.
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
# Keep temporary hash storage alive until lookup finishes reading it.
|
||||
hash_ids.record_stream(stream)
|
||||
with torch.cuda.stream(stream):
|
||||
self.embed_tokens.lookup(hash_ids, rows, background=True)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _finish_prefetch(self, stream: torch.cuda.Stream) -> None:
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
|
||||
if self._prefetch_stream is not None:
|
||||
self._finish_prefetch(self._prefetch_stream)
|
||||
if self.embed_tokens.dp_size > 1:
|
||||
slot = engram_gathered_num_tokens()
|
||||
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
|
||||
return _gather_engram_rows(staged, num_tokens)
|
||||
return super()._ready_rows(num_tokens)
|
||||
@@ -12,6 +12,7 @@ import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.kernel import MEGA_MOE_BACKENDS
|
||||
from vllm.distributed import (
|
||||
get_engram_dp_size,
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -74,8 +75,9 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.engram import Engram, EngramLayout, NgramHashState
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, gather_engram_hashes
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
@@ -466,6 +468,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
# layer's sliding-window KV cache. Only PP ranks owning an engram
|
||||
# layer need it.
|
||||
self.engram_hash: NgramHashState | None = None
|
||||
self.engram_dp_shared_memory = bool(
|
||||
vllm_config.engram_config and vllm_config.engram_config.dp_shared_memory
|
||||
)
|
||||
self.engram_swa_prefix: str | None = None
|
||||
if self.engram_layout is not None:
|
||||
local_engram = any(
|
||||
@@ -594,12 +599,21 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
swa_metadata.slot_mapping,
|
||||
swa_metadata.block_table,
|
||||
)
|
||||
elif not self.engram_dp_shared_memory and get_engram_dp_size() > 1:
|
||||
# DP-sharded lookups are collective, so a replica skipping the
|
||||
# hash still has to reach them.
|
||||
engram_hashes, engram_mask = self.engram_hash.dummy_hashes(input_ids)
|
||||
if engram_hashes is not None:
|
||||
# Gather all Engram rows before entering the decoder layers.
|
||||
# One gather feeds every layer sharing the DP-split table.
|
||||
gathered_hashes = gather_engram_hashes(
|
||||
engram_hashes, dp_shared_memory=self.engram_dp_shared_memory
|
||||
)
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
engram = getattr(layer, "engram", None)
|
||||
if engram is not None:
|
||||
engram.prepare_embeddings(
|
||||
engram_hashes[:, engram.layer_hash_index]
|
||||
gathered_hashes[:, engram.layer_hash_index]
|
||||
)
|
||||
|
||||
full_num_tokens = positions.shape[0]
|
||||
|
||||
Reference in New Issue
Block a user