[Perf][DSv4.1] Faster Engram host lookups: sorted rows, inline big lookups (#59327)
Signed-off-by: Juntian Liu <Juntianl777@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4ed438b79c
commit
4c2d277643
@@ -39,10 +39,10 @@ from vllm.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.models.deepseek_v41.common.engram import EngramLayout
|
||||
from vllm.models.deepseek_v41.nvidia import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.nvidia.engram import (
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
EngramLayout,
|
||||
ParallelEngramEmbedding,
|
||||
engram_head_shard_rank,
|
||||
gather_engram_hashes,
|
||||
@@ -686,11 +686,30 @@ def test_engram_dp_shared_memory_runtime_requirements(
|
||||
)
|
||||
|
||||
|
||||
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch):
|
||||
"""A /dev/shm smaller than the tables must fall back instead of failing startup."""
|
||||
@pytest.mark.parametrize("query_fails", [False, True])
|
||||
def test_engram_tables_too_large_for_shm_are_not_shared(monkeypatch, query_fails):
|
||||
"""A /dev/shm that is too small or cannot be queried must fall back, and the
|
||||
leader must still broadcast that decision so its peers do not hang."""
|
||||
broadcasts = []
|
||||
|
||||
def broadcast(obj):
|
||||
broadcasts.append(obj)
|
||||
return obj
|
||||
|
||||
def query(*args, **kwargs):
|
||||
raise OSError("statvfs failed")
|
||||
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: 2)
|
||||
monkeypatch.setattr(
|
||||
engram_ops,
|
||||
"get_engram_dp_group",
|
||||
lambda: SimpleNamespace(rank_in_group=0, broadcast_object=broadcast),
|
||||
)
|
||||
if query_fails:
|
||||
monkeypatch.setattr(engram_ops, "check_shm_free_space", query)
|
||||
layout = SimpleNamespace(num_embeddings=(1 << 50,), head_dim=DIM)
|
||||
assert not engram_ops.can_share_engram_tables(layout)
|
||||
assert len(broadcasts) == 1 and broadcasts[0] is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -9,13 +9,10 @@ import torch
|
||||
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram as CommonEngram,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
NgramHashState,
|
||||
ParallelEngramEmbedding,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops
|
||||
from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -111,7 +108,7 @@ def test_fused_engram_post_wkv_matches_reference(
|
||||
)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
|
||||
module = CommonEngram.__new__(CommonEngram)
|
||||
module = Engram.__new__(Engram)
|
||||
torch.nn.Module.__init__(module)
|
||||
module.dim = dim
|
||||
module.hc_mult = hc_mult
|
||||
@@ -121,6 +118,7 @@ def test_fused_engram_post_wkv_matches_reference(
|
||||
module.embed_tokens = torch.nn.Identity()
|
||||
# `forward` reads rows staged by `prepare_embeddings`, so inject kv there.
|
||||
module.embed_tokens.tp_size = module.embed_tokens.n_hash_cols = 1
|
||||
module.embed_tokens.dp_size = 1
|
||||
module.staged_rows = kv.unsqueeze(1)
|
||||
if use_sequence_parallel:
|
||||
padded = torch.nn.functional.pad(kv, (0, 0, 0, (-num_kv_tokens) % tp_size))
|
||||
@@ -592,9 +590,8 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
monkeypatch.setattr(nvidia_engram_ops, "get_engram_dp_size", lambda: dp_size)
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(nvidia_engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
with pytest.raises(AssertionError, match="ranks without hash heads"):
|
||||
ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads)
|
||||
|
||||
@@ -602,10 +599,19 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="CUDA or ROCm required"
|
||||
)
|
||||
@pytest.mark.parametrize("cpu_offload", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"cpu_offload,sort_rows", [(False, False), (True, False), (True, True)]
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
|
||||
def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch):
|
||||
"""Keep complete buckets and reconstruct head order, including TP padding."""
|
||||
def test_engram_head_shards_reconstruct_checkpoint(
|
||||
cpu_offload, sort_rows, tp_size, monkeypatch
|
||||
):
|
||||
"""Keep complete buckets and reconstruct head order, including TP padding,
|
||||
whether or not the host lookup sorts its rows by table offset."""
|
||||
if sort_rows:
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "_engram_lookup_thresholds", lambda _: (0, None)
|
||||
)
|
||||
head_sizes = (17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73)
|
||||
num_rows, dim = sum(head_sizes), 64
|
||||
torch.manual_seed(0)
|
||||
@@ -796,10 +802,8 @@ def test_engram_shared_prefetch_stream_waits_per_layer():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)]
|
||||
)
|
||||
def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
|
||||
@pytest.mark.parametrize("cpu_offload", [None, False])
|
||||
def test_engram_constructor_honors_offload(monkeypatch, cpu_offload):
|
||||
"""Default offload uses pinned storage and the given stream; False uses HBM."""
|
||||
from vllm.config import EngramConfig
|
||||
|
||||
@@ -817,23 +821,18 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
|
||||
else EngramConfig(cpu_offload=cpu_offload),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
)
|
||||
offloaded = backend == "nvidia" and vllm_config.engram_config.cpu_offload
|
||||
if backend == "common":
|
||||
vllm_config.engram_config = None
|
||||
offloaded = vllm_config.engram_config.cpu_offload
|
||||
monkeypatch.setattr(engram_ops, "get_current_vllm_config", lambda: vllm_config)
|
||||
monkeypatch.setattr(
|
||||
nvidia_engram_ops, "get_current_vllm_config", lambda: vllm_config
|
||||
)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity()
|
||||
)
|
||||
stream = torch.cuda.Stream()
|
||||
kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {}
|
||||
with torch.device("cuda"):
|
||||
cls = Engram if backend == "nvidia" else CommonEngram
|
||||
module = cls(config, None, layout, 0, False, "engram", **kwargs)
|
||||
module = Engram(
|
||||
config, None, layout, 0, False, "engram", prefetch_stream=stream
|
||||
)
|
||||
layer = module.embed_tokens
|
||||
if offloaded:
|
||||
assert layer.weight.device.type == "cpu" and layer.weight.is_pinned()
|
||||
@@ -897,7 +896,8 @@ def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_depende
|
||||
"The allocator reuse negative control requires the native allocator"
|
||||
)
|
||||
|
||||
def start(self, hash_ids, rows, stream):
|
||||
def start(self, hash_ids, rows):
|
||||
stream = self._prefetch_stream
|
||||
if missing_dependency != "producer_wait":
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
if missing_dependency != "record_stream":
|
||||
|
||||
@@ -65,13 +65,9 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.engram import Engram, EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
|
||||
# Engram host offload and its prefetch stream are neither ROCm- nor
|
||||
# NVIDIA-specific, so they are imported rather than duplicated.
|
||||
from ..nvidia.engram import Engram
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
|
||||
|
||||
|
||||
@@ -34,23 +34,41 @@ chunk-by-chunk while an n-gram at position ``p`` needs the token ids at
|
||||
cache for the rest.
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import mmap
|
||||
import os
|
||||
import tempfile
|
||||
import weakref
|
||||
from contextlib import ExitStack
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import VllmConfig, get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_engram_dp_group,
|
||||
get_engram_dp_size,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
SHM_PATH,
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -58,6 +76,18 @@ logger = init_logger(__name__)
|
||||
DEAD_ID = -1
|
||||
|
||||
|
||||
def _engram_lookup_thresholds(device: torch.device) -> tuple[int | None, int | None]:
|
||||
"""Host lookup thresholds on `device`: the rows from which to sort, and the
|
||||
tokens (max over DP ranks) from which to run inline. None disables that path."""
|
||||
if current_platform.is_cuda() and current_platform.is_device_capability_family(
|
||||
100, device.index or 0
|
||||
):
|
||||
# Measured on GB200.
|
||||
return 36864, 4096
|
||||
# TODO: Add hand-tuned thresholds for other GPU families.
|
||||
return None, None
|
||||
|
||||
|
||||
def _is_prime(n: int) -> bool:
|
||||
"""Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import)."""
|
||||
if n < 2:
|
||||
@@ -583,12 +613,14 @@ def _engram_head_shard_weight_loader(
|
||||
"ids_stride_t",
|
||||
"ids_stride_h",
|
||||
"GRID",
|
||||
"SORTED",
|
||||
]
|
||||
)
|
||||
def _engram_lookup_kernel(
|
||||
weight,
|
||||
scales,
|
||||
ids,
|
||||
sorted_dst,
|
||||
out,
|
||||
vocab_start,
|
||||
vocab_end,
|
||||
@@ -602,27 +634,37 @@ def _engram_lookup_kernel(
|
||||
QUANT_BLOCK: tl.constexpr,
|
||||
BLOCK_R: tl.constexpr,
|
||||
GRID,
|
||||
SORTED,
|
||||
):
|
||||
"""Gather fp8 rows, apply their ue8m0 block scales, write bf16.
|
||||
|
||||
Only this rank's heads are read; padded heads write zeros for all-gather.
|
||||
`weight`/`scales` may address pinned host memory through UVA.
|
||||
`weight`/`scales` may address pinned host memory through UVA. If SORTED,
|
||||
`ids` holds rows relative to vocab_start in table order and `sorted_dst`
|
||||
their output rows.
|
||||
"""
|
||||
cols = tl.arange(0, DIM)
|
||||
scale_cols = cols // QUANT_BLOCK
|
||||
for base in tl.range(tl.program_id(0) * BLOCK_R, num_rows, GRID * BLOCK_R):
|
||||
rows = base + tl.arange(0, BLOCK_R)
|
||||
valid = rows < num_rows
|
||||
head = HEAD_START + rows % LOCAL_HEADS
|
||||
token = (rows // LOCAL_HEADS).to(tl.int64)
|
||||
index = tl.load(
|
||||
ids + token * ids_stride_t + head * ids_stride_h,
|
||||
mask=valid & (head < TOTAL_HEADS),
|
||||
other=-1,
|
||||
).to(tl.int64)
|
||||
owned = valid & (head < TOTAL_HEADS)
|
||||
owned &= (index >= vocab_start) & (index < vocab_end)
|
||||
local = tl.where(owned, index - vocab_start, 0)
|
||||
if SORTED:
|
||||
local = tl.load(ids + rows, mask=valid, other=-1).to(tl.int64)
|
||||
dst = tl.load(sorted_dst + rows, mask=valid, other=0).to(tl.int32)
|
||||
owned = (local >= 0) & (local < vocab_end - vocab_start)
|
||||
else:
|
||||
head = HEAD_START + rows % LOCAL_HEADS
|
||||
token = (rows // LOCAL_HEADS).to(tl.int64)
|
||||
index = tl.load(
|
||||
ids + token * ids_stride_t + head * ids_stride_h,
|
||||
mask=valid & (head < TOTAL_HEADS),
|
||||
other=-1,
|
||||
).to(tl.int64)
|
||||
dst = rows
|
||||
owned = valid & (head < TOTAL_HEADS)
|
||||
owned &= (index >= vocab_start) & (index < vocab_end)
|
||||
local = index - vocab_start
|
||||
local = tl.where(owned, local, 0)
|
||||
values = tl.load(
|
||||
weight + local[:, None] * DIM + cols[None, :],
|
||||
mask=owned[:, None],
|
||||
@@ -636,16 +678,240 @@ def _engram_lookup_kernel(
|
||||
# ue8m0 is a power of two, so its byte *is* the fp32 exponent field.
|
||||
scale = (scale.to(tl.int32) << 23).to(tl.float32, bitcast=True)
|
||||
tl.store(
|
||||
out + rows[:, None] * DIM + cols[None, :],
|
||||
out + dst[:, None] * DIM + cols[None, :],
|
||||
(values.to(tl.float32) * scale).to(tl.bfloat16),
|
||||
mask=valid[:, None],
|
||||
)
|
||||
|
||||
|
||||
def engram_head_shard_rank() -> int:
|
||||
"""This rank's slot among the hash-head shards of one engram table.
|
||||
|
||||
TP-major, so the shards a DP gather brings in are contiguous heads and
|
||||
the following TP gather completes the head order.
|
||||
"""
|
||||
dp_group = get_engram_dp_group()
|
||||
dp_size = dp_group.world_size if dp_group is not None else 1
|
||||
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
|
||||
return get_tensor_model_parallel_rank() * dp_size + dp_rank
|
||||
|
||||
|
||||
def engram_gathered_num_tokens() -> int:
|
||||
"""Per-replica token slot for the node-local Engram DP group."""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
if dp_metadata is None:
|
||||
raise RuntimeError("a DP-shared engram table needs DP token metadata")
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
# Engram groups are contiguous slices of the full DP group.
|
||||
start = get_dp_group().rank_in_group - group.rank_in_group
|
||||
return int(
|
||||
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
|
||||
)
|
||||
|
||||
|
||||
def gather_engram_hashes(
|
||||
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""Collect the n-gram ids of every DP replica sharing one table.
|
||||
|
||||
Replicas are padded to a common token slot, so the gathered shape is
|
||||
static under CUDA graph capture (where DP already pads alike).
|
||||
"""
|
||||
dp_group = get_engram_dp_group()
|
||||
if dp_group is None or dp_shared_memory:
|
||||
return hash_ids
|
||||
slot = engram_gathered_num_tokens()
|
||||
if hash_ids.shape[0] > slot:
|
||||
raise ValueError("Engram token count exceeds the DP token slot")
|
||||
if hash_ids.shape[0] < slot:
|
||||
pad = hash_ids.new_full(
|
||||
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
|
||||
)
|
||||
hash_ids = torch.cat((hash_ids, pad))
|
||||
return dp_group.all_gather(hash_ids, dim=0)
|
||||
|
||||
|
||||
class DPSharedEngramStorage:
|
||||
"""Registered host weights shared by a node-local DP group with one writer."""
|
||||
|
||||
def __init__(
|
||||
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
|
||||
) -> None:
|
||||
self.group = group
|
||||
weight_bytes = num_rows * dim
|
||||
storage = self._allocate(weight_bytes + weight_bytes // block_size)
|
||||
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
|
||||
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
|
||||
def _allocate(self, num_bytes: int) -> torch.Tensor:
|
||||
"""Map and register one physical allocation across a node-local DP group."""
|
||||
group = self.group
|
||||
with ExitStack() as stack:
|
||||
path = error = None
|
||||
if group.rank_in_group == 0:
|
||||
try:
|
||||
check_shm_free_space(num_bytes)
|
||||
backing_file = stack.enter_context(
|
||||
tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
|
||||
)
|
||||
backing_file.truncate(num_bytes)
|
||||
path = backing_file.name
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
path, error = group.broadcast_object((path, error))
|
||||
if error is not None:
|
||||
raise RuntimeError(
|
||||
"Engram shared-memory creation failed on EDP rank 0: " + error
|
||||
)
|
||||
|
||||
mapping = owner = tensor = None
|
||||
try:
|
||||
with open(path, "r+b") as file:
|
||||
mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
|
||||
owner = np.frombuffer(mapping, dtype=np.uint8)
|
||||
pointer = owner.ctypes.data
|
||||
tensor = torch.from_numpy(owner)
|
||||
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
||||
if result.value != 0:
|
||||
raise RuntimeError(f"cudaHostRegister failed: {result}")
|
||||
finalizer = weakref.finalize(owner, self._unregister, mapping, pointer)
|
||||
finalizer.atexit = False # type: ignore[misc]
|
||||
# The UVA helper otherwise allocates a private pinned copy.
|
||||
if not tensor.is_pinned():
|
||||
raise RuntimeError(
|
||||
"cudaHostRegister did not pin the shared Engram mapping"
|
||||
)
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
# Also fences peer mappings before the leader unlinks the file.
|
||||
errors: list[str | None] = [None] * group.world_size
|
||||
torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
|
||||
failures = "; ".join(
|
||||
f"EDP rank {rank}: {error}"
|
||||
for rank, error in enumerate(errors)
|
||||
if error is not None
|
||||
)
|
||||
if failures:
|
||||
# Dropping the owner runs the finalizer; the mapping then unmaps.
|
||||
del tensor, owner, mapping
|
||||
raise RuntimeError(
|
||||
"Engram shared-memory initialization failed: " + failures
|
||||
)
|
||||
assert tensor is not None
|
||||
return tensor
|
||||
|
||||
@staticmethod
|
||||
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
|
||||
# Torch storage retains the numpy owner, including through cached UVA views.
|
||||
# Keep its mmap alive until CUDA has released the registration.
|
||||
result = torch.cuda.cudart().cudaHostUnregister(pointer)
|
||||
if result.value != 0:
|
||||
logger.warning("Engram cudaHostUnregister failed: %s", result)
|
||||
|
||||
def load_weight(
|
||||
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
if self.group.rank_in_group == 0:
|
||||
_engram_head_shard_weight_loader(param, loaded_weight)
|
||||
# Read order may differ across ranks. Equal load counts ensure all shared
|
||||
# weights are ready after the last weight-loader call returns.
|
||||
torch.distributed.barrier(group=self.group.cpu_group)
|
||||
|
||||
def get_views(
|
||||
self, weight: torch.Tensor, scales: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if (weight.data_ptr(), scales.data_ptr()) != (
|
||||
self.weight.data_ptr(),
|
||||
self.weight_scale_inv.data_ptr(),
|
||||
):
|
||||
raise RuntimeError("Shared Engram parameter storage must not be replaced")
|
||||
if self._views is None:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
|
||||
)
|
||||
return self._views
|
||||
|
||||
|
||||
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
|
||||
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
|
||||
if get_engram_dp_size() == 1:
|
||||
logger.warning_once(
|
||||
"Engram DP replicas are not co-located on one node; "
|
||||
"storing the offloaded tables per rank instead of sharing them."
|
||||
)
|
||||
return False
|
||||
num_bytes = sum(layout.num_embeddings) * (
|
||||
layout.head_dim + layout.head_dim // block_size
|
||||
)
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
# Only the leader allocates, so every replica follows its check.
|
||||
error = None
|
||||
if group.rank_in_group == 0:
|
||||
if not os.path.isdir(SHM_PATH):
|
||||
error = f"{SHM_PATH} is not mounted"
|
||||
else:
|
||||
try:
|
||||
check_shm_free_space(num_bytes, allocation_name="Engram tables")
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
error = group.broadcast_object(error)
|
||||
if error is not None:
|
||||
logger.warning_once(
|
||||
"Sharding the offloaded Engram tables across DP replicas: %s", error
|
||||
)
|
||||
return error is None
|
||||
|
||||
|
||||
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
|
||||
"""Register prefaulted huge pages, or return None for pinned-memory fallback."""
|
||||
try:
|
||||
mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
|
||||
mapping.madvise(mmap.MADV_HUGEPAGE)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Engram huge-page allocation failed (%s); using pinned memory.", exc
|
||||
)
|
||||
return None
|
||||
|
||||
owner = np.frombuffer(mapping, dtype=np.uint8)
|
||||
# Fault the pages in before CUDA pins them, so they can be huge.
|
||||
owner[:: mmap.PAGESIZE] = 0
|
||||
tensor = torch.from_numpy(owner)
|
||||
pointer = tensor.data_ptr()
|
||||
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"Engram cudaHostRegister failed (%s); using pinned memory.", result
|
||||
)
|
||||
return None
|
||||
# Tensor views retain owner; its finalizer retains mapping until unregister.
|
||||
finalizer = weakref.finalize(
|
||||
owner, DPSharedEngramStorage._unregister, mapping, pointer
|
||||
)
|
||||
finalizer.atexit = False # type: ignore[misc]
|
||||
if not tensor.is_pinned():
|
||||
logger.warning(
|
||||
"CUDA did not recognize Engram registration; using pinned memory."
|
||||
)
|
||||
return None
|
||||
return tensor
|
||||
|
||||
|
||||
class ParallelEngramEmbedding(nn.Module):
|
||||
"""TP-sharded hash heads with FP8 rows and per-block E8M0 scales."""
|
||||
"""Hash heads with FP8 rows and per-block E8M0 scales.
|
||||
|
||||
Heads are TP-sharded, and additionally DP-sharded across a node-local
|
||||
group unless the table lives in (DP-shared) pinned host memory.
|
||||
"""
|
||||
|
||||
_weight_loader = staticmethod(_engram_head_shard_weight_loader)
|
||||
_shared_memory: DPSharedEngramStorage | None = None
|
||||
_packed: torch.Tensor | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -653,10 +919,32 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
dim: int,
|
||||
head_sizes: tuple[int, ...],
|
||||
block_size: int = 32,
|
||||
cpu_offload: bool = False,
|
||||
dp_shared_memory: bool = False,
|
||||
use_thp: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
assert head_sizes and all(size > 0 for size in head_sizes)
|
||||
assert sum(head_sizes) <= num_embeddings
|
||||
self.cpu_offload = cpu_offload
|
||||
self.dp_shared_memory = dp_shared_memory
|
||||
self.use_thp = use_thp
|
||||
self.dp_size = get_engram_dp_size()
|
||||
if dp_shared_memory:
|
||||
if not cpu_offload:
|
||||
raise ValueError("dp_shared_memory requires cpu_offload=True")
|
||||
if self.dp_size <= 1:
|
||||
raise ValueError(
|
||||
"dp_shared_memory requires a node-local Engram DP "
|
||||
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
|
||||
"Check that the node layout and rank placement allow complete "
|
||||
"DP replicas to be co-located."
|
||||
)
|
||||
self.dp_size = 1
|
||||
if cpu_offload and not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._view_src: tuple[int, int] | None = None
|
||||
self.num_embeddings = num_embeddings
|
||||
self.dim = dim
|
||||
self.block_size = block_size
|
||||
@@ -688,20 +976,99 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
"engram_vocab_start": self.vocab_start_idx,
|
||||
},
|
||||
)
|
||||
if cpu_offload:
|
||||
# Constant dummy values avoid randomizing huge CPU lookup tables.
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
|
||||
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
|
||||
logger.info(
|
||||
"Engram table offloaded to pinned host memory: %d rows x %d, "
|
||||
"%.2f GiB %s",
|
||||
self.part_num_embeddings,
|
||||
dim,
|
||||
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
|
||||
"shared across DP replicas" if dp_shared_memory else "per rank",
|
||||
)
|
||||
|
||||
def _get_shard_info(self) -> tuple[int, int]:
|
||||
return self.tp_size, get_tensor_model_parallel_rank()
|
||||
if self.dp_size == 1:
|
||||
return self.tp_size, get_tensor_model_parallel_rank()
|
||||
return self.tp_size * self.dp_size, engram_head_shard_rank()
|
||||
|
||||
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.dp_shared_memory:
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
storage = DPSharedEngramStorage(
|
||||
self.part_num_embeddings, self.dim, self.block_size, group
|
||||
)
|
||||
self._shared_memory = storage
|
||||
self._weight_loader = storage.load_weight
|
||||
return storage.weight, storage.weight_scale_inv
|
||||
scale_dim = self.dim // self.block_size
|
||||
if not self.cpu_offload:
|
||||
return (
|
||||
torch.empty(
|
||||
self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn
|
||||
),
|
||||
torch.empty(self.part_num_embeddings, scale_dim, dtype=torch.uint8),
|
||||
)
|
||||
if self.use_thp:
|
||||
weight_bytes = self.part_num_embeddings * self.dim
|
||||
packed = _allocate_huge_page_storage(
|
||||
weight_bytes + weight_bytes // self.block_size
|
||||
)
|
||||
if packed is not None:
|
||||
self._packed = packed
|
||||
return (
|
||||
packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim),
|
||||
packed[weight_bytes:].view(-1, scale_dim),
|
||||
)
|
||||
# Model initialization may be inside a CUDA device context.
|
||||
return (
|
||||
torch.empty(self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn),
|
||||
torch.empty(
|
||||
self.part_num_embeddings, self.dim // self.block_size, dtype=torch.uint8
|
||||
self.part_num_embeddings,
|
||||
self.dim,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
),
|
||||
torch.empty(
|
||||
self.part_num_embeddings,
|
||||
scale_dim,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
),
|
||||
)
|
||||
|
||||
def collapse_huge_pages(self) -> None:
|
||||
"""Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
|
||||
if self._packed is None:
|
||||
return
|
||||
addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
|
||||
if libc.madvise(addr, num_bytes, 25) != 0:
|
||||
logger.warning(
|
||||
"Engram MADV_COLLAPSE failed; keeping existing pages: %s",
|
||||
os.strerror(ctypes.get_errno()),
|
||||
)
|
||||
|
||||
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self.weight.data, self.weight_scale_inv.data
|
||||
if self._shared_memory is not None:
|
||||
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
|
||||
if not self.cpu_offload:
|
||||
return self.weight.data, self.weight_scale_inv.data
|
||||
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
|
||||
if self._view_src != src:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight.data),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
|
||||
)
|
||||
self._view_src = src
|
||||
assert self._views is not None
|
||||
return self._views
|
||||
|
||||
def lookup(
|
||||
self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
|
||||
@@ -714,6 +1081,24 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
if not rows:
|
||||
return
|
||||
weight, scales = self._storage()
|
||||
ids_stride_t, ids_stride_h = indices.stride()
|
||||
sort_min_rows, _ = _engram_lookup_thresholds(indices.device)
|
||||
sort = (
|
||||
self.cpu_offload
|
||||
and self._packed is None
|
||||
and sort_min_rows is not None
|
||||
and rows >= sort_min_rows
|
||||
)
|
||||
dst = indices
|
||||
if sort:
|
||||
# Small-page host tables miss the TLB per row; in table order, rows on
|
||||
# one page share it.
|
||||
head_end = self.head_start + self.part_n_hash_cols
|
||||
local = indices[:, self.head_start : head_end] - self.vocab_start_idx
|
||||
if (pad := self.part_n_hash_cols - local.shape[1]) > 0:
|
||||
local = nn.functional.pad(local, (0, pad), value=-1)
|
||||
indices, dst = local.flatten().sort()
|
||||
dst = dst.int()
|
||||
# The table dwarfs TLB reach, so a persistent grid near the SM count
|
||||
# beats one program per row; halve it to leave SMs for the main stream.
|
||||
tiles = triton.cdiv(rows, 16)
|
||||
@@ -722,12 +1107,13 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
weight,
|
||||
scales,
|
||||
indices,
|
||||
dst,
|
||||
out,
|
||||
self.vocab_start_idx,
|
||||
self.vocab_end_idx,
|
||||
rows,
|
||||
indices.stride(0),
|
||||
indices.stride(1),
|
||||
ids_stride_t,
|
||||
ids_stride_h,
|
||||
HEAD_START=self.head_start,
|
||||
LOCAL_HEADS=self.part_n_hash_cols,
|
||||
TOTAL_HEADS=self.n_hash_cols,
|
||||
@@ -735,17 +1121,23 @@ class ParallelEngramEmbedding(nn.Module):
|
||||
QUANT_BLOCK=self.block_size,
|
||||
BLOCK_R=16,
|
||||
GRID=grid,
|
||||
SORTED=sort,
|
||||
)
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
|
||||
bf16, gathered from all shards for this replica's tokens."""
|
||||
num_tokens = indices.shape[0]
|
||||
if self.dp_size > 1:
|
||||
indices = gather_engram_hashes(indices)
|
||||
out = torch.empty(
|
||||
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=indices.device,
|
||||
)
|
||||
self.lookup(indices, out)
|
||||
if self.dp_size > 1:
|
||||
out = _gather_engram_rows(out, num_tokens)
|
||||
if self.tp_size > 1:
|
||||
out = tensor_model_parallel_all_gather(out, dim=1)
|
||||
return out[:, : self.n_hash_cols]
|
||||
@@ -891,6 +1283,25 @@ def _engram_select_rows(
|
||||
)
|
||||
|
||||
|
||||
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
|
||||
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
|
||||
dp_group = get_engram_dp_group()
|
||||
assert dp_group is not None
|
||||
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
|
||||
assert remainder == 0 and 0 <= num_tokens <= slot
|
||||
gathered = dp_group.all_gather(staged, dim=0)
|
||||
local_heads, dim = staged.shape[1:]
|
||||
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
|
||||
_engram_select_rows(
|
||||
gathered,
|
||||
rows,
|
||||
staged.shape[0],
|
||||
dp_group.rank_in_group * slot,
|
||||
local_heads * dim,
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
class Engram(nn.Module):
|
||||
"""Writes an n-gram lookup into the residual stream, gated by how well it
|
||||
matches that stream.
|
||||
@@ -901,6 +1312,9 @@ class Engram(nn.Module):
|
||||
training kernel).
|
||||
"""
|
||||
|
||||
_prefetch_stream: torch.cuda.Stream | None = None
|
||||
_prefetch_done: torch.cuda.Event | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
@@ -909,8 +1323,12 @@ class Engram(nn.Module):
|
||||
layer_hash_index: int,
|
||||
use_sequence_parallel: bool,
|
||||
prefix: str,
|
||||
prefetch_stream: torch.cuda.Stream | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Layers sharing one stream serialize their offloaded lookups, so they
|
||||
# take turns instead of jointly starving decoder compute of SMs.
|
||||
self._prefetch_stream = prefetch_stream
|
||||
self.layer_hash_index = layer_hash_index
|
||||
self.dim = config.hidden_size
|
||||
self.hc_mult = config.hc_mult
|
||||
@@ -950,28 +1368,78 @@ class Engram(nn.Module):
|
||||
def _create_embedding(
|
||||
self, layout: EngramLayout, layer_hash_index: int
|
||||
) -> ParallelEngramEmbedding:
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
assert engram_config is not None
|
||||
return ParallelEngramEmbedding(
|
||||
layout.num_embeddings[layer_hash_index],
|
||||
layout.head_dim,
|
||||
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
||||
cpu_offload=engram_config.cpu_offload,
|
||||
dp_shared_memory=bool(engram_config.dp_shared_memory),
|
||||
use_thp=engram_config.use_thp,
|
||||
)
|
||||
|
||||
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
|
||||
# Persistent storage keeps lookup addresses stable across graph replays.
|
||||
self.staged_rows = torch.empty(
|
||||
max_tokens,
|
||||
max_tokens * self.embed_tokens.dp_size,
|
||||
self.embed_tokens.part_n_hash_cols,
|
||||
head_dim,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if not self.embed_tokens.cpu_offload:
|
||||
self._prefetch_stream = None
|
||||
return
|
||||
if self._prefetch_stream is None:
|
||||
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
|
||||
self._prefetch_done = torch.cuda.Event()
|
||||
|
||||
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
|
||||
"""Look up rows before the decoder layers consume them."""
|
||||
"""Look up, or prefetch from host memory, rows before the decoder layers
|
||||
consume them; DP-sharded tables take the group's gathered hash IDs."""
|
||||
rows = self.staged_rows[: hash_ids.shape[0]]
|
||||
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
|
||||
self.embed_tokens.lookup(hash_ids, rows)
|
||||
if self._prefetch_stream is None:
|
||||
self.embed_tokens.lookup(hash_ids, rows)
|
||||
else:
|
||||
self._start_prefetch(hash_ids, rows)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _start_prefetch(self, hash_ids: torch.Tensor, rows: torch.Tensor) -> None:
|
||||
# Eager boundaries let the lookup span piecewise graph segments, and
|
||||
# decide per replay (on the replay stream) whether it runs inline.
|
||||
num_tokens = hash_ids.shape[0]
|
||||
# All DP ranks must agree, or EP collectives wait on the slowest one.
|
||||
if is_forward_context_available() and (dp := get_forward_context().dp_metadata):
|
||||
num_tokens = int(dp.num_tokens_across_dp_cpu.max())
|
||||
current = stream = torch.cuda.current_stream()
|
||||
# Big lookups stall persistent main-stream kernels anyway: run them inline.
|
||||
_, inline_min_tokens = _engram_lookup_thresholds(hash_ids.device)
|
||||
if inline_min_tokens is not None and num_tokens >= inline_min_tokens:
|
||||
self.embed_tokens.lookup(hash_ids, rows)
|
||||
else:
|
||||
stream = self._prefetch_stream
|
||||
assert stream is not None
|
||||
stream.wait_stream(current)
|
||||
# Keep temporary hash storage alive until lookup finishes reading it.
|
||||
hash_ids.record_stream(stream)
|
||||
with torch.cuda.stream(stream):
|
||||
self.embed_tokens.lookup(hash_ids, rows, background=True)
|
||||
assert self._prefetch_done is not None
|
||||
self._prefetch_done.record(stream)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _finish_prefetch(self, event: torch.cuda.Event) -> None:
|
||||
# Wait for this layer's lookup only, not for later layers on the stream.
|
||||
torch.cuda.current_stream().wait_event(event)
|
||||
|
||||
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
|
||||
if self._prefetch_done is not None:
|
||||
self._finish_prefetch(self._prefetch_done)
|
||||
if self.embed_tokens.dp_size > 1:
|
||||
slot = engram_gathered_num_tokens()
|
||||
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
|
||||
return _gather_engram_rows(staged, num_tokens)
|
||||
return self.staged_rows[:num_tokens]
|
||||
|
||||
def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -1,497 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NVIDIA Engram DP sharding, shared host storage, and asynchronous prefetch."""
|
||||
|
||||
import ctypes
|
||||
import mmap
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import weakref
|
||||
from contextlib import ExitStack
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_dp_group,
|
||||
get_engram_dp_group,
|
||||
get_engram_dp_size,
|
||||
get_tensor_model_parallel_rank,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
SHM_PATH,
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.distributed.parallel_state import GroupCoordinator
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
DEAD_ID,
|
||||
EngramLayout,
|
||||
_engram_head_shard_weight_loader,
|
||||
_engram_select_rows,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram as BaseEngram,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
ParallelEngramEmbedding as BaseParallelEngramEmbedding,
|
||||
)
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
|
||||
"""Register prefaulted huge pages, or return None for pinned-memory fallback."""
|
||||
try:
|
||||
mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
|
||||
mapping.madvise(mmap.MADV_HUGEPAGE)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Engram huge-page allocation failed (%s); using pinned memory.", exc
|
||||
)
|
||||
return None
|
||||
|
||||
owner = np.frombuffer(mapping, dtype=np.uint8)
|
||||
# Fault the pages in before CUDA pins them, so they can be huge.
|
||||
owner[:: mmap.PAGESIZE] = 0
|
||||
tensor = torch.from_numpy(owner)
|
||||
pointer = tensor.data_ptr()
|
||||
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"Engram cudaHostRegister failed (%s); using pinned memory.", result
|
||||
)
|
||||
return None
|
||||
# Tensor views retain owner; its finalizer retains mapping until unregister.
|
||||
finalizer = weakref.finalize(
|
||||
owner, DPSharedEngramStorage._unregister, mapping, pointer
|
||||
)
|
||||
finalizer.atexit = False # type: ignore[misc]
|
||||
if not tensor.is_pinned():
|
||||
logger.warning(
|
||||
"CUDA did not recognize Engram registration; using pinned memory."
|
||||
)
|
||||
return None
|
||||
return tensor
|
||||
|
||||
|
||||
def engram_head_shard_rank() -> int:
|
||||
"""This rank's slot among the hash-head shards of one engram table.
|
||||
|
||||
TP-major, so the shards a DP gather brings in are contiguous heads and
|
||||
the following TP gather completes the head order.
|
||||
"""
|
||||
dp_group = get_engram_dp_group()
|
||||
dp_size = dp_group.world_size if dp_group is not None else 1
|
||||
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
|
||||
return get_tensor_model_parallel_rank() * dp_size + dp_rank
|
||||
|
||||
|
||||
def engram_gathered_num_tokens() -> int:
|
||||
"""Per-replica token slot for the node-local Engram DP group."""
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
if dp_metadata is None:
|
||||
raise RuntimeError("a DP-shared engram table needs DP token metadata")
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
# Engram groups are contiguous slices of the full DP group.
|
||||
start = get_dp_group().rank_in_group - group.rank_in_group
|
||||
return int(
|
||||
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
|
||||
)
|
||||
|
||||
|
||||
def gather_engram_hashes(
|
||||
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""Collect the n-gram ids of every DP replica sharing one table.
|
||||
|
||||
Replicas are padded to a common token slot, so the gathered shape is
|
||||
static under CUDA graph capture (where DP already pads alike).
|
||||
"""
|
||||
dp_group = get_engram_dp_group()
|
||||
if dp_group is None or dp_shared_memory:
|
||||
return hash_ids
|
||||
slot = engram_gathered_num_tokens()
|
||||
if hash_ids.shape[0] > slot:
|
||||
raise ValueError("Engram token count exceeds the DP token slot")
|
||||
if hash_ids.shape[0] < slot:
|
||||
pad = hash_ids.new_full(
|
||||
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
|
||||
)
|
||||
hash_ids = torch.cat((hash_ids, pad))
|
||||
return dp_group.all_gather(hash_ids, dim=0)
|
||||
|
||||
|
||||
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
|
||||
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
|
||||
if not os.path.isdir(SHM_PATH):
|
||||
logger.warning_once(
|
||||
"%s is not mounted; storing the offloaded Engram tables per rank "
|
||||
"instead of sharing them across DP replicas.",
|
||||
SHM_PATH,
|
||||
)
|
||||
return False
|
||||
if get_engram_dp_size() == 1:
|
||||
logger.warning_once(
|
||||
"Engram DP replicas are not co-located on one node; "
|
||||
"storing the offloaded tables per rank instead of sharing them."
|
||||
)
|
||||
return False
|
||||
num_bytes = sum(layout.num_embeddings) * (
|
||||
layout.head_dim + layout.head_dim // block_size
|
||||
)
|
||||
if shutil.disk_usage(SHM_PATH).total < num_bytes:
|
||||
logger.warning_once(
|
||||
"Sharing Engram tables across DP replicas needs %.1f GiB of %s "
|
||||
"(--shm-size or --ipc=host); sharding them across replicas instead.",
|
||||
num_bytes / 1024**3,
|
||||
SHM_PATH,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class DPSharedEngramStorage:
|
||||
"""Registered host weights shared by a node-local DP group with one writer."""
|
||||
|
||||
def __init__(
|
||||
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
|
||||
) -> None:
|
||||
self.group = group
|
||||
weight_bytes = num_rows * dim
|
||||
storage = self._allocate(weight_bytes + weight_bytes // block_size)
|
||||
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
|
||||
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
|
||||
def _allocate(self, num_bytes: int) -> torch.Tensor:
|
||||
"""Map and register one physical allocation across a node-local DP group."""
|
||||
group = self.group
|
||||
with ExitStack() as stack:
|
||||
path = error = None
|
||||
if group.rank_in_group == 0:
|
||||
try:
|
||||
check_shm_free_space(num_bytes)
|
||||
backing_file = stack.enter_context(
|
||||
tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
|
||||
)
|
||||
backing_file.truncate(num_bytes)
|
||||
path = backing_file.name
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
path, error = group.broadcast_object((path, error))
|
||||
if error is not None:
|
||||
raise RuntimeError(
|
||||
"Engram shared-memory creation failed on EDP rank 0: " + error
|
||||
)
|
||||
|
||||
mapping = owner = tensor = None
|
||||
try:
|
||||
with open(path, "r+b") as file:
|
||||
mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
|
||||
owner = np.frombuffer(mapping, dtype=np.uint8)
|
||||
pointer = owner.ctypes.data
|
||||
tensor = torch.from_numpy(owner)
|
||||
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
||||
if result.value != 0:
|
||||
raise RuntimeError(f"cudaHostRegister failed: {result}")
|
||||
finalizer = weakref.finalize(owner, self._unregister, mapping, pointer)
|
||||
finalizer.atexit = False # type: ignore[misc]
|
||||
# The UVA helper otherwise allocates a private pinned copy.
|
||||
if not tensor.is_pinned():
|
||||
raise RuntimeError(
|
||||
"cudaHostRegister did not pin the shared Engram mapping"
|
||||
)
|
||||
except Exception as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
# Also fences peer mappings before the leader unlinks the file.
|
||||
errors: list[str | None] = [None] * group.world_size
|
||||
torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
|
||||
failures = "; ".join(
|
||||
f"EDP rank {rank}: {error}"
|
||||
for rank, error in enumerate(errors)
|
||||
if error is not None
|
||||
)
|
||||
if failures:
|
||||
# Dropping the owner runs the finalizer; the mapping then unmaps.
|
||||
del tensor, owner, mapping
|
||||
raise RuntimeError(
|
||||
"Engram shared-memory initialization failed: " + failures
|
||||
)
|
||||
assert tensor is not None
|
||||
return tensor
|
||||
|
||||
@staticmethod
|
||||
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
|
||||
# Torch storage retains the numpy owner, including through cached UVA views.
|
||||
# Keep its mmap alive until CUDA has released the registration.
|
||||
result = torch.cuda.cudart().cudaHostUnregister(pointer)
|
||||
if result.value != 0:
|
||||
logger.warning("Engram cudaHostUnregister failed: %s", result)
|
||||
|
||||
def load_weight(
|
||||
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
||||
) -> None:
|
||||
if self.group.rank_in_group == 0:
|
||||
_engram_head_shard_weight_loader(param, loaded_weight)
|
||||
# Read order may differ across ranks. Equal load counts ensure all shared
|
||||
# weights are ready after the last weight-loader call returns.
|
||||
torch.distributed.barrier(group=self.group.cpu_group)
|
||||
|
||||
def get_views(
|
||||
self, weight: torch.Tensor, scales: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if (weight.data_ptr(), scales.data_ptr()) != (
|
||||
self.weight.data_ptr(),
|
||||
self.weight_scale_inv.data_ptr(),
|
||||
):
|
||||
raise RuntimeError("Shared Engram parameter storage must not be replaced")
|
||||
if self._views is None:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
|
||||
)
|
||||
return self._views
|
||||
|
||||
|
||||
class ParallelEngramEmbedding(BaseParallelEngramEmbedding):
|
||||
"""Extend TP lookup with DP head sharding or shared, CPU-offloaded TP slices."""
|
||||
|
||||
_shared_memory: DPSharedEngramStorage | None = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_embeddings: int,
|
||||
dim: int,
|
||||
head_sizes: tuple[int, ...],
|
||||
block_size: int = 32,
|
||||
cpu_offload: bool = False,
|
||||
dp_shared_memory: bool = False,
|
||||
use_thp: bool = False,
|
||||
) -> None:
|
||||
self.cpu_offload = cpu_offload
|
||||
self.dp_shared_memory = dp_shared_memory
|
||||
self.use_thp = use_thp
|
||||
self._packed: torch.Tensor | None = None
|
||||
self.dp_size = get_engram_dp_size()
|
||||
if dp_shared_memory:
|
||||
if not cpu_offload:
|
||||
raise ValueError("dp_shared_memory requires cpu_offload=True")
|
||||
if self.dp_size <= 1:
|
||||
raise ValueError(
|
||||
"dp_shared_memory requires a node-local Engram DP "
|
||||
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
|
||||
"Check that the node layout and rank placement allow complete "
|
||||
"DP replicas to be co-located."
|
||||
)
|
||||
self.dp_size = 1
|
||||
if cpu_offload and not is_uva_available():
|
||||
raise RuntimeError("Engram CPU offload requires UVA support")
|
||||
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
||||
self._view_src: tuple[int, int] | None = None
|
||||
super().__init__(num_embeddings, dim, head_sizes, block_size)
|
||||
if cpu_offload:
|
||||
# Constant dummy values avoid randomizing huge CPU lookup tables.
|
||||
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
||||
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
|
||||
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
|
||||
logger.info(
|
||||
"Engram table offloaded to pinned host memory: %d rows x %d, "
|
||||
"%.2f GiB %s",
|
||||
self.part_num_embeddings,
|
||||
dim,
|
||||
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
|
||||
"shared across DP replicas" if dp_shared_memory else "per rank",
|
||||
)
|
||||
|
||||
def _get_shard_info(self) -> tuple[int, int]:
|
||||
if self.dp_size == 1:
|
||||
return super()._get_shard_info()
|
||||
return self.tp_size * self.dp_size, engram_head_shard_rank()
|
||||
|
||||
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.dp_shared_memory:
|
||||
group = get_engram_dp_group()
|
||||
assert group is not None
|
||||
storage = DPSharedEngramStorage(
|
||||
self.part_num_embeddings, self.dim, self.block_size, group
|
||||
)
|
||||
self._shared_memory = storage
|
||||
self._weight_loader = storage.load_weight
|
||||
return storage.weight, storage.weight_scale_inv
|
||||
if not self.cpu_offload:
|
||||
return super()._allocate_weights()
|
||||
if self.use_thp:
|
||||
weight_bytes = self.part_num_embeddings * self.dim
|
||||
packed = _allocate_huge_page_storage(
|
||||
weight_bytes + weight_bytes // self.block_size
|
||||
)
|
||||
if packed is not None:
|
||||
self._packed = packed
|
||||
return (
|
||||
packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim),
|
||||
packed[weight_bytes:].view(-1, self.dim // self.block_size),
|
||||
)
|
||||
# Model initialization may be inside a CUDA device context.
|
||||
return (
|
||||
torch.empty(
|
||||
self.part_num_embeddings,
|
||||
self.dim,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
),
|
||||
torch.empty(
|
||||
self.part_num_embeddings,
|
||||
self.dim // self.block_size,
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
),
|
||||
)
|
||||
|
||||
def collapse_huge_pages(self) -> None:
|
||||
"""Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
|
||||
if self._packed is None:
|
||||
return
|
||||
addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
|
||||
if libc.madvise(addr, num_bytes, 25) != 0:
|
||||
logger.warning(
|
||||
"Engram MADV_COLLAPSE failed; keeping existing pages: %s",
|
||||
os.strerror(ctypes.get_errno()),
|
||||
)
|
||||
|
||||
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self._shared_memory is not None:
|
||||
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
|
||||
if not self.cpu_offload:
|
||||
return super()._storage()
|
||||
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
|
||||
if self._view_src != src:
|
||||
self._views = (
|
||||
get_accelerator_view_from_cpu_tensor(self.weight.data),
|
||||
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
|
||||
)
|
||||
self._view_src = src
|
||||
assert self._views is not None
|
||||
return self._views
|
||||
|
||||
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
if self.dp_size == 1:
|
||||
return super().forward(indices)
|
||||
num_tokens = indices.shape[0]
|
||||
indices = gather_engram_hashes(indices)
|
||||
out = torch.empty(
|
||||
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
||||
dtype=torch.bfloat16,
|
||||
device=indices.device,
|
||||
)
|
||||
self.lookup(indices, out)
|
||||
out = _gather_engram_rows(out, num_tokens)
|
||||
if self.tp_size > 1:
|
||||
out = tensor_model_parallel_all_gather(out, dim=1)
|
||||
return out[:, : self.n_hash_cols]
|
||||
|
||||
|
||||
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
|
||||
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
|
||||
dp_group = get_engram_dp_group()
|
||||
assert dp_group is not None
|
||||
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
|
||||
assert remainder == 0 and 0 <= num_tokens <= slot
|
||||
gathered = dp_group.all_gather(staged, dim=0)
|
||||
local_heads, dim = staged.shape[1:]
|
||||
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
|
||||
_engram_select_rows(
|
||||
gathered,
|
||||
rows,
|
||||
staged.shape[0],
|
||||
dp_group.rank_in_group * slot,
|
||||
local_heads * dim,
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
class Engram(BaseEngram):
|
||||
"""NVIDIA Engram with asynchronous offload and node-local DP lookup."""
|
||||
|
||||
_prefetch_stream: torch.cuda.Stream | None = None
|
||||
_prefetch_done: torch.cuda.Event | None = None
|
||||
|
||||
def __init__(
|
||||
self, *args, prefetch_stream: torch.cuda.Stream | None = None, **kwargs
|
||||
) -> None:
|
||||
# Layers sharing one stream serialize their offloaded lookups, so they
|
||||
# take turns instead of jointly starving decoder compute of SMs.
|
||||
self._prefetch_stream = prefetch_stream
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def _create_embedding(
|
||||
self, layout: EngramLayout, layer_hash_index: int
|
||||
) -> ParallelEngramEmbedding:
|
||||
engram_config = get_current_vllm_config().engram_config
|
||||
assert engram_config is not None
|
||||
return ParallelEngramEmbedding(
|
||||
layout.num_embeddings[layer_hash_index],
|
||||
layout.head_dim,
|
||||
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
||||
cpu_offload=engram_config.cpu_offload,
|
||||
dp_shared_memory=bool(engram_config.dp_shared_memory),
|
||||
use_thp=engram_config.use_thp,
|
||||
)
|
||||
|
||||
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
|
||||
super()._init_staging(max_tokens * self.embed_tokens.dp_size, head_dim)
|
||||
if not self.embed_tokens.cpu_offload:
|
||||
self._prefetch_stream = None
|
||||
return
|
||||
if self._prefetch_stream is None:
|
||||
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
|
||||
self._prefetch_done = torch.cuda.Event()
|
||||
|
||||
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
|
||||
"""Prefetch local shared rows or the DP group's gathered hash IDs."""
|
||||
if self._prefetch_stream is None:
|
||||
return super().prepare_embeddings(hash_ids)
|
||||
rows = self.staged_rows[: hash_ids.shape[0]]
|
||||
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
|
||||
self._start_prefetch(hash_ids, rows, self._prefetch_stream)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _start_prefetch(
|
||||
self, hash_ids: torch.Tensor, rows: torch.Tensor, stream: torch.cuda.Stream
|
||||
) -> None:
|
||||
# Eager boundaries let the lookup span piecewise graph segments.
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
# Keep temporary hash storage alive until lookup finishes reading it.
|
||||
hash_ids.record_stream(stream)
|
||||
with torch.cuda.stream(stream):
|
||||
self.embed_tokens.lookup(hash_ids, rows, background=True)
|
||||
assert self._prefetch_done is not None
|
||||
self._prefetch_done.record(stream)
|
||||
|
||||
@eager_break_during_capture
|
||||
def _finish_prefetch(self, event: torch.cuda.Event) -> None:
|
||||
# Wait for this layer's lookup only, not for later layers on the stream.
|
||||
torch.cuda.current_stream().wait_event(event)
|
||||
|
||||
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
|
||||
if self._prefetch_done is not None:
|
||||
self._finish_prefetch(self._prefetch_done)
|
||||
if self.embed_tokens.dp_size > 1:
|
||||
slot = engram_gathered_num_tokens()
|
||||
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
|
||||
return _gather_engram_rows(staged, num_tokens)
|
||||
return super()._ready_rows(num_tokens)
|
||||
@@ -98,9 +98,14 @@ from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.engram import (
|
||||
Engram,
|
||||
EngramLayout,
|
||||
NgramHashState,
|
||||
can_share_engram_tables,
|
||||
gather_engram_hashes,
|
||||
)
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, can_share_engram_tables, gather_engram_hashes
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
init_mhc_all_reduce,
|
||||
|
||||
Reference in New Issue
Block a user