1741 lines
66 KiB
Python
1741 lines
66 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Engram: n-gram hash lookups gated into the hyper-connection stream.
|
|
|
|
Port of the reference ``inference/engram.py`` + ``Engram`` /
|
|
``ParallelEngramEmbedding`` from ``inference/model.py`` (DeepSeek V4.1
|
|
checkpoint layout). Engram modules live on the backbone layers listed in
|
|
``engram_layer_ids`` only.
|
|
|
|
Two pieces of cross-forward state are needed because vLLM streams tokens
|
|
chunk-by-chunk while an n-gram at position ``p`` needs the token ids at
|
|
``p-1..p-3``:
|
|
|
|
- ``token_map``: token id -> compressed vocab id, built once from the
|
|
model's tokenizer at init (deterministic; asserted against
|
|
``engram_compressed_vocab_size``).
|
|
- ``hash_cache``: one int32 slot per KV slot of the first local layer's
|
|
sliding-window cache, holding the compressed id (or DEAD) of the token
|
|
last written to that slot. Slots are stable per (request, position) —
|
|
the block table pins a position to a physical slot, prefix-cache hits
|
|
reuse both the physical blocks and the identical token ids, and
|
|
spec-decode rollbacks rewrite the same slots — so lookbacks read back
|
|
exactly what the owning request wrote. Lookback depth (3) is far inside
|
|
the sliding window (128), so window eviction never frees a block a
|
|
live lookback still needs.
|
|
|
|
Slots are not part of the KV cache, so KV loaded from another instance
|
|
(P/D, offload connectors) leaves them unwritten. The runner therefore
|
|
passes ``lookback_token_ids``, the ids just before each request's chunk
|
|
start, which take precedence over the slots. The V2 runner reads them
|
|
from its device-resident token history and needs no slot cache; the V1
|
|
runner's CPU token table holds placeholders for generated tokens under
|
|
async scheduling, so it passes prompt positions only and keeps the slot
|
|
cache for the rest.
|
|
"""
|
|
|
|
import ctypes
|
|
import mmap
|
|
import os
|
|
import tempfile
|
|
import weakref
|
|
from contextlib import ExitStack
|
|
|
|
import numpy as np
|
|
import torch
|
|
from torch import nn
|
|
|
|
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
|
from vllm.config import VllmConfig, get_current_vllm_config
|
|
from vllm.distributed import (
|
|
get_dp_group,
|
|
get_engram_dp_group,
|
|
get_engram_dp_size,
|
|
get_tensor_model_parallel_rank,
|
|
get_tensor_model_parallel_world_size,
|
|
tensor_model_parallel_all_gather,
|
|
)
|
|
from vllm.distributed.device_communicators.shm_broadcast import (
|
|
SHM_PATH,
|
|
check_shm_free_space,
|
|
)
|
|
from vllm.distributed.parallel_state import GroupCoordinator
|
|
from vllm.forward_context import get_forward_context, is_forward_context_available
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
|
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
|
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
|
from vllm.model_executor.utils import set_weight_attrs
|
|
from vllm.models.deepseek_v41.common.ops.query_quant import (
|
|
can_fuse_query_quant,
|
|
mxfp8_scale_bytes,
|
|
mxfp8_scale_offset,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.triton_utils import tl, triton
|
|
from vllm.utils.platform_utils import is_uva_available
|
|
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# Per-rank token slot from which the Engram DP exchange switches from peer
|
|
# reads to all-to-all.
|
|
ENGRAM_A2A_MIN_SLOT = 128
|
|
|
|
# Cache value for tokens that take no part in an n-gram (image spans).
|
|
DEAD_ID = -1
|
|
|
|
|
|
def _engram_lookup_thresholds(device: torch.device) -> tuple[int | None, int | None]:
|
|
"""Host lookup thresholds on `device`: the rows from which to sort, and the
|
|
tokens (max over DP ranks) from which to run inline. None disables that path."""
|
|
if current_platform.is_cuda() and current_platform.is_device_capability_family(
|
|
100, device.index or 0
|
|
):
|
|
# Measured on GB200.
|
|
return 36864, 4096
|
|
# TODO: Add hand-tuned thresholds for other GPU families.
|
|
return None, None
|
|
|
|
|
|
def _is_prime(n: int) -> bool:
|
|
"""Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import)."""
|
|
if n < 2:
|
|
return False
|
|
for p in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37):
|
|
if n % p == 0:
|
|
return n == p
|
|
d = n - 1
|
|
r = 0
|
|
while d % 2 == 0:
|
|
d //= 2
|
|
r += 1
|
|
for a in (2, 7, 61):
|
|
x = pow(a, d, n)
|
|
if x in (1, n - 1):
|
|
continue
|
|
for _ in range(r - 1):
|
|
x = x * x % n
|
|
if x == n - 1:
|
|
break
|
|
else:
|
|
return False
|
|
return True
|
|
|
|
|
|
def find_next_prime(start: int, seen_primes: set[int]) -> int:
|
|
"""The smallest prime above `start` that has not been handed out yet."""
|
|
candidate = start + 1
|
|
while not _is_prime(candidate) or candidate in seen_primes:
|
|
candidate += 1
|
|
return candidate
|
|
|
|
|
|
def build_compressed_token_map(tokenizer) -> tuple[list[int], int]:
|
|
"""Map every token id onto a smaller id space where tokens that normalize
|
|
alike collapse together.
|
|
|
|
N-grams are hashed over these compressed ids, so " The", "the" and "THE"
|
|
all hash the same way. The compressed size matters beyond bounds checking:
|
|
every hash multiplier is derived from it.
|
|
"""
|
|
from tokenizers import Regex, normalizers
|
|
|
|
# A private-use char, so a token that is exactly one space survives
|
|
# Strip() instead of collapsing to the empty string and merging with
|
|
# unrelated tokens.
|
|
sentinel = "\ue000"
|
|
normalizer = normalizers.Sequence(
|
|
[
|
|
normalizers.NFKC(),
|
|
normalizers.NFD(),
|
|
normalizers.StripAccents(),
|
|
normalizers.Lowercase(),
|
|
normalizers.Replace(Regex(r"[ \t\r\n]+"), " "),
|
|
normalizers.Replace(Regex(r"^ $"), sentinel),
|
|
normalizers.Strip(),
|
|
normalizers.Replace(sentinel, " "),
|
|
]
|
|
)
|
|
|
|
# The raw Rust tokenizer, matching what training decodes with
|
|
# (no clean_up_tokenization_spaces).
|
|
backend = tokenizer.backend_tokenizer
|
|
key_to_new: dict[str, int] = {}
|
|
lookup = [0] * len(tokenizer)
|
|
for token_id in range(len(tokenizer)):
|
|
text = backend.decode([token_id], skip_special_tokens=False)
|
|
if "\ufffd" in text:
|
|
# A partial UTF-8 byte token: nothing to normalize, so key it
|
|
# by its raw form.
|
|
key = backend.id_to_token(token_id)
|
|
else:
|
|
normalized = normalizer.normalize_str(text)
|
|
key = normalized if normalized else text
|
|
|
|
new_id = key_to_new.get(key)
|
|
if new_id is None:
|
|
new_id = len(key_to_new)
|
|
key_to_new[key] = new_id
|
|
lookup[token_id] = new_id
|
|
|
|
return lookup, len(key_to_new)
|
|
|
|
|
|
def compute_hash_multipliers(
|
|
layer_ids: tuple[int, ...], max_ngram_size: int, compressed_vocab_size: int
|
|
) -> torch.Tensor:
|
|
"""One multiplier per (layer, lookback), from a per-layer RNG so layers
|
|
hash differently. Kept odd and bounded so `token_id * multiplier` cannot
|
|
overflow int64.
|
|
"""
|
|
max_long = np.iinfo(np.int64).max
|
|
multiplier_bound = max(1, (max_long // compressed_vocab_size) // 2)
|
|
rows = []
|
|
for layer_id in layer_ids:
|
|
generator = np.random.default_rng(10007 * layer_id)
|
|
values = generator.integers(
|
|
low=0,
|
|
high=multiplier_bound,
|
|
size=(max_ngram_size,),
|
|
dtype=np.int64,
|
|
)
|
|
rows.append(torch.tensor(values * 2 + 1))
|
|
return torch.stack(rows)
|
|
|
|
|
|
class EngramLayout:
|
|
"""Bucket layout of the n-gram hash tables.
|
|
|
|
A position is hashed as `max_ngram_size - 1` n-grams (2-gram .. max), each
|
|
split over `n_heads` heads. Every (n-gram size, head) pair owns its own
|
|
prime-sized bucket range in the layer's table; the primes are drawn in
|
|
order and never reused, which keeps the ranges disjoint.
|
|
"""
|
|
|
|
def __init__(self, config) -> None:
|
|
self.layer_ids: tuple[int, ...] = tuple(config.engram_layer_ids)
|
|
self.num_embeddings: tuple[int, ...] = tuple(config.engram_num_embeddings)
|
|
self.max_ngram_size: int = config.engram_max_ngram_size
|
|
self.n_heads: int = config.engram_n_heads
|
|
self.head_dim: int = config.engram_head_dim
|
|
self.compressed_vocab_size: int = config.engram_compressed_vocab_size
|
|
self.pad_token_id: int = config.engram_pad_token_id
|
|
assert len(self.layer_ids) == len(self.num_embeddings)
|
|
|
|
primes = []
|
|
seen: set[int] = set()
|
|
for _ in self.layer_ids:
|
|
per_ngram = []
|
|
for _ in range(self.max_ngram_size - 1):
|
|
sizes, current = [], config.engram_vocab_size - 1
|
|
for _ in range(self.n_heads):
|
|
current = find_next_prime(current, seen)
|
|
seen.add(current)
|
|
sizes.append(current)
|
|
per_ngram.append(tuple(sizes))
|
|
primes.append(tuple(per_ngram))
|
|
self.primes: tuple[tuple[tuple[int, ...], ...], ...] = tuple(primes)
|
|
self.n_hash_cols = (self.max_ngram_size - 1) * self.n_heads
|
|
flat = [[p for per_ngram in layer for p in per_ngram] for layer in primes]
|
|
offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat]
|
|
self.offsets = torch.tensor(np.array(offsets)) # [n_layers, n_hash_cols]
|
|
|
|
@classmethod
|
|
def from_config(cls, config) -> "EngramLayout | None":
|
|
if not getattr(config, "engram_layer_ids", None):
|
|
return None
|
|
return cls(config)
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_tokens"])
|
|
def _write_hash_cache_kernel(
|
|
input_ids,
|
|
token_map,
|
|
dead_mask,
|
|
slot_mapping,
|
|
cache,
|
|
num_tokens,
|
|
input_stride,
|
|
mask_stride,
|
|
slot_stride,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
dead_id,
|
|
):
|
|
token_idx = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
|
slot = tl.load(
|
|
slot_mapping + token_idx * slot_stride, token_idx < num_tokens, other=-1
|
|
).to(tl.int64)
|
|
valid = (token_idx < num_tokens) & (slot >= 0)
|
|
token = tl.load(input_ids + token_idx * input_stride, valid, other=0)
|
|
value = tl.load(token_map + token, valid, other=0)
|
|
dead = tl.load(dead_mask + token_idx * mask_stride, valid, other=False)
|
|
value = tl.where(dead, dead_id, value)
|
|
tl.store(cache + slot, value, valid)
|
|
|
|
|
|
# Keep request shapes out of the cache key so one warmed variant covers runtime batches.
|
|
@triton.jit(
|
|
do_not_specialize=[
|
|
"num_tokens",
|
|
"num_slots",
|
|
"num_query_rows",
|
|
"num_table_rows",
|
|
"max_blocks",
|
|
]
|
|
)
|
|
def _hash_ids_kernel(
|
|
input_ids,
|
|
token_map,
|
|
dead_mask,
|
|
positions,
|
|
block_table,
|
|
query_start_loc,
|
|
multipliers,
|
|
primes,
|
|
offsets,
|
|
cache,
|
|
lookback_token_ids,
|
|
lookback_dead_mask,
|
|
output,
|
|
num_tokens,
|
|
num_slots,
|
|
pad_id,
|
|
input_stride,
|
|
mask_stride,
|
|
position_stride,
|
|
table_stride,
|
|
table_col_stride,
|
|
query_stride,
|
|
num_query_rows,
|
|
num_table_rows,
|
|
max_blocks,
|
|
cache_block_size,
|
|
MAX_NGRAM: tl.constexpr,
|
|
num_heads,
|
|
BLOCK_T: tl.constexpr,
|
|
BLOCK_H: tl.constexpr,
|
|
dead_id,
|
|
lookback_depth,
|
|
lookback_row_stride,
|
|
lookback_col_stride,
|
|
lookback_mask_row_stride,
|
|
lookback_mask_col_stride,
|
|
):
|
|
token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T)
|
|
layer = tl.program_id(1)
|
|
num_layers = tl.num_programs(1)
|
|
valid = token < num_tokens
|
|
# Upper bound in query_start_loc[1:], including repeated padding boundaries.
|
|
lo = tl.full((BLOCK_T,), 0, tl.int32)
|
|
hi = tl.full((BLOCK_T,), num_query_rows, tl.int32)
|
|
while tl.sum((lo < hi).to(tl.int32), 0) > 0:
|
|
mid = (lo + hi) // 2
|
|
end = tl.load(
|
|
query_start_loc + (mid + 1) * query_stride,
|
|
lo < hi,
|
|
other=0,
|
|
)
|
|
right = token >= end
|
|
active = lo < hi
|
|
lo = tl.where(active & right, mid + 1, lo)
|
|
hi = tl.where(active & ~right, mid, hi)
|
|
req = tl.minimum(lo, num_query_rows - 1).to(tl.int64)
|
|
chunk_idx = tl.load(query_start_loc + req * query_stride)
|
|
chunk_idx = tl.minimum(chunk_idx, num_tokens - 1).to(tl.int64)
|
|
chunk_start = tl.load(positions + chunk_idx * position_stride)
|
|
position = tl.load(positions + token * position_stride, valid, other=0).to(tl.int64)
|
|
head = tl.arange(0, BLOCK_H)
|
|
blocked = tl.full((BLOCK_T,), False, tl.int1)
|
|
rolling = tl.full((BLOCK_T,), 0, tl.int64)
|
|
for shift in tl.static_range(MAX_NGRAM):
|
|
lookback = position - shift
|
|
in_batch = lookback >= chunk_start
|
|
batch_idx = tl.maximum(token - shift, 0)
|
|
batch_token = tl.load(
|
|
input_ids + batch_idx * input_stride, valid & in_batch, other=0
|
|
)
|
|
batch_source = tl.load(token_map + batch_token, valid & in_batch, other=0)
|
|
batch_dead = tl.load(
|
|
dead_mask + batch_idx * mask_stride, valid & in_batch, other=False
|
|
)
|
|
batch_source = tl.where(batch_dead, dead_id, batch_source)
|
|
|
|
col = chunk_start - 1 - lookback
|
|
in_window = valid & ~in_batch & (col >= 0) & (col < lookback_depth)
|
|
col = tl.minimum(tl.maximum(col, 0), lookback_depth - 1)
|
|
window_token = tl.load(
|
|
lookback_token_ids + req * lookback_row_stride + col * lookback_col_stride,
|
|
in_window,
|
|
other=-1,
|
|
)
|
|
known = in_window & (window_token >= 0)
|
|
window_source = tl.load(token_map + window_token, known, other=0)
|
|
window_dead = tl.load(
|
|
lookback_dead_mask
|
|
+ req * lookback_mask_row_stride
|
|
+ col * lookback_mask_col_stride,
|
|
known,
|
|
other=False,
|
|
)
|
|
window_source = tl.where(window_dead, dead_id, window_source)
|
|
|
|
if cache is not None:
|
|
clamped = tl.minimum(
|
|
tl.maximum(lookback, 0), max_blocks * cache_block_size - 1
|
|
)
|
|
block_row = tl.minimum(req, num_table_rows - 1)
|
|
needs_cache = valid & ~in_batch & ~known
|
|
block = tl.load(
|
|
block_table
|
|
+ block_row * table_stride
|
|
+ (clamped // cache_block_size) * table_col_stride,
|
|
needs_cache,
|
|
other=0,
|
|
).to(tl.int64)
|
|
slot = tl.minimum(
|
|
tl.maximum(block * cache_block_size + clamped % cache_block_size, 0),
|
|
num_slots - 1,
|
|
)
|
|
fallback = tl.load(cache + slot, needs_cache, other=0)
|
|
else:
|
|
fallback = tl.full((BLOCK_T,), pad_id, tl.int32)
|
|
source = tl.where(
|
|
in_batch, batch_source, tl.where(known, window_source, fallback)
|
|
).to(tl.int64)
|
|
blocked |= (lookback < 0) | (source == dead_id)
|
|
value = tl.where(blocked, pad_id, source)
|
|
multiplier = tl.load(multipliers + layer * MAX_NGRAM + shift)
|
|
rolling ^= value * multiplier
|
|
if shift > 0:
|
|
col = (shift - 1) * num_heads + head
|
|
param_offset = layer * (MAX_NGRAM - 1) * num_heads + col
|
|
prime = tl.load(primes + param_offset, head < num_heads, other=1)
|
|
offset = tl.load(offsets + param_offset, head < num_heads, other=0)
|
|
hashed = rolling[:, None] % prime[None, :] + offset[None, :]
|
|
out_offset = (token.to(tl.int64) * num_layers + layer)[:, None] * (
|
|
(MAX_NGRAM - 1) * num_heads
|
|
) + col[None, :]
|
|
tl.store(output + out_offset, hashed, valid[:, None] & (head < num_heads))
|
|
|
|
|
|
class NgramHashState(nn.Module):
|
|
"""Maps each position to the hash ids of the n-grams ending there.
|
|
|
|
Stateless on the V2 runner, which supplies every lookback token id. On
|
|
the V1 runner it also keeps `hash_cache`, the slot-keyed rolling store
|
|
of compressed ids (see module docstring), for generated tokens.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
vllm_config: VllmConfig,
|
|
layout: EngramLayout,
|
|
swa_cache_module: nn.Module,
|
|
) -> None:
|
|
super().__init__()
|
|
self.layout = layout
|
|
self.swa_cache_module = swa_cache_module
|
|
self.block_size: int = swa_cache_module.block_size
|
|
self.lookback_depth: int = layout.max_ngram_size - 1
|
|
self.use_slot_cache: bool = not vllm_config.use_v2_model_runner
|
|
self._cache: torch.Tensor | None = None
|
|
self._kv_cache_ref: weakref.ReferenceType[torch.Tensor] | None = None
|
|
|
|
model_config = vllm_config.model_config
|
|
from transformers import AutoTokenizer
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
model_config.tokenizer,
|
|
trust_remote_code=model_config.trust_remote_code,
|
|
revision=model_config.revision,
|
|
)
|
|
token_map, vocab_size = build_compressed_token_map(tokenizer)
|
|
if vocab_size != layout.compressed_vocab_size:
|
|
raise ValueError(
|
|
f"Compressed vocab size mismatch: built {vocab_size} from the "
|
|
f"tokenizer, config expects {layout.compressed_vocab_size}; "
|
|
"every hash multiplier derives from it, so the engram tables "
|
|
"would be silently rehashed."
|
|
)
|
|
self.pad_id = token_map[layout.pad_token_id]
|
|
multipliers = compute_hash_multipliers(
|
|
layout.layer_ids, layout.max_ngram_size, vocab_size
|
|
)
|
|
self.register_buffer(
|
|
"token_map", torch.tensor(token_map, dtype=torch.int32), persistent=False
|
|
)
|
|
self.register_buffer("primes", torch.tensor(layout.primes), persistent=False)
|
|
self.register_buffer("offsets", layout.offsets, persistent=False)
|
|
self.register_buffer("multipliers", multipliers, persistent=False)
|
|
logger.info(
|
|
"Built engram token map (%d -> %d ids) for layers %s",
|
|
len(token_map),
|
|
vocab_size,
|
|
layout.layer_ids,
|
|
)
|
|
|
|
def ensure_cache(self) -> bool:
|
|
"""Lazily size the slot-keyed cache from the bound SWA KV cache.
|
|
|
|
Returns False while the KV cache is unbound (profile run); the caller
|
|
skips engram hashing then. Without the slot cache only that check
|
|
remains.
|
|
"""
|
|
kv_cache = self.swa_cache_module.kv_cache
|
|
if kv_cache.numel() == 0:
|
|
self._cache = None
|
|
self._kv_cache_ref = None
|
|
return False
|
|
if not self.use_slot_cache:
|
|
return True
|
|
if self._kv_cache_ref is not None and self._kv_cache_ref() is kv_cache:
|
|
return True
|
|
# Graph memory profiling binds a temporary, smaller KV cache first.
|
|
# Rebinding must discard its hash history without retaining KV storage.
|
|
self._cache = torch.zeros(
|
|
kv_cache.shape[0] * self.block_size,
|
|
dtype=torch.int32,
|
|
device=kv_cache.device,
|
|
)
|
|
self._kv_cache_ref = weakref.ref(kv_cache)
|
|
return True
|
|
|
|
def dummy_hashes(
|
|
self, input_ids: torch.Tensor
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Participate in DP lookups without valid rows or hash-cache updates."""
|
|
num_tokens = input_ids.shape[0]
|
|
num_layers, max_ngram = self.multipliers.shape
|
|
num_heads = self.primes.shape[-1]
|
|
hashes = input_ids.new_full(
|
|
(num_tokens, num_layers, (max_ngram - 1) * num_heads),
|
|
DEAD_ID,
|
|
dtype=torch.int32,
|
|
)
|
|
keep = torch.zeros(num_tokens, dtype=torch.bool, device=input_ids.device)
|
|
return hashes, keep
|
|
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor,
|
|
positions: torch.Tensor,
|
|
query_start_loc: torch.Tensor,
|
|
dead_mask: torch.Tensor,
|
|
lookback_token_ids: torch.Tensor,
|
|
lookback_dead_mask: torch.Tensor,
|
|
slot_mapping: torch.Tensor | None,
|
|
block_table: torch.Tensor | None,
|
|
) -> torch.Tensor:
|
|
"""Compute [tokens, layers, hash columns] int32 n-gram hashes.
|
|
|
|
History comes from the current chunk, then the runner's lookback
|
|
window, then the optional V1 slot cache. V2 needs only one launch.
|
|
"""
|
|
cache = self._cache if self.use_slot_cache else None
|
|
num_tokens = input_ids.shape[0]
|
|
num_layers, max_ngram = self.multipliers.shape
|
|
num_heads = self.primes.shape[-1]
|
|
output = input_ids.new_empty(
|
|
(num_tokens, num_layers, (max_ngram - 1) * num_heads), dtype=torch.int32
|
|
)
|
|
if num_tokens == 0:
|
|
return output
|
|
if self.use_slot_cache:
|
|
assert cache is not None and slot_mapping is not None
|
|
assert block_table is not None
|
|
# Finish writes before other thread blocks read fallback history.
|
|
_write_hash_cache_kernel[(triton.cdiv(num_tokens, 256),)](
|
|
input_ids,
|
|
self.token_map,
|
|
dead_mask,
|
|
slot_mapping,
|
|
cache,
|
|
num_tokens,
|
|
input_ids.stride(0),
|
|
dead_mask.stride(0),
|
|
slot_mapping.stride(0),
|
|
256,
|
|
DEAD_ID,
|
|
)
|
|
_hash_ids_kernel[(triton.cdiv(num_tokens, 32), num_layers)](
|
|
input_ids,
|
|
self.token_map,
|
|
dead_mask,
|
|
positions,
|
|
block_table,
|
|
query_start_loc,
|
|
self.multipliers,
|
|
self.primes,
|
|
self.offsets,
|
|
cache,
|
|
lookback_token_ids,
|
|
lookback_dead_mask,
|
|
output,
|
|
num_tokens,
|
|
cache.shape[0] if cache is not None else 0,
|
|
self.pad_id,
|
|
input_stride=input_ids.stride(0),
|
|
mask_stride=dead_mask.stride(0),
|
|
position_stride=positions.stride(0),
|
|
table_stride=block_table.stride(0) if block_table is not None else 0,
|
|
table_col_stride=block_table.stride(1) if block_table is not None else 0,
|
|
query_stride=query_start_loc.stride(0),
|
|
num_query_rows=query_start_loc.numel() - 1,
|
|
num_table_rows=block_table.shape[0] if block_table is not None else 0,
|
|
max_blocks=block_table.shape[1] if block_table is not None else 0,
|
|
cache_block_size=self.block_size,
|
|
MAX_NGRAM=max_ngram,
|
|
num_heads=num_heads,
|
|
BLOCK_T=32,
|
|
BLOCK_H=triton.next_power_of_2(num_heads),
|
|
dead_id=DEAD_ID,
|
|
lookback_depth=lookback_token_ids.shape[1],
|
|
lookback_row_stride=lookback_token_ids.stride(0),
|
|
lookback_col_stride=lookback_token_ids.stride(1),
|
|
lookback_mask_row_stride=lookback_dead_mask.stride(0),
|
|
lookback_mask_col_stride=lookback_dead_mask.stride(1),
|
|
num_warps=4,
|
|
)
|
|
return output
|
|
|
|
|
|
def _engram_head_shard_weight_loader(
|
|
param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
|
) -> None:
|
|
"""Load this rank's complete head buckets. ue8m0 scales arrive as
|
|
float8_e8m0fnu; keep the raw bytes (the param stores uint8)."""
|
|
part_rows = param.shape[0]
|
|
if loaded_weight.dtype == torch.float8_e8m0fnu:
|
|
loaded_weight = loaded_weight.view(torch.uint8)
|
|
shard = loaded_weight.narrow(0, param.engram_vocab_start, part_rows)
|
|
assert shard.shape == param.shape, (
|
|
f"engram shard {tuple(shard.shape)} does not fit param {tuple(param.shape)}"
|
|
)
|
|
param.data.copy_(shard)
|
|
|
|
|
|
# Branching on SORTED at runtime intermittently crashes Triton 3.8's
|
|
# RemoveLayoutConversions.
|
|
# TODO: Remove the 3.8 check once Triton ships
|
|
# https://github.com/triton-lang/triton/pull/10706.
|
|
_SORTED_IS_CONSTEXPR = tl.constexpr(
|
|
triton.__version__.startswith("3.8") and current_platform.is_rocm()
|
|
)
|
|
|
|
|
|
@triton.jit(
|
|
do_not_specialize=[
|
|
"vocab_start",
|
|
"vocab_end",
|
|
"num_rows",
|
|
"ids_stride_t",
|
|
"ids_stride_h",
|
|
"GRID",
|
|
*(() if _SORTED_IS_CONSTEXPR else ("SORTED",)),
|
|
]
|
|
)
|
|
def _engram_lookup_kernel(
|
|
weight,
|
|
scales,
|
|
ids,
|
|
sorted_dst,
|
|
out,
|
|
vocab_start,
|
|
vocab_end,
|
|
num_rows,
|
|
ids_stride_t,
|
|
ids_stride_h,
|
|
HEAD_START: tl.constexpr,
|
|
LOCAL_HEADS: tl.constexpr,
|
|
TOTAL_HEADS: tl.constexpr,
|
|
DIM: tl.constexpr,
|
|
QUANT_BLOCK: tl.constexpr,
|
|
BLOCK_R: tl.constexpr,
|
|
GRID,
|
|
SORTED: tl.constexpr if _SORTED_IS_CONSTEXPR else None, # type: ignore[valid-type]
|
|
PACKED: tl.constexpr,
|
|
OUT_STRIDE: tl.constexpr,
|
|
):
|
|
"""Gather FP8 rows as packed values/scales or dequantized BF16.
|
|
|
|
Only this rank's heads are read; padded heads write zeros for all-gather.
|
|
`weight`/`scales` may address pinned host memory through UVA. If SORTED,
|
|
`ids` holds rows relative to vocab_start in table order and `sorted_dst`
|
|
their output rows.
|
|
"""
|
|
cols = tl.arange(0, DIM)
|
|
scale_cols = cols // QUANT_BLOCK
|
|
for base in tl.range(tl.program_id(0) * BLOCK_R, num_rows, GRID * BLOCK_R):
|
|
rows = base + tl.arange(0, BLOCK_R)
|
|
valid = rows < num_rows
|
|
if SORTED:
|
|
local = tl.load(ids + rows, mask=valid, other=-1).to(tl.int64)
|
|
dst = tl.load(sorted_dst + rows, mask=valid, other=0).to(tl.int32)
|
|
owned = (local >= 0) & (local < vocab_end - vocab_start)
|
|
else:
|
|
head = HEAD_START + rows % LOCAL_HEADS
|
|
token = (rows // LOCAL_HEADS).to(tl.int64)
|
|
index = tl.load(
|
|
ids + token * ids_stride_t + head * ids_stride_h,
|
|
mask=valid & (head < TOTAL_HEADS),
|
|
other=-1,
|
|
).to(tl.int64)
|
|
dst = rows
|
|
owned = valid & (head < TOTAL_HEADS)
|
|
owned &= (index >= vocab_start) & (index < vocab_end)
|
|
local = index - vocab_start
|
|
local = tl.where(owned, local, 0)
|
|
values = tl.load(
|
|
weight + local[:, None] * DIM + cols[None, :],
|
|
mask=owned[:, None],
|
|
other=0.0,
|
|
)
|
|
scale = tl.load(
|
|
scales + local[:, None] * (DIM // QUANT_BLOCK) + scale_cols[None, :],
|
|
mask=owned[:, None],
|
|
other=0,
|
|
)
|
|
if PACKED:
|
|
width: tl.constexpr = DIM + DIM // QUANT_BLOCK
|
|
dst = dst.to(tl.int64)
|
|
dst = dst // LOCAL_HEADS * OUT_STRIDE + dst % LOCAL_HEADS * width
|
|
tl.store(
|
|
out + dst[:, None] + cols[None, :],
|
|
values.to(tl.uint8, bitcast=True),
|
|
mask=valid[:, None],
|
|
)
|
|
tl.store(
|
|
out + dst[:, None] + DIM + scale_cols[None, :],
|
|
scale,
|
|
mask=valid[:, None] & (cols[None, :] % QUANT_BLOCK == 0),
|
|
)
|
|
else:
|
|
# ue8m0 is a power of two, so its byte *is* the fp32 exponent field.
|
|
scale = (scale.to(tl.int32) << 23).to(tl.float32, bitcast=True)
|
|
tl.store(
|
|
out + dst[:, None] * DIM + cols[None, :],
|
|
(values.to(tl.float32) * scale).to(tl.bfloat16),
|
|
mask=valid[:, None],
|
|
)
|
|
|
|
|
|
def engram_head_shard_rank() -> int:
|
|
"""This rank's slot among the hash-head shards of one engram table.
|
|
|
|
TP-major, so the shards a DP gather brings in are contiguous heads and
|
|
the following TP gather completes the head order.
|
|
"""
|
|
dp_group = get_engram_dp_group()
|
|
dp_size = dp_group.world_size if dp_group is not None else 1
|
|
dp_rank = dp_group.rank_in_group if dp_group is not None else 0
|
|
return get_tensor_model_parallel_rank() * dp_size + dp_rank
|
|
|
|
|
|
def engram_gathered_num_tokens() -> int:
|
|
"""Per-replica token slot for the node-local Engram DP group."""
|
|
dp_metadata = get_forward_context().dp_metadata
|
|
if dp_metadata is None:
|
|
raise RuntimeError("a DP-shared engram table needs DP token metadata")
|
|
group = get_engram_dp_group()
|
|
assert group is not None
|
|
# Engram groups are contiguous slices of the full DP group.
|
|
start = get_dp_group().rank_in_group - group.rank_in_group
|
|
return int(
|
|
dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max()
|
|
)
|
|
|
|
|
|
def gather_engram_hashes(
|
|
hash_ids: torch.Tensor, *, dp_shared_memory: bool = False
|
|
) -> torch.Tensor:
|
|
"""Collect the n-gram ids of every DP replica sharing one table.
|
|
|
|
Replicas are padded to a common token slot, so the gathered shape is
|
|
static under CUDA graph capture (where DP already pads alike).
|
|
"""
|
|
dp_group = get_engram_dp_group()
|
|
if dp_group is None or dp_shared_memory:
|
|
return hash_ids
|
|
slot = engram_gathered_num_tokens()
|
|
if hash_ids.shape[0] > slot:
|
|
raise ValueError("Engram token count exceeds the DP token slot")
|
|
if hash_ids.shape[0] < slot:
|
|
pad = hash_ids.new_full(
|
|
(slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID
|
|
)
|
|
hash_ids = torch.cat((hash_ids, pad))
|
|
return dp_group.all_gather(hash_ids, dim=0)
|
|
|
|
|
|
class DPSharedEngramStorage:
|
|
"""Registered host weights shared by a node-local DP group with one writer."""
|
|
|
|
def __init__(
|
|
self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator
|
|
) -> None:
|
|
self.group = group
|
|
weight_bytes = num_rows * dim
|
|
storage = self._allocate(weight_bytes + weight_bytes // block_size)
|
|
self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim)
|
|
self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size)
|
|
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
|
|
|
def _allocate(self, num_bytes: int) -> torch.Tensor:
|
|
"""Map and register one physical allocation across a node-local DP group."""
|
|
group = self.group
|
|
with ExitStack() as stack:
|
|
path = error = None
|
|
if group.rank_in_group == 0:
|
|
try:
|
|
check_shm_free_space(num_bytes)
|
|
backing_file = stack.enter_context(
|
|
tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH)
|
|
)
|
|
backing_file.truncate(num_bytes)
|
|
path = backing_file.name
|
|
except Exception as exc:
|
|
error = f"{type(exc).__name__}: {exc}"
|
|
path, error = group.broadcast_object((path, error))
|
|
if error is not None:
|
|
raise RuntimeError(
|
|
"Engram shared-memory creation failed on EDP rank 0: " + error
|
|
)
|
|
|
|
mapping = owner = tensor = None
|
|
try:
|
|
with open(path, "r+b") as file:
|
|
mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED)
|
|
owner = np.frombuffer(mapping, dtype=np.uint8)
|
|
pointer = owner.ctypes.data
|
|
tensor = torch.from_numpy(owner)
|
|
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
|
if result.value != 0:
|
|
raise RuntimeError(f"cudaHostRegister failed: {result}")
|
|
finalizer = weakref.finalize(owner, self._unregister, mapping, pointer)
|
|
finalizer.atexit = False # type: ignore[misc]
|
|
# The UVA helper otherwise allocates a private pinned copy.
|
|
if not tensor.is_pinned():
|
|
raise RuntimeError(
|
|
"cudaHostRegister did not pin the shared Engram mapping"
|
|
)
|
|
except Exception as exc:
|
|
error = f"{type(exc).__name__}: {exc}"
|
|
|
|
# Also fences peer mappings before the leader unlinks the file.
|
|
errors: list[str | None] = [None] * group.world_size
|
|
torch.distributed.all_gather_object(errors, error, group=group.cpu_group)
|
|
failures = "; ".join(
|
|
f"EDP rank {rank}: {error}"
|
|
for rank, error in enumerate(errors)
|
|
if error is not None
|
|
)
|
|
if failures:
|
|
# Dropping the owner runs the finalizer; the mapping then unmaps.
|
|
del tensor, owner, mapping
|
|
raise RuntimeError(
|
|
"Engram shared-memory initialization failed: " + failures
|
|
)
|
|
assert tensor is not None
|
|
return tensor
|
|
|
|
@staticmethod
|
|
def _unregister(mapping: mmap.mmap, pointer: int) -> None:
|
|
# Torch storage retains the numpy owner, including through cached UVA views.
|
|
# Keep its mmap alive until CUDA has released the registration.
|
|
result = torch.cuda.cudart().cudaHostUnregister(pointer)
|
|
if result.value != 0:
|
|
logger.warning("Engram cudaHostUnregister failed: %s", result)
|
|
|
|
def load_weight(
|
|
self, param: torch.nn.Parameter, loaded_weight: torch.Tensor
|
|
) -> None:
|
|
if self.group.rank_in_group == 0:
|
|
_engram_head_shard_weight_loader(param, loaded_weight)
|
|
# Read order may differ across ranks. Equal load counts ensure all shared
|
|
# weights are ready after the last weight-loader call returns.
|
|
torch.distributed.barrier(group=self.group.cpu_group)
|
|
|
|
def get_views(
|
|
self, weight: torch.Tensor, scales: torch.Tensor
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
if (weight.data_ptr(), scales.data_ptr()) != (
|
|
self.weight.data_ptr(),
|
|
self.weight_scale_inv.data_ptr(),
|
|
):
|
|
raise RuntimeError("Shared Engram parameter storage must not be replaced")
|
|
if self._views is None:
|
|
self._views = (
|
|
get_accelerator_view_from_cpu_tensor(self.weight),
|
|
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv),
|
|
)
|
|
return self._views
|
|
|
|
|
|
def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool:
|
|
"""Whether co-located DP replicas exist and /dev/shm can hold the full tables."""
|
|
if get_engram_dp_size() == 1:
|
|
logger.warning_once(
|
|
"Engram DP replicas are not co-located on one node; "
|
|
"storing the offloaded tables per rank instead of sharing them."
|
|
)
|
|
return False
|
|
num_bytes = sum(layout.num_embeddings) * (
|
|
layout.head_dim + layout.head_dim // block_size
|
|
)
|
|
group = get_engram_dp_group()
|
|
assert group is not None
|
|
# Only the leader allocates, so every replica follows its check.
|
|
error = None
|
|
if group.rank_in_group == 0:
|
|
if not os.path.isdir(SHM_PATH):
|
|
error = f"{SHM_PATH} is not mounted"
|
|
else:
|
|
try:
|
|
check_shm_free_space(num_bytes, allocation_name="Engram tables")
|
|
except Exception as exc:
|
|
error = f"{type(exc).__name__}: {exc}"
|
|
error = group.broadcast_object(error)
|
|
if error is not None:
|
|
logger.warning_once(
|
|
"Sharding the offloaded Engram tables across DP replicas: %s", error
|
|
)
|
|
return error is None
|
|
|
|
|
|
def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None:
|
|
"""Register prefaulted huge pages, or return None for pinned-memory fallback."""
|
|
try:
|
|
mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS)
|
|
mapping.madvise(mmap.MADV_HUGEPAGE)
|
|
except OSError as exc:
|
|
logger.warning(
|
|
"Engram huge-page allocation failed (%s); using pinned memory.", exc
|
|
)
|
|
return None
|
|
|
|
owner = np.frombuffer(mapping, dtype=np.uint8)
|
|
# Fault the pages in before CUDA pins them, so they can be huge.
|
|
owner[:: mmap.PAGESIZE] = 0
|
|
tensor = torch.from_numpy(owner)
|
|
pointer = tensor.data_ptr()
|
|
result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0)
|
|
if result.value != 0:
|
|
logger.warning(
|
|
"Engram cudaHostRegister failed (%s); using pinned memory.", result
|
|
)
|
|
return None
|
|
# Tensor views retain owner; its finalizer retains mapping until unregister.
|
|
finalizer = weakref.finalize(
|
|
owner, DPSharedEngramStorage._unregister, mapping, pointer
|
|
)
|
|
finalizer.atexit = False # type: ignore[misc]
|
|
if not tensor.is_pinned():
|
|
logger.warning(
|
|
"CUDA did not recognize Engram registration; using pinned memory."
|
|
)
|
|
return None
|
|
return tensor
|
|
|
|
|
|
class ParallelEngramEmbedding(nn.Module):
|
|
"""Hash heads with FP8 rows and per-block E8M0 scales.
|
|
|
|
Heads are TP-sharded, and additionally DP-sharded across a node-local
|
|
group unless the table lives in (DP-shared) pinned host memory.
|
|
"""
|
|
|
|
_weight_loader = staticmethod(_engram_head_shard_weight_loader)
|
|
_shared_memory: DPSharedEngramStorage | None = None
|
|
_packed: torch.Tensor | None = None
|
|
|
|
def __init__(
|
|
self,
|
|
num_embeddings: int,
|
|
dim: int,
|
|
head_sizes: tuple[int, ...],
|
|
block_size: int = 32,
|
|
cpu_offload: bool = False,
|
|
dp_shared_memory: bool = False,
|
|
use_thp: bool = False,
|
|
) -> None:
|
|
super().__init__()
|
|
assert head_sizes and all(size > 0 for size in head_sizes)
|
|
assert sum(head_sizes) <= num_embeddings
|
|
self.cpu_offload = cpu_offload
|
|
self.dp_shared_memory = dp_shared_memory
|
|
self.use_thp = use_thp
|
|
self.dp_size = get_engram_dp_size()
|
|
if dp_shared_memory:
|
|
if not cpu_offload:
|
|
raise ValueError("dp_shared_memory requires cpu_offload=True")
|
|
if self.dp_size <= 1:
|
|
raise ValueError(
|
|
"dp_shared_memory requires a node-local Engram DP "
|
|
f"group with size > 1; effective Engram DP size is {self.dp_size}. "
|
|
"Check that the node layout and rank placement allow complete "
|
|
"DP replicas to be co-located."
|
|
)
|
|
self.dp_size = 1
|
|
if cpu_offload and not is_uva_available():
|
|
raise RuntimeError("Engram CPU offload requires UVA support")
|
|
self._views: tuple[torch.Tensor, torch.Tensor] | None = None
|
|
self._view_src: tuple[int, int] | None = None
|
|
self.num_embeddings = num_embeddings
|
|
self.dim = dim
|
|
self.block_size = block_size
|
|
self.n_hash_cols = len(head_sizes)
|
|
self.tp_size = get_tensor_model_parallel_world_size()
|
|
num_shards, head_rank = self._get_shard_info()
|
|
self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, num_shards)
|
|
# TODO: Support row-wise sharding when there are too few hash heads.
|
|
assert (num_shards - 1) * self.part_n_hash_cols < self.n_hash_cols, (
|
|
f"Engram sharding leaves ranks without hash heads: "
|
|
f"{self.n_hash_cols} heads over {num_shards} shards"
|
|
)
|
|
self.head_start = head_rank * self.part_n_hash_cols
|
|
head_end = self.head_start + self.part_n_hash_cols
|
|
self.vocab_start_idx = sum(head_sizes[: self.head_start])
|
|
self.vocab_end_idx = sum(head_sizes[:head_end])
|
|
self.part_num_embeddings = self.vocab_end_idx - self.vocab_start_idx
|
|
self._num_sms = torch.cuda.get_device_properties(
|
|
torch.accelerator.current_device_index()
|
|
).multi_processor_count
|
|
weight, scales = self._allocate_weights()
|
|
self.weight = nn.Parameter(weight, requires_grad=False)
|
|
self.weight_scale_inv = nn.Parameter(scales, requires_grad=False)
|
|
for param in (self.weight, self.weight_scale_inv):
|
|
set_weight_attrs(
|
|
param,
|
|
{
|
|
"weight_loader": self._weight_loader,
|
|
"engram_vocab_start": self.vocab_start_idx,
|
|
},
|
|
)
|
|
if cpu_offload:
|
|
# Constant dummy values avoid randomizing huge CPU lookup tables.
|
|
set_weight_attrs(self.weight, {"dummy_weight_value": 1.0})
|
|
# The ue8m0 encoding of scale 1.0 is exponent byte 127.
|
|
set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127})
|
|
logger.info(
|
|
"Engram table offloaded to pinned host memory: %d rows x %d, "
|
|
"%.2f GiB %s",
|
|
self.part_num_embeddings,
|
|
dim,
|
|
self.part_num_embeddings * (dim + dim // block_size) / 1024**3,
|
|
"shared across DP replicas" if dp_shared_memory else "per rank",
|
|
)
|
|
|
|
def _get_shard_info(self) -> tuple[int, int]:
|
|
if self.dp_size == 1:
|
|
return self.tp_size, get_tensor_model_parallel_rank()
|
|
return self.tp_size * self.dp_size, engram_head_shard_rank()
|
|
|
|
def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]:
|
|
if self.dp_shared_memory:
|
|
group = get_engram_dp_group()
|
|
assert group is not None
|
|
storage = DPSharedEngramStorage(
|
|
self.part_num_embeddings, self.dim, self.block_size, group
|
|
)
|
|
self._shared_memory = storage
|
|
self._weight_loader = storage.load_weight
|
|
return storage.weight, storage.weight_scale_inv
|
|
scale_dim = self.dim // self.block_size
|
|
if not self.cpu_offload:
|
|
return (
|
|
torch.empty(
|
|
self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn
|
|
),
|
|
torch.empty(self.part_num_embeddings, scale_dim, dtype=torch.uint8),
|
|
)
|
|
if self.use_thp:
|
|
weight_bytes = self.part_num_embeddings * self.dim
|
|
packed = _allocate_huge_page_storage(
|
|
weight_bytes + weight_bytes // self.block_size
|
|
)
|
|
if packed is not None:
|
|
self._packed = packed
|
|
return (
|
|
packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim),
|
|
packed[weight_bytes:].view(-1, scale_dim),
|
|
)
|
|
# Model initialization may be inside a CUDA device context.
|
|
return (
|
|
torch.empty(
|
|
self.part_num_embeddings,
|
|
self.dim,
|
|
dtype=torch.float8_e4m3fn,
|
|
device="cpu",
|
|
pin_memory=True,
|
|
),
|
|
torch.empty(
|
|
self.part_num_embeddings,
|
|
scale_dim,
|
|
dtype=torch.uint8,
|
|
device="cpu",
|
|
pin_memory=True,
|
|
),
|
|
)
|
|
|
|
def collapse_huge_pages(self) -> None:
|
|
"""Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small."""
|
|
if self._packed is None:
|
|
return
|
|
addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes
|
|
libc = ctypes.CDLL(None, use_errno=True)
|
|
libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int)
|
|
if libc.madvise(addr, num_bytes, 25) != 0:
|
|
logger.warning(
|
|
"Engram MADV_COLLAPSE failed; keeping existing pages: %s",
|
|
os.strerror(ctypes.get_errno()),
|
|
)
|
|
|
|
def _storage(self) -> tuple[torch.Tensor, torch.Tensor]:
|
|
if self._shared_memory is not None:
|
|
return self._shared_memory.get_views(self.weight, self.weight_scale_inv)
|
|
if not self.cpu_offload:
|
|
return self.weight.data, self.weight_scale_inv.data
|
|
src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr())
|
|
if self._view_src != src:
|
|
self._views = (
|
|
get_accelerator_view_from_cpu_tensor(self.weight.data),
|
|
get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data),
|
|
)
|
|
self._view_src = src
|
|
assert self._views is not None
|
|
return self._views
|
|
|
|
def lookup(
|
|
self, indices: torch.Tensor, out: torch.Tensor, background: bool = False
|
|
) -> None:
|
|
"""Look up local heads into BF16, or packed FP8 bytes followed by scales.
|
|
|
|
`background` limits the grid to leave SMs for concurrent work.
|
|
"""
|
|
rows = indices.shape[0] * self.part_n_hash_cols
|
|
if not rows:
|
|
return
|
|
weight, scales = self._storage()
|
|
ids_stride_t, ids_stride_h = indices.stride()
|
|
sort_min_rows, _ = _engram_lookup_thresholds(indices.device)
|
|
sort = (
|
|
self.cpu_offload
|
|
and self._packed is None
|
|
and sort_min_rows is not None
|
|
and rows >= sort_min_rows
|
|
)
|
|
dst = indices
|
|
if sort:
|
|
# Small-page host tables miss the TLB per row; in table order, rows on
|
|
# one page share it.
|
|
head_end = self.head_start + self.part_n_hash_cols
|
|
local = indices[:, self.head_start : head_end] - self.vocab_start_idx
|
|
if (pad := self.part_n_hash_cols - local.shape[1]) > 0:
|
|
local = nn.functional.pad(local, (0, pad), value=-1)
|
|
indices, dst = local.flatten().sort()
|
|
dst = dst.int()
|
|
# The table dwarfs TLB reach, so a persistent grid near the SM count
|
|
# beats one program per row; halve it to leave SMs for the main stream.
|
|
tiles = triton.cdiv(rows, 16)
|
|
grid = min(tiles, self._num_sms // 2 if background else self._num_sms)
|
|
_engram_lookup_kernel[(grid,)](
|
|
weight,
|
|
scales,
|
|
indices,
|
|
dst,
|
|
out,
|
|
self.vocab_start_idx,
|
|
self.vocab_end_idx,
|
|
rows,
|
|
ids_stride_t,
|
|
ids_stride_h,
|
|
HEAD_START=self.head_start,
|
|
LOCAL_HEADS=self.part_n_hash_cols,
|
|
TOTAL_HEADS=self.n_hash_cols,
|
|
DIM=self.dim,
|
|
QUANT_BLOCK=self.block_size,
|
|
BLOCK_R=16,
|
|
GRID=grid,
|
|
SORTED=sort,
|
|
PACKED=out.dtype == torch.uint8,
|
|
OUT_STRIDE=out.stride(0),
|
|
)
|
|
|
|
def forward(self, indices: torch.Tensor) -> torch.Tensor:
|
|
"""indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim]
|
|
bf16, gathered from all shards for this replica's tokens."""
|
|
num_tokens = indices.shape[0]
|
|
if self.dp_size > 1:
|
|
indices = gather_engram_hashes(indices)
|
|
out = torch.empty(
|
|
(indices.shape[0], self.part_n_hash_cols, self.dim),
|
|
dtype=torch.bfloat16,
|
|
device=indices.device,
|
|
)
|
|
self.lookup(indices, out)
|
|
if self.dp_size > 1:
|
|
out = _gather_engram_rows(out, num_tokens)
|
|
if self.tp_size > 1:
|
|
out = tensor_model_parallel_all_gather(out, dim=1)
|
|
return out[:, : self.n_hash_cols]
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_kv_tokens"])
|
|
def _fused_engram_post_wkv_kernel(
|
|
hidden_states,
|
|
kv,
|
|
q_weight,
|
|
k_weight,
|
|
token_mask,
|
|
output,
|
|
num_kv_tokens,
|
|
hidden_stride_t,
|
|
hidden_stride_h,
|
|
hidden_stride_d,
|
|
kv_stride_t,
|
|
kv_stride_d,
|
|
q_stride_h,
|
|
q_stride_d,
|
|
k_stride_h,
|
|
k_stride_d,
|
|
mask_stride,
|
|
output_stride_t,
|
|
output_stride_h,
|
|
output_stride_d,
|
|
eps,
|
|
clamp_value,
|
|
DIM: tl.constexpr,
|
|
HC_MULT: tl.constexpr,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
HAS_MASK: tl.constexpr,
|
|
):
|
|
program_idx = tl.program_id(0)
|
|
token_idx = program_idx // HC_MULT
|
|
hc_idx = program_idx % HC_MULT
|
|
token_idx = token_idx.to(tl.int64)
|
|
source_idx = token_idx
|
|
source_valid = source_idx < num_kv_tokens
|
|
|
|
dim_offsets = tl.arange(0, BLOCK_SIZE)
|
|
dim_valid = dim_offsets < DIM
|
|
hidden = tl.load(
|
|
hidden_states
|
|
+ token_idx * hidden_stride_t
|
|
+ hc_idx * hidden_stride_h
|
|
+ dim_offsets * hidden_stride_d,
|
|
mask=dim_valid,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
key = tl.load(
|
|
kv + source_idx * kv_stride_t + (hc_idx * DIM + dim_offsets) * kv_stride_d,
|
|
mask=source_valid & dim_valid,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
q = tl.load(
|
|
q_weight + hc_idx * q_stride_h + dim_offsets * q_stride_d,
|
|
mask=dim_valid,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
k = tl.load(
|
|
k_weight + hc_idx * k_stride_h + dim_offsets * k_stride_d,
|
|
mask=dim_valid,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
|
|
hidden_rms = tl.rsqrt(tl.sum(hidden * hidden, axis=0) / DIM + eps)
|
|
key_rms = tl.rsqrt(tl.sum(key * key, axis=0) / DIM + eps)
|
|
dot = tl.sum(hidden * q * k * key, axis=0)
|
|
dot *= hidden_rms * key_rms * tl.rsqrt(DIM * 1.0)
|
|
gate_input = tl.sqrt(tl.maximum(tl.abs(dot), clamp_value))
|
|
gate_input = tl.where(dot < 0.0, -gate_input, gate_input)
|
|
gate = tl.sigmoid(gate_input)
|
|
if HAS_MASK:
|
|
active = tl.load(
|
|
token_mask + source_idx * mask_stride,
|
|
mask=source_valid,
|
|
other=0,
|
|
)
|
|
gate = tl.where(active, gate, 0.0)
|
|
|
|
value = tl.load(
|
|
kv + source_idx * kv_stride_t + (HC_MULT * DIM + dim_offsets) * kv_stride_d,
|
|
mask=source_valid & dim_valid,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
tl.store(
|
|
output
|
|
+ token_idx * output_stride_t
|
|
+ hc_idx * output_stride_h
|
|
+ dim_offsets * output_stride_d,
|
|
hidden + gate * value,
|
|
mask=dim_valid,
|
|
)
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_tokens", "token_start", "num_elements"])
|
|
def _engram_select_rows_kernel(
|
|
gathered,
|
|
output,
|
|
num_tokens,
|
|
token_start,
|
|
num_elements,
|
|
LOCAL_WIDTH: tl.constexpr,
|
|
WIDTH: tl.constexpr,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
):
|
|
offsets = tl.program_id(0).to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
|
tokens = token_start + offsets // WIDTH
|
|
cols = offsets % WIDTH
|
|
source = (cols // LOCAL_WIDTH * num_tokens + tokens) * LOCAL_WIDTH
|
|
source += cols % LOCAL_WIDTH
|
|
values = tl.load(
|
|
gathered + source, (offsets < num_elements) & (tokens < num_tokens), other=0
|
|
)
|
|
tl.store(output + offsets, values, offsets < num_elements)
|
|
|
|
|
|
def _engram_select_rows(
|
|
gathered: torch.Tensor,
|
|
output: torch.Tensor,
|
|
source_tokens: int,
|
|
token_start: int,
|
|
local_width: int,
|
|
) -> None:
|
|
"""Copy one token window out of a rank-major gathered buffer.
|
|
|
|
Both gathers land rank-major (`[rank][token][local width]`); this walks the
|
|
window the rank keeps and lays its ranks out side by side as width.
|
|
"""
|
|
if output.numel() == 0:
|
|
return
|
|
_engram_select_rows_kernel[(triton.cdiv(output.numel(), 1024),)](
|
|
gathered,
|
|
output,
|
|
source_tokens,
|
|
token_start,
|
|
output.numel(),
|
|
local_width,
|
|
output.shape[1] * output.shape[2],
|
|
BLOCK_SIZE=1024,
|
|
)
|
|
|
|
|
|
def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor:
|
|
"""Exchange DP tokens for heads, retaining only this replica's tokens."""
|
|
dp_group = get_engram_dp_group()
|
|
assert dp_group is not None
|
|
slot, remainder = divmod(staged.shape[0], dp_group.world_size)
|
|
assert remainder == 0 and 0 <= num_tokens <= slot
|
|
gathered = dp_group.all_gather(staged, dim=0)
|
|
local_heads, dim = staged.shape[1:]
|
|
rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim))
|
|
_engram_select_rows(
|
|
gathered,
|
|
rows,
|
|
staged.shape[0],
|
|
dp_group.rank_in_group * slot,
|
|
local_heads * dim,
|
|
)
|
|
return rows
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_tokens"])
|
|
def _engram_unpack_fp8_kernel(
|
|
packed,
|
|
values,
|
|
scales,
|
|
num_tokens,
|
|
values_stride,
|
|
scales_stride,
|
|
HEADS: tl.constexpr,
|
|
LOCAL_HEADS: tl.constexpr,
|
|
LAYERS: tl.constexpr,
|
|
RANK: tl.constexpr,
|
|
DIM: tl.constexpr,
|
|
BLOCK: tl.constexpr,
|
|
PEER: tl.constexpr,
|
|
):
|
|
"""Split `[rank][token][layer][local head]` packed rows into MXFP8 values and
|
|
F8_128x4 scales, one layer per grid column; `packed` holds per-rank
|
|
pointers if PEER. Scales past `num_tokens` zero-pad the 128-row tile.
|
|
"""
|
|
row = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
|
|
layer = tl.program_id(1).to(tl.int64)
|
|
token, head = row // HEADS, row % HEADS
|
|
if PEER:
|
|
src = tl.load(packed + head // LOCAL_HEADS)
|
|
src = tl.multiple_of(src, DIM // 32).to(tl.pointer_type(tl.uint8))
|
|
source = RANK * num_tokens + token
|
|
else:
|
|
src = packed
|
|
source = head // LOCAL_HEADS * num_tokens + token
|
|
source = (source * LAYERS + layer) * LOCAL_HEADS + head % LOCAL_HEADS
|
|
# Packed rows are only scale-width aligned.
|
|
src += tl.multiple_of(source * (DIM + DIM // 32), DIM // 32)
|
|
valid = token < num_tokens
|
|
cols = tl.arange(0, DIM)
|
|
data = tl.load(src[:, None] + cols[None, :], mask=valid[:, None], other=0)
|
|
values += layer * values_stride + row * DIM
|
|
tl.store(values[:, None] + cols[None, :], data, mask=valid[:, None])
|
|
scale_cols = tl.arange(0, DIM // 32)
|
|
scale = tl.load(
|
|
src[:, None] + DIM + scale_cols[None, :], mask=valid[:, None], other=0
|
|
)
|
|
group = head[:, None] * (DIM // 32) + scale_cols[None, :]
|
|
offset = mxfp8_scale_offset(token[:, None], group, HEADS * DIM // 32)
|
|
tl.store(scales + layer * scales_stride + offset, scale)
|
|
|
|
|
|
class EngramBatch(nn.Module):
|
|
"""Stage every Engram layer's WKV input as MXFP8 with one exchange.
|
|
|
|
A module so the model owns the peer exchange's pointer buffers.
|
|
"""
|
|
|
|
def __init__(self, layers: list["Engram"], max_tokens: int) -> None:
|
|
super().__init__()
|
|
first = layers[0].layer_hash_index
|
|
hash_indices = [layer.layer_hash_index for layer in layers]
|
|
assert hash_indices == list(range(first, first + len(layers)))
|
|
self.layers = layers
|
|
table = layers[0].embed_tokens
|
|
device = layers[0].staged_rows.device
|
|
self.packed_rows = torch.empty(
|
|
max_tokens * table.dp_size,
|
|
len(layers),
|
|
table.part_n_hash_cols,
|
|
table.dim + table.dim // 32,
|
|
dtype=torch.uint8,
|
|
device=device,
|
|
)
|
|
width = table.n_hash_cols * table.dim
|
|
self.fp8_values = torch.empty(
|
|
len(layers), max_tokens, width, dtype=torch.uint8, device=device
|
|
)
|
|
self.fp8_scales = torch.empty(
|
|
len(layers),
|
|
mxfp8_scale_bytes(max_tokens, width),
|
|
dtype=torch.uint8,
|
|
device=device,
|
|
)
|
|
max_peer_slot = min(max_tokens, ENGRAM_A2A_MIN_SLOT - 1)
|
|
self.peer_exchange = None
|
|
if table.cpu_offload:
|
|
from vllm.models.deepseek_v41.nvidia.ops.engram_peer import (
|
|
EngramPeerExchange,
|
|
)
|
|
|
|
self.peer_exchange = EngramPeerExchange.create(layers, max_peer_slot)
|
|
for layer in layers:
|
|
# Owned by the model; keep it out of each layer's submodules.
|
|
object.__setattr__(layer, "_batch", self)
|
|
# The layers read this batch's staging instead of their BF16 rows.
|
|
layer.staged_rows = layer.staged_rows.new_empty(
|
|
(0, *layer.staged_rows.shape[1:])
|
|
)
|
|
|
|
@classmethod
|
|
def create(cls, layers: list["Engram"], max_tokens: int) -> "EngramBatch | None":
|
|
"""Batch DP-sharded tables whose WKV takes FlashInfer MXFP8 input."""
|
|
if not layers:
|
|
return None
|
|
table = layers[0].embed_tokens
|
|
if (
|
|
table.dp_size == 1
|
|
or table.tp_size > 1
|
|
or table.dim % 128
|
|
or get_current_vllm_config().lora_config is not None
|
|
or not can_fuse_query_quant([layer.wkv for layer in layers])
|
|
):
|
|
return None
|
|
return cls(layers, max_tokens)
|
|
|
|
def prepare_embeddings(self, hashes: torch.Tensor) -> None:
|
|
"""Stage every layer's WKV input before the decoder layers.
|
|
|
|
DP-sharded tables decide on the group's slot, so every rank agrees.
|
|
"""
|
|
slot = engram_gathered_num_tokens()
|
|
peer = self.peer_exchange is not None and slot < ENGRAM_A2A_MIN_SLOT
|
|
self._stage(hashes if peer else gather_engram_hashes(hashes), slot, peer)
|
|
|
|
@eager_break_during_capture
|
|
def _stage(self, hashes: torch.Tensor, slot: int, peer: bool) -> None:
|
|
if slot == 0:
|
|
return
|
|
if peer:
|
|
assert self.peer_exchange is not None
|
|
rows = self.peer_exchange.exchange(hashes, slot)
|
|
rank = self.peer_exchange.dp_rank
|
|
else:
|
|
rows, rank = self.packed_rows[: hashes.shape[0]], 0
|
|
for index, layer in enumerate(self.layers):
|
|
layer.embed_tokens.lookup(
|
|
hashes[:, layer.layer_hash_index], rows[:, index]
|
|
)
|
|
group = get_engram_dp_group()
|
|
assert group is not None
|
|
sent, rows = rows, torch.empty_like(rows)
|
|
torch.distributed.all_to_all_single(rows, sent, group=group.device_group)
|
|
table = self.layers[0].embed_tokens
|
|
heads = table.n_hash_cols
|
|
grid = (triton.cdiv(slot, 128) * 128 * heads // 16, len(self.layers))
|
|
_engram_unpack_fp8_kernel[grid](
|
|
rows,
|
|
self.fp8_values,
|
|
self.fp8_scales,
|
|
slot,
|
|
self.fp8_values.stride(0),
|
|
self.fp8_scales.stride(0),
|
|
HEADS=heads,
|
|
LOCAL_HEADS=table.part_n_hash_cols,
|
|
LAYERS=len(self.layers),
|
|
RANK=rank,
|
|
DIM=table.dim,
|
|
BLOCK=16,
|
|
PEER=peer,
|
|
)
|
|
|
|
def wkv_input(self, layer: "Engram", num_tokens: int) -> QuantizedActivation:
|
|
index = layer.layer_hash_index - self.layers[0].layer_hash_index
|
|
values = self.fp8_values[index, :num_tokens].view(torch.float8_e4m3fn)
|
|
return QuantizedActivation(
|
|
values,
|
|
self.fp8_scales[index, : mxfp8_scale_bytes(num_tokens, values.shape[1])],
|
|
torch.bfloat16,
|
|
values.shape,
|
|
kMxfp8Dynamic,
|
|
)
|
|
|
|
|
|
class Engram(nn.Module):
|
|
"""Writes an n-gram lookup into the residual stream, gated by how well it
|
|
matches that stream.
|
|
|
|
The hash ids fetch `n_hash_cols` rows; `wkv` turns them into one key per
|
|
hc copy plus a shared value. The gate is a normalized dot product of the
|
|
stream against the key, signed-sqrt'ed before the sigmoid (matching the
|
|
training kernel).
|
|
"""
|
|
|
|
_prefetch_stream: torch.cuda.Stream | None = None
|
|
_prefetch_done: torch.cuda.Event | None = None
|
|
_batch: EngramBatch | None = None
|
|
|
|
def __init__(
|
|
self,
|
|
config,
|
|
quant_config: QuantizationConfig | None,
|
|
layout: EngramLayout,
|
|
layer_hash_index: int,
|
|
use_sequence_parallel: bool,
|
|
prefix: str,
|
|
prefetch_stream: torch.cuda.Stream | None = None,
|
|
) -> None:
|
|
super().__init__()
|
|
# Layers sharing one stream serialize their offloaded lookups, so they
|
|
# take turns instead of jointly starving decoder compute of SMs.
|
|
self._prefetch_stream = prefetch_stream
|
|
self.layer_hash_index = layer_hash_index
|
|
self.dim = config.hidden_size
|
|
self.hc_mult = config.hc_mult
|
|
self.eps = config.rms_norm_eps
|
|
self.clamp_value = 1e-6
|
|
self.use_sequence_parallel = use_sequence_parallel
|
|
|
|
# Named ``embed_tokens`` so the checkpoint's ``engram.embed.weight``
|
|
# survives the mapper's ``embed.weight`` -> ``embed_tokens.weight``
|
|
# suffix rule.
|
|
self.embed_tokens = self._create_embedding(layout, layer_hash_index)
|
|
n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads
|
|
# Without sequence parallelism every TP rank holds every token, so
|
|
# shard the output columns instead of replicating the projection.
|
|
self.wkv = ColumnParallelLinear(
|
|
n_hash_cols * layout.head_dim,
|
|
self.dim * (self.hc_mult + 1),
|
|
bias=False,
|
|
gather_output=True,
|
|
quant_config=quant_config,
|
|
return_bias=False,
|
|
prefix=f"{prefix}.wkv",
|
|
disable_tp=use_sequence_parallel,
|
|
)
|
|
self.q_weight = nn.Parameter(
|
|
torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),
|
|
requires_grad=False,
|
|
)
|
|
self.k_weight = nn.Parameter(
|
|
torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16),
|
|
requires_grad=False,
|
|
)
|
|
|
|
max_tokens = get_current_vllm_config().scheduler_config.max_num_batched_tokens
|
|
self._init_staging(max_tokens, layout.head_dim)
|
|
|
|
def _create_embedding(
|
|
self, layout: EngramLayout, layer_hash_index: int
|
|
) -> ParallelEngramEmbedding:
|
|
engram_config = get_current_vllm_config().engram_config
|
|
assert engram_config is not None
|
|
return ParallelEngramEmbedding(
|
|
layout.num_embeddings[layer_hash_index],
|
|
layout.head_dim,
|
|
tuple(size for order in layout.primes[layer_hash_index] for size in order),
|
|
cpu_offload=engram_config.cpu_offload,
|
|
dp_shared_memory=bool(engram_config.dp_shared_memory),
|
|
use_thp=engram_config.use_thp,
|
|
)
|
|
|
|
def _init_staging(self, max_tokens: int, head_dim: int) -> None:
|
|
# Persistent storage keeps lookup addresses stable across graph replays.
|
|
self.staged_rows = torch.empty(
|
|
max_tokens * self.embed_tokens.dp_size,
|
|
self.embed_tokens.part_n_hash_cols,
|
|
head_dim,
|
|
dtype=torch.bfloat16,
|
|
)
|
|
if not self.embed_tokens.cpu_offload:
|
|
self._prefetch_stream = None
|
|
return
|
|
if self._prefetch_stream is None:
|
|
self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device)
|
|
self._prefetch_done = torch.cuda.Event()
|
|
|
|
def prepare_embeddings(self, hash_ids: torch.Tensor) -> None:
|
|
"""Look up, or prefetch from host memory, rows before the decoder layers
|
|
consume them; DP-sharded tables take the group's gathered hash IDs."""
|
|
rows = self.staged_rows[: hash_ids.shape[0]]
|
|
assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small"
|
|
if self._prefetch_stream is None:
|
|
self.embed_tokens.lookup(hash_ids, rows)
|
|
else:
|
|
self._start_prefetch(hash_ids, rows)
|
|
|
|
@eager_break_during_capture
|
|
def _start_prefetch(self, hash_ids: torch.Tensor, rows: torch.Tensor) -> None:
|
|
# Eager boundaries let the lookup span piecewise graph segments, and
|
|
# decide per replay (on the replay stream) whether it runs inline.
|
|
num_tokens = hash_ids.shape[0]
|
|
# All DP ranks must agree, or EP collectives wait on the slowest one.
|
|
if is_forward_context_available() and (dp := get_forward_context().dp_metadata):
|
|
num_tokens = int(dp.num_tokens_across_dp_cpu.max())
|
|
current = stream = torch.cuda.current_stream()
|
|
# Big lookups stall persistent main-stream kernels anyway: run them inline.
|
|
_, inline_min_tokens = _engram_lookup_thresholds(hash_ids.device)
|
|
if inline_min_tokens is not None and num_tokens >= inline_min_tokens:
|
|
self.embed_tokens.lookup(hash_ids, rows)
|
|
else:
|
|
stream = self._prefetch_stream
|
|
assert stream is not None
|
|
stream.wait_stream(current)
|
|
# Keep temporary hash storage alive until lookup finishes reading it.
|
|
hash_ids.record_stream(stream)
|
|
with torch.cuda.stream(stream):
|
|
self.embed_tokens.lookup(hash_ids, rows, background=True)
|
|
assert self._prefetch_done is not None
|
|
self._prefetch_done.record(stream)
|
|
|
|
@eager_break_during_capture
|
|
def _finish_prefetch(self, event: torch.cuda.Event) -> None:
|
|
# Wait for this layer's lookup only, not for later layers on the stream.
|
|
torch.cuda.current_stream().wait_event(event)
|
|
|
|
def _ready_rows(self, num_tokens: int) -> torch.Tensor:
|
|
if self._prefetch_done is not None:
|
|
self._finish_prefetch(self._prefetch_done)
|
|
if self.embed_tokens.dp_size > 1:
|
|
slot = engram_gathered_num_tokens()
|
|
staged = self.staged_rows[: slot * self.embed_tokens.dp_size]
|
|
return _gather_engram_rows(staged, num_tokens)
|
|
return self.staged_rows[:num_tokens]
|
|
|
|
def embed(self, hash_ids: torch.Tensor) -> torch.Tensor:
|
|
"""Gather heads, returning only local tokens when SP is enabled."""
|
|
rows = self._ready_rows(hash_ids.shape[0])
|
|
if self.embed_tokens.tp_size == 1:
|
|
return rows[:, : self.embed_tokens.n_hash_cols]
|
|
if self.use_sequence_parallel:
|
|
tp_size = self.embed_tokens.tp_size
|
|
num_tokens, local_heads, dim = rows.shape
|
|
gathered = tensor_model_parallel_all_gather(rows, dim=0)
|
|
chunk = (num_tokens + tp_size - 1) // tp_size
|
|
out = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim))
|
|
_engram_select_rows(
|
|
gathered,
|
|
out,
|
|
num_tokens,
|
|
get_tensor_model_parallel_rank() * chunk,
|
|
local_heads * dim,
|
|
)
|
|
return out
|
|
rows = tensor_model_parallel_all_gather(rows, dim=1)
|
|
return rows[:, : self.embed_tokens.n_hash_cols]
|
|
|
|
def forward(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
hash_ids: torch.Tensor,
|
|
token_mask: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all
|
|
tokens, pre sequence-parallel shard); token_mask: [T], False shuts
|
|
the gate so those positions pass through untouched."""
|
|
if self._batch is not None:
|
|
kv = self.wkv(self._batch.wkv_input(self, hash_ids.shape[0]))
|
|
else:
|
|
kv = self.wkv(self.embed(hash_ids).flatten(-2))
|
|
num_kv_tokens = hash_ids.shape[0]
|
|
assert token_mask is None or token_mask.shape == (num_kv_tokens,)
|
|
if self.use_sequence_parallel:
|
|
tp_size = get_tensor_model_parallel_world_size()
|
|
tp_rank = get_tensor_model_parallel_rank()
|
|
shard_size = (num_kv_tokens + tp_size - 1) // tp_size
|
|
assert hidden_states.shape[0] == shard_size
|
|
start = min(tp_rank * shard_size, num_kv_tokens)
|
|
num_kv_tokens = min(shard_size, num_kv_tokens - start)
|
|
if token_mask is not None:
|
|
token_mask = token_mask[start : start + num_kv_tokens]
|
|
|
|
num_tokens, hc_mult, dim = hidden_states.shape
|
|
assert hc_mult == self.hc_mult and dim == self.dim
|
|
assert kv.ndim == 2 and kv.shape[1] == (hc_mult + 1) * dim
|
|
output = torch.empty_like(hidden_states)
|
|
if num_tokens == 0:
|
|
return output
|
|
|
|
block_size = triton.next_power_of_2(dim)
|
|
num_warps = 8 if block_size >= 2048 else 4
|
|
mask = token_mask if token_mask is not None else hidden_states
|
|
_fused_engram_post_wkv_kernel[(num_tokens * hc_mult,)](
|
|
hidden_states,
|
|
kv,
|
|
self.q_weight,
|
|
self.k_weight,
|
|
mask,
|
|
output,
|
|
num_kv_tokens,
|
|
hidden_states.stride(0),
|
|
hidden_states.stride(1),
|
|
hidden_states.stride(2),
|
|
kv.stride(0),
|
|
kv.stride(1),
|
|
self.q_weight.stride(0),
|
|
self.q_weight.stride(1),
|
|
self.k_weight.stride(0),
|
|
self.k_weight.stride(1),
|
|
token_mask.stride(0) if token_mask is not None else 0,
|
|
output.stride(0),
|
|
output.stride(1),
|
|
output.stride(2),
|
|
self.eps,
|
|
self.clamp_value,
|
|
DIM=dim,
|
|
HC_MULT=hc_mult,
|
|
BLOCK_SIZE=block_size,
|
|
HAS_MASK=token_mask is not None,
|
|
num_warps=num_warps,
|
|
)
|
|
return output
|