# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Engram: n-gram hash lookups gated into the hyper-connection stream. Port of the reference ``inference/engram.py`` + ``Engram`` / ``ParallelEngramEmbedding`` from ``inference/model.py`` (DeepSeek V4.1 checkpoint layout). Engram modules live on the backbone layers listed in ``engram_layer_ids`` only. Two pieces of cross-forward state are needed because vLLM streams tokens chunk-by-chunk while an n-gram at position ``p`` needs the token ids at ``p-1..p-3``: - ``token_map``: token id -> compressed vocab id, built once from the model's tokenizer at init (deterministic; asserted against ``engram_compressed_vocab_size``). - ``hash_cache``: one int32 slot per KV slot of the first local layer's sliding-window cache, holding the compressed id (or DEAD) of the token last written to that slot. Slots are stable per (request, position) — the block table pins a position to a physical slot, prefix-cache hits reuse both the physical blocks and the identical token ids, and spec-decode rollbacks rewrite the same slots — so lookbacks read back exactly what the owning request wrote. Lookback depth (3) is far inside the sliding window (128), so window eviction never frees a block a live lookback still needs. Slots are not part of the KV cache, so KV loaded from another instance (P/D, offload connectors) leaves them unwritten. The runner therefore passes ``lookback_token_ids``, the ids just before each request's chunk start, which take precedence over the slots. The V2 runner reads them from its device-resident token history and needs no slot cache; the V1 runner's CPU token table holds placeholders for generated tokens under async scheduling, so it passes prompt positions only and keeps the slot cache for the rest. """ import ctypes import mmap import os import tempfile import weakref from contextlib import ExitStack import numpy as np import torch from torch import nn from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.config import VllmConfig, get_current_vllm_config from vllm.distributed import ( get_dp_group, get_engram_dp_group, get_engram_dp_size, get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, tensor_model_parallel_all_gather, ) from vllm.distributed.device_communicators.shm_broadcast import ( SHM_PATH, check_shm_free_space, ) from vllm.distributed.parallel_state import GroupCoordinator from vllm.forward_context import get_forward_context, is_forward_context_available from vllm.logger import init_logger from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic from vllm.model_executor.utils import set_weight_attrs from vllm.models.deepseek_v41.common.ops.query_quant import ( can_fuse_query_quant, mxfp8_scale_bytes, mxfp8_scale_offset, ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.platform_utils import is_uva_available from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor logger = init_logger(__name__) # Per-rank token slot from which the Engram DP exchange switches from peer # reads to all-to-all. ENGRAM_A2A_MIN_SLOT = 128 # Cache value for tokens that take no part in an n-gram (image spans). DEAD_ID = -1 def _engram_lookup_thresholds(device: torch.device) -> tuple[int | None, int | None]: """Host lookup thresholds on `device`: the rows from which to sort, and the tokens (max over DP ranks) from which to run inline. None disables that path.""" if current_platform.is_cuda() and current_platform.is_device_capability_family( 100, device.index or 0 ): # Measured on GB200. return 36864, 4096 # TODO: Add hand-tuned thresholds for other GPU families. return None, None def _is_prime(n: int) -> bool: """Deterministic Miller-Rabin for n < 2**32 (avoids a sympy import).""" if n < 2: return False for p in (2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37): if n % p == 0: return n == p d = n - 1 r = 0 while d % 2 == 0: d //= 2 r += 1 for a in (2, 7, 61): x = pow(a, d, n) if x in (1, n - 1): continue for _ in range(r - 1): x = x * x % n if x == n - 1: break else: return False return True def find_next_prime(start: int, seen_primes: set[int]) -> int: """The smallest prime above `start` that has not been handed out yet.""" candidate = start + 1 while not _is_prime(candidate) or candidate in seen_primes: candidate += 1 return candidate def build_compressed_token_map(tokenizer) -> tuple[list[int], int]: """Map every token id onto a smaller id space where tokens that normalize alike collapse together. N-grams are hashed over these compressed ids, so " The", "the" and "THE" all hash the same way. The compressed size matters beyond bounds checking: every hash multiplier is derived from it. """ from tokenizers import Regex, normalizers # A private-use char, so a token that is exactly one space survives # Strip() instead of collapsing to the empty string and merging with # unrelated tokens. sentinel = "\ue000" normalizer = normalizers.Sequence( [ normalizers.NFKC(), normalizers.NFD(), normalizers.StripAccents(), normalizers.Lowercase(), normalizers.Replace(Regex(r"[ \t\r\n]+"), " "), normalizers.Replace(Regex(r"^ $"), sentinel), normalizers.Strip(), normalizers.Replace(sentinel, " "), ] ) # The raw Rust tokenizer, matching what training decodes with # (no clean_up_tokenization_spaces). backend = tokenizer.backend_tokenizer key_to_new: dict[str, int] = {} lookup = [0] * len(tokenizer) for token_id in range(len(tokenizer)): text = backend.decode([token_id], skip_special_tokens=False) if "\ufffd" in text: # A partial UTF-8 byte token: nothing to normalize, so key it # by its raw form. key = backend.id_to_token(token_id) else: normalized = normalizer.normalize_str(text) key = normalized if normalized else text new_id = key_to_new.get(key) if new_id is None: new_id = len(key_to_new) key_to_new[key] = new_id lookup[token_id] = new_id return lookup, len(key_to_new) def compute_hash_multipliers( layer_ids: tuple[int, ...], max_ngram_size: int, compressed_vocab_size: int ) -> torch.Tensor: """One multiplier per (layer, lookback), from a per-layer RNG so layers hash differently. Kept odd and bounded so `token_id * multiplier` cannot overflow int64. """ max_long = np.iinfo(np.int64).max multiplier_bound = max(1, (max_long // compressed_vocab_size) // 2) rows = [] for layer_id in layer_ids: generator = np.random.default_rng(10007 * layer_id) values = generator.integers( low=0, high=multiplier_bound, size=(max_ngram_size,), dtype=np.int64, ) rows.append(torch.tensor(values * 2 + 1)) return torch.stack(rows) class EngramLayout: """Bucket layout of the n-gram hash tables. A position is hashed as `max_ngram_size - 1` n-grams (2-gram .. max), each split over `n_heads` heads. Every (n-gram size, head) pair owns its own prime-sized bucket range in the layer's table; the primes are drawn in order and never reused, which keeps the ranges disjoint. """ def __init__(self, config) -> None: self.layer_ids: tuple[int, ...] = tuple(config.engram_layer_ids) self.num_embeddings: tuple[int, ...] = tuple(config.engram_num_embeddings) self.max_ngram_size: int = config.engram_max_ngram_size self.n_heads: int = config.engram_n_heads self.head_dim: int = config.engram_head_dim self.compressed_vocab_size: int = config.engram_compressed_vocab_size self.pad_token_id: int = config.engram_pad_token_id assert len(self.layer_ids) == len(self.num_embeddings) primes = [] seen: set[int] = set() for _ in self.layer_ids: per_ngram = [] for _ in range(self.max_ngram_size - 1): sizes, current = [], config.engram_vocab_size - 1 for _ in range(self.n_heads): current = find_next_prime(current, seen) seen.add(current) sizes.append(current) per_ngram.append(tuple(sizes)) primes.append(tuple(per_ngram)) self.primes: tuple[tuple[tuple[int, ...], ...], ...] = tuple(primes) self.n_hash_cols = (self.max_ngram_size - 1) * self.n_heads flat = [[p for per_ngram in layer for p in per_ngram] for layer in primes] offsets = [np.cumsum([0, *sizes[:-1]]) for sizes in flat] self.offsets = torch.tensor(np.array(offsets)) # [n_layers, n_hash_cols] @classmethod def from_config(cls, config) -> "EngramLayout | None": if not getattr(config, "engram_layer_ids", None): return None return cls(config) @triton.jit(do_not_specialize=["num_tokens"]) def _write_hash_cache_kernel( input_ids, token_map, dead_mask, slot_mapping, cache, num_tokens, input_stride, mask_stride, slot_stride, BLOCK_SIZE: tl.constexpr, dead_id, ): token_idx = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) slot = tl.load( slot_mapping + token_idx * slot_stride, token_idx < num_tokens, other=-1 ).to(tl.int64) valid = (token_idx < num_tokens) & (slot >= 0) token = tl.load(input_ids + token_idx * input_stride, valid, other=0) value = tl.load(token_map + token, valid, other=0) dead = tl.load(dead_mask + token_idx * mask_stride, valid, other=False) value = tl.where(dead, dead_id, value) tl.store(cache + slot, value, valid) # Keep request shapes out of the cache key so one warmed variant covers runtime batches. @triton.jit( do_not_specialize=[ "num_tokens", "num_slots", "num_query_rows", "num_table_rows", "max_blocks", ] ) def _hash_ids_kernel( input_ids, token_map, dead_mask, positions, block_table, query_start_loc, multipliers, primes, offsets, cache, lookback_token_ids, lookback_dead_mask, output, num_tokens, num_slots, pad_id, input_stride, mask_stride, position_stride, table_stride, table_col_stride, query_stride, num_query_rows, num_table_rows, max_blocks, cache_block_size, MAX_NGRAM: tl.constexpr, num_heads, BLOCK_T: tl.constexpr, BLOCK_H: tl.constexpr, dead_id, lookback_depth, lookback_row_stride, lookback_col_stride, lookback_mask_row_stride, lookback_mask_col_stride, ): token = tl.program_id(0) * BLOCK_T + tl.arange(0, BLOCK_T) layer = tl.program_id(1) num_layers = tl.num_programs(1) valid = token < num_tokens # Upper bound in query_start_loc[1:], including repeated padding boundaries. lo = tl.full((BLOCK_T,), 0, tl.int32) hi = tl.full((BLOCK_T,), num_query_rows, tl.int32) while tl.sum((lo < hi).to(tl.int32), 0) > 0: mid = (lo + hi) // 2 end = tl.load( query_start_loc + (mid + 1) * query_stride, lo < hi, other=0, ) right = token >= end active = lo < hi lo = tl.where(active & right, mid + 1, lo) hi = tl.where(active & ~right, mid, hi) req = tl.minimum(lo, num_query_rows - 1).to(tl.int64) chunk_idx = tl.load(query_start_loc + req * query_stride) chunk_idx = tl.minimum(chunk_idx, num_tokens - 1).to(tl.int64) chunk_start = tl.load(positions + chunk_idx * position_stride) position = tl.load(positions + token * position_stride, valid, other=0).to(tl.int64) head = tl.arange(0, BLOCK_H) blocked = tl.full((BLOCK_T,), False, tl.int1) rolling = tl.full((BLOCK_T,), 0, tl.int64) for shift in tl.static_range(MAX_NGRAM): lookback = position - shift in_batch = lookback >= chunk_start batch_idx = tl.maximum(token - shift, 0) batch_token = tl.load( input_ids + batch_idx * input_stride, valid & in_batch, other=0 ) batch_source = tl.load(token_map + batch_token, valid & in_batch, other=0) batch_dead = tl.load( dead_mask + batch_idx * mask_stride, valid & in_batch, other=False ) batch_source = tl.where(batch_dead, dead_id, batch_source) col = chunk_start - 1 - lookback in_window = valid & ~in_batch & (col >= 0) & (col < lookback_depth) col = tl.minimum(tl.maximum(col, 0), lookback_depth - 1) window_token = tl.load( lookback_token_ids + req * lookback_row_stride + col * lookback_col_stride, in_window, other=-1, ) known = in_window & (window_token >= 0) window_source = tl.load(token_map + window_token, known, other=0) window_dead = tl.load( lookback_dead_mask + req * lookback_mask_row_stride + col * lookback_mask_col_stride, known, other=False, ) window_source = tl.where(window_dead, dead_id, window_source) if cache is not None: clamped = tl.minimum( tl.maximum(lookback, 0), max_blocks * cache_block_size - 1 ) block_row = tl.minimum(req, num_table_rows - 1) needs_cache = valid & ~in_batch & ~known block = tl.load( block_table + block_row * table_stride + (clamped // cache_block_size) * table_col_stride, needs_cache, other=0, ).to(tl.int64) slot = tl.minimum( tl.maximum(block * cache_block_size + clamped % cache_block_size, 0), num_slots - 1, ) fallback = tl.load(cache + slot, needs_cache, other=0) else: fallback = tl.full((BLOCK_T,), pad_id, tl.int32) source = tl.where( in_batch, batch_source, tl.where(known, window_source, fallback) ).to(tl.int64) blocked |= (lookback < 0) | (source == dead_id) value = tl.where(blocked, pad_id, source) multiplier = tl.load(multipliers + layer * MAX_NGRAM + shift) rolling ^= value * multiplier if shift > 0: col = (shift - 1) * num_heads + head param_offset = layer * (MAX_NGRAM - 1) * num_heads + col prime = tl.load(primes + param_offset, head < num_heads, other=1) offset = tl.load(offsets + param_offset, head < num_heads, other=0) hashed = rolling[:, None] % prime[None, :] + offset[None, :] out_offset = (token.to(tl.int64) * num_layers + layer)[:, None] * ( (MAX_NGRAM - 1) * num_heads ) + col[None, :] tl.store(output + out_offset, hashed, valid[:, None] & (head < num_heads)) class NgramHashState(nn.Module): """Maps each position to the hash ids of the n-grams ending there. Stateless on the V2 runner, which supplies every lookback token id. On the V1 runner it also keeps `hash_cache`, the slot-keyed rolling store of compressed ids (see module docstring), for generated tokens. """ def __init__( self, vllm_config: VllmConfig, layout: EngramLayout, swa_cache_module: nn.Module, ) -> None: super().__init__() self.layout = layout self.swa_cache_module = swa_cache_module self.block_size: int = swa_cache_module.block_size self.lookback_depth: int = layout.max_ngram_size - 1 self.use_slot_cache: bool = not vllm_config.use_v2_model_runner self._cache: torch.Tensor | None = None self._kv_cache_ref: weakref.ReferenceType[torch.Tensor] | None = None model_config = vllm_config.model_config from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained( model_config.tokenizer, trust_remote_code=model_config.trust_remote_code, revision=model_config.revision, ) token_map, vocab_size = build_compressed_token_map(tokenizer) if vocab_size != layout.compressed_vocab_size: raise ValueError( f"Compressed vocab size mismatch: built {vocab_size} from the " f"tokenizer, config expects {layout.compressed_vocab_size}; " "every hash multiplier derives from it, so the engram tables " "would be silently rehashed." ) self.pad_id = token_map[layout.pad_token_id] multipliers = compute_hash_multipliers( layout.layer_ids, layout.max_ngram_size, vocab_size ) self.register_buffer( "token_map", torch.tensor(token_map, dtype=torch.int32), persistent=False ) self.register_buffer("primes", torch.tensor(layout.primes), persistent=False) self.register_buffer("offsets", layout.offsets, persistent=False) self.register_buffer("multipliers", multipliers, persistent=False) logger.info( "Built engram token map (%d -> %d ids) for layers %s", len(token_map), vocab_size, layout.layer_ids, ) def ensure_cache(self) -> bool: """Lazily size the slot-keyed cache from the bound SWA KV cache. Returns False while the KV cache is unbound (profile run); the caller skips engram hashing then. Without the slot cache only that check remains. """ kv_cache = self.swa_cache_module.kv_cache if kv_cache.numel() == 0: self._cache = None self._kv_cache_ref = None return False if not self.use_slot_cache: return True if self._kv_cache_ref is not None and self._kv_cache_ref() is kv_cache: return True # Graph memory profiling binds a temporary, smaller KV cache first. # Rebinding must discard its hash history without retaining KV storage. self._cache = torch.zeros( kv_cache.shape[0] * self.block_size, dtype=torch.int32, device=kv_cache.device, ) self._kv_cache_ref = weakref.ref(kv_cache) return True def dummy_hashes( self, input_ids: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: """Participate in DP lookups without valid rows or hash-cache updates.""" num_tokens = input_ids.shape[0] num_layers, max_ngram = self.multipliers.shape num_heads = self.primes.shape[-1] hashes = input_ids.new_full( (num_tokens, num_layers, (max_ngram - 1) * num_heads), DEAD_ID, dtype=torch.int32, ) keep = torch.zeros(num_tokens, dtype=torch.bool, device=input_ids.device) return hashes, keep def forward( self, input_ids: torch.Tensor, positions: torch.Tensor, query_start_loc: torch.Tensor, dead_mask: torch.Tensor, lookback_token_ids: torch.Tensor, lookback_dead_mask: torch.Tensor, slot_mapping: torch.Tensor | None, block_table: torch.Tensor | None, ) -> torch.Tensor: """Compute [tokens, layers, hash columns] int32 n-gram hashes. History comes from the current chunk, then the runner's lookback window, then the optional V1 slot cache. V2 needs only one launch. """ cache = self._cache if self.use_slot_cache else None num_tokens = input_ids.shape[0] num_layers, max_ngram = self.multipliers.shape num_heads = self.primes.shape[-1] output = input_ids.new_empty( (num_tokens, num_layers, (max_ngram - 1) * num_heads), dtype=torch.int32 ) if num_tokens == 0: return output if self.use_slot_cache: assert cache is not None and slot_mapping is not None assert block_table is not None # Finish writes before other thread blocks read fallback history. _write_hash_cache_kernel[(triton.cdiv(num_tokens, 256),)]( input_ids, self.token_map, dead_mask, slot_mapping, cache, num_tokens, input_ids.stride(0), dead_mask.stride(0), slot_mapping.stride(0), 256, DEAD_ID, ) _hash_ids_kernel[(triton.cdiv(num_tokens, 32), num_layers)]( input_ids, self.token_map, dead_mask, positions, block_table, query_start_loc, self.multipliers, self.primes, self.offsets, cache, lookback_token_ids, lookback_dead_mask, output, num_tokens, cache.shape[0] if cache is not None else 0, self.pad_id, input_stride=input_ids.stride(0), mask_stride=dead_mask.stride(0), position_stride=positions.stride(0), table_stride=block_table.stride(0) if block_table is not None else 0, table_col_stride=block_table.stride(1) if block_table is not None else 0, query_stride=query_start_loc.stride(0), num_query_rows=query_start_loc.numel() - 1, num_table_rows=block_table.shape[0] if block_table is not None else 0, max_blocks=block_table.shape[1] if block_table is not None else 0, cache_block_size=self.block_size, MAX_NGRAM=max_ngram, num_heads=num_heads, BLOCK_T=32, BLOCK_H=triton.next_power_of_2(num_heads), dead_id=DEAD_ID, lookback_depth=lookback_token_ids.shape[1], lookback_row_stride=lookback_token_ids.stride(0), lookback_col_stride=lookback_token_ids.stride(1), lookback_mask_row_stride=lookback_dead_mask.stride(0), lookback_mask_col_stride=lookback_dead_mask.stride(1), num_warps=4, ) return output def _engram_head_shard_weight_loader( param: torch.nn.Parameter, loaded_weight: torch.Tensor ) -> None: """Load this rank's complete head buckets. ue8m0 scales arrive as float8_e8m0fnu; keep the raw bytes (the param stores uint8).""" part_rows = param.shape[0] if loaded_weight.dtype == torch.float8_e8m0fnu: loaded_weight = loaded_weight.view(torch.uint8) shard = loaded_weight.narrow(0, param.engram_vocab_start, part_rows) assert shard.shape == param.shape, ( f"engram shard {tuple(shard.shape)} does not fit param {tuple(param.shape)}" ) param.data.copy_(shard) # Branching on SORTED at runtime intermittently crashes Triton 3.8's # RemoveLayoutConversions. # TODO: Remove the 3.8 check once Triton ships # https://github.com/triton-lang/triton/pull/10706. _SORTED_IS_CONSTEXPR = tl.constexpr( triton.__version__.startswith("3.8") and current_platform.is_rocm() ) @triton.jit( do_not_specialize=[ "vocab_start", "vocab_end", "num_rows", "ids_stride_t", "ids_stride_h", "GRID", *(() if _SORTED_IS_CONSTEXPR else ("SORTED",)), ] ) def _engram_lookup_kernel( weight, scales, ids, sorted_dst, out, vocab_start, vocab_end, num_rows, ids_stride_t, ids_stride_h, HEAD_START: tl.constexpr, LOCAL_HEADS: tl.constexpr, TOTAL_HEADS: tl.constexpr, DIM: tl.constexpr, QUANT_BLOCK: tl.constexpr, BLOCK_R: tl.constexpr, GRID, SORTED: tl.constexpr if _SORTED_IS_CONSTEXPR else None, # type: ignore[valid-type] PACKED: tl.constexpr, OUT_STRIDE: tl.constexpr, ): """Gather FP8 rows as packed values/scales or dequantized BF16. Only this rank's heads are read; padded heads write zeros for all-gather. `weight`/`scales` may address pinned host memory through UVA. If SORTED, `ids` holds rows relative to vocab_start in table order and `sorted_dst` their output rows. """ cols = tl.arange(0, DIM) scale_cols = cols // QUANT_BLOCK for base in tl.range(tl.program_id(0) * BLOCK_R, num_rows, GRID * BLOCK_R): rows = base + tl.arange(0, BLOCK_R) valid = rows < num_rows if SORTED: local = tl.load(ids + rows, mask=valid, other=-1).to(tl.int64) dst = tl.load(sorted_dst + rows, mask=valid, other=0).to(tl.int32) owned = (local >= 0) & (local < vocab_end - vocab_start) else: head = HEAD_START + rows % LOCAL_HEADS token = (rows // LOCAL_HEADS).to(tl.int64) index = tl.load( ids + token * ids_stride_t + head * ids_stride_h, mask=valid & (head < TOTAL_HEADS), other=-1, ).to(tl.int64) dst = rows owned = valid & (head < TOTAL_HEADS) owned &= (index >= vocab_start) & (index < vocab_end) local = index - vocab_start local = tl.where(owned, local, 0) values = tl.load( weight + local[:, None] * DIM + cols[None, :], mask=owned[:, None], other=0.0, ) scale = tl.load( scales + local[:, None] * (DIM // QUANT_BLOCK) + scale_cols[None, :], mask=owned[:, None], other=0, ) if PACKED: width: tl.constexpr = DIM + DIM // QUANT_BLOCK dst = dst.to(tl.int64) dst = dst // LOCAL_HEADS * OUT_STRIDE + dst % LOCAL_HEADS * width tl.store( out + dst[:, None] + cols[None, :], values.to(tl.uint8, bitcast=True), mask=valid[:, None], ) tl.store( out + dst[:, None] + DIM + scale_cols[None, :], scale, mask=valid[:, None] & (cols[None, :] % QUANT_BLOCK == 0), ) else: # ue8m0 is a power of two, so its byte *is* the fp32 exponent field. scale = (scale.to(tl.int32) << 23).to(tl.float32, bitcast=True) tl.store( out + dst[:, None] * DIM + cols[None, :], (values.to(tl.float32) * scale).to(tl.bfloat16), mask=valid[:, None], ) def engram_head_shard_rank() -> int: """This rank's slot among the hash-head shards of one engram table. TP-major, so the shards a DP gather brings in are contiguous heads and the following TP gather completes the head order. """ dp_group = get_engram_dp_group() dp_size = dp_group.world_size if dp_group is not None else 1 dp_rank = dp_group.rank_in_group if dp_group is not None else 0 return get_tensor_model_parallel_rank() * dp_size + dp_rank def engram_gathered_num_tokens() -> int: """Per-replica token slot for the node-local Engram DP group.""" dp_metadata = get_forward_context().dp_metadata if dp_metadata is None: raise RuntimeError("a DP-shared engram table needs DP token metadata") group = get_engram_dp_group() assert group is not None # Engram groups are contiguous slices of the full DP group. start = get_dp_group().rank_in_group - group.rank_in_group return int( dp_metadata.num_tokens_across_dp_cpu[start : start + group.world_size].max() ) def gather_engram_hashes( hash_ids: torch.Tensor, *, dp_shared_memory: bool = False ) -> torch.Tensor: """Collect the n-gram ids of every DP replica sharing one table. Replicas are padded to a common token slot, so the gathered shape is static under CUDA graph capture (where DP already pads alike). """ dp_group = get_engram_dp_group() if dp_group is None or dp_shared_memory: return hash_ids slot = engram_gathered_num_tokens() if hash_ids.shape[0] > slot: raise ValueError("Engram token count exceeds the DP token slot") if hash_ids.shape[0] < slot: pad = hash_ids.new_full( (slot - hash_ids.shape[0], *hash_ids.shape[1:]), DEAD_ID ) hash_ids = torch.cat((hash_ids, pad)) return dp_group.all_gather(hash_ids, dim=0) class DPSharedEngramStorage: """Registered host weights shared by a node-local DP group with one writer.""" def __init__( self, num_rows: int, dim: int, block_size: int, group: GroupCoordinator ) -> None: self.group = group weight_bytes = num_rows * dim storage = self._allocate(weight_bytes + weight_bytes // block_size) self.weight = storage[:weight_bytes].view(torch.float8_e4m3fn).view(-1, dim) self.weight_scale_inv = storage[weight_bytes:].view(-1, dim // block_size) self._views: tuple[torch.Tensor, torch.Tensor] | None = None def _allocate(self, num_bytes: int) -> torch.Tensor: """Map and register one physical allocation across a node-local DP group.""" group = self.group with ExitStack() as stack: path = error = None if group.rank_in_group == 0: try: check_shm_free_space(num_bytes) backing_file = stack.enter_context( tempfile.NamedTemporaryFile(prefix="vllm_engram_", dir=SHM_PATH) ) backing_file.truncate(num_bytes) path = backing_file.name except Exception as exc: error = f"{type(exc).__name__}: {exc}" path, error = group.broadcast_object((path, error)) if error is not None: raise RuntimeError( "Engram shared-memory creation failed on EDP rank 0: " + error ) mapping = owner = tensor = None try: with open(path, "r+b") as file: mapping = mmap.mmap(file.fileno(), num_bytes, flags=mmap.MAP_SHARED) owner = np.frombuffer(mapping, dtype=np.uint8) pointer = owner.ctypes.data tensor = torch.from_numpy(owner) result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) if result.value != 0: raise RuntimeError(f"cudaHostRegister failed: {result}") finalizer = weakref.finalize(owner, self._unregister, mapping, pointer) finalizer.atexit = False # type: ignore[misc] # The UVA helper otherwise allocates a private pinned copy. if not tensor.is_pinned(): raise RuntimeError( "cudaHostRegister did not pin the shared Engram mapping" ) except Exception as exc: error = f"{type(exc).__name__}: {exc}" # Also fences peer mappings before the leader unlinks the file. errors: list[str | None] = [None] * group.world_size torch.distributed.all_gather_object(errors, error, group=group.cpu_group) failures = "; ".join( f"EDP rank {rank}: {error}" for rank, error in enumerate(errors) if error is not None ) if failures: # Dropping the owner runs the finalizer; the mapping then unmaps. del tensor, owner, mapping raise RuntimeError( "Engram shared-memory initialization failed: " + failures ) assert tensor is not None return tensor @staticmethod def _unregister(mapping: mmap.mmap, pointer: int) -> None: # Torch storage retains the numpy owner, including through cached UVA views. # Keep its mmap alive until CUDA has released the registration. result = torch.cuda.cudart().cudaHostUnregister(pointer) if result.value != 0: logger.warning("Engram cudaHostUnregister failed: %s", result) def load_weight( self, param: torch.nn.Parameter, loaded_weight: torch.Tensor ) -> None: if self.group.rank_in_group == 0: _engram_head_shard_weight_loader(param, loaded_weight) # Read order may differ across ranks. Equal load counts ensure all shared # weights are ready after the last weight-loader call returns. torch.distributed.barrier(group=self.group.cpu_group) def get_views( self, weight: torch.Tensor, scales: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor]: if (weight.data_ptr(), scales.data_ptr()) != ( self.weight.data_ptr(), self.weight_scale_inv.data_ptr(), ): raise RuntimeError("Shared Engram parameter storage must not be replaced") if self._views is None: self._views = ( get_accelerator_view_from_cpu_tensor(self.weight), get_accelerator_view_from_cpu_tensor(self.weight_scale_inv), ) return self._views def can_share_engram_tables(layout: EngramLayout, block_size: int = 32) -> bool: """Whether co-located DP replicas exist and /dev/shm can hold the full tables.""" if get_engram_dp_size() == 1: logger.warning_once( "Engram DP replicas are not co-located on one node; " "storing the offloaded tables per rank instead of sharing them." ) return False num_bytes = sum(layout.num_embeddings) * ( layout.head_dim + layout.head_dim // block_size ) group = get_engram_dp_group() assert group is not None # Only the leader allocates, so every replica follows its check. error = None if group.rank_in_group == 0: if not os.path.isdir(SHM_PATH): error = f"{SHM_PATH} is not mounted" else: try: check_shm_free_space(num_bytes, allocation_name="Engram tables") except Exception as exc: error = f"{type(exc).__name__}: {exc}" error = group.broadcast_object(error) if error is not None: logger.warning_once( "Sharding the offloaded Engram tables across DP replicas: %s", error ) return error is None def _allocate_huge_page_storage(num_bytes: int) -> torch.Tensor | None: """Register prefaulted huge pages, or return None for pinned-memory fallback.""" try: mapping = mmap.mmap(-1, num_bytes, flags=mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS) mapping.madvise(mmap.MADV_HUGEPAGE) except OSError as exc: logger.warning( "Engram huge-page allocation failed (%s); using pinned memory.", exc ) return None owner = np.frombuffer(mapping, dtype=np.uint8) # Fault the pages in before CUDA pins them, so they can be huge. owner[:: mmap.PAGESIZE] = 0 tensor = torch.from_numpy(owner) pointer = tensor.data_ptr() result = torch.cuda.cudart().cudaHostRegister(pointer, num_bytes, 0) if result.value != 0: logger.warning( "Engram cudaHostRegister failed (%s); using pinned memory.", result ) return None # Tensor views retain owner; its finalizer retains mapping until unregister. finalizer = weakref.finalize( owner, DPSharedEngramStorage._unregister, mapping, pointer ) finalizer.atexit = False # type: ignore[misc] if not tensor.is_pinned(): logger.warning( "CUDA did not recognize Engram registration; using pinned memory." ) return None return tensor class ParallelEngramEmbedding(nn.Module): """Hash heads with FP8 rows and per-block E8M0 scales. Heads are TP-sharded, and additionally DP-sharded across a node-local group unless the table lives in (DP-shared) pinned host memory. """ _weight_loader = staticmethod(_engram_head_shard_weight_loader) _shared_memory: DPSharedEngramStorage | None = None _packed: torch.Tensor | None = None def __init__( self, num_embeddings: int, dim: int, head_sizes: tuple[int, ...], block_size: int = 32, cpu_offload: bool = False, dp_shared_memory: bool = False, use_thp: bool = False, ) -> None: super().__init__() assert head_sizes and all(size > 0 for size in head_sizes) assert sum(head_sizes) <= num_embeddings self.cpu_offload = cpu_offload self.dp_shared_memory = dp_shared_memory self.use_thp = use_thp self.dp_size = get_engram_dp_size() if dp_shared_memory: if not cpu_offload: raise ValueError("dp_shared_memory requires cpu_offload=True") if self.dp_size <= 1: raise ValueError( "dp_shared_memory requires a node-local Engram DP " f"group with size > 1; effective Engram DP size is {self.dp_size}. " "Check that the node layout and rank placement allow complete " "DP replicas to be co-located." ) self.dp_size = 1 if cpu_offload and not is_uva_available(): raise RuntimeError("Engram CPU offload requires UVA support") self._views: tuple[torch.Tensor, torch.Tensor] | None = None self._view_src: tuple[int, int] | None = None self.num_embeddings = num_embeddings self.dim = dim self.block_size = block_size self.n_hash_cols = len(head_sizes) self.tp_size = get_tensor_model_parallel_world_size() num_shards, head_rank = self._get_shard_info() self.part_n_hash_cols = triton.cdiv(self.n_hash_cols, num_shards) # TODO: Support row-wise sharding when there are too few hash heads. assert (num_shards - 1) * self.part_n_hash_cols < self.n_hash_cols, ( f"Engram sharding leaves ranks without hash heads: " f"{self.n_hash_cols} heads over {num_shards} shards" ) self.head_start = head_rank * self.part_n_hash_cols head_end = self.head_start + self.part_n_hash_cols self.vocab_start_idx = sum(head_sizes[: self.head_start]) self.vocab_end_idx = sum(head_sizes[:head_end]) self.part_num_embeddings = self.vocab_end_idx - self.vocab_start_idx self._num_sms = torch.cuda.get_device_properties( torch.accelerator.current_device_index() ).multi_processor_count weight, scales = self._allocate_weights() self.weight = nn.Parameter(weight, requires_grad=False) self.weight_scale_inv = nn.Parameter(scales, requires_grad=False) for param in (self.weight, self.weight_scale_inv): set_weight_attrs( param, { "weight_loader": self._weight_loader, "engram_vocab_start": self.vocab_start_idx, }, ) if cpu_offload: # Constant dummy values avoid randomizing huge CPU lookup tables. set_weight_attrs(self.weight, {"dummy_weight_value": 1.0}) # The ue8m0 encoding of scale 1.0 is exponent byte 127. set_weight_attrs(self.weight_scale_inv, {"dummy_weight_value": 127}) logger.info( "Engram table offloaded to pinned host memory: %d rows x %d, " "%.2f GiB %s", self.part_num_embeddings, dim, self.part_num_embeddings * (dim + dim // block_size) / 1024**3, "shared across DP replicas" if dp_shared_memory else "per rank", ) def _get_shard_info(self) -> tuple[int, int]: if self.dp_size == 1: return self.tp_size, get_tensor_model_parallel_rank() return self.tp_size * self.dp_size, engram_head_shard_rank() def _allocate_weights(self) -> tuple[torch.Tensor, torch.Tensor]: if self.dp_shared_memory: group = get_engram_dp_group() assert group is not None storage = DPSharedEngramStorage( self.part_num_embeddings, self.dim, self.block_size, group ) self._shared_memory = storage self._weight_loader = storage.load_weight return storage.weight, storage.weight_scale_inv scale_dim = self.dim // self.block_size if not self.cpu_offload: return ( torch.empty( self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn ), torch.empty(self.part_num_embeddings, scale_dim, dtype=torch.uint8), ) if self.use_thp: weight_bytes = self.part_num_embeddings * self.dim packed = _allocate_huge_page_storage( weight_bytes + weight_bytes // self.block_size ) if packed is not None: self._packed = packed return ( packed[:weight_bytes].view(torch.float8_e4m3fn).view(-1, self.dim), packed[weight_bytes:].view(-1, scale_dim), ) # Model initialization may be inside a CUDA device context. return ( torch.empty( self.part_num_embeddings, self.dim, dtype=torch.float8_e4m3fn, device="cpu", pin_memory=True, ), torch.empty( self.part_num_embeddings, scale_dim, dtype=torch.uint8, device="cpu", pin_memory=True, ), ) def collapse_huge_pages(self) -> None: """Best-effort MADV_COLLAPSE (Linux >= 6.1) of pages that faulted small.""" if self._packed is None: return addr, num_bytes = self._packed.data_ptr(), self._packed.nbytes libc = ctypes.CDLL(None, use_errno=True) libc.madvise.argtypes = (ctypes.c_void_p, ctypes.c_size_t, ctypes.c_int) if libc.madvise(addr, num_bytes, 25) != 0: logger.warning( "Engram MADV_COLLAPSE failed; keeping existing pages: %s", os.strerror(ctypes.get_errno()), ) def _storage(self) -> tuple[torch.Tensor, torch.Tensor]: if self._shared_memory is not None: return self._shared_memory.get_views(self.weight, self.weight_scale_inv) if not self.cpu_offload: return self.weight.data, self.weight_scale_inv.data src = (self.weight.data_ptr(), self.weight_scale_inv.data_ptr()) if self._view_src != src: self._views = ( get_accelerator_view_from_cpu_tensor(self.weight.data), get_accelerator_view_from_cpu_tensor(self.weight_scale_inv.data), ) self._view_src = src assert self._views is not None return self._views def lookup( self, indices: torch.Tensor, out: torch.Tensor, background: bool = False ) -> None: """Look up local heads into BF16, or packed FP8 bytes followed by scales. `background` limits the grid to leave SMs for concurrent work. """ rows = indices.shape[0] * self.part_n_hash_cols if not rows: return weight, scales = self._storage() ids_stride_t, ids_stride_h = indices.stride() sort_min_rows, _ = _engram_lookup_thresholds(indices.device) sort = ( self.cpu_offload and self._packed is None and sort_min_rows is not None and rows >= sort_min_rows ) dst = indices if sort: # Small-page host tables miss the TLB per row; in table order, rows on # one page share it. head_end = self.head_start + self.part_n_hash_cols local = indices[:, self.head_start : head_end] - self.vocab_start_idx if (pad := self.part_n_hash_cols - local.shape[1]) > 0: local = nn.functional.pad(local, (0, pad), value=-1) indices, dst = local.flatten().sort() dst = dst.int() # The table dwarfs TLB reach, so a persistent grid near the SM count # beats one program per row; halve it to leave SMs for the main stream. tiles = triton.cdiv(rows, 16) grid = min(tiles, self._num_sms // 2 if background else self._num_sms) _engram_lookup_kernel[(grid,)]( weight, scales, indices, dst, out, self.vocab_start_idx, self.vocab_end_idx, rows, ids_stride_t, ids_stride_h, HEAD_START=self.head_start, LOCAL_HEADS=self.part_n_hash_cols, TOTAL_HEADS=self.n_hash_cols, DIM=self.dim, QUANT_BLOCK=self.block_size, BLOCK_R=16, GRID=grid, SORTED=sort, PACKED=out.dtype == torch.uint8, OUT_STRIDE=out.stride(0), ) def forward(self, indices: torch.Tensor) -> torch.Tensor: """indices: [num_tokens, n_hash_cols] -> [num_tokens, n_hash_cols, dim] bf16, gathered from all shards for this replica's tokens.""" num_tokens = indices.shape[0] if self.dp_size > 1: indices = gather_engram_hashes(indices) out = torch.empty( (indices.shape[0], self.part_n_hash_cols, self.dim), dtype=torch.bfloat16, device=indices.device, ) self.lookup(indices, out) if self.dp_size > 1: out = _gather_engram_rows(out, num_tokens) if self.tp_size > 1: out = tensor_model_parallel_all_gather(out, dim=1) return out[:, : self.n_hash_cols] @triton.jit(do_not_specialize=["num_kv_tokens"]) def _fused_engram_post_wkv_kernel( hidden_states, kv, q_weight, k_weight, token_mask, output, num_kv_tokens, hidden_stride_t, hidden_stride_h, hidden_stride_d, kv_stride_t, kv_stride_d, q_stride_h, q_stride_d, k_stride_h, k_stride_d, mask_stride, output_stride_t, output_stride_h, output_stride_d, eps, clamp_value, DIM: tl.constexpr, HC_MULT: tl.constexpr, BLOCK_SIZE: tl.constexpr, HAS_MASK: tl.constexpr, ): program_idx = tl.program_id(0) token_idx = program_idx // HC_MULT hc_idx = program_idx % HC_MULT token_idx = token_idx.to(tl.int64) source_idx = token_idx source_valid = source_idx < num_kv_tokens dim_offsets = tl.arange(0, BLOCK_SIZE) dim_valid = dim_offsets < DIM hidden = tl.load( hidden_states + token_idx * hidden_stride_t + hc_idx * hidden_stride_h + dim_offsets * hidden_stride_d, mask=dim_valid, other=0.0, ).to(tl.float32) key = tl.load( kv + source_idx * kv_stride_t + (hc_idx * DIM + dim_offsets) * kv_stride_d, mask=source_valid & dim_valid, other=0.0, ).to(tl.float32) q = tl.load( q_weight + hc_idx * q_stride_h + dim_offsets * q_stride_d, mask=dim_valid, other=0.0, ).to(tl.float32) k = tl.load( k_weight + hc_idx * k_stride_h + dim_offsets * k_stride_d, mask=dim_valid, other=0.0, ).to(tl.float32) hidden_rms = tl.rsqrt(tl.sum(hidden * hidden, axis=0) / DIM + eps) key_rms = tl.rsqrt(tl.sum(key * key, axis=0) / DIM + eps) dot = tl.sum(hidden * q * k * key, axis=0) dot *= hidden_rms * key_rms * tl.rsqrt(DIM * 1.0) gate_input = tl.sqrt(tl.maximum(tl.abs(dot), clamp_value)) gate_input = tl.where(dot < 0.0, -gate_input, gate_input) gate = tl.sigmoid(gate_input) if HAS_MASK: active = tl.load( token_mask + source_idx * mask_stride, mask=source_valid, other=0, ) gate = tl.where(active, gate, 0.0) value = tl.load( kv + source_idx * kv_stride_t + (HC_MULT * DIM + dim_offsets) * kv_stride_d, mask=source_valid & dim_valid, other=0.0, ).to(tl.float32) tl.store( output + token_idx * output_stride_t + hc_idx * output_stride_h + dim_offsets * output_stride_d, hidden + gate * value, mask=dim_valid, ) @triton.jit(do_not_specialize=["num_tokens", "token_start", "num_elements"]) def _engram_select_rows_kernel( gathered, output, num_tokens, token_start, num_elements, LOCAL_WIDTH: tl.constexpr, WIDTH: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): offsets = tl.program_id(0).to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) tokens = token_start + offsets // WIDTH cols = offsets % WIDTH source = (cols // LOCAL_WIDTH * num_tokens + tokens) * LOCAL_WIDTH source += cols % LOCAL_WIDTH values = tl.load( gathered + source, (offsets < num_elements) & (tokens < num_tokens), other=0 ) tl.store(output + offsets, values, offsets < num_elements) def _engram_select_rows( gathered: torch.Tensor, output: torch.Tensor, source_tokens: int, token_start: int, local_width: int, ) -> None: """Copy one token window out of a rank-major gathered buffer. Both gathers land rank-major (`[rank][token][local width]`); this walks the window the rank keeps and lays its ranks out side by side as width. """ if output.numel() == 0: return _engram_select_rows_kernel[(triton.cdiv(output.numel(), 1024),)]( gathered, output, source_tokens, token_start, output.numel(), local_width, output.shape[1] * output.shape[2], BLOCK_SIZE=1024, ) def _gather_engram_rows(staged: torch.Tensor, num_tokens: int) -> torch.Tensor: """Exchange DP tokens for heads, retaining only this replica's tokens.""" dp_group = get_engram_dp_group() assert dp_group is not None slot, remainder = divmod(staged.shape[0], dp_group.world_size) assert remainder == 0 and 0 <= num_tokens <= slot gathered = dp_group.all_gather(staged, dim=0) local_heads, dim = staged.shape[1:] rows = staged.new_empty((num_tokens, dp_group.world_size * local_heads, dim)) _engram_select_rows( gathered, rows, staged.shape[0], dp_group.rank_in_group * slot, local_heads * dim, ) return rows @triton.jit(do_not_specialize=["num_tokens"]) def _engram_unpack_fp8_kernel( packed, values, scales, num_tokens, values_stride, scales_stride, HEADS: tl.constexpr, LOCAL_HEADS: tl.constexpr, LAYERS: tl.constexpr, RANK: tl.constexpr, DIM: tl.constexpr, BLOCK: tl.constexpr, PEER: tl.constexpr, ): """Split `[rank][token][layer][local head]` packed rows into MXFP8 values and F8_128x4 scales, one layer per grid column; `packed` holds per-rank pointers if PEER. Scales past `num_tokens` zero-pad the 128-row tile. """ row = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK) layer = tl.program_id(1).to(tl.int64) token, head = row // HEADS, row % HEADS if PEER: src = tl.load(packed + head // LOCAL_HEADS) src = tl.multiple_of(src, DIM // 32).to(tl.pointer_type(tl.uint8)) source = RANK * num_tokens + token else: src = packed source = head // LOCAL_HEADS * num_tokens + token source = (source * LAYERS + layer) * LOCAL_HEADS + head % LOCAL_HEADS # Packed rows are only scale-width aligned. src += tl.multiple_of(source * (DIM + DIM // 32), DIM // 32) valid = token < num_tokens cols = tl.arange(0, DIM) data = tl.load(src[:, None] + cols[None, :], mask=valid[:, None], other=0) values += layer * values_stride + row * DIM tl.store(values[:, None] + cols[None, :], data, mask=valid[:, None]) scale_cols = tl.arange(0, DIM // 32) scale = tl.load( src[:, None] + DIM + scale_cols[None, :], mask=valid[:, None], other=0 ) group = head[:, None] * (DIM // 32) + scale_cols[None, :] offset = mxfp8_scale_offset(token[:, None], group, HEADS * DIM // 32) tl.store(scales + layer * scales_stride + offset, scale) class EngramBatch(nn.Module): """Stage every Engram layer's WKV input as MXFP8 with one exchange. A module so the model owns the peer exchange's pointer buffers. """ def __init__(self, layers: list["Engram"], max_tokens: int) -> None: super().__init__() first = layers[0].layer_hash_index hash_indices = [layer.layer_hash_index for layer in layers] assert hash_indices == list(range(first, first + len(layers))) self.layers = layers table = layers[0].embed_tokens device = layers[0].staged_rows.device self.packed_rows = torch.empty( max_tokens * table.dp_size, len(layers), table.part_n_hash_cols, table.dim + table.dim // 32, dtype=torch.uint8, device=device, ) width = table.n_hash_cols * table.dim self.fp8_values = torch.empty( len(layers), max_tokens, width, dtype=torch.uint8, device=device ) self.fp8_scales = torch.empty( len(layers), mxfp8_scale_bytes(max_tokens, width), dtype=torch.uint8, device=device, ) max_peer_slot = min(max_tokens, ENGRAM_A2A_MIN_SLOT - 1) self.peer_exchange = None if table.cpu_offload: from vllm.models.deepseek_v41.nvidia.ops.engram_peer import ( EngramPeerExchange, ) self.peer_exchange = EngramPeerExchange.create(layers, max_peer_slot) for layer in layers: # Owned by the model; keep it out of each layer's submodules. object.__setattr__(layer, "_batch", self) # The layers read this batch's staging instead of their BF16 rows. layer.staged_rows = layer.staged_rows.new_empty( (0, *layer.staged_rows.shape[1:]) ) @classmethod def create(cls, layers: list["Engram"], max_tokens: int) -> "EngramBatch | None": """Batch DP-sharded tables whose WKV takes FlashInfer MXFP8 input.""" if not layers: return None table = layers[0].embed_tokens if ( table.dp_size == 1 or table.tp_size > 1 or table.dim % 128 or get_current_vllm_config().lora_config is not None or not can_fuse_query_quant([layer.wkv for layer in layers]) ): return None return cls(layers, max_tokens) def prepare_embeddings(self, hashes: torch.Tensor) -> None: """Stage every layer's WKV input before the decoder layers. DP-sharded tables decide on the group's slot, so every rank agrees. """ slot = engram_gathered_num_tokens() peer = self.peer_exchange is not None and slot < ENGRAM_A2A_MIN_SLOT self._stage(hashes if peer else gather_engram_hashes(hashes), slot, peer) @eager_break_during_capture def _stage(self, hashes: torch.Tensor, slot: int, peer: bool) -> None: if slot == 0: return if peer: assert self.peer_exchange is not None rows = self.peer_exchange.exchange(hashes, slot) rank = self.peer_exchange.dp_rank else: rows, rank = self.packed_rows[: hashes.shape[0]], 0 for index, layer in enumerate(self.layers): layer.embed_tokens.lookup( hashes[:, layer.layer_hash_index], rows[:, index] ) group = get_engram_dp_group() assert group is not None sent, rows = rows, torch.empty_like(rows) torch.distributed.all_to_all_single(rows, sent, group=group.device_group) table = self.layers[0].embed_tokens heads = table.n_hash_cols grid = (triton.cdiv(slot, 128) * 128 * heads // 16, len(self.layers)) _engram_unpack_fp8_kernel[grid]( rows, self.fp8_values, self.fp8_scales, slot, self.fp8_values.stride(0), self.fp8_scales.stride(0), HEADS=heads, LOCAL_HEADS=table.part_n_hash_cols, LAYERS=len(self.layers), RANK=rank, DIM=table.dim, BLOCK=16, PEER=peer, ) def wkv_input(self, layer: "Engram", num_tokens: int) -> QuantizedActivation: index = layer.layer_hash_index - self.layers[0].layer_hash_index values = self.fp8_values[index, :num_tokens].view(torch.float8_e4m3fn) return QuantizedActivation( values, self.fp8_scales[index, : mxfp8_scale_bytes(num_tokens, values.shape[1])], torch.bfloat16, values.shape, kMxfp8Dynamic, ) class Engram(nn.Module): """Writes an n-gram lookup into the residual stream, gated by how well it matches that stream. The hash ids fetch `n_hash_cols` rows; `wkv` turns them into one key per hc copy plus a shared value. The gate is a normalized dot product of the stream against the key, signed-sqrt'ed before the sigmoid (matching the training kernel). """ _prefetch_stream: torch.cuda.Stream | None = None _prefetch_done: torch.cuda.Event | None = None _batch: EngramBatch | None = None def __init__( self, config, quant_config: QuantizationConfig | None, layout: EngramLayout, layer_hash_index: int, use_sequence_parallel: bool, prefix: str, prefetch_stream: torch.cuda.Stream | None = None, ) -> None: super().__init__() # Layers sharing one stream serialize their offloaded lookups, so they # take turns instead of jointly starving decoder compute of SMs. self._prefetch_stream = prefetch_stream self.layer_hash_index = layer_hash_index self.dim = config.hidden_size self.hc_mult = config.hc_mult self.eps = config.rms_norm_eps self.clamp_value = 1e-6 self.use_sequence_parallel = use_sequence_parallel # Named ``embed_tokens`` so the checkpoint's ``engram.embed.weight`` # survives the mapper's ``embed.weight`` -> ``embed_tokens.weight`` # suffix rule. self.embed_tokens = self._create_embedding(layout, layer_hash_index) n_hash_cols = (layout.max_ngram_size - 1) * layout.n_heads # Without sequence parallelism every TP rank holds every token, so # shard the output columns instead of replicating the projection. self.wkv = ColumnParallelLinear( n_hash_cols * layout.head_dim, self.dim * (self.hc_mult + 1), bias=False, gather_output=True, quant_config=quant_config, return_bias=False, prefix=f"{prefix}.wkv", disable_tp=use_sequence_parallel, ) self.q_weight = nn.Parameter( torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16), requires_grad=False, ) self.k_weight = nn.Parameter( torch.empty(self.hc_mult, self.dim, dtype=torch.bfloat16), requires_grad=False, ) max_tokens = get_current_vllm_config().scheduler_config.max_num_batched_tokens self._init_staging(max_tokens, layout.head_dim) def _create_embedding( self, layout: EngramLayout, layer_hash_index: int ) -> ParallelEngramEmbedding: engram_config = get_current_vllm_config().engram_config assert engram_config is not None return ParallelEngramEmbedding( layout.num_embeddings[layer_hash_index], layout.head_dim, tuple(size for order in layout.primes[layer_hash_index] for size in order), cpu_offload=engram_config.cpu_offload, dp_shared_memory=bool(engram_config.dp_shared_memory), use_thp=engram_config.use_thp, ) def _init_staging(self, max_tokens: int, head_dim: int) -> None: # Persistent storage keeps lookup addresses stable across graph replays. self.staged_rows = torch.empty( max_tokens * self.embed_tokens.dp_size, self.embed_tokens.part_n_hash_cols, head_dim, dtype=torch.bfloat16, ) if not self.embed_tokens.cpu_offload: self._prefetch_stream = None return if self._prefetch_stream is None: self._prefetch_stream = torch.cuda.Stream(device=self.staged_rows.device) self._prefetch_done = torch.cuda.Event() def prepare_embeddings(self, hash_ids: torch.Tensor) -> None: """Look up, or prefetch from host memory, rows before the decoder layers consume them; DP-sharded tables take the group's gathered hash IDs.""" rows = self.staged_rows[: hash_ids.shape[0]] assert rows.shape[0] == hash_ids.shape[0], "engram staging buffer too small" if self._prefetch_stream is None: self.embed_tokens.lookup(hash_ids, rows) else: self._start_prefetch(hash_ids, rows) @eager_break_during_capture def _start_prefetch(self, hash_ids: torch.Tensor, rows: torch.Tensor) -> None: # Eager boundaries let the lookup span piecewise graph segments, and # decide per replay (on the replay stream) whether it runs inline. num_tokens = hash_ids.shape[0] # All DP ranks must agree, or EP collectives wait on the slowest one. if is_forward_context_available() and (dp := get_forward_context().dp_metadata): num_tokens = int(dp.num_tokens_across_dp_cpu.max()) current = stream = torch.cuda.current_stream() # Big lookups stall persistent main-stream kernels anyway: run them inline. _, inline_min_tokens = _engram_lookup_thresholds(hash_ids.device) if inline_min_tokens is not None and num_tokens >= inline_min_tokens: self.embed_tokens.lookup(hash_ids, rows) else: stream = self._prefetch_stream assert stream is not None stream.wait_stream(current) # Keep temporary hash storage alive until lookup finishes reading it. hash_ids.record_stream(stream) with torch.cuda.stream(stream): self.embed_tokens.lookup(hash_ids, rows, background=True) assert self._prefetch_done is not None self._prefetch_done.record(stream) @eager_break_during_capture def _finish_prefetch(self, event: torch.cuda.Event) -> None: # Wait for this layer's lookup only, not for later layers on the stream. torch.cuda.current_stream().wait_event(event) def _ready_rows(self, num_tokens: int) -> torch.Tensor: if self._prefetch_done is not None: self._finish_prefetch(self._prefetch_done) if self.embed_tokens.dp_size > 1: slot = engram_gathered_num_tokens() staged = self.staged_rows[: slot * self.embed_tokens.dp_size] return _gather_engram_rows(staged, num_tokens) return self.staged_rows[:num_tokens] def embed(self, hash_ids: torch.Tensor) -> torch.Tensor: """Gather heads, returning only local tokens when SP is enabled.""" rows = self._ready_rows(hash_ids.shape[0]) if self.embed_tokens.tp_size == 1: return rows[:, : self.embed_tokens.n_hash_cols] if self.use_sequence_parallel: tp_size = self.embed_tokens.tp_size num_tokens, local_heads, dim = rows.shape gathered = tensor_model_parallel_all_gather(rows, dim=0) chunk = (num_tokens + tp_size - 1) // tp_size out = rows.new_empty((chunk, self.embed_tokens.n_hash_cols, dim)) _engram_select_rows( gathered, out, num_tokens, get_tensor_model_parallel_rank() * chunk, local_heads * dim, ) return out rows = tensor_model_parallel_all_gather(rows, dim=1) return rows[:, : self.embed_tokens.n_hash_cols] def forward( self, hidden_states: torch.Tensor, hash_ids: torch.Tensor, token_mask: torch.Tensor | None = None, ) -> torch.Tensor: """hidden_states: [T, hc_mult, dim]; hash_ids: [T, n_hash_cols] (all tokens, pre sequence-parallel shard); token_mask: [T], False shuts the gate so those positions pass through untouched.""" if self._batch is not None: kv = self.wkv(self._batch.wkv_input(self, hash_ids.shape[0])) else: kv = self.wkv(self.embed(hash_ids).flatten(-2)) num_kv_tokens = hash_ids.shape[0] assert token_mask is None or token_mask.shape == (num_kv_tokens,) if self.use_sequence_parallel: tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() shard_size = (num_kv_tokens + tp_size - 1) // tp_size assert hidden_states.shape[0] == shard_size start = min(tp_rank * shard_size, num_kv_tokens) num_kv_tokens = min(shard_size, num_kv_tokens - start) if token_mask is not None: token_mask = token_mask[start : start + num_kv_tokens] num_tokens, hc_mult, dim = hidden_states.shape assert hc_mult == self.hc_mult and dim == self.dim assert kv.ndim == 2 and kv.shape[1] == (hc_mult + 1) * dim output = torch.empty_like(hidden_states) if num_tokens == 0: return output block_size = triton.next_power_of_2(dim) num_warps = 8 if block_size >= 2048 else 4 mask = token_mask if token_mask is not None else hidden_states _fused_engram_post_wkv_kernel[(num_tokens * hc_mult,)]( hidden_states, kv, self.q_weight, self.k_weight, mask, output, num_kv_tokens, hidden_states.stride(0), hidden_states.stride(1), hidden_states.stride(2), kv.stride(0), kv.stride(1), self.q_weight.stride(0), self.q_weight.stride(1), self.k_weight.stride(0), self.k_weight.stride(1), token_mask.stride(0) if token_mask is not None else 0, output.stride(0), output.stride(1), output.stride(2), self.eps, self.clamp_value, DIM=dim, HC_MULT=hc_mult, BLOCK_SIZE=block_size, HAS_MASK=token_mask is not None, num_warps=num_warps, ) return output