# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """DeepseekV4 MLA Attention Layer""" import math from abc import ABC, abstractmethod from collections.abc import Callable from functools import cached_property from typing import TYPE_CHECKING, Any, ClassVar, cast import regex as re import torch import torch.nn as nn import torch.nn.functional as F from transformers import DeepseekV2Config, DeepseekV3Config import vllm.envs as envs from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation from vllm.model_executor.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer from vllm.model_executor.model_loader.weight_utils import make_attention_sink from vllm.models.common.ops import fused_q_kv_rmsnorm from vllm.models.common.ops.sequence_parallel import ( sp_all_gather, sp_reduce_scatter, sp_shard, ) from vllm.models.deepseek_v41.common.ops import ( MXFP4_BLOCK_SIZE, fused_indexer_q_rope_quant, indexer_k_norm_rope_store, ) if TYPE_CHECKING: from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import GemmRsAr from vllm.model_executor.layers.rocm_paged_mxfp4_indexer import ( RocmSparseMQAIndexer, ) from vllm.v1.attention.backends.mla.sparse_swa import ( DeepseekSparseSWAMetadata, ) from vllm.config import ( CacheConfig, VllmConfig, get_current_vllm_config, ) from vllm.config.cache import CacheDType from vllm.distributed import get_pcp_group, get_tensor_model_parallel_world_size from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.models.utils import extract_layer_index from vllm.models.deepseek_v41.common.rope import build_deepseek_v4_rope from vllm.models.deepseek_v41.compressor import DeepseekCompressor from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.multi_stream_utils import ( execute_in_parallel, maybe_execute_in_parallel, ) from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata from vllm.v1.attention.backends.mla.indexer import ( DeepseekV41IndexerBackend, dsa_indexer_uses_fp4, get_max_prefill_buffer_size, ) from vllm.v1.attention.backends.mla.sparse_indexer import ( DeepseekV41SparseIndexerBackend, ) from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache from vllm.v1.kv_cache_interface import ( KVCacheSpec, MLAAttentionSpec, get_kv_quant_mode, ) logger = init_logger(__name__) # From this many tokens, a layer with shard_fused_wqa_wkv set projects one token # shard per TP rank and all-gathers the result. _SHARD_FUSED_WQA_WKV_MIN_TOKENS = 2048 def _replace_layer_index(prefix: str, layer_id: int) -> str: """Swap the layer index inside a ``...layers....`` prefix.""" new_prefix, n = re.subn(r"\.layers\.\d+\.", f".layers.{layer_id}.", prefix) assert n == 1, f"Cannot locate layer index in prefix {prefix}" return new_prefix def _indexer_k_cache_head_dim(index_head_dim: int, use_fp4_kv: bool) -> int: """Per-token byte width of the paged indexer K cache row.""" if use_fp4_kv: # MXFP4 stores two values per byte plus one UE8M0 byte per 32 values. # head_dim bytes = 64 packed values + 4 UE8M0 scales = 68. return index_head_dim // 2 + index_head_dim // MXFP4_BLOCK_SIZE # NOTE(yifan): FP8 indexer cache uses the same layout as V3.2: # head_dim bytes = 128 fp8 + 4 fp32 scale = 132. return index_head_dim + index_head_dim // 128 * 4 @triton.jit def _fill_short_context_topk_indices( output, positions, TOP_K: tl.constexpr, COMPRESS_RATIO: tl.constexpr, PADDED_TOP_K: tl.constexpr, ): # small triton kernel that selects every candidate, -1 otherwise row = tl.program_id(0) offsets = tl.arange(0, PADDED_TOP_K) num_compressed = (tl.load(positions + row) + 1) // COMPRESS_RATIO tl.store( output + row * TOP_K + offsets, tl.where(offsets < num_compressed, offsets, -1), mask=offsets < TOP_K, ) # Which packed fp8_ds_mla record a V4.1 layer writes. FlashMLA decodes # DeepSeek's V4.1 record -- all 512 dims (RoPE included) as fp8 e4m3 with one # UE8M0 scale per 32 dims, each token's 512 data bytes followed by its 16 scale # bytes -- only in its SM100 sparse-decode kernels. Every other arch keeps the # V4 record: 448 fp8 NoPE + 64 bf16 RoPE plus 7 UE8M0 scales of 64 dims and a # pad byte (584 B, 576 B pages). def _use_v41_mxfp8_kv_record() -> bool: return current_platform.is_device_capability_family(100) def _resolve_dsv4_kv_cache_dtype( use_fp8_ds_mla_layout: bool, kv_cache_dtype: CacheDType, cache_config: CacheConfig | None, packed_kv_cache_dtype: CacheDType = "fp8_ds_mla", ) -> tuple[CacheDType, torch.dtype]: """Map ``(layout, --kv-cache-dtype)`` to ``(cache_dtype_str, torch_dtype)``. Both layouts are paged; they differ in the per-token block format. The packed formats are ``uint8``-backed: ``fp8_ds_mla`` is UE8M0 block-scaled fp8 throughout, ``nvfp4_ds_mla`` keeps that sliding-window record and stores the compressed cache as NVFP4. An unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves to ``packed_kv_cache_dtype``, the record this layer's kernel prefers, and the canonical string is written back onto ``cache_config`` so the page-size specs pick the right per-token slot. Plain-row backends store each token's KV row in its element dtype: bf16 or per-tensor FP8 E4M3. """ if use_fp8_ds_mla_layout: if kv_cache_dtype in ("auto", "fp8"): kv_cache_dtype = packed_kv_cache_dtype elif not kv_cache_dtype.endswith("_ds_mla"): raise ValueError( "DeepseekV4 packed KV layouts only support fp8 kv-cache, got " f"{kv_cache_dtype}. Please set `--kv-cache-dtype fp8` or " "select a backend that supports bfloat16 KV cache." ) if cache_config is not None and cache_config.cache_dtype != kv_cache_dtype: cache_config.cache_dtype = kv_cache_dtype logger.info_once("Using DeepSeek's %s KV cache format.", kv_cache_dtype) return kv_cache_dtype, torch.uint8 if kv_cache_dtype.endswith("_ds_mla"): raise ValueError( f"{kv_cache_dtype} is a packed FlashMLA DeepSeek V4.1 KV cache " "format; the selected backend stores plain KV rows." ) # Plain bf16 / per-tensor fp8 KV row (FlashInfer). if kv_cache_dtype.startswith("fp8"): return kv_cache_dtype, torch.float8_e4m3fn # auto / bfloat16 -> plain bf16 KV row. return kv_cache_dtype, torch.bfloat16 class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): """DeepseekV4 MLA attention layer. The platform-specific sparse-MLA forward (``forward_mqa`` / ``get_padded_num_q_heads`` / ``_o_proj`` / ``backend_cls``) is provided by a subclass — ``DeepseekV4FlashMLAAttention`` / ``DeepseekV4FlashInferSM120Attention`` / ``DeepseekV4FlashInferMLAAttention`` (CUDA) or ``DeepseekV41ROCMAiterMLAAttention`` (ROCm) — selected by the platform-specific deepseek_v41 model module. The base is never instantiated directly. """ # Provided by the platform subclass. backend_cls: ClassVar[type[AttentionBackend]] # Backend for the SWA cache layer; None uses the default SWA backend. swa_backend_cls: ClassVar[type[AttentionBackend] | None] = None # KV-cache per-token block format (both layouts are paged). True (default) # = fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8); False = plain # bf16 / per-tensor fp8 KV row. Backends can override the instance hook when # a single attention class dispatches across arch-specific layouts. use_fp8_ds_mla_layout: ClassVar[bool] = True # Prefill is processed in fixed-size chunks; this bounds the bf16 kv-gather # workspace allocated in _forward_prefill and is also read by the dummy-run # path to pre-reserve that workspace. PREFILL_CHUNK_SIZE: ClassVar[int] = 4 # ---- attention-interface contract, declared by the platform subclass ---- @property def accepts_unnormed_unroped_query(self) -> bool: """Whether ``forward_mqa``'s ``q`` is the raw ``wq_b`` output. True when the attention kernel applies the Q norm and RoPE itself, and reads Q in its own chunk-interleaved layout, so the layer only zero-pads Q to ``padded_heads`` and inserts KV before calling it. """ return False @property def packed_kv_cache_dtype(self) -> CacheDType: """The packed KV record this layer's kernel prefers. What an unspecific ``--kv-cache-dtype`` (``auto`` / ``fp8``) resolves to. Mega attention overrides it: its kernel is the one that can read an NVFP4 compressed cache. """ return "fp8_ds_mla" @classmethod @abstractmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: """Q head count the q/output buffers are allocated at. The layer allocates the q/output buffers at ``[N, get_padded_num_q_heads(n_local_heads), head_dim]``. Must satisfy ``result >= num_heads``. Backends with no padding constraint return ``num_heads``. """ raise NotImplementedError @abstractmethod def forward_mqa( self, q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, output: torch.Tensor, ) -> None: """Platform-specific sparse MLA forward; writes attention into ``output``.""" raise NotImplementedError @abstractmethod def _o_proj( self, attn_out: "torch.Tensor | QuantizedActivation", positions: torch.Tensor, ) -> torch.Tensor: """Project whatever ``_alloc_attn_out`` produced through wo_a and wo_b. Takes the buffer whole, so each layer owns the shape it allocated: the bf16 layers slice off their padding heads and apply the inverse RoPE, while a layer whose attention kernel already did the inverse RoPE and the FP8 cast gets a QuantizedActivation and has only wo_a and wo_b left. """ raise NotImplementedError def _uses_fp8_ds_mla_layout(self) -> bool: """Return whether this instance stores fp8 KV in fp8_ds_mla layout.""" return self.use_fp8_ds_mla_layout def _indexer_cls( self, k_cache: "DeepseekV4IndexerCache | None" ) -> type["DeepseekV4Indexer"]: """The indexer for this layer's index K cache. A platform subclass returns one that writes and scores its own cache layout.""" return DeepseekV4Indexer def __init__( self, vllm_config: VllmConfig, prefix: str, topk_indices_buffer: torch.Tensor | None = None, aux_stream_list: list[torch.cuda.Stream] | None = None, candidate_block_buffer: torch.Tensor | None = None, ) -> None: super().__init__() config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config cache_config = vllm_config.cache_config tp_size = get_tensor_model_parallel_world_size() layer_id = extract_layer_index(prefix) self.layer_id = layer_id self.prefix = prefix # Alias for compatibility with compressor self.use_pcp = vllm_config.parallel_config.prefill_context_parallel_size > 1 self.hidden_size = config.hidden_size self.n_heads = config.num_attention_heads assert self.n_heads % tp_size == 0 self.n_local_heads = self.n_heads // tp_size self.q_lora_rank = config.q_lora_rank self.o_lora_rank = config.o_lora_rank self.head_dim = config.head_dim self.rope_head_dim = config.qk_rope_head_dim self.nope_head_dim = self.head_dim - self.rope_head_dim self.n_groups = config.o_groups self.n_local_groups = self.n_groups // tp_size self.window_size = config.sliding_window # ---- v4.1 sparse-attention topology ---- # compress_ratios has one entry per layer (MTP layers included): # 0 = pure sliding window, 1 = full-length compressed cache, # 2 = ratio-2 compressed. Compressors and compressed-KV caches live # only on ``kv_source_layer_ids``; indexers only on # ``index_source_layer_ids``. Consumers reuse the most recently # published source below them. compress_ratios = getattr(config, "compress_ratios", None) if compress_ratios is not None and layer_id < len(compress_ratios): self.compress_ratio = int(compress_ratios[layer_id]) else: # MTP layers past the configured list are pure sliding-window. self.compress_ratio = 0 if self.compress_ratio not in (0, 1, 2): raise ValueError( f"DeepSeek V4.1 layer {layer_id} has compress_ratio=" f"{self.compress_ratio}; only 0 (sliding window), 1 and 2 are " "supported." ) self.kv_source_layers = tuple( getattr(config, "kv_source_layer_ids", None) or () ) self.index_source_layers = tuple( getattr(config, "index_source_layer_ids", None) or () ) self.candidate_source_layer = getattr(config, "candidate_source_layer_id", -1) self.candidate_topk_blocks = getattr(config, "candidate_topk_blocks", 0) self.candidate_block_size = getattr(config, "candidate_block_size", 0) is_backbone = layer_id < config.num_hidden_layers self.is_kv_source = is_backbone and layer_id in self.kv_source_layers self.is_index_source = is_backbone and layer_id in self.index_source_layers if self.compress_ratio > 0: if not self.kv_source_layers or not self.index_source_layers: raise ValueError( "DeepSeek V4.1 requires kv_source_layer_ids / " "index_source_layer_ids in the config for compressed " f"layers (layer {layer_id} has " f"compress_ratio={self.compress_ratio})." ) self.kv_source_layer_id = max( s for s in self.kv_source_layers if s <= layer_id ) self.index_source_layer_id = max( s for s in self.index_source_layers if s <= layer_id ) else: self.kv_source_layer_id = None self.index_source_layer_id = None self.eps = config.rms_norm_eps self.scale = self.head_dim**-0.5 # Padded Q head count is dictated by the platform subclass. self.padded_heads = self.get_padded_num_q_heads(self.n_local_heads) # Sink padded to the same head count, initialized to -inf (no sink # effect). Weight loading fills the first n_local_heads slots. self.attn_sink = make_attention_sink(self.padded_heads, torch.float32) self.fused_wqa_wkv = MergedColumnParallelLinear( self.hidden_size, [self.q_lora_rank, self.head_dim], bias=False, quant_config=quant_config, prefix=f"{prefix}.fused_wqa_wkv", disable_tp=True, # fused ReplicatedLinear ) self.q_norm = RMSNorm(self.q_lora_rank, self.eps) self.wq_b = ColumnParallelLinear( self.q_lora_rank, self.n_heads * self.head_dim, bias=False, quant_config=quant_config, return_bias=False, prefix=f"{prefix}.wq_b", ) self.kv_norm = RMSNorm(self.head_dim, self.eps) self.wo_a = ColumnParallelLinear( self.n_heads * self.head_dim // self.n_groups, self.n_groups * self.o_lora_rank, bias=False, quant_config=quant_config, return_bias=False, prefix=f"{prefix}.wo_a", ) self.wo_a.is_bmm = True self.wo_a.bmm_batch_size = self.n_local_groups self._o_proj_block_size = ( 32 if getattr(self.wo_a, "weight_block_size", None) == [1, 32] else 128 ) self.wo_b = RowParallelLinear( self.n_groups * self.o_lora_rank, self.hidden_size, bias=False, quant_config=quant_config, return_bias=False, prefix=f"{prefix}.wo_b", ) # Set by ``bind_gemm_rs`` when the decoder layer runs sequence # parallel and the fused GEMM + reduce-scatter kernel accepts wo_b. self.gemm_rs: GemmRsAr | None = None # Set by the NVIDIA decoder layer from _shard_fused_wqa_wkv. self.shard_fused_wqa_wkv = False # Initialize rotary embedding before the indexer/compressor consume it. self.rotary_emb = build_deepseek_v4_rope( config, head_dim=self.head_dim, rope_head_dim=self.rope_head_dim, max_position_embeddings=config.max_position_embeddings, compress_ratio=self.compress_ratio, ) self.indexer_rotary_emb = self.rotary_emb self.topk_indices_buffer = topk_indices_buffer self.candidate_block_buffer = candidate_block_buffer # Register with compilation context for metadata lookup. Done before # indexer/compressor creation so consumers can resolve their source # layers through it. compilation_config = vllm_config.compilation_config if prefix and prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") if prefix: compilation_config.static_forward_context[prefix] = self self.kv_cache = torch.tensor([]) self._static_forward_context = compilation_config.static_forward_context self.indexer = None if self.is_index_source: index_k_cache: DeepseekV4IndexerCache | None # Index K cache: owned only by kv-source layers (their indexer has # wk/k_norm); non-owning index sources share the K cache of the # latest kv source below them (its indexer produces the keys). if self.is_kv_source: index_k_cache = DeepseekV4IndexerCache( head_dim=_indexer_k_cache_head_dim( config.index_head_dim, dsa_indexer_uses_fp4(vllm_config) ), dtype=torch.uint8, prefix=f"{prefix}.indexer.k_cache", cache_config=cache_config, compress_ratio=self.compress_ratio, ) else: assert self.kv_source_layer_id is not None k_cache_prefix = ( f"{_replace_layer_index(prefix, self.kv_source_layer_id)}" ".indexer.k_cache" ) index_k_cache = self._static_forward_context.get(k_cache_prefix) if index_k_cache is None: raise NotImplementedError( f"Indexer K cache source {k_cache_prefix} not found on " "this rank; PP splits inside a v4.1 kv-sharing group " "are not supported." ) is_candidate_source = layer_id == self.candidate_source_layer uses_candidates = 0 <= self.candidate_source_layer < layer_id self.indexer = self._indexer_cls(index_k_cache)( vllm_config, config=config, hidden_size=self.hidden_size, q_lora_rank=self.q_lora_rank, quant_config=quant_config, cache_config=cache_config, topk_indices_buffer=topk_indices_buffer, compress_ratio=self.compress_ratio, prefix=f"{prefix}.indexer", owns_k=self.is_kv_source, k_cache=index_k_cache, main_head_dim=self.head_dim, candidate_block_buffer=( candidate_block_buffer if (is_candidate_source or uses_candidates) else None ), candidate_block_size=self.candidate_block_size, candidate_write=is_candidate_source, ) self._prepare_and_attn_fn = self._prepare_and_attn if not vllm_config.use_v2_model_runner: # MRV1's piecewise capture only tolerates the wide eager region: with # the narrow one the attention input preparation stays in the captured # graph and MRV1 produces garbage (#51430). self._prepare_and_attn_fn = self._prepare_and_attn_eager self.aux_stream_list = aux_stream_list # [0]: GEMM start / post-GEMM event0. [1..3]: GEMM done events; # [1] doubles as post-GEMM event1. Reuse is safe: GEMM fully joins # before post-GEMM starts. self.ln_events = [torch.cuda.Event() for _ in range(4)] assert cache_config is not None, "DeepseekV4 attention requires cache_config" # ---- Attention / KV-cache setup ---- self.max_num_batched_tokens = ( vllm_config.scheduler_config.max_num_batched_tokens ) self.max_model_len = vllm_config.model_config.max_model_len # Resolve the kv-cache dtype from this backend's block format. The same # resolution drives the SWA cache tensor dtype below. self.kv_cache_dtype, self.kv_cache_torch_dtype = _resolve_dsv4_kv_cache_dtype( self._uses_fp8_ds_mla_layout(), cache_config.cache_dtype, cache_config, self.packed_kv_cache_dtype, ) self.kv_mxfp8 = _use_v41_mxfp8_kv_record() self.swa_bytes_per_token = 528 if self.kv_mxfp8 else 584 # nvfp4_ds_mla keeps the MXFP8 sliding-window record and stores the # compressed cache as NVFP4 (256 B of e2m1 pairs + 32 e4m3 scales). self.compressed_bytes_per_token = ( 288 if self.kv_cache_dtype == "nvfp4_ds_mla" else self.swa_bytes_per_token ) # One alignment for every page in the block: the block stride is their # sum. V4 needs its 576 B TMA stride; V4.1's is on the SWA block # stride, and 512 is free (its pages are already 512 B multiples). self.kv_page_alignment = 512 if self.kv_mxfp8 else 576 if self.kv_cache_dtype == "nvfp4_ds_mla" and not self.kv_mxfp8: raise ValueError( "nvfp4_ds_mla needs the V4.1 KV records, which FlashMLA " "decodes only on SM100." ) swa_bounded_replay = cache_config.swa_bounded_replay if swa_bounded_replay and not vllm_config.use_v2_model_runner: logger.warning_once( "SWA bounded replay needs model runner V2 (only it skips the " "paged-KV writes of replayed tokens); the sliding-window cache " "takes part in prefix caching instead." ) swa_bounded_replay = False if ( swa_bounded_replay and vllm_config.parallel_config.prefill_context_parallel_size > 1 ): logger.warning_once( "SWA bounded replay is off under prefill context parallelism " "(the replayed tokens' slot padding knows the rank-local batch " "only); the sliding-window cache takes part in prefix caching " "instead." ) swa_bounded_replay = False if swa_bounded_replay and current_platform.is_rocm(): logger.warning_once( "Prefix-cache SWA replay is off on ROCm (the sparse SWA " "metadata builders forward replay_start, but the window clamp " "it relies on lives in the FlashInfer and FlashMLA prefill " "kernels, so the padded slots fault); the sliding-window cache " "takes part in prefix caching instead." ) swa_bounded_replay = False self.swa_cache_layer = DeepseekV4SWACache( head_dim=self.head_dim, window_size=self.window_size, dtype=self.kv_cache_torch_dtype, prefix=f"{prefix}.swa_cache", cache_config=cache_config, backend_cls=self.swa_backend_cls, block_size=32, packed_bytes_per_token=self.swa_bytes_per_token, packed_page_alignment=self.kv_page_alignment, # FlashMLA reads the interleaved V4.1 records as rows of the whole # cache, so the packed block has to hold whole rows of both. packed_block_stride_alignment=( math.lcm(self.swa_bytes_per_token, self.compressed_bytes_per_token) if self.kv_mxfp8 else None ), bounded_replay=swa_bounded_replay, ) # The attention layer itself was already registered with the # compilation context above (before indexer/compressor creation). # Compressors live only on kv-source layers; consumers read the # source's compressed cache through the forward context. self.compressor = None if self.is_kv_source: self.compressor = DeepseekCompressor( vllm_config=vllm_config, compress_ratio=self.compress_ratio, hidden_size=self.hidden_size, head_dim=self.head_dim, rotate=True, prefix=f"{prefix}.compressor", k_cache_prefix=self.prefix, ) # Prefix of the attention layer owning this layer's compressed KV # cache (self for kv sources). if self.compress_ratio > 0: assert self.kv_source_layer_id is not None self.compressed_cache_prefix: str | None = _replace_layer_index( prefix, self.kv_source_layer_id ) if ( not self.is_kv_source and self.compressed_cache_prefix not in self._static_forward_context ): raise NotImplementedError( f"Compressed-KV source {self.compressed_cache_prefix} not " "found on this rank; PP splits inside a v4.1 kv-sharing " "group are not supported." ) else: self.compressed_cache_prefix = None if vllm_config.kernel_config.enable_jit_warmup: from vllm.v1.attention.backends.mla.sparse_swa import ( _COMPUTE_PREFILL_METADATA_KERNEL, _COMPUTE_SWA_INDICES_AND_LENS_KERNEL, ) _COMPUTE_PREFILL_METADATA_KERNEL.register_warmup() _COMPUTE_SWA_INDICES_AND_LENS_KERNEL.register_warmup( window_size=self.window_size, block_size=self.swa_cache_layer.block_size, ) if self.compress_ratio > 1: from vllm.v1.attention.backends.mla.compressor_utils import ( _COMPRESSED_SLOT_MAPPING_KERNEL, ) _COMPRESSED_SLOT_MAPPING_KERNEL.register_warmup() if self.indexer is not None: from vllm.v1.attention.backends.mla.indexer import ( _BUILD_PREFILL_CHUNK_METADATA_KERNEL, _PREPARE_UNIFORM_DECODE_KERNEL, ) _PREPARE_UNIFORM_DECODE_KERNEL.register_warmup() _BUILD_PREFILL_CHUNK_METADATA_KERNEL.register_warmup() spec_config = vllm_config.speculative_config if spec_config is not None and spec_config.use_dspark(): from vllm.v1.attention.backends.mla.sparse_swa import ( _COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL, ) _COMPUTE_DSPARK_NONCAUSAL_SWA_INDICES_KERNEL.register_warmup( window_size=self.window_size, num_speculative_tokens=spec_config.num_speculative_tokens, block_size=self.swa_cache_layer.block_size, ) # Every backend that gathers a chunk's KV through # combine_topk_swa_indices needs its Triton kernel warmed, mega # attention included -- it calls it from _forward_prefill_mega. if self.backend_cls.get_name() in ( "FLASHMLA_SPARSE_DSV41", "FLASHMLA_MEGA_ATTN_DSV41", "ROCM_FLASHMLA_SPARSE_DSV41", ): from vllm.models.deepseek_v41.common.ops.cache_utils import ( _COMBINE_TOPK_SWA_INDICES_KERNEL, ) _COMBINE_TOPK_SWA_INDICES_KERNEL.register_warmup() def forward( self, positions: torch.Tensor, hidden_states: torch.Tensor, llama_4_scaling: torch.Tensor | None = None, ) -> torch.Tensor: # The eager attention region writes into a caller-owned buffer # (breakable_cudagraph needs in-place outputs); its shape and how it is # projected afterwards follow the interface contract above. attn_out = self._alloc_attn_out(hidden_states.shape[0], hidden_states) # Keep the attention input preparation in the captured graph. Only the # sparse indexer and MLA attention run in the eager break below. qr_kv, kv_score, indexer_weights = self._run_parallel_input_projections( hidden_states ) qr, qr_scale, kv = self._split_qkv_and_norm(qr_kv) self._prepare_and_attn_fn( hidden_states, qr, kv, qr_scale, kv_score, indexer_weights, positions, attn_out, ) return self._o_proj(attn_out, positions) def forward_kv(self, positions: torch.Tensor, hidden_states: torch.Tensor) -> None: """Write only the KV ``forward`` writes (SWA KV, compressed KV, index K), skipping wq_b, the indexer's top-k and attention.""" attn_metadata = get_forward_context().attn_metadata qr_kv, kv_score, _ = self._run_parallel_input_projections(hidden_states) _, _, kv = self._split_qkv_and_norm(qr_kv) if isinstance(attn_metadata, dict): # The KV half of _fused_qnorm_rope_kv_insert, bit for bit, without # the unused query; same as dspark's _insert_context_kv. cache = self.swa_cache_layer.kv_cache torch.ops._C.fused_deepseek_v4_kv_rope_insert( kv, cache, cast(Any, attn_metadata[self.swa_cache_layer.prefix]).slot_mapping, positions, self.rotary_emb.cos_sin_cache, self.swa_cache_layer.block_size, self._flashinfer_fp8_kv_scale if cache.dtype == torch.float8_e4m3fn else None, self.kv_mxfp8, ) if self.compressor is not None: latent = self.compressor(kv_score, positions) self.compressor.insert_cache(latent, positions, self.rotary_emb) if self.indexer is not None and self.indexer.owns_k: self.indexer._produce_k(latent, positions, self.indexer_rotary_emb) def bind_gemm_rs(self) -> None: """Fuse ``wo_b`` with the sequence-parallel TP reduce-scatter. The decoder layer calls this after the model initialized the process-wide GEMM-RS workspace and before weights load: the eligibility check inspects the projection's linear kernel and weight shape, which online quantization may later re-layout. """ from vllm.model_executor.kernels.linear.cute_dsl.gemm_rs_ar import ( get_gemm_rs_ar, ) # The layer owns the reduction under sequence parallel. assert not self.wo_b.reduce_results gemm_rs = get_gemm_rs_ar() if gemm_rs.can_run(self.wo_b): self.gemm_rs = gemm_rs else: gemm_rs.warn_incompatible_projection() def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor: """Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result. Every ``_o_proj`` implementation projects through this so the decoder layer sees one contract: when ``gemm_rs`` is bound the output is already the local sequence-parallel shard, otherwise it is the unreduced TP partial. """ if self.gemm_rs is None: return self.wo_b(z) if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z): return self.gemm_rs.apply(z, self.wo_b) # Small batches stay on the unfused path, which is faster there. return sp_reduce_scatter(self.wo_b(z)) def _alloc_attn_out( self, num_tokens: int, hidden_states: torch.Tensor ) -> "torch.Tensor | QuantizedActivation": """The buffer ``forward_mqa`` fills, per the interface contract.""" return torch.empty( (num_tokens, self.padded_heads, self.head_dim), dtype=hidden_states.dtype, device=hidden_states.device, ) @cached_property def _can_fuse_query_quant(self) -> bool: from vllm.models.deepseek_v41.common.ops.query_quant import ( can_fuse_query_quant, ) linears = [self.wq_b] if self.indexer is not None: linears.append(self.indexer.wq_b) return can_fuse_query_quant(linears) def _split_qkv_and_norm( self, qr_kv: torch.Tensor ) -> tuple[torch.Tensor | QuantizedActivation, torch.Tensor | None, torch.Tensor]: """Split the fused q-lora / kv projection and RMSNorm both halves. Compatible MXFP8 projections share the quantized Q and scales; other projection backends consume the normalized Q directly. """ qr, kv = qr_kv.split([self.q_lora_rank, self.head_dim], dim=-1) if self.q_lora_rank % 32 == 0 and self._can_fuse_query_quant: from vllm.models.deepseek_v41.common.ops.query_quant import ( fused_q_kv_rmsnorm_quant, ) qr_quant, kv = fused_q_kv_rmsnorm_quant( qr, kv, self.q_norm.weight.data, self.kv_norm.weight.data, self.eps, ) return qr_quant, None, kv qr, kv = fused_q_kv_rmsnorm( qr, kv, self.q_norm.weight.data, self.kv_norm.weight.data, self.eps, ) return qr, None, kv @eager_break_during_capture def _prepare_and_attn_eager( self, hidden_states: torch.Tensor, qr: torch.Tensor | QuantizedActivation, kv: torch.Tensor, qr_scale: torch.Tensor | None, kv_score: torch.Tensor, indexer_weights: torch.Tensor, positions: torch.Tensor, attn_out: "torch.Tensor | QuantizedActivation", ) -> None: """Wide eager region: the whole of ``_prepare_and_attn`` runs eagerly. The nested ``_sparse_indexer_and_attn`` break runs inline, since ``add_eager`` clears ``_capturing`` before invoking this. """ self._prepare_and_attn( hidden_states, qr, kv, qr_scale, kv_score, indexer_weights, positions, attn_out, ) def _prepare_and_attn( self, hidden_states: torch.Tensor, qr: torch.Tensor | QuantizedActivation, kv: torch.Tensor, qr_scale: torch.Tensor | None, kv_score: torch.Tensor, indexer_weights: torch.Tensor, positions: torch.Tensor, attn_out: "torch.Tensor | QuantizedActivation", ) -> None: """Attention input preparation followed by the sparse indexer and MLA. Only the latter runs in the eager break. Q/SWA preparation overlaps state saving and compression. Once the latent is ready, main-cache insertion overlaps indexer preparation; both cache writes finish before sparse attention reads either cache. """ attn_metadata = get_forward_context().attn_metadata indexer = self.indexer compressor = self.compressor aux_streams = self.aux_stream_list def project_query_and_cache_kv() -> torch.Tensor: q = self._wq_b_proj(qr, qr_scale).view( -1, self.n_local_heads, self.head_dim ) return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata) index_q: torch.Tensor | None = None index_q_scale: torch.Tensor | None = None index_weights_out: torch.Tensor | None = None latent: torch.Tensor | None = None # Keep PCP collectives on one stream in the same order on every rank. aux_stream = ( aux_streams[0] if aux_streams is not None and not self.use_pcp else None ) if compressor is not None: # Q projection / KV insertion on the default stream overlaps the # compressor on aux stream 0 (sequential on ROCm). q, latent = maybe_execute_in_parallel( project_query_and_cache_kv, lambda: compressor(kv_score, positions), self.ln_events[0], self.ln_events[1], aux_stream, ) else: q = project_query_and_cache_kv() cache_latent = latent cache_positions = positions if self.use_pcp and latent is not None: assert isinstance(attn_metadata, dict) swa_metadata = cast( "DeepseekSparseSWAMetadata", attn_metadata[self.swa_cache_layer.prefix] ) assert swa_metadata.cache_positions is not None cache_latent = get_pcp_group().all_gather(latent, dim=0) cache_positions = swa_metadata.cache_positions def prepare_indexer(): if indexer is None: return None, None, None return indexer( qr, latent, indexer_weights, positions, self.indexer_rotary_emb, qr_scale, cache_latent=cache_latent, cache_positions=cache_positions, ) if compressor is not None: indexer_result, _ = maybe_execute_in_parallel( prepare_indexer, lambda: compressor.insert_cache( cache_latent, cache_positions, self.rotary_emb ), self.ln_events[0], self.ln_events[1], aux_stream, ) else: indexer_result = prepare_indexer() index_q, index_q_scale, index_weights_out = indexer_result self._sparse_indexer_and_attn( hidden_states, index_q, index_q_scale, index_weights_out, q, kv, positions, attn_out, ) def _fused_wqa_wkv_gemm(self, hidden_states: torch.Tensor) -> torch.Tensor: # Override point: the ROCm layer preshuffles this weight in place, so # it cannot go through fused_wqa_wkv directly. num_tokens = hidden_states.shape[0] # MergedColumnParallelLinear returns (output, bias); bias is None. if self.shard_fused_wqa_wkv and num_tokens >= _SHARD_FUSED_WQA_WKV_MIN_TOKENS: qr_kv, _ = self.fused_wqa_wkv(sp_shard(hidden_states)) return sp_all_gather(qr_kv)[:num_tokens] qr_kv, _ = self.fused_wqa_wkv(hidden_states) return qr_kv def _wq_b_proj( self, qr: torch.Tensor | QuantizedActivation, qr_scale: torch.Tensor | None = None, ) -> torch.Tensor: """Project normalized Q, bypassing quantization when already fused.""" assert qr_scale is None, "ROCm-only path" return self.wq_b(qr) def _run_parallel_input_projections( self, hidden_states: torch.Tensor ) -> tuple[ torch.Tensor, torch.Tensor | None, torch.Tensor | None, ]: aux_streams = self.aux_stream_list if aux_streams is not None: aux_streams = aux_streams[:2] # fused_wqa_wkv (heaviest) on default; the two lighter input GEMMs on # aux streams 0/1 when their owning module exists. ln_events[0] is the # fan-out start event; ln_events[1..2] are per-aux done events. The # v4.1 indexer derives K from the kv-source compressor's latent, so # unlike v4.0 there is no indexer K GEMM over hidden_states here. aux_fns: list[Callable[[], Any] | None] = [None, None] if self.compressor is not None: # Local ref so the closure keeps a non-None type for mypy. compressor = self.compressor def compressor_kv_score() -> torch.Tensor: return torch.mm( hidden_states, compressor.fused_wkv_wgate.weight.T, out_dtype=torch.float32, ) aux_fns[0] = compressor_kv_score if self.indexer is not None: indexer = self.indexer def indexer_weights_proj() -> torch.Tensor: # ReplicatedLinear returns (output, bias); bias is None. weights, _ = indexer.weights_proj(hidden_states) return weights aux_fns[1] = indexer_weights_proj qr_kv, (kv_score, indexer_weights) = execute_in_parallel( lambda: self._fused_wqa_wkv_gemm(hidden_states), aux_fns, self.ln_events[0], self.ln_events[1:3], aux_streams, enable=hidden_states.shape[0] <= envs.VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD, ) return qr_kv, kv_score, indexer_weights @eager_break_during_capture def _sparse_indexer_and_attn( self, hidden_states: torch.Tensor, index_q: torch.Tensor | None, index_q_scale: torch.Tensor | None, index_weights: torch.Tensor | None, q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, out: torch.Tensor, ) -> None: if self.indexer is not None and index_q is not None: assert index_weights is not None q_quant = (index_q, index_q_scale) if index_q_scale is not None else index_q self.indexer.indexer_op( hidden_states, q_quant, None, index_weights, ) # MLA attention writes into the pre-allocated `out` buffer # ([num_tokens, padded_heads, head_dim]). self.forward_mqa(q, kv, positions, out) def _fused_qnorm_rope_kv_insert( self, q: torch.Tensor, kv: torch.Tensor, positions: torch.Tensor, attn_metadata: ( dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]] | None ), ) -> torch.Tensor: """Ready ``q`` for the attention kernel and publish this step's KV. One launch does both. With ``accepts_unnormed_unroped_query`` the attention kernel norms and rotates Q itself and reads it in its own chunk-interleaved layout, so the Q half of the launch is a zero-pad to ``padded_heads`` -- and nothing at all once the shard is that wide. """ if not isinstance(attn_metadata, dict): # Profile run: kernel doesn't fire; produce a padded tensor so # downstream FlashMLA gets the right shape. if self.n_local_heads >= self.padded_heads: return q if self.accepts_unnormed_unroped_query: # Padding heads sit at the tail of every head-dim chunk in # that layout, so no head-major pad of `q` reproduces it -- # and nothing reads it on a profile run. return q.new_zeros((q.shape[0], self.padded_heads, q.shape[2])) return F.pad( q, (0, 0, 0, self.padded_heads - self.n_local_heads), value=0.0, ) swa_metadata = cast( "DeepseekSparseSWAMetadata | None", attn_metadata.get(self.swa_cache_layer.prefix), ) assert swa_metadata is not None swa_kv_cache = self.swa_cache_layer.kv_cache # The fused insert ops require int64 position_ids; the runner's positions # buffer is already int64, so no cast is needed. assert positions.dtype == torch.int64 cos_sin_cache = self.rotary_emb.cos_sin_cache cache_dtype = swa_kv_cache.dtype slot_mapping = swa_metadata.slot_mapping if self.use_pcp: cache_kv = get_pcp_group().all_gather(kv.contiguous(), dim=0) assert swa_metadata.cache_slot_mapping is not None assert swa_metadata.cache_positions is not None torch.ops._C.fused_deepseek_v4_kv_rope_insert( cache_kv, swa_kv_cache, swa_metadata.cache_slot_mapping, swa_metadata.cache_positions, cos_sin_cache, swa_metadata.block_size, self._flashinfer_fp8_kv_scale if cache_dtype == torch.float8_e4m3fn else None, self.kv_mxfp8, ) # The fused op now only prepares the rank-local queries. slot_mapping = slot_mapping[:0] # kv is unchanged; attention reads kv solely via swa_kv_cache. if cache_dtype == torch.uint8: # fp8_ds_mla UE8M0 paged path. Horizontally fused: # Q side: GPT-J RoPE, zero-filling the padding head slots; the # kernel allocates and returns the padded q tensor. An # interleaved Q skips the RoPE its attention kernel owns # and keeps only the pad, which q_head_padded=0 drops # too once the shard is already padded_heads wide. # KV side: GPT-J RoPE + UE8M0 FP8 quant + paged cache insert. swa_kv_cache_2d = swa_kv_cache.view(swa_kv_cache.shape[0], -1) pad_to = ( 0 if self.accepts_unnormed_unroped_query and self.n_local_heads == self.padded_heads else self.padded_heads ) q_padded = torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert( q, kv, swa_kv_cache_2d, slot_mapping, positions, cos_sin_cache, pad_to, self.eps, swa_metadata.block_size, False, # apply_q_norm: qr is normed before wq_b self.kv_mxfp8, not self.accepts_unnormed_unroped_query, # apply_q_rope self.accepts_unnormed_unroped_query, # is_q_interleaved ) return q if pad_to == 0 else q_padded assert not self.accepts_unnormed_unroped_query, ( "the chunk-interleaved Q layout only pairs with a packed KV record" ) # Plain-row path: the [num_blocks, block_size, 512] cache stores the KV # row in its element dtype (no Q padding). bf16 rewrites q in place; # per-tensor fp8 writes a separately-allocated fp8 q and quantizes the # KV row. block_size = swa_metadata.block_size assert swa_kv_cache.shape[1:] == (block_size, self.head_dim) swa_kv_cache_3d = swa_kv_cache if cache_dtype == torch.bfloat16: torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert( q, kv, swa_kv_cache_3d, slot_mapping, positions, cos_sin_cache, self.eps, block_size, False, ) return q # per-tensor fp8 (torch.float8_e4m3fn) q_fp8 = torch.empty_like(q, dtype=torch.float8_e4m3fn) torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert( q, kv, q_fp8, swa_kv_cache_3d, slot_mapping, positions, cos_sin_cache, self._flashinfer_fp8_kv_scale, self._flashinfer_fp8_q_scale_inv, self.eps, block_size, False, ) return q_fp8 def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: # [B, H=1, N, C] -> [B, N, C] self.kv_cache = kv_cache.squeeze(1) def get_attn_backend(self) -> type[AttentionBackend]: return self.backend_cls def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: # Only kv-source layers own a compressed-KV cache; consumers read the # source's cache through the forward context, and cr==0 layers are # pure SWA. The SWA cache is allocated separately as # DeepseekV4SWACache. if not self.is_kv_source: return None # fp8_ds_mla is a UE8M0 block-scaled uint8 layout whose page rounds up # to the decode kernel's TMA stride; plain bf16 / per-tensor fp8 rows # use natural element-size pages. uses_fp8_ds_mla_layout = self.kv_cache_dtype in ("fp8_ds_mla", "nvfp4_ds_mla") return MLAAttentionSpec( block_size=vllm_config.cache_config.block_size, num_kv_heads=1, head_size=self.head_dim, dtype=torch.uint8 if uses_fp8_ds_mla_layout else self.kv_cache_torch_dtype, tokens_per_state=self.compress_ratio, cache_dtype_str=self.kv_cache_dtype, alignment=self.kv_page_alignment if uses_fp8_ds_mla_layout else 512, model_version="deepseek_v4", kv_quant_mode=get_kv_quant_mode(self.kv_cache_dtype), # Packed record width; head_size stays semantic (512). state_content_bytes=( self.compressed_bytes_per_token if uses_fp8_ds_mla_layout else None ), ) def _compressed_kv_cache(self) -> torch.Tensor: """The compressed-KV cache tensor of this layer's kv source (own cache for kv-source layers).""" if self.is_kv_source: return self.kv_cache assert self.compressed_cache_prefix is not None source = self._static_forward_context[self.compressed_cache_prefix] return source.kv_cache class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase): def __init__( self, head_dim: int, dtype: torch.dtype, prefix: str, cache_config: CacheConfig, compress_ratio: int = 1, ): super().__init__() self.kv_cache = torch.tensor([]) self.head_dim = head_dim self.prefix = prefix self.cache_config = cache_config self.dtype = dtype self.compress_ratio = compress_ratio vllm_config = get_current_vllm_config() self.sparse_logits = vllm_config.attention_config.indexer_sparse_logits # aiter's paged MXFP4 kernels write this cache and read it in place. self.rocm_mxfp4 = current_platform.is_rocm() and dsa_indexer_uses_fp4( vllm_config ) compilation_config = vllm_config.compilation_config if prefix in compilation_config.static_forward_context: raise ValueError(f"Duplicate layer name: {prefix}") compilation_config.static_forward_context[prefix] = self def bind_kv_cache(self, kv_cache: torch.Tensor) -> None: # [B, H=1, N, C] -> [B, N, C] self.kv_cache = kv_cache.squeeze(1) def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: # head_dim already carries the fp8 scale padding # tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout. # nvfp4_ds_mla is packed too: its compressed record is NVFP4 but the # sliding-window record stays the V4.1 MXFP8 one, so the indexer page # takes the same alignment either way. uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype in ( "fp8_ds_mla", "nvfp4_ds_mla", ) page_alignment = ( 576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512 ) return MLAAttentionSpec( block_size=self.cache_config.block_size, num_kv_heads=1, head_size=self.head_dim, dtype=self.dtype, tokens_per_state=self.compress_ratio, # The indexer page shares a physical block with the main KV cache # (the block stride is the sum of every page in the group), so it # has to carry the same padding: the packed KV pages' alignment, # else 512B for FlashInfer sparse (#44577). alignment=page_alignment, # DeepGEMM's paged sparse logits need a 256B-aligned block stride; # V4 records keep their TMA stride (V4.1's is on the SWA spec). block_stride_alignment=( math.lcm( 256, 1 if uses_fp8_ds_mla_layout and _use_v41_mxfp8_kv_record() else page_alignment, ) if self.sparse_logits and not self.rocm_mxfp4 else None ), ) def forward(self): ... def get_attn_backend(self) -> type[AttentionBackend]: if self.rocm_mxfp4: from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( DeepseekV41RocmMxfp4IndexerBackend, ) return DeepseekV41RocmMxfp4IndexerBackend if self.sparse_logits: return DeepseekV41SparseIndexerBackend return DeepseekV41IndexerBackend class DeepseekV4Indexer(nn.Module): """DeepSeek V4.1 sparse-attention indexer. Exists only on ``index_source_layer_ids``; consumers reuse the topk indices it publishes into the shared ``topk_indices_buffer``. Unlike v4.0 the index key is derived from the kv-source layer's compressor latent (``k = k_norm(wk(latent))``, ``owns_k``) instead of an indexer-local compressor over hidden states, so there is no hidden-state K GEMM here. Non-owning index sources share the kv source's paged K cache. Two-level candidate filtering: the indexer at ``candidate_source_layer_id`` additionally publishes the top ``candidate_topk_blocks`` blocks of ``candidate_block_size`` compressed positions (``candidate_write``); later indexers mask their scores to those blocks before their own top-k. """ # The scoring layers; a platform subclass swaps in its own. mqa_cls: ClassVar[type["SparseMQAIndexer | RocmSparseMQAIndexer"]] = ( SparseMQAIndexer ) attn_cls: ClassVar[type[SparseAttnIndexer]] = SparseAttnIndexer def __init__( self, vllm_config: VllmConfig, config: DeepseekV2Config | DeepseekV3Config, hidden_size: int, q_lora_rank: int, quant_config: QuantizationConfig | None, cache_config: CacheConfig | None, topk_indices_buffer: torch.Tensor | None, compress_ratio: int, prefix: str, *, owns_k: bool, k_cache: DeepseekV4IndexerCache, main_head_dim: int, candidate_block_buffer: torch.Tensor | None = None, candidate_block_size: int = 0, candidate_write: bool = False, ): super().__init__() self.vllm_config = vllm_config self.config = config self.quant_config = quant_config self.topk_tokens = config.index_topk self.n_head = config.index_n_heads # 32 self.head_dim = config.index_head_dim # 128 self.rope_dim = config.qk_rope_head_dim # 64 self.q_lora_rank = q_lora_rank # 1280 self.compress_ratio = compress_ratio self.owns_k = owns_k self.use_fp4_kv = dsa_indexer_uses_fp4(vllm_config) logger.info_once( "Using %s indexer cache for Lightning Indexer.", "MXFP4" if self.use_fp4_kv else "FP8", ) # no tensor parallel, just replicated self.wq_b = ReplicatedLinear( self.q_lora_rank, self.head_dim * self.n_head, bias=False, quant_config=quant_config, prefix=f"{prefix}.wq_b", ) self.weights_proj = ReplicatedLinear( hidden_size, self.n_head, bias=False, quant_config=None, prefix=f"{prefix}.weights_proj", ) self.softmax_scale = self.head_dim**-0.5 self.scale_fmt = "ue8m0" self.quant_block_size = 128 # TODO: get from config self.topk_indices_buffer = topk_indices_buffer self.max_model_len = ( vllm_config.model_config.max_model_len // self.compress_ratio ) self.prefix = prefix self.max_total_seq_len = ( get_max_prefill_buffer_size(vllm_config) // self.compress_ratio ) assert cache_config is not None, "Deepseek V4 indexer requires cache_config" if owns_k: # wk maps the main compressor's (pre-RoPE) latent to index keys. # The checkpoint stores it in bf16 with no quantization scales. self.wk = ReplicatedLinear( main_head_dim, self.head_dim, bias=False, quant_config=None, prefix=f"{prefix}.wk", ) self.k_norm = RMSNorm(self.head_dim, config.rms_norm_eps) self.k_cache = k_cache # Candidate consumers can score only the candidate blocks (opt-in); # the candidate source and pre-candidate indexers stay dense. use_sparse_logits = ( vllm_config.attention_config.indexer_sparse_logits and candidate_block_buffer is not None and not candidate_write ) self.indexer_op: SparseAttnIndexer | SparseMQAIndexer | RocmSparseMQAIndexer if use_sparse_logits: if not self.use_fp4_kv: raise ValueError( "attention_config.indexer_sparse_logits requires the MXFP4 " "indexer cache (attention_config.indexer_kv_dtype='mxfp4')." ) assert candidate_block_buffer is not None assert topk_indices_buffer is not None self.indexer_op = self.mqa_cls( self.k_cache, self.topk_tokens, self.head_dim, self.max_total_seq_len, topk_indices_buffer, candidate_block_buffer, candidate_block_size, ) else: self.indexer_op = self.attn_cls( self.k_cache, self.quant_block_size, self.scale_fmt, self.topk_tokens, self.head_dim, self.max_model_len, self.max_total_seq_len, self.topk_indices_buffer, skip_k_cache_insert=True, use_fp4_cache=self.use_fp4_kv, compress_ratio=self.compress_ratio, candidate_blocks=candidate_block_buffer, candidate_block_size=candidate_block_size, candidate_write=candidate_write, ) # The fused Q kernel writes the per-head weights in the dtype the # scoring kernels take, so no cast runs per step. self.indexer_weights_dtype = ( self.mqa_cls.weights_dtype if use_sparse_logits else torch.float32 ) def _produce_k( self, latent: torch.Tensor | None, positions: torch.Tensor, rotary_emb: nn.Module, ) -> None: """Turn the compressor's pre-RoPE latent into paged index keys. ``k_norm(wk(latent))`` at group-boundary tokens, RoPE'd at the group position and MXFP4/FP8-quantized into the indexer K cache. """ attn_metadata = get_forward_context().attn_metadata if not isinstance(attn_metadata, dict) or latent is None: # Profile run: the indexer K cache is not bound yet and the # compressor skipped its latent output. return assert self.owns_k indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix]) # ReplicatedLinear returns (output, bias); bias is None. Rows at # non-boundary tokens hold garbage latent and are skipped by the # store kernel. k_pre, _ = self.wk(latent) indexer_k_norm_rope_store( k_pre, positions, rotary_emb.cos_sin_cache, self.k_norm.weight, self.k_norm.variance_epsilon, self.k_cache.kv_cache, indexer_metadata.slot_mapping, self.compress_ratio, self.use_fp4_kv, ) def forward_q( self, qr: torch.Tensor | QuantizedActivation, qr_scale: torch.Tensor | None, indexer_weights: torch.Tensor, positions: torch.Tensor, rotary_emb: nn.Module, ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: """Build the indexer queries: wq_b over qr plus fused RoPE/quant. Split out so the ROCm layer can schedule it apart from ``_produce_k`` (e.g. on an aux stream once ``qr`` is ready). """ q = self._wq_b_proj(qr, qr_scale) q = q.view(-1, self.n_head, self.head_dim) q_quant, weights = fused_indexer_q_rope_quant( positions, q, rotary_emb.cos_sin_cache, indexer_weights, self.softmax_scale, self.n_head**-0.5, use_fp4=self.use_fp4_kv, weights_out_dtype=self.indexer_weights_dtype, ) if isinstance(q_quant, tuple): q, q_scale = q_quant else: q, q_scale = q_quant, None return q, q_scale, weights def forward( self, qr: torch.Tensor | QuantizedActivation, latent: torch.Tensor | None, indexer_weights: torch.Tensor, positions: torch.Tensor, rotary_emb: nn.Module, qr_scale: torch.Tensor | None = None, *, cache_latent: torch.Tensor | None = None, cache_positions: torch.Tensor | None = None, ) -> tuple[torch.Tensor | None, torch.Tensor | None, torch.Tensor | None]: if cache_latent is None: cache_latent = latent if cache_positions is None: cache_positions = positions attn_metadata = get_forward_context().attn_metadata if isinstance(attn_metadata, dict): indexer_metadata = cast(Any, attn_metadata[self.k_cache.prefix]) if ( indexer_metadata.max_seq_len // self.compress_ratio <= self.topk_tokens and not torch.cuda.is_current_stream_capturing() ): # candidates num smaller than topk, every candidate is selected # but we still need to build k cache if self.owns_k: self._produce_k(cache_latent, cache_positions, rotary_emb) assert self.topk_indices_buffer is not None num_tokens = ( indexer_metadata.num_decode_tokens + indexer_metadata.num_prefill_tokens ) if num_tokens > 0: _fill_short_context_topk_indices[(num_tokens,)]( self.topk_indices_buffer, positions, TOP_K=self.topk_tokens, COMPRESS_RATIO=self.compress_ratio, PADDED_TOP_K=triton.next_power_of_2(self.topk_tokens), num_warps=8, ) return None, None, None if self.owns_k: # K write must land before indexer_op reads the cache # (skip_k_cache_insert=True). self._produce_k(cache_latent, cache_positions, rotary_emb) return self.forward_q(qr, qr_scale, indexer_weights, positions, rotary_emb) def _wq_b_proj( self, qr: torch.Tensor | QuantizedActivation, qr_scale: torch.Tensor | None = None, ) -> torch.Tensor: """Project normalized Q with the main attention's shared quantization.""" assert qr_scale is None, "ROCm-only path" # ReplicatedLinear returns (output, bias); bias is None. q, _ = self.wq_b(qr) return q