[DSA] Wire DeepGEMM sparse MQA logits into the DeepSeek V4.1 indexer (#56254)

Signed-off-by: Jared Wen <w13431838023@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jared Wen
2026-09-15 14:07:14 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent f2aad6aa70
commit 1257512305
13 changed files with 1830 additions and 26 deletions
@@ -309,12 +309,22 @@ class FusedIndexerQRopeQuantTritonKernel(
)
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
(`SparseMQAIndexer`) is enabled."""
if vllm_config.attention_config.indexer_sparse_logits:
return (torch.float32, torch.bfloat16)
return (torch.float32,)
class FusedIndexerQRopeMxFp4TritonKernel(
VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"]
):
@dataclass(frozen=True)
class CompileKey:
dtype: torch.dtype
weights_out_dtype: torch.dtype
num_heads: int
index_q_half_rot_dim: int
index_q_head_dim: int
@@ -434,12 +444,14 @@ class FusedIndexerQRopeMxFp4TritonKernel(
self,
*,
dtype: torch.dtype,
weights_out_dtype: torch.dtype,
num_heads: int,
head_dim: int,
rope_dim: int,
) -> CompileKey:
return self.CompileKey(
dtype=dtype,
weights_out_dtype=weights_out_dtype,
num_heads=num_heads,
index_q_half_rot_dim=rope_dim // 2,
index_q_head_dim=head_dim,
@@ -456,6 +468,7 @@ class FusedIndexerQRopeMxFp4TritonKernel(
return self._trace_dispatch(self.dispatch)(
dtype=vllm_config.model_config.dtype,
weights_out_dtype=_indexer_weights_out_dtypes(vllm_config),
num_heads=num_heads,
head_dim=head_dim,
rope_dim=rope_dim,
@@ -491,7 +504,7 @@ class FusedIndexerQRopeMxFp4TritonKernel(
shape=(1, compile_key.num_heads, scale_head_dim),
),
index_weights_out=TritonWarmupTensor(
torch.float32,
compile_key.weights_out_dtype,
shape=(1, compile_key.num_heads),
),
)
@@ -540,12 +553,17 @@ def fused_indexer_q_rope_quant(
index_weights_softmax_scale: float,
index_weights_head_scale: float,
use_fp4: bool = False,
weights_out_dtype: torch.dtype = torch.float32,
) -> tuple[
torch.Tensor | tuple[torch.Tensor, torch.Tensor],
torch.Tensor,
]:
"""Fused RoPE + quantize Q for the sparse indexer.
``weights_out_dtype`` is the dtype the downstream scoring kernel takes:
fp32 for the dense MQA-logits kernels, bf16 for DeepGEMM's sparse
MQA-logits kernels (CUDA MXFP4 path only).
Weight-fold semantics (important — the two paths differ):
FP8 path (use_fp4=False, default):
@@ -577,7 +595,7 @@ def fused_indexer_q_rope_quant(
num_index_q_heads = index_q.shape[1]
index_q_head_dim = index_q.shape[2]
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
index_weights_out = torch.empty_like(index_weights, dtype=weights_out_dtype)
if use_fp4:
assert index_q_head_dim % MXFP4_BLOCK_SIZE == 0, (
@@ -613,6 +631,7 @@ def fused_indexer_q_rope_quant(
weights_out=index_weights_out,
)
elif current_platform.is_xpu():
assert weights_out_dtype == torch.float32, weights_out_dtype
torch.ops.vllm.xpu_deepseek_fused_indexer_q_rope_mxfp4(
index_q,
positions,
@@ -647,6 +666,7 @@ def fused_indexer_q_rope_quant(
index_q_scale.view(torch.int32).squeeze(-1),
), index_weights_out
assert weights_out_dtype == torch.float32, weights_out_dtype
fp8_dtype = current_platform.fp8_dtype()
use_fnuz = fp8_dtype == torch.float8_e4m3fnuz
fp8_max = 224.0 if use_fnuz else 448.0
@@ -165,6 +165,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
rope_dim: int
num_heads: int
cos_sin_dtype: type[cutlass.Numeric]
weights_out_dtype: type[cutlass.Numeric]
coarsen: int
@staticmethod
@@ -177,6 +178,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
# process multiple heads at the same time to armotize RoPE load costs
coarsen = compile_key.coarsen
tb_size = IndexerQMxFp4Kernel.tb_size
weights_out_dtype = compile_key.weights_out_dtype
assert num_heads % coarsen == 0
# later we will use 32B load = 16 BF16 elems
@@ -287,7 +289,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
weight_head_id = global_tid % num_heads
weights_out[weight_token_id, weight_head_id] = (
weights[weight_token_id, weight_head_id].to(Float32) * scale
)
).to(weights_out_dtype)
@cute.jit
def host_entrypoint(
@@ -320,11 +322,13 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
self,
*,
cos_sin_dtype: type[cutlass.Numeric],
weights_out_dtype: type[cutlass.Numeric],
**compile_key_fields: int,
) -> CompileKey:
return self.CompileKey(
**compile_key_fields,
cos_sin_dtype=cos_sin_dtype,
weights_out_dtype=weights_out_dtype,
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
@@ -340,11 +344,17 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
return []
# The DeepSeek V4.1 sparse-logits indexer takes bf16 weights; the
# dense scoring kernels take fp32.
weights_out_dtypes: tuple[type[cutlass.Numeric], ...] = (Float32,)
if vllm_config.attention_config.indexer_sparse_logits:
weights_out_dtypes = (Float32, BFloat16)
return self._trace_dispatch(self.dispatch)(
head_dim=head_dim,
rope_dim=rope_dim,
num_heads=num_heads,
cos_sin_dtype=Float32,
weights_out_dtype=weights_out_dtypes,
coarsen=(1, 4),
)
@@ -380,7 +390,9 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
divisibility=4,
)
weights_out = make_fake_tensor(
Float32, (num_tokens, compile_key.num_heads), divisibility=4
compile_key.weights_out_dtype,
(num_tokens, compile_key.num_heads),
divisibility=4,
)
return (
positions,
@@ -413,6 +425,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
rope_dim=cos_sin_cache.shape[-1],
num_heads=num_heads,
cos_sin_dtype=torch_to_cute_dtype(cos_sin_cache.dtype),
weights_out_dtype=torch_to_cute_dtype(weights_out.dtype),
coarsen=1 if num_tokens < 512 else 4,
)
launch_args = (
+67 -20
View File
@@ -4,6 +4,7 @@
DeepseekV4 MLA Attention Layer
"""
import math
from abc import ABC, abstractmethod
from collections.abc import Callable
from functools import cached_property
@@ -25,6 +26,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer
from vllm.models.common.ops import fused_q_kv_rmsnorm
from vllm.models.deepseek_v41.common.ops import (
MXFP4_BLOCK_SIZE,
@@ -63,6 +65,9 @@ from vllm.v1.attention.backends.mla.indexer import (
dsa_indexer_uses_fp4,
get_max_prefill_buffer_size,
)
from vllm.v1.attention.backends.mla.sparse_indexer import (
DeepseekV41SparseIndexerBackend,
)
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache
from vllm.v1.kv_cache_interface import (
KVCacheSpec,
@@ -997,7 +1002,9 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
self.cache_config = cache_config
self.dtype = dtype
self.compress_ratio = compress_ratio
compilation_config = get_current_vllm_config().compilation_config
vllm_config = get_current_vllm_config()
self.sparse_logits = vllm_config.attention_config.indexer_sparse_logits
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@@ -1010,6 +1017,9 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
# head_dim already carries the fp8 scale padding
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
page_alignment = (
576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512
)
return MLAAttentionSpec(
block_size=self.cache_config.block_size,
num_kv_heads=1,
@@ -1020,16 +1030,21 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
# (the block stride is the sum of every page in the group), so it
# has to carry the same padding: the packed record's TMA stride,
# else 512B for FlashInfer sparse (#44577).
alignment=(
576
if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record()
else 512
alignment=page_alignment,
# DeepGEMM's paged sparse MQA logits address pages by the block
# stride, which packs every layer's page in this layout, and need
# it 512B-aligned; the main KV record read from the same blocks
# needs its own TMA stride, so keep both.
block_stride_alignment=(
math.lcm(512, page_alignment) if self.sparse_logits else None
),
)
def forward(self): ...
def get_attn_backend(self) -> type[AttentionBackend]:
if self.sparse_logits:
return DeepseekV41SparseIndexerBackend
return DeepseekV41IndexerBackend
@@ -1129,21 +1144,52 @@ class DeepseekV4Indexer(nn.Module):
self.k_norm = RMSNorm(self.head_dim, config.rms_norm_eps)
self.k_cache = k_cache
self.indexer_op = SparseAttnIndexer(
self.k_cache,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=True,
use_fp4_cache=self.use_fp4_kv,
compress_ratio=self.compress_ratio,
candidate_blocks=candidate_block_buffer,
candidate_block_size=candidate_block_size,
candidate_write=candidate_write,
# Candidate consumers can score only the candidate blocks (opt-in);
# the candidate source and pre-candidate indexers stay dense.
use_sparse_logits = (
vllm_config.attention_config.indexer_sparse_logits
and candidate_block_buffer is not None
and not candidate_write
)
self.indexer_op: SparseAttnIndexer | SparseMQAIndexer
if use_sparse_logits:
if not self.use_fp4_kv:
raise ValueError(
"attention_config.indexer_sparse_logits requires the MXFP4 "
"indexer cache (attention_config.indexer_kv_dtype='mxfp4')."
)
assert candidate_block_buffer is not None
assert topk_indices_buffer is not None
self.indexer_op = SparseMQAIndexer(
self.k_cache,
self.topk_tokens,
self.head_dim,
self.max_total_seq_len,
topk_indices_buffer,
candidate_block_buffer,
candidate_block_size,
)
else:
self.indexer_op = SparseAttnIndexer(
self.k_cache,
self.quant_block_size,
self.scale_fmt,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.max_total_seq_len,
self.topk_indices_buffer,
skip_k_cache_insert=True,
use_fp4_cache=self.use_fp4_kv,
compress_ratio=self.compress_ratio,
candidate_blocks=candidate_block_buffer,
candidate_block_size=candidate_block_size,
candidate_write=candidate_write,
)
# The fused Q kernel writes the per-head weights in the dtype the
# scoring kernels take, so no cast runs per step.
self.indexer_weights_dtype = (
SparseMQAIndexer.weights_dtype if use_sparse_logits else torch.float32
)
def _produce_k(
@@ -1231,6 +1277,7 @@ class DeepseekV4Indexer(nn.Module):
self.softmax_scale,
self.n_head**-0.5,
use_fp4=self.use_fp4_kv,
weights_out_dtype=self.indexer_weights_dtype,
)
if isinstance(q_quant, tuple):
q, q_scale = q_quant