[DSA] Wire DeepGEMM sparse MQA logits into the DeepSeek V4.1 indexer (#56254)
Signed-off-by: Jared Wen <w13431838023@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f2aad6aa70
commit
1257512305
@@ -309,12 +309,22 @@ class FusedIndexerQRopeQuantTritonKernel(
|
||||
)
|
||||
|
||||
|
||||
def _indexer_weights_out_dtypes(vllm_config: Any) -> tuple[torch.dtype, ...]:
|
||||
"""Weights dtypes the model's indexer layers ask for: fp32 for the dense
|
||||
scoring kernels, plus bf16 when the DeepSeek V4.1 sparse-logits indexer
|
||||
(`SparseMQAIndexer`) is enabled."""
|
||||
if vllm_config.attention_config.indexer_sparse_logits:
|
||||
return (torch.float32, torch.bfloat16)
|
||||
return (torch.float32,)
|
||||
|
||||
|
||||
class FusedIndexerQRopeMxFp4TritonKernel(
|
||||
VllmTritonJitKernel["FusedIndexerQRopeMxFp4TritonKernel.CompileKey"]
|
||||
):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
dtype: torch.dtype
|
||||
weights_out_dtype: torch.dtype
|
||||
num_heads: int
|
||||
index_q_half_rot_dim: int
|
||||
index_q_head_dim: int
|
||||
@@ -434,12 +444,14 @@ class FusedIndexerQRopeMxFp4TritonKernel(
|
||||
self,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
weights_out_dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
rope_dim: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
dtype=dtype,
|
||||
weights_out_dtype=weights_out_dtype,
|
||||
num_heads=num_heads,
|
||||
index_q_half_rot_dim=rope_dim // 2,
|
||||
index_q_head_dim=head_dim,
|
||||
@@ -456,6 +468,7 @@ class FusedIndexerQRopeMxFp4TritonKernel(
|
||||
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
weights_out_dtype=_indexer_weights_out_dtypes(vllm_config),
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
@@ -491,7 +504,7 @@ class FusedIndexerQRopeMxFp4TritonKernel(
|
||||
shape=(1, compile_key.num_heads, scale_head_dim),
|
||||
),
|
||||
index_weights_out=TritonWarmupTensor(
|
||||
torch.float32,
|
||||
compile_key.weights_out_dtype,
|
||||
shape=(1, compile_key.num_heads),
|
||||
),
|
||||
)
|
||||
@@ -540,12 +553,17 @@ def fused_indexer_q_rope_quant(
|
||||
index_weights_softmax_scale: float,
|
||||
index_weights_head_scale: float,
|
||||
use_fp4: bool = False,
|
||||
weights_out_dtype: torch.dtype = torch.float32,
|
||||
) -> tuple[
|
||||
torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""Fused RoPE + quantize Q for the sparse indexer.
|
||||
|
||||
``weights_out_dtype`` is the dtype the downstream scoring kernel takes:
|
||||
fp32 for the dense MQA-logits kernels, bf16 for DeepGEMM's sparse
|
||||
MQA-logits kernels (CUDA MXFP4 path only).
|
||||
|
||||
Weight-fold semantics (important — the two paths differ):
|
||||
|
||||
FP8 path (use_fp4=False, default):
|
||||
@@ -577,7 +595,7 @@ def fused_indexer_q_rope_quant(
|
||||
num_index_q_heads = index_q.shape[1]
|
||||
index_q_head_dim = index_q.shape[2]
|
||||
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=torch.float32)
|
||||
index_weights_out = torch.empty_like(index_weights, dtype=weights_out_dtype)
|
||||
|
||||
if use_fp4:
|
||||
assert index_q_head_dim % MXFP4_BLOCK_SIZE == 0, (
|
||||
@@ -613,6 +631,7 @@ def fused_indexer_q_rope_quant(
|
||||
weights_out=index_weights_out,
|
||||
)
|
||||
elif current_platform.is_xpu():
|
||||
assert weights_out_dtype == torch.float32, weights_out_dtype
|
||||
torch.ops.vllm.xpu_deepseek_fused_indexer_q_rope_mxfp4(
|
||||
index_q,
|
||||
positions,
|
||||
@@ -647,6 +666,7 @@ def fused_indexer_q_rope_quant(
|
||||
index_q_scale.view(torch.int32).squeeze(-1),
|
||||
), index_weights_out
|
||||
|
||||
assert weights_out_dtype == torch.float32, weights_out_dtype
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
use_fnuz = fp8_dtype == torch.float8_e4m3fnuz
|
||||
fp8_max = 224.0 if use_fnuz else 448.0
|
||||
|
||||
@@ -165,6 +165,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
rope_dim: int
|
||||
num_heads: int
|
||||
cos_sin_dtype: type[cutlass.Numeric]
|
||||
weights_out_dtype: type[cutlass.Numeric]
|
||||
coarsen: int
|
||||
|
||||
@staticmethod
|
||||
@@ -177,6 +178,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
# process multiple heads at the same time to armotize RoPE load costs
|
||||
coarsen = compile_key.coarsen
|
||||
tb_size = IndexerQMxFp4Kernel.tb_size
|
||||
weights_out_dtype = compile_key.weights_out_dtype
|
||||
assert num_heads % coarsen == 0
|
||||
|
||||
# later we will use 32B load = 16 BF16 elems
|
||||
@@ -287,7 +289,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
weight_head_id = global_tid % num_heads
|
||||
weights_out[weight_token_id, weight_head_id] = (
|
||||
weights[weight_token_id, weight_head_id].to(Float32) * scale
|
||||
)
|
||||
).to(weights_out_dtype)
|
||||
|
||||
@cute.jit
|
||||
def host_entrypoint(
|
||||
@@ -320,11 +322,13 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
self,
|
||||
*,
|
||||
cos_sin_dtype: type[cutlass.Numeric],
|
||||
weights_out_dtype: type[cutlass.Numeric],
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
cos_sin_dtype=cos_sin_dtype,
|
||||
weights_out_dtype=weights_out_dtype,
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
@@ -340,11 +344,17 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
if num_heads <= 0 or head_dim <= 0 or rope_dim <= 0:
|
||||
return []
|
||||
|
||||
# The DeepSeek V4.1 sparse-logits indexer takes bf16 weights; the
|
||||
# dense scoring kernels take fp32.
|
||||
weights_out_dtypes: tuple[type[cutlass.Numeric], ...] = (Float32,)
|
||||
if vllm_config.attention_config.indexer_sparse_logits:
|
||||
weights_out_dtypes = (Float32, BFloat16)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
head_dim=head_dim,
|
||||
rope_dim=rope_dim,
|
||||
num_heads=num_heads,
|
||||
cos_sin_dtype=Float32,
|
||||
weights_out_dtype=weights_out_dtypes,
|
||||
coarsen=(1, 4),
|
||||
)
|
||||
|
||||
@@ -380,7 +390,9 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
divisibility=4,
|
||||
)
|
||||
weights_out = make_fake_tensor(
|
||||
Float32, (num_tokens, compile_key.num_heads), divisibility=4
|
||||
compile_key.weights_out_dtype,
|
||||
(num_tokens, compile_key.num_heads),
|
||||
divisibility=4,
|
||||
)
|
||||
return (
|
||||
positions,
|
||||
@@ -413,6 +425,7 @@ class IndexerQMxFp4Kernel(VllmCuTeDSLJitKernel["IndexerQMxFp4Kernel.CompileKey"]
|
||||
rope_dim=cos_sin_cache.shape[-1],
|
||||
num_heads=num_heads,
|
||||
cos_sin_dtype=torch_to_cute_dtype(cos_sin_cache.dtype),
|
||||
weights_out_dtype=torch_to_cute_dtype(weights_out.dtype),
|
||||
coarsen=1 if num_tokens < 512 else 4,
|
||||
)
|
||||
launch_args = (
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
DeepseekV4 MLA Attention Layer
|
||||
"""
|
||||
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from functools import cached_property
|
||||
@@ -25,6 +26,7 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
|
||||
from vllm.model_executor.layers.sparse_mqa_indexer import SparseMQAIndexer
|
||||
from vllm.models.common.ops import fused_q_kv_rmsnorm
|
||||
from vllm.models.deepseek_v41.common.ops import (
|
||||
MXFP4_BLOCK_SIZE,
|
||||
@@ -63,6 +65,9 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
dsa_indexer_uses_fp4,
|
||||
get_max_prefill_buffer_size,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_indexer import (
|
||||
DeepseekV41SparseIndexerBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import DeepseekV4SWACache
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheSpec,
|
||||
@@ -997,7 +1002,9 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
self.cache_config = cache_config
|
||||
self.dtype = dtype
|
||||
self.compress_ratio = compress_ratio
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
vllm_config = get_current_vllm_config()
|
||||
self.sparse_logits = vllm_config.attention_config.indexer_sparse_logits
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
@@ -1010,6 +1017,9 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
# head_dim already carries the fp8 scale padding
|
||||
# tokens_per_state=1 for V3.2, >1 for DeepseekV4; same cache layout.
|
||||
uses_fp8_ds_mla_layout = vllm_config.cache_config.cache_dtype == "fp8_ds_mla"
|
||||
page_alignment = (
|
||||
576 if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record() else 512
|
||||
)
|
||||
return MLAAttentionSpec(
|
||||
block_size=self.cache_config.block_size,
|
||||
num_kv_heads=1,
|
||||
@@ -1020,16 +1030,21 @@ class DeepseekV4IndexerCache(torch.nn.Module, AttentionLayerBase):
|
||||
# (the block stride is the sum of every page in the group), so it
|
||||
# has to carry the same padding: the packed record's TMA stride,
|
||||
# else 512B for FlashInfer sparse (#44577).
|
||||
alignment=(
|
||||
576
|
||||
if uses_fp8_ds_mla_layout and not _use_v41_mxfp8_kv_record()
|
||||
else 512
|
||||
alignment=page_alignment,
|
||||
# DeepGEMM's paged sparse MQA logits address pages by the block
|
||||
# stride, which packs every layer's page in this layout, and need
|
||||
# it 512B-aligned; the main KV record read from the same blocks
|
||||
# needs its own TMA stride, so keep both.
|
||||
block_stride_alignment=(
|
||||
math.lcm(512, page_alignment) if self.sparse_logits else None
|
||||
),
|
||||
)
|
||||
|
||||
def forward(self): ...
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
if self.sparse_logits:
|
||||
return DeepseekV41SparseIndexerBackend
|
||||
return DeepseekV41IndexerBackend
|
||||
|
||||
|
||||
@@ -1129,21 +1144,52 @@ class DeepseekV4Indexer(nn.Module):
|
||||
self.k_norm = RMSNorm(self.head_dim, config.rms_norm_eps)
|
||||
self.k_cache = k_cache
|
||||
|
||||
self.indexer_op = SparseAttnIndexer(
|
||||
self.k_cache,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=True,
|
||||
use_fp4_cache=self.use_fp4_kv,
|
||||
compress_ratio=self.compress_ratio,
|
||||
candidate_blocks=candidate_block_buffer,
|
||||
candidate_block_size=candidate_block_size,
|
||||
candidate_write=candidate_write,
|
||||
# Candidate consumers can score only the candidate blocks (opt-in);
|
||||
# the candidate source and pre-candidate indexers stay dense.
|
||||
use_sparse_logits = (
|
||||
vllm_config.attention_config.indexer_sparse_logits
|
||||
and candidate_block_buffer is not None
|
||||
and not candidate_write
|
||||
)
|
||||
self.indexer_op: SparseAttnIndexer | SparseMQAIndexer
|
||||
if use_sparse_logits:
|
||||
if not self.use_fp4_kv:
|
||||
raise ValueError(
|
||||
"attention_config.indexer_sparse_logits requires the MXFP4 "
|
||||
"indexer cache (attention_config.indexer_kv_dtype='mxfp4')."
|
||||
)
|
||||
assert candidate_block_buffer is not None
|
||||
assert topk_indices_buffer is not None
|
||||
self.indexer_op = SparseMQAIndexer(
|
||||
self.k_cache,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_total_seq_len,
|
||||
topk_indices_buffer,
|
||||
candidate_block_buffer,
|
||||
candidate_block_size,
|
||||
)
|
||||
else:
|
||||
self.indexer_op = SparseAttnIndexer(
|
||||
self.k_cache,
|
||||
self.quant_block_size,
|
||||
self.scale_fmt,
|
||||
self.topk_tokens,
|
||||
self.head_dim,
|
||||
self.max_model_len,
|
||||
self.max_total_seq_len,
|
||||
self.topk_indices_buffer,
|
||||
skip_k_cache_insert=True,
|
||||
use_fp4_cache=self.use_fp4_kv,
|
||||
compress_ratio=self.compress_ratio,
|
||||
candidate_blocks=candidate_block_buffer,
|
||||
candidate_block_size=candidate_block_size,
|
||||
candidate_write=candidate_write,
|
||||
)
|
||||
# The fused Q kernel writes the per-head weights in the dtype the
|
||||
# scoring kernels take, so no cast runs per step.
|
||||
self.indexer_weights_dtype = (
|
||||
SparseMQAIndexer.weights_dtype if use_sparse_logits else torch.float32
|
||||
)
|
||||
|
||||
def _produce_k(
|
||||
@@ -1231,6 +1277,7 @@ class DeepseekV4Indexer(nn.Module):
|
||||
self.softmax_scale,
|
||||
self.n_head**-0.5,
|
||||
use_fp4=self.use_fp4_kv,
|
||||
weights_out_dtype=self.indexer_weights_dtype,
|
||||
)
|
||||
if isinstance(q_quant, tuple):
|
||||
q, q_scale = q_quant
|
||||
|
||||
Reference in New Issue
Block a user