[DSA] Wire DeepGEMM sparse MQA logits into the DeepSeek V4.1 indexer (#56254)

Signed-off-by: Jared Wen <w13431838023@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jared Wen
2026-09-15 14:07:14 -07:00
committed by GitHub
co-authored by Claude Fable 5.1
parent f2aad6aa70
commit 1257512305
13 changed files with 1830 additions and 26 deletions
+183
View File
@@ -158,6 +158,10 @@ _grouped_fp4_impl: Callable[..., Any] | None = None
_fp8_fp4_mqa_logits_impl: Callable[..., Any] | None = None
_fp8_fp4_paged_mqa_logits_impl: Callable[..., Any] | None = None
_get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_fp8_fp4_sparse_mqa_logits_impl: Callable[..., Any] | None = None
_fp8_fp4_paged_sparse_mqa_logits_impl: Callable[..., Any] | None = None
_get_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_get_paged_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
_mega_mhc_impl: Callable[..., Any] | None = None
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
@@ -232,6 +236,9 @@ def _lazy_init() -> None:
global _grouped_impl, _grouped_masked_impl, _grouped_fp4_impl
global _fp8_fp4_mqa_logits_impl, _fp8_fp4_paged_mqa_logits_impl
global _get_paged_mqa_logits_metadata_impl
global _fp8_fp4_sparse_mqa_logits_impl, _fp8_fp4_paged_sparse_mqa_logits_impl
global _get_sparse_mqa_logits_metadata_impl
global _get_paged_sparse_mqa_logits_metadata_impl
global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl
global _get_mn_major_tma_aligned_tensor_impl
global _get_mk_alignment_for_contiguous_layout_impl
@@ -295,6 +302,17 @@ def _lazy_init() -> None:
_get_paged_mqa_logits_metadata_impl = getattr(
_dg, "get_paged_mqa_logits_metadata", None
)
# Sparse-indexer kernels (DeepGEMM >= 2.8, SM100 only).
_fp8_fp4_sparse_mqa_logits_impl = getattr(_dg, "fp8_fp4_sparse_mqa_logits", None)
_fp8_fp4_paged_sparse_mqa_logits_impl = getattr(
_dg, "fp8_fp4_paged_sparse_mqa_logits", None
)
_get_sparse_mqa_logits_metadata_impl = getattr(
_dg, "get_sparse_mqa_logits_metadata", None
)
_get_paged_sparse_mqa_logits_metadata_impl = getattr(
_dg, "get_paged_sparse_mqa_logits_metadata", None
)
_tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None)
_mega_mhc_impl = getattr(_dg, "mega_mhc", None)
_get_mn_major_tma_aligned_tensor_impl = getattr(
@@ -681,6 +699,171 @@ def fp8_fp4_paged_mqa_logits(
)
def has_deep_gemm_sparse_mqa() -> bool:
"""Whether the installed DeepGEMM provides the sparse-indexer kernels
(``fp8_fp4_(paged_)sparse_mqa_logits``, added in DeepGEMM 2.8, SM100-only)."""
_lazy_init()
return (
_fp8_fp4_sparse_mqa_logits_impl is not None
and _fp8_fp4_paged_sparse_mqa_logits_impl is not None
and _get_sparse_mqa_logits_metadata_impl is not None
and _get_paged_sparse_mqa_logits_metadata_impl is not None
)
def get_sparse_mqa_logits_metadata(
cu_seqlen_ks: torch.Tensor,
cu_seqlen_ke: torch.Tensor,
num_kv_tokens: int,
sparse_kv_block_indices: torch.Tensor,
qk_dtype: torch.dtype,
sparse_block_kv: int,
) -> torch.Tensor:
"""Build scheduling metadata for `fp8_fp4_sparse_mqa_logits`.
Always uses the unaligned-ks variant, which degenerates to the aligned
variant when ``cu_seqlen_ks % sparse_block_kv == 0``, so callers never
need a host-side sync to pick one.
Args:
cu_seqlen_ks/cu_seqlen_ke: Per-row K range bounds in the packed KV
workspace, shape [num_q_tokens], dtype int32.
num_kv_tokens: Total KV tokens in the packed workspace.
sparse_kv_block_indices: Per-row candidate block ids, shape
[num_q_tokens, num_max_sparse_blocks], dtype int32. Each row's
valid prefix must be sorted ascending (repeats are tolerated but
wasteful); pad by repeating the last valid block. Block ``i``
covers tokens
``[i * sparse_block_kv + ks % sparse_block_kv, ...)``.
qk_dtype: dtype of the packed Q values (``torch.int8`` for MXFP4,
``torch.float8_e4m3fn`` for FP8).
sparse_block_kv: Tokens per sparse block, 8 or 16.
"""
_lazy_init()
if _get_sparse_mqa_logits_metadata_impl is None:
return _missing()
return _get_sparse_mqa_logits_metadata_impl(
cu_seqlen_ks,
cu_seqlen_ke,
num_kv_tokens,
sparse_kv_block_indices,
qk_dtype,
sparse_block_kv,
use_unaligned_ks=True,
)
def get_paged_sparse_mqa_logits_metadata(
context_lens: torch.Tensor,
block_table: torch.Tensor,
indices: torch.Tensor,
page_kv: int,
sparse_kv_block_indices: torch.Tensor,
qk_dtype: torch.dtype,
sparse_block_kv: int,
) -> torch.Tensor:
"""Build scheduling metadata for `fp8_fp4_paged_sparse_mqa_logits`.
Args:
context_lens: Per-row context length, shape [num_q_tokens], int32.
block_table: Per-row page table, shape [num_q_tokens, max_pages],
int32; rows of queries paired with the same request must be
identical (expand it per query row before calling).
indices: Request index for each query row, shape [num_q_tokens].
page_kv: Tokens per KV page; must be a multiple of sparse_block_kv.
sparse_kv_block_indices: Per-row logical (context-relative) candidate
block ids, [num_q_tokens, num_max_sparse_blocks], int32.
qk_dtype: dtype of the packed Q values.
sparse_block_kv: Tokens per sparse block, 8 or 16.
"""
_lazy_init()
if _get_paged_sparse_mqa_logits_metadata_impl is None:
return _missing()
return _get_paged_sparse_mqa_logits_metadata_impl(
context_lens,
block_table,
indices,
page_kv,
sparse_kv_block_indices,
qk_dtype,
sparse_block_kv,
)
def fp8_fp4_sparse_mqa_logits(
q: tuple[torch.Tensor, torch.Tensor],
kv: tuple[torch.Tensor, torch.Tensor],
weights: torch.Tensor,
metadata: torch.Tensor,
num_max_sparse_blocks: int,
sparse_block_kv: int,
) -> torch.Tensor:
"""Compute MQA logits only at the candidate blocks (prefill path).
Args:
q: ``(q_values, q_scale)``. q_values is [M, H, D] (packed FP4 viewed
as int8, or FP8); q_scale is the packed UE8M0 scale tensor [M, H]
int32 (mandatory, unlike the dense kernels).
kv: ``(kv_values, kv_scale)`` — the packed KV workspace with UE8M0
int32 scales.
weights: [M, H] ``torch.bfloat16`` (the Q scale is NOT folded in).
metadata: From `get_sparse_mqa_logits_metadata`.
num_max_sparse_blocks: Width of each sparse-index row.
sparse_block_kv: Tokens per sparse block, 8 or 16.
Returns:
bf16 logits of shape [M, num_max_sparse_blocks * sparse_block_kv];
column ``j * sparse_block_kv + o`` scores the token at
``sparse_kv_block_indices[row, j] * sparse_block_kv + ks % sbk + o``.
"""
_lazy_init()
if _fp8_fp4_sparse_mqa_logits_impl is None:
return _missing()
return _fp8_fp4_sparse_mqa_logits_impl(
q,
kv,
weights,
metadata,
num_max_sparse_blocks,
sparse_block_kv,
use_unaligned_ks=True,
)
def fp8_fp4_paged_sparse_mqa_logits(
q: tuple[torch.Tensor, torch.Tensor],
kv_cache: torch.Tensor,
weights: torch.Tensor,
metadata: torch.Tensor,
num_max_sparse_blocks: int,
sparse_block_kv: int,
) -> torch.Tensor:
"""Compute MQA logits only at the candidate blocks (paged decode path).
Args:
q: ``(q_values, q_scale)``; q_values is [num_q_tokens, 1, H, D].
kv_cache: Fused paged cache [num_pages, page_kv, 1, D' + 4] uint8,
page stride 512B-aligned.
weights: [num_q_tokens, H] ``torch.bfloat16``.
metadata: From `get_paged_sparse_mqa_logits_metadata`.
Returns:
bf16 logits of shape
[num_q_tokens, num_max_sparse_blocks * sparse_block_kv].
"""
_lazy_init()
if _fp8_fp4_paged_sparse_mqa_logits_impl is None:
return _missing()
return _fp8_fp4_paged_sparse_mqa_logits_impl(
q,
kv_cache,
weights,
metadata,
num_max_sparse_blocks,
sparse_block_kv,
)
def tf32_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,