[DSA] Wire DeepGEMM sparse MQA logits into the DeepSeek V4.1 indexer (#56254)
Signed-off-by: Jared Wen <w13431838023@gmail.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
f2aad6aa70
commit
1257512305
@@ -158,6 +158,10 @@ _grouped_fp4_impl: Callable[..., Any] | None = None
|
||||
_fp8_fp4_mqa_logits_impl: Callable[..., Any] | None = None
|
||||
_fp8_fp4_paged_mqa_logits_impl: Callable[..., Any] | None = None
|
||||
_get_paged_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_fp8_fp4_sparse_mqa_logits_impl: Callable[..., Any] | None = None
|
||||
_fp8_fp4_paged_sparse_mqa_logits_impl: Callable[..., Any] | None = None
|
||||
_get_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_get_paged_sparse_mqa_logits_metadata_impl: Callable[..., Any] | None = None
|
||||
_tf32_hc_prenorm_gemm_impl: Callable[..., Any] | None = None
|
||||
_mega_mhc_impl: Callable[..., Any] | None = None
|
||||
_get_mn_major_tma_aligned_tensor_impl: Callable[..., Any] | None = None
|
||||
@@ -232,6 +236,9 @@ def _lazy_init() -> None:
|
||||
global _grouped_impl, _grouped_masked_impl, _grouped_fp4_impl
|
||||
global _fp8_fp4_mqa_logits_impl, _fp8_fp4_paged_mqa_logits_impl
|
||||
global _get_paged_mqa_logits_metadata_impl
|
||||
global _fp8_fp4_sparse_mqa_logits_impl, _fp8_fp4_paged_sparse_mqa_logits_impl
|
||||
global _get_sparse_mqa_logits_metadata_impl
|
||||
global _get_paged_sparse_mqa_logits_metadata_impl
|
||||
global _tf32_hc_prenorm_gemm_impl, _mega_mhc_impl
|
||||
global _get_mn_major_tma_aligned_tensor_impl
|
||||
global _get_mk_alignment_for_contiguous_layout_impl
|
||||
@@ -295,6 +302,17 @@ def _lazy_init() -> None:
|
||||
_get_paged_mqa_logits_metadata_impl = getattr(
|
||||
_dg, "get_paged_mqa_logits_metadata", None
|
||||
)
|
||||
# Sparse-indexer kernels (DeepGEMM >= 2.8, SM100 only).
|
||||
_fp8_fp4_sparse_mqa_logits_impl = getattr(_dg, "fp8_fp4_sparse_mqa_logits", None)
|
||||
_fp8_fp4_paged_sparse_mqa_logits_impl = getattr(
|
||||
_dg, "fp8_fp4_paged_sparse_mqa_logits", None
|
||||
)
|
||||
_get_sparse_mqa_logits_metadata_impl = getattr(
|
||||
_dg, "get_sparse_mqa_logits_metadata", None
|
||||
)
|
||||
_get_paged_sparse_mqa_logits_metadata_impl = getattr(
|
||||
_dg, "get_paged_sparse_mqa_logits_metadata", None
|
||||
)
|
||||
_tf32_hc_prenorm_gemm_impl = getattr(_dg, "tf32_hc_prenorm_gemm", None)
|
||||
_mega_mhc_impl = getattr(_dg, "mega_mhc", None)
|
||||
_get_mn_major_tma_aligned_tensor_impl = getattr(
|
||||
@@ -681,6 +699,171 @@ def fp8_fp4_paged_mqa_logits(
|
||||
)
|
||||
|
||||
|
||||
def has_deep_gemm_sparse_mqa() -> bool:
|
||||
"""Whether the installed DeepGEMM provides the sparse-indexer kernels
|
||||
(``fp8_fp4_(paged_)sparse_mqa_logits``, added in DeepGEMM 2.8, SM100-only)."""
|
||||
_lazy_init()
|
||||
return (
|
||||
_fp8_fp4_sparse_mqa_logits_impl is not None
|
||||
and _fp8_fp4_paged_sparse_mqa_logits_impl is not None
|
||||
and _get_sparse_mqa_logits_metadata_impl is not None
|
||||
and _get_paged_sparse_mqa_logits_metadata_impl is not None
|
||||
)
|
||||
|
||||
|
||||
def get_sparse_mqa_logits_metadata(
|
||||
cu_seqlen_ks: torch.Tensor,
|
||||
cu_seqlen_ke: torch.Tensor,
|
||||
num_kv_tokens: int,
|
||||
sparse_kv_block_indices: torch.Tensor,
|
||||
qk_dtype: torch.dtype,
|
||||
sparse_block_kv: int,
|
||||
) -> torch.Tensor:
|
||||
"""Build scheduling metadata for `fp8_fp4_sparse_mqa_logits`.
|
||||
|
||||
Always uses the unaligned-ks variant, which degenerates to the aligned
|
||||
variant when ``cu_seqlen_ks % sparse_block_kv == 0``, so callers never
|
||||
need a host-side sync to pick one.
|
||||
|
||||
Args:
|
||||
cu_seqlen_ks/cu_seqlen_ke: Per-row K range bounds in the packed KV
|
||||
workspace, shape [num_q_tokens], dtype int32.
|
||||
num_kv_tokens: Total KV tokens in the packed workspace.
|
||||
sparse_kv_block_indices: Per-row candidate block ids, shape
|
||||
[num_q_tokens, num_max_sparse_blocks], dtype int32. Each row's
|
||||
valid prefix must be sorted ascending (repeats are tolerated but
|
||||
wasteful); pad by repeating the last valid block. Block ``i``
|
||||
covers tokens
|
||||
``[i * sparse_block_kv + ks % sparse_block_kv, ...)``.
|
||||
qk_dtype: dtype of the packed Q values (``torch.int8`` for MXFP4,
|
||||
``torch.float8_e4m3fn`` for FP8).
|
||||
sparse_block_kv: Tokens per sparse block, 8 or 16.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_sparse_mqa_logits_metadata_impl is None:
|
||||
return _missing()
|
||||
return _get_sparse_mqa_logits_metadata_impl(
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
num_kv_tokens,
|
||||
sparse_kv_block_indices,
|
||||
qk_dtype,
|
||||
sparse_block_kv,
|
||||
use_unaligned_ks=True,
|
||||
)
|
||||
|
||||
|
||||
def get_paged_sparse_mqa_logits_metadata(
|
||||
context_lens: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
indices: torch.Tensor,
|
||||
page_kv: int,
|
||||
sparse_kv_block_indices: torch.Tensor,
|
||||
qk_dtype: torch.dtype,
|
||||
sparse_block_kv: int,
|
||||
) -> torch.Tensor:
|
||||
"""Build scheduling metadata for `fp8_fp4_paged_sparse_mqa_logits`.
|
||||
|
||||
Args:
|
||||
context_lens: Per-row context length, shape [num_q_tokens], int32.
|
||||
block_table: Per-row page table, shape [num_q_tokens, max_pages],
|
||||
int32; rows of queries paired with the same request must be
|
||||
identical (expand it per query row before calling).
|
||||
indices: Request index for each query row, shape [num_q_tokens].
|
||||
page_kv: Tokens per KV page; must be a multiple of sparse_block_kv.
|
||||
sparse_kv_block_indices: Per-row logical (context-relative) candidate
|
||||
block ids, [num_q_tokens, num_max_sparse_blocks], int32.
|
||||
qk_dtype: dtype of the packed Q values.
|
||||
sparse_block_kv: Tokens per sparse block, 8 or 16.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _get_paged_sparse_mqa_logits_metadata_impl is None:
|
||||
return _missing()
|
||||
return _get_paged_sparse_mqa_logits_metadata_impl(
|
||||
context_lens,
|
||||
block_table,
|
||||
indices,
|
||||
page_kv,
|
||||
sparse_kv_block_indices,
|
||||
qk_dtype,
|
||||
sparse_block_kv,
|
||||
)
|
||||
|
||||
|
||||
def fp8_fp4_sparse_mqa_logits(
|
||||
q: tuple[torch.Tensor, torch.Tensor],
|
||||
kv: tuple[torch.Tensor, torch.Tensor],
|
||||
weights: torch.Tensor,
|
||||
metadata: torch.Tensor,
|
||||
num_max_sparse_blocks: int,
|
||||
sparse_block_kv: int,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits only at the candidate blocks (prefill path).
|
||||
|
||||
Args:
|
||||
q: ``(q_values, q_scale)``. q_values is [M, H, D] (packed FP4 viewed
|
||||
as int8, or FP8); q_scale is the packed UE8M0 scale tensor [M, H]
|
||||
int32 (mandatory, unlike the dense kernels).
|
||||
kv: ``(kv_values, kv_scale)`` — the packed KV workspace with UE8M0
|
||||
int32 scales.
|
||||
weights: [M, H] ``torch.bfloat16`` (the Q scale is NOT folded in).
|
||||
metadata: From `get_sparse_mqa_logits_metadata`.
|
||||
num_max_sparse_blocks: Width of each sparse-index row.
|
||||
sparse_block_kv: Tokens per sparse block, 8 or 16.
|
||||
|
||||
Returns:
|
||||
bf16 logits of shape [M, num_max_sparse_blocks * sparse_block_kv];
|
||||
column ``j * sparse_block_kv + o`` scores the token at
|
||||
``sparse_kv_block_indices[row, j] * sparse_block_kv + ks % sbk + o``.
|
||||
"""
|
||||
_lazy_init()
|
||||
if _fp8_fp4_sparse_mqa_logits_impl is None:
|
||||
return _missing()
|
||||
return _fp8_fp4_sparse_mqa_logits_impl(
|
||||
q,
|
||||
kv,
|
||||
weights,
|
||||
metadata,
|
||||
num_max_sparse_blocks,
|
||||
sparse_block_kv,
|
||||
use_unaligned_ks=True,
|
||||
)
|
||||
|
||||
|
||||
def fp8_fp4_paged_sparse_mqa_logits(
|
||||
q: tuple[torch.Tensor, torch.Tensor],
|
||||
kv_cache: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
metadata: torch.Tensor,
|
||||
num_max_sparse_blocks: int,
|
||||
sparse_block_kv: int,
|
||||
) -> torch.Tensor:
|
||||
"""Compute MQA logits only at the candidate blocks (paged decode path).
|
||||
|
||||
Args:
|
||||
q: ``(q_values, q_scale)``; q_values is [num_q_tokens, 1, H, D].
|
||||
kv_cache: Fused paged cache [num_pages, page_kv, 1, D' + 4] uint8,
|
||||
page stride 512B-aligned.
|
||||
weights: [num_q_tokens, H] ``torch.bfloat16``.
|
||||
metadata: From `get_paged_sparse_mqa_logits_metadata`.
|
||||
|
||||
Returns:
|
||||
bf16 logits of shape
|
||||
[num_q_tokens, num_max_sparse_blocks * sparse_block_kv].
|
||||
"""
|
||||
_lazy_init()
|
||||
if _fp8_fp4_paged_sparse_mqa_logits_impl is None:
|
||||
return _missing()
|
||||
return _fp8_fp4_paged_sparse_mqa_logits_impl(
|
||||
q,
|
||||
kv_cache,
|
||||
weights,
|
||||
metadata,
|
||||
num_max_sparse_blocks,
|
||||
sparse_block_kv,
|
||||
)
|
||||
|
||||
|
||||
def tf32_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user