[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-10-12 23:24:43 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 86b1c2ef15
commit a6009ea85b
23 changed files with 516 additions and 208 deletions
@@ -771,6 +771,8 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
# The logits are compressed: row r starts at K position ks[r].
max_len = int(lens.max())
dense = deep_gemm.fp8_fp4_mqa_logits(
(
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
@@ -780,19 +782,13 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
weights,
ks,
ke,
clean_logits=False,
logits_dtype=torch.bfloat16,
max_len,
)
cols = torch.arange(total, device="cuda")
in_range = (cols - ks.unsqueeze(1) >= 0) & (
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
)
masked = dense.masked_fill(
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
float("-inf"),
).float()
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
in_range = torch.arange(max_len, device="cuda") < lens.unsqueeze(1)
# A row's last candidate block may extend past max_len.
in_cand = _candidate_token_mask(candidates, cbk, max_len + cbk)[:, :max_len]
masked = dense.masked_fill(~(in_range & in_cand), float("-inf")).float()
ref = _reference_topk(masked, topk)
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
scratch = _sparse_scratch(rows, num_candidates, topk)
@@ -827,11 +823,11 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
graph.replay()
else:
kernel_metadata = run_sparse()
_assert_same_selection(dense, out, ref, row_starts=ks)
_assert_same_selection(dense, out, ref)
out.fill_(-1)
run_sparse(kernel_metadata)
_assert_same_selection(dense, out, ref, row_starts=ks)
_assert_same_selection(dense, out, ref)
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
@@ -902,21 +898,22 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
def reference_topk():
context_lens_2d = row_lens.view(batch, next_n)
# SM100 paged MQA logits take varlen rows: one Q token per row.
context_lens_2d = row_lens.view(rows, 1)
indices = torch.arange(batch, dtype=torch.int32, device="cuda")
indices = indices.repeat_interleave(next_n)
sched = deep_gemm.get_paged_mqa_logits_metadata(
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
context_lens_2d, page_kv, deep_gemm.get_num_sms(), indices
)
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
(q, q_scale),
(q.view(rows, 1, num_heads, -1), q_scale.view(rows, 1, num_heads)),
kv_cache,
weights,
context_lens_2d,
block_table,
block_table.repeat_interleave(next_n, dim=0),
sched,
int(req_lens.max()),
False,
torch.bfloat16,
None,
indices,
)
lens_flat = row_lens.flatten()
cols = torch.arange(logits.shape[1], device="cuda")