[Kernel][Build] Bump DeepGEMM to the 26/09/30 release and adapt the sparse indexer (#61149)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
86b1c2ef15
commit
a6009ea85b
@@ -771,6 +771,8 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
candidates = _make_candidates(lens, num_candidates, cbk).cuda()
|
||||
|
||||
# Reference: dense bf16 logits masked to the candidates, then torch.topk.
|
||||
# The logits are compressed: row r starts at K position ks[r].
|
||||
max_len = int(lens.max())
|
||||
dense = deep_gemm.fp8_fp4_mqa_logits(
|
||||
(
|
||||
q_fp.view(torch.int8).view(rows, num_heads, head_dim // 2),
|
||||
@@ -780,19 +782,13 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
weights,
|
||||
ks,
|
||||
ke,
|
||||
clean_logits=False,
|
||||
logits_dtype=torch.bfloat16,
|
||||
max_len,
|
||||
)
|
||||
cols = torch.arange(total, device="cuda")
|
||||
in_range = (cols - ks.unsqueeze(1) >= 0) & (
|
||||
cols - ks.unsqueeze(1) < lens.unsqueeze(1)
|
||||
)
|
||||
masked = dense.masked_fill(
|
||||
~(in_range & _candidate_token_mask(candidates, cbk, total, ks)),
|
||||
float("-inf"),
|
||||
).float()
|
||||
ref_abs = _reference_topk(masked, topk) # packed-workspace positions
|
||||
ref = torch.where(ref_abs >= 0, ref_abs - ks.unsqueeze(1), -1)
|
||||
in_range = torch.arange(max_len, device="cuda") < lens.unsqueeze(1)
|
||||
# A row's last candidate block may extend past max_len.
|
||||
in_cand = _candidate_token_mask(candidates, cbk, max_len + cbk)[:, :max_len]
|
||||
masked = dense.masked_fill(~(in_range & in_cand), float("-inf")).float()
|
||||
ref = _reference_topk(masked, topk)
|
||||
|
||||
out = torch.full((rows, topk), -1, dtype=torch.int32, device="cuda")
|
||||
scratch = _sparse_scratch(rows, num_candidates, topk)
|
||||
@@ -827,11 +823,11 @@ def test_sparse_mqa_logits_prefill_matches_dense_masked(capture: bool):
|
||||
graph.replay()
|
||||
else:
|
||||
kernel_metadata = run_sparse()
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
_assert_same_selection(dense, out, ref)
|
||||
out.fill_(-1)
|
||||
run_sparse(kernel_metadata)
|
||||
|
||||
_assert_same_selection(dense, out, ref, row_starts=ks)
|
||||
_assert_same_selection(dense, out, ref)
|
||||
|
||||
|
||||
def _build_fp4_paged_cache(req_lens: torch.Tensor, page_kv: int, head_dim: int):
|
||||
@@ -902,21 +898,22 @@ def test_sparse_mqa_logits_paged_matches_dense_masked(next_n: int, capture: bool
|
||||
candidates = _make_candidates(row_lens.flatten(), num_candidates, cbk).cuda()
|
||||
|
||||
def reference_topk():
|
||||
context_lens_2d = row_lens.view(batch, next_n)
|
||||
# SM100 paged MQA logits take varlen rows: one Q token per row.
|
||||
context_lens_2d = row_lens.view(rows, 1)
|
||||
indices = torch.arange(batch, dtype=torch.int32, device="cuda")
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
sched = deep_gemm.get_paged_mqa_logits_metadata(
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), None
|
||||
context_lens_2d, page_kv, deep_gemm.get_num_sms(), indices
|
||||
)
|
||||
logits = deep_gemm.fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
(q.view(rows, 1, num_heads, -1), q_scale.view(rows, 1, num_heads)),
|
||||
kv_cache,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_table,
|
||||
block_table.repeat_interleave(next_n, dim=0),
|
||||
sched,
|
||||
int(req_lens.max()),
|
||||
False,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
indices,
|
||||
)
|
||||
lens_flat = row_lens.flatten()
|
||||
cols = torch.arange(logits.shape[1], device="cuda")
|
||||
|
||||
Reference in New Issue
Block a user