[M3] Tune Triton indexer score decode for spec-decode (#45743)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Thien Tran
2026-06-16 21:07:34 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 20a5f8b43b
commit 5bdc01bcc3
3 changed files with 72 additions and 49 deletions
+11 -7
View File
@@ -134,7 +134,6 @@ def _reference_index_topk(
topk: int,
init_blocks: int,
local_blocks: int,
sm_scale: float,
) -> torch.Tensor:
total_q, num_idx_heads, _ = idx_q.shape
out = torch.full(
@@ -150,7 +149,7 @@ def _reference_index_topk(
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
pages = block_table[req_id, :num_blocks]
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
score = torch.einsum("qhd,kd->hqk", q.float(), k.float())
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
k_pos = torch.arange(k.shape[0], device=idx_q.device)
@@ -221,7 +220,6 @@ def test_prefill_index_topk_correctness():
max_query_len=q_lens.max().item(),
max_seq_len=max_seq_len,
num_kv_heads=num_idx_heads,
sm_scale=head_dim**-0.5,
)
actual = minimax_m3_index_topk(
score,
@@ -242,15 +240,22 @@ def test_prefill_index_topk_correctness():
topk,
init_blocks,
local_blocks,
head_dim**-0.5,
)
_assert_topk_indices_equal_unordered(actual, expected)
@pytest.mark.parametrize("decode_query_len", [1, 4])
@pytest.mark.parametrize(
("decode_query_len", "max_decode_query_len"),
[
(1, 1),
(1, 4),
(4, 4),
],
)
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
def test_decode_index_topk_correctness(
decode_query_len: int,
max_decode_query_len: int,
num_padded_reqs: int,
):
topk = 6
@@ -293,8 +298,8 @@ def test_decode_index_topk_correctness(
init_blocks=init_blocks,
local_blocks=local_blocks,
num_kv_heads=num_idx_heads,
sm_scale=head_dim**-0.5,
decode_query_len=decode_query_len,
max_decode_query_len=max_decode_query_len,
)
expected = torch.full_like(actual, -1)
active_tokens = active_batch * decode_query_len
@@ -308,7 +313,6 @@ def test_decode_index_topk_correctness(
topk,
init_blocks,
local_blocks,
head_dim**-0.5,
)
_assert_topk_indices_equal_unordered(actual, expected)