[M3] Tune Triton indexer score decode for spec-decode (#45743)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
20a5f8b43b
commit
5bdc01bcc3
@@ -134,7 +134,6 @@ def _reference_index_topk(
|
||||
topk: int,
|
||||
init_blocks: int,
|
||||
local_blocks: int,
|
||||
sm_scale: float,
|
||||
) -> torch.Tensor:
|
||||
total_q, num_idx_heads, _ = idx_q.shape
|
||||
out = torch.full(
|
||||
@@ -150,7 +149,7 @@ def _reference_index_topk(
|
||||
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
pages = block_table[req_id, :num_blocks]
|
||||
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
|
||||
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
|
||||
score = torch.einsum("qhd,kd->hqk", q.float(), k.float())
|
||||
|
||||
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
|
||||
k_pos = torch.arange(k.shape[0], device=idx_q.device)
|
||||
@@ -221,7 +220,6 @@ def test_prefill_index_topk_correctness():
|
||||
max_query_len=q_lens.max().item(),
|
||||
max_seq_len=max_seq_len,
|
||||
num_kv_heads=num_idx_heads,
|
||||
sm_scale=head_dim**-0.5,
|
||||
)
|
||||
actual = minimax_m3_index_topk(
|
||||
score,
|
||||
@@ -242,15 +240,22 @@ def test_prefill_index_topk_correctness():
|
||||
topk,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
head_dim**-0.5,
|
||||
)
|
||||
_assert_topk_indices_equal_unordered(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_query_len", [1, 4])
|
||||
@pytest.mark.parametrize(
|
||||
("decode_query_len", "max_decode_query_len"),
|
||||
[
|
||||
(1, 1),
|
||||
(1, 4),
|
||||
(4, 4),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
|
||||
def test_decode_index_topk_correctness(
|
||||
decode_query_len: int,
|
||||
max_decode_query_len: int,
|
||||
num_padded_reqs: int,
|
||||
):
|
||||
topk = 6
|
||||
@@ -293,8 +298,8 @@ def test_decode_index_topk_correctness(
|
||||
init_blocks=init_blocks,
|
||||
local_blocks=local_blocks,
|
||||
num_kv_heads=num_idx_heads,
|
||||
sm_scale=head_dim**-0.5,
|
||||
decode_query_len=decode_query_len,
|
||||
max_decode_query_len=max_decode_query_len,
|
||||
)
|
||||
expected = torch.full_like(actual, -1)
|
||||
active_tokens = active_batch * decode_query_len
|
||||
@@ -308,7 +313,6 @@ def test_decode_index_topk_correctness(
|
||||
topk,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
head_dim**-0.5,
|
||||
)
|
||||
_assert_topk_indices_equal_unordered(actual, expected)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user