[Perf][MiniMax-M3] Triton indexer: decode grid retune + SM12.0 split-K (#56151)
Signed-off-by: venkywonka <23023424+venkywonka@users.noreply.github.com> Signed-off-by: Zijing Liu <liuzijing2014@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Zijing Liu <liuzijing2014@gmail.com> Co-authored-by: Kyle Liang <199154931+kyleliang-nv@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Zijing Liu
Kyle Liang
parent
b56b54bc54
commit
84f738f187
@@ -614,14 +614,22 @@ def _reference_decode_index_score(
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def test_prefill_index_topk_correctness():
|
# 3600: 29 blocks over 12 splits of 3, so the last busy split has 2 blocks.
|
||||||
|
@pytest.mark.parametrize("prefix_len", [1024, 3600])
|
||||||
|
@pytest.mark.parametrize("force_split_k", [False, True])
|
||||||
|
def test_prefill_index_topk_correctness(monkeypatch, prefix_len, force_split_k):
|
||||||
|
if force_split_k:
|
||||||
|
# Take the SM12.0-only split-K launch on any CUDA GPU.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
current_platform, "is_device_capability", lambda cap, *_: cap == (12, 0)
|
||||||
|
)
|
||||||
topk = 6
|
topk = 6
|
||||||
init_blocks = 0
|
init_blocks = 0
|
||||||
local_blocks = 1
|
local_blocks = 1
|
||||||
num_idx_heads = 2
|
num_idx_heads = 2
|
||||||
head_dim = 16
|
head_dim = 16
|
||||||
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
|
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
|
||||||
prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
|
prefix_lens = torch.tensor((0, prefix_len), device="cuda", dtype=torch.int32)
|
||||||
seq_lens = prefix_lens + q_lens
|
seq_lens = prefix_lens + q_lens
|
||||||
batch = q_lens.numel()
|
batch = q_lens.numel()
|
||||||
max_seq_len = seq_lens.max().item()
|
max_seq_len = seq_lens.max().item()
|
||||||
|
|||||||
@@ -101,6 +101,7 @@ def _index_block_score_kernel(
|
|||||||
stride_bt_b,
|
stride_bt_b,
|
||||||
BLOCK_SIZE_Q: tl.constexpr,
|
BLOCK_SIZE_Q: tl.constexpr,
|
||||||
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
|
||||||
|
USE_SPLIT_K: tl.constexpr,
|
||||||
):
|
):
|
||||||
pid_q = tl.program_id(0)
|
pid_q = tl.program_id(0)
|
||||||
pid_bh = tl.program_id(1)
|
pid_bh = tl.program_id(1)
|
||||||
@@ -136,8 +137,19 @@ def _index_block_score_kernel(
|
|||||||
bt_row = block_table_ptr + pid_b * stride_bt_b
|
bt_row = block_table_ptr + pid_b * stride_bt_b
|
||||||
# Causal window: only blocks up to the last query token's position.
|
# Causal window: only blocks up to the last query token's position.
|
||||||
hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
|
hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
|
||||||
for i in tl.range(0, hi, BLOCK_SIZE_K):
|
num_blocks = tl.cdiv(hi, BLOCK_SIZE_K)
|
||||||
blk = i // BLOCK_SIZE_K
|
if USE_SPLIT_K:
|
||||||
|
pid_k = tl.program_id(2)
|
||||||
|
blocks_per_split = tl.cdiv(num_blocks, tl.num_programs(2))
|
||||||
|
block_start = pid_k * blocks_per_split
|
||||||
|
block_end = tl.minimum(block_start + blocks_per_split, num_blocks)
|
||||||
|
if block_start >= block_end:
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
block_start = 0
|
||||||
|
block_end = num_blocks
|
||||||
|
for blk in tl.range(block_start, block_end):
|
||||||
|
i = blk * BLOCK_SIZE_K
|
||||||
page = tl.load(bt_row + blk).to(tl.int64)
|
page = tl.load(bt_row + blk).to(tl.int64)
|
||||||
pos = i + off_k
|
pos = i + off_k
|
||||||
# index-K for this page: [BLOCK_SIZE_D, BLOCK_SIZE_K] (transposed)
|
# index-K for this page: [BLOCK_SIZE_D, BLOCK_SIZE_K] (transposed)
|
||||||
@@ -686,7 +698,17 @@ def minimax_m3_index_score(
|
|||||||
device=idx_q.device,
|
device=idx_q.device,
|
||||||
)
|
)
|
||||||
BLOCK_SIZE_Q = 64
|
BLOCK_SIZE_Q = 64
|
||||||
grid_score = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads)
|
n_q_tiles = triton.cdiv(max_query_len, BLOCK_SIZE_Q)
|
||||||
|
SCORE_TARGET_GRID = 48
|
||||||
|
split_k = max(
|
||||||
|
1,
|
||||||
|
min(max_block, SCORE_TARGET_GRID // max(1, n_q_tiles * batch * num_idx_heads)),
|
||||||
|
)
|
||||||
|
if not (
|
||||||
|
current_platform.is_cuda() and current_platform.is_device_capability((12, 0))
|
||||||
|
):
|
||||||
|
split_k = 1
|
||||||
|
grid_score = (n_q_tiles, batch * num_idx_heads, split_k)
|
||||||
_index_block_score_kernel[grid_score](
|
_index_block_score_kernel[grid_score](
|
||||||
idx_q,
|
idx_q,
|
||||||
index_kv_cache,
|
index_kv_cache,
|
||||||
@@ -709,6 +731,7 @@ def minimax_m3_index_score(
|
|||||||
block_table.stride(0),
|
block_table.stride(0),
|
||||||
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
|
BLOCK_SIZE_Q=BLOCK_SIZE_Q,
|
||||||
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
|
BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
|
||||||
|
USE_SPLIT_K=split_k > 1,
|
||||||
)
|
)
|
||||||
return score
|
return score
|
||||||
|
|
||||||
@@ -822,7 +845,7 @@ def minimax_m3_index_decode_score(
|
|||||||
)
|
)
|
||||||
# split-K over seq blocks; chunk count depends only on shape constants so
|
# split-K over seq blocks; chunk count depends only on shape constants so
|
||||||
# the grid is fixed within a cuda graph.
|
# the grid is fixed within a cuda graph.
|
||||||
TARGET_GRID = 512
|
TARGET_GRID = 4096
|
||||||
MAX_NUM_KV_CHUNKS = 256
|
MAX_NUM_KV_CHUNKS = 256
|
||||||
# Use the configured max decode length to avoid Triton recompiles when
|
# Use the configured max decode length to avoid Triton recompiles when
|
||||||
# switching between qlen=1 and spec-decode verification batches.
|
# switching between qlen=1 and spec-decode verification batches.
|
||||||
@@ -921,7 +944,7 @@ def minimax_m3_index_decode(
|
|||||||
)
|
)
|
||||||
# Chunk count is shape-constant (cudagraph-safe), capped so the merge sorts
|
# Chunk count is shape-constant (cudagraph-safe), capped so the merge sorts
|
||||||
# pow2(num_topk_chunks * pow2(topk)) candidates.
|
# pow2(num_topk_chunks * pow2(topk)) candidates.
|
||||||
TOPK_TARGET_GRID = 64
|
TOPK_TARGET_GRID = 512
|
||||||
MAX_NUM_TOPK_CHUNKS = 16
|
MAX_NUM_TOPK_CHUNKS = 16
|
||||||
topk_target = max(
|
topk_target = max(
|
||||||
1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, batch * num_idx_heads))
|
1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, batch * num_idx_heads))
|
||||||
|
|||||||
Reference in New Issue
Block a user