[ROCm][Perf] Tune MiniMax-M3 decode top-k for short contexts (#55235)
Signed-off-by: fai <fangzhouai@gmail.com>
This commit is contained in:
@@ -37,6 +37,7 @@ MIN_DECODE_SCORE_GFX950_HIGH_BATCH_REQUESTS = 20
|
||||
MAX_DECODE_SCORE_GFX950_HIGH_BATCH_REQUESTS = 64
|
||||
MAX_DECODE_SCORE_KV_CHUNKS = 256
|
||||
DECODE_TOPK_BLOCKS_PER_CHUNK = 512
|
||||
DECODE_TOPK_SHORT_BLOCKS_PER_CHUNK = 128
|
||||
MAX_DECODE_TOPK_FAST_CHUNKS = 16
|
||||
DECODE_TOPK_TARGET_GRID = 64
|
||||
|
||||
@@ -103,14 +104,23 @@ def _decode_topk_launch_policy(
|
||||
topk: int,
|
||||
*,
|
||||
is_gfx950: bool,
|
||||
) -> tuple[int, bool, bool]:
|
||||
"""Choose the graph grid and bounded-context selector specialization."""
|
||||
) -> tuple[int, int, int, int, bool, bool]:
|
||||
"""Choose the selector grid and compile-time launch configuration."""
|
||||
if (
|
||||
is_gfx950
|
||||
and topk == 16
|
||||
and 0 < max_block <= MAX_DECODE_TOPK_FAST_CHUNKS * DECODE_TOPK_BLOCKS_PER_CHUNK
|
||||
):
|
||||
return MAX_DECODE_TOPK_FAST_CHUNKS, True, True
|
||||
if max_block <= DECODE_TOPK_SHORT_BLOCKS_PER_CHUNK:
|
||||
return 1, DECODE_TOPK_SHORT_BLOCKS_PER_CHUNK, 2, 1, True, True
|
||||
return (
|
||||
MAX_DECODE_TOPK_FAST_CHUNKS,
|
||||
DECODE_TOPK_BLOCKS_PER_CHUNK,
|
||||
4,
|
||||
2,
|
||||
True,
|
||||
True,
|
||||
)
|
||||
|
||||
target = max(
|
||||
1,
|
||||
@@ -119,7 +129,14 @@ def _decode_topk_launch_policy(
|
||||
DECODE_TOPK_TARGET_GRID // max(1, total_q * num_idx_heads),
|
||||
),
|
||||
)
|
||||
return 1 << (target.bit_length() - 1), False, False
|
||||
return (
|
||||
1 << (target.bit_length() - 1),
|
||||
DECODE_TOPK_BLOCKS_PER_CHUNK,
|
||||
8,
|
||||
2,
|
||||
False,
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1525,6 +1542,9 @@ def minimax_m3_index_decode(
|
||||
# only the number of chunks needed for its live context.
|
||||
(
|
||||
num_topk_chunks,
|
||||
topk_block_size,
|
||||
selector_num_warps,
|
||||
selector_num_stages,
|
||||
single_tile_guaranteed,
|
||||
adaptive_final_merge,
|
||||
) = _decode_topk_launch_policy(
|
||||
@@ -1608,12 +1628,12 @@ def minimax_m3_index_decode(
|
||||
pages_per_sparse_block=PAGES_PER_SPARSE_BLOCK,
|
||||
block_page_stride=selector_block_page_stride,
|
||||
NUM_TOPK_CHUNKS=num_topk_chunks,
|
||||
BLOCK_SIZE_K=512,
|
||||
BLOCK_SIZE_K=topk_block_size,
|
||||
BLOCK_SIZE_T=block_size_t,
|
||||
EMIT_SPARSE_TABLE=emit_sparse_table,
|
||||
SINGLE_TILE_GUARANTEED=single_tile_guaranteed,
|
||||
ADAPTIVE_FINAL_MERGE=adaptive_final_merge,
|
||||
num_warps=4 if adaptive_final_merge else 8,
|
||||
num_stages=2,
|
||||
num_warps=selector_num_warps,
|
||||
num_stages=selector_num_stages,
|
||||
)
|
||||
return topk_idx
|
||||
|
||||
Reference in New Issue
Block a user