diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index e3e4897e15..83eef72e31 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -614,14 +614,22 @@ def _reference_decode_index_score( return out -def test_prefill_index_topk_correctness(): +# 3600: 29 blocks over 12 splits of 3, so the last busy split has 2 blocks. +@pytest.mark.parametrize("prefix_len", [1024, 3600]) +@pytest.mark.parametrize("force_split_k", [False, True]) +def test_prefill_index_topk_correctness(monkeypatch, prefix_len, force_split_k): + if force_split_k: + # Take the SM12.0-only split-K launch on any CUDA GPU. + monkeypatch.setattr( + current_platform, "is_device_capability", lambda cap, *_: cap == (12, 0) + ) topk = 6 init_blocks = 0 local_blocks = 1 num_idx_heads = 2 head_dim = 16 q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32) - prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32) + prefix_lens = torch.tensor((0, prefix_len), device="cuda", dtype=torch.int32) seq_lens = prefix_lens + q_lens batch = q_lens.numel() max_seq_len = seq_lens.max().item() diff --git a/vllm/models/minimax_m3/common/ops/index_topk.py b/vllm/models/minimax_m3/common/ops/index_topk.py index 5123dabe90..94b74363e7 100644 --- a/vllm/models/minimax_m3/common/ops/index_topk.py +++ b/vllm/models/minimax_m3/common/ops/index_topk.py @@ -101,6 +101,7 @@ def _index_block_score_kernel( stride_bt_b, BLOCK_SIZE_Q: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128) + USE_SPLIT_K: tl.constexpr, ): pid_q = tl.program_id(0) pid_bh = tl.program_id(1) @@ -136,8 +137,19 @@ def _index_block_score_kernel( bt_row = block_table_ptr + pid_b * stride_bt_b # Causal window: only blocks up to the last query token's position. hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q) - for i in tl.range(0, hi, BLOCK_SIZE_K): - blk = i // BLOCK_SIZE_K + num_blocks = tl.cdiv(hi, BLOCK_SIZE_K) + if USE_SPLIT_K: + pid_k = tl.program_id(2) + blocks_per_split = tl.cdiv(num_blocks, tl.num_programs(2)) + block_start = pid_k * blocks_per_split + block_end = tl.minimum(block_start + blocks_per_split, num_blocks) + if block_start >= block_end: + return + else: + block_start = 0 + block_end = num_blocks + for blk in tl.range(block_start, block_end): + i = blk * BLOCK_SIZE_K page = tl.load(bt_row + blk).to(tl.int64) pos = i + off_k # index-K for this page: [BLOCK_SIZE_D, BLOCK_SIZE_K] (transposed) @@ -686,7 +698,17 @@ def minimax_m3_index_score( device=idx_q.device, ) BLOCK_SIZE_Q = 64 - grid_score = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads) + n_q_tiles = triton.cdiv(max_query_len, BLOCK_SIZE_Q) + SCORE_TARGET_GRID = 48 + split_k = max( + 1, + min(max_block, SCORE_TARGET_GRID // max(1, n_q_tiles * batch * num_idx_heads)), + ) + if not ( + current_platform.is_cuda() and current_platform.is_device_capability((12, 0)) + ): + split_k = 1 + grid_score = (n_q_tiles, batch * num_idx_heads, split_k) _index_block_score_kernel[grid_score]( idx_q, index_kv_cache, @@ -709,6 +731,7 @@ def minimax_m3_index_score( block_table.stride(0), BLOCK_SIZE_Q=BLOCK_SIZE_Q, BLOCK_SIZE_K=SPARSE_BLOCK_SIZE, + USE_SPLIT_K=split_k > 1, ) return score @@ -822,7 +845,7 @@ def minimax_m3_index_decode_score( ) # split-K over seq blocks; chunk count depends only on shape constants so # the grid is fixed within a cuda graph. - TARGET_GRID = 512 + TARGET_GRID = 4096 MAX_NUM_KV_CHUNKS = 256 # Use the configured max decode length to avoid Triton recompiles when # switching between qlen=1 and spec-decode verification batches. @@ -921,7 +944,7 @@ def minimax_m3_index_decode( ) # Chunk count is shape-constant (cudagraph-safe), capped so the merge sorts # pow2(num_topk_chunks * pow2(topk)) candidates. - TOPK_TARGET_GRID = 64 + TOPK_TARGET_GRID = 512 MAX_NUM_TOPK_CHUNKS = 16 topk_target = max( 1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, batch * num_idx_heads))