[ROCm][Perf] Optimize MiniMax-M3 decode indexer and top-k (#54682)

Signed-off-by: fai <fangzhouai@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Fangzhou Ai
2026-09-03 13:20:18 -07:00
committed by GitHub
co-authored by Andreas Karatzas OpenAI Codex
parent 2a336d8239
commit cee0f92c02
6 changed files with 2028 additions and 345 deletions
+840
View File
@@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.ops.sparse_attn import (
)
from vllm.models.minimax_m3.common.sparse_attention import MiniMaxM3SparseTritonImpl
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.utils import record_kv_cache_layout
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
@@ -92,6 +93,382 @@ SM_SCALE = HEAD_DIM**-0.5
TOPK = 16
@pytest.mark.parametrize(
(
"num_reqs",
"head_dim",
"query_dtype",
"cache_dtype",
"is_gfx950",
"expected",
),
[
(19, 128, torch.bfloat16, torch.bfloat16, True, (16, False)),
(20, 128, torch.bfloat16, torch.bfloat16, True, (32, True)),
(32, 128, torch.bfloat16, torch.bfloat16, True, (32, True)),
(33, 128, torch.bfloat16, torch.bfloat16, True, (16, True)),
(64, 128, torch.bfloat16, torch.bfloat16, True, (16, True)),
(65, 128, torch.bfloat16, torch.bfloat16, True, (4, False)),
(20, 128, torch.bfloat16, torch.bfloat16, False, (16, False)),
(20, 64, torch.bfloat16, torch.bfloat16, True, (16, False)),
(20, 128, torch.float16, torch.bfloat16, True, (16, False)),
(20, 128, torch.bfloat16, torch.float16, True, (16, False)),
],
)
def test_amd_decode_score_split_launch_policy(
num_reqs: int,
head_dim: int,
query_dtype: torch.dtype,
cache_dtype: torch.dtype,
is_gfx950: bool,
expected: tuple[int, bool],
):
from vllm.models.minimax_m3.amd.ops.index_topk import (
_decode_score_split_launch_policy,
)
assert (
_decode_score_split_launch_policy(
num_reqs,
head_dim,
query_dtype,
cache_dtype,
is_gfx950=is_gfx950,
)
== expected
)
@pytest.mark.parametrize("num_reqs", range(1, 12))
def test_amd_balanced_decode_score_mapping_exact_coverage(num_reqs: int):
from vllm.models.minimax_m3.amd.ops.index_topk import (
_decode_score_program_budget,
)
budget = _decode_score_program_budget(
num_reqs,
128,
torch.bfloat16,
torch.bfloat16,
is_gfx950=True,
)
assert budget is not None
patterns = (
(0,) * num_reqs,
tuple((request * 7919 + 17) % 65535 for request in range(num_reqs)),
(65534,) * num_reqs,
)
for request_blocks in patterns:
width = max(1, (sum(request_blocks) + budget - 1) // budget)
grid = budget + len(request_blocks) - 1
intervals = []
for request, blocks in enumerate(request_blocks):
programs = (blocks + width - 1) // width
for local_program in range(programs):
begin = local_program * width
intervals.append((request, begin, min(begin + width, blocks)))
assert len(intervals) <= grid
by_request: list[list[tuple[int, int]]] = [[] for _ in request_blocks]
for request, begin, end in intervals:
assert 0 <= request < len(request_blocks)
assert 0 <= begin < end <= request_blocks[request]
by_request[request].append((begin, end))
for blocks, request_intervals in zip(
request_blocks,
by_request,
strict=True,
):
cursor = 0
for begin, end in request_intervals:
assert begin == cursor
cursor = end
assert cursor == blocks
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="MiniMax-M3 balanced decode score is ROCm-only",
)
@pytest.mark.parametrize(
(
"num_reqs",
"max_block",
"num_idx_heads",
"decode_query_len",
"max_decode_query_len",
"init_blocks",
"local_blocks",
),
[
(1, 37, 1, 1, 1, 1, 0),
(8, 37, 1, 1, 4, 2, 1),
(9, 37, 1, 4, 4, 0, 2),
(11, 37, 1, 3, 5, 1, 3),
(1, 37, 2, 1, 1, 2, 0),
(8, 37, 2, 1, 4, 0, 1),
(9, 37, 2, 4, 4, 1, 2),
(11, 37, 2, 3, 5, 2, 3),
(1, 37, 4, 1, 1, 0, 0),
(8, 37, 4, 1, 4, 1, 1),
(9, 37, 4, 4, 4, 2, 2),
(11, 37, 4, 3, 5, 0, 3),
(20, 65, 1, 4, 4, 1, 3),
(36, 65, 1, 4, 4, 2, 2),
(64, 65, 1, 4, 4, 0, 1),
],
)
def test_amd_optimized_decode_score_bitwise_and_graph_replay(
num_reqs: int,
max_block: int,
num_idx_heads: int,
decode_query_len: int,
max_decode_query_len: int,
init_blocks: int,
local_blocks: int,
):
from vllm.models.minimax_m3.amd.ops.index_topk import (
_decode_index_score_balanced_kernel,
_decode_index_score_kernel,
_decode_score_program_budget,
_decode_score_split_launch_policy,
)
set_random_seed(0)
head_dim = 128
block_size_q = 1 << (max_decode_query_len - 1).bit_length()
score_stride = (max_block + 15) // 16 * 16
num_pages = num_reqs * max_block
block_table = torch.arange(
num_pages,
device="cuda",
dtype=torch.int32,
).reshape(num_reqs, max_block)
idx_q = torch.randn(
(num_reqs * decode_query_len, num_idx_heads, head_dim),
device="cuda",
dtype=torch.bfloat16,
)
index_kv_cache = torch.randn(
(num_pages, BLOCK_SIZE, head_dim),
device="cuda",
dtype=torch.bfloat16,
)
seq_lens = torch.empty(num_reqs, device="cuda", dtype=torch.int32)
deployed_score = torch.empty(
(num_idx_heads, num_reqs * decode_query_len, score_stride),
device="cuda",
dtype=torch.float32,
)
optimized_score = torch.empty_like(deployed_score)
budget = _decode_score_program_budget(
num_reqs,
head_dim,
idx_q.dtype,
index_kv_cache.dtype,
is_gfx950=True,
)
optimized_chunks = None
if budget is None:
optimized_chunks, use_high_batch_config = _decode_score_split_launch_policy(
num_reqs,
head_dim,
idx_q.dtype,
index_kv_cache.dtype,
is_gfx950=True,
)
assert use_high_batch_config
target = max(1, min(256, 512 // num_reqs))
deployed_chunks = 1 << (target.bit_length() - 1)
def launch_deployed() -> None:
_decode_index_score_kernel[(num_reqs, deployed_chunks)](
idx_q,
index_kv_cache,
deployed_score,
block_table,
seq_lens,
num_idx_heads,
head_dim,
init_blocks,
local_blocks,
decode_query_len,
idx_q.stride(0),
idx_q.stride(1),
idx_q.stride(2),
index_kv_cache.stride(0),
index_kv_cache.stride(1),
index_kv_cache.stride(2),
deployed_score.stride(0),
deployed_score.stride(1),
deployed_score.stride(2),
block_table.stride(0),
BLOCK_SIZE_K=BLOCK_SIZE,
BLOCK_SIZE_Q=block_size_q,
num_kv_chunks=deployed_chunks,
USE_PDL=False,
)
def launch_optimized() -> None:
if budget is not None:
_decode_index_score_balanced_kernel[(budget + num_reqs - 1,)](
idx_q,
index_kv_cache,
optimized_score,
block_table,
seq_lens,
num_idx_heads,
head_dim,
init_blocks,
local_blocks,
num_reqs,
budget,
decode_query_len,
idx_q.stride(0),
idx_q.stride(1),
idx_q.stride(2),
index_kv_cache.stride(0),
index_kv_cache.stride(1),
index_kv_cache.stride(2),
optimized_score.stride(0),
optimized_score.stride(1),
optimized_score.stride(2),
block_table.stride(0),
BLOCK_SIZE_K=BLOCK_SIZE,
BLOCK_SIZE_Q=block_size_q,
num_warps=2,
num_stages=1,
)
else:
assert optimized_chunks is not None
_decode_index_score_kernel[(num_reqs, optimized_chunks)](
idx_q,
index_kv_cache,
optimized_score,
block_table,
seq_lens,
num_idx_heads,
head_dim,
init_blocks,
local_blocks,
decode_query_len,
idx_q.stride(0),
idx_q.stride(1),
idx_q.stride(2),
index_kv_cache.stride(0),
index_kv_cache.stride(1),
index_kv_cache.stride(2),
optimized_score.stride(0),
optimized_score.stride(1),
optimized_score.stride(2),
block_table.stride(0),
BLOCK_SIZE_K=BLOCK_SIZE,
BLOCK_SIZE_Q=block_size_q,
num_kv_chunks=optimized_chunks,
USE_PDL=False,
num_warps=2,
num_stages=1,
)
def score_masks(
expected_seq_lens: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
query_offsets = torch.arange(decode_query_len, device="cuda")
kv_lens = torch.clamp(
expected_seq_lens[:, None] - decode_query_len + query_offsets + 1,
min=0,
)
query_blocks = (kv_lens + BLOCK_SIZE - 1) // BLOCK_SIZE
block_ids = torch.arange(score_stride, device="cuda")
topk_mask = block_ids[None, :] < query_blocks.reshape(-1, 1)
request_blocks = query_blocks.max(dim=1).values
write_mask = (
block_ids[None, :]
< request_blocks.repeat_interleave(decode_query_len)[:, None]
)
return write_mask, topk_mask
def packed_keys(score: torch.Tensor, live: torch.Tensor) -> torch.Tensor:
normalized = torch.where(torch.isnan(score), -1e30, score)
normalized = torch.where(normalized == 0.0, 0.0, normalized)
bits = normalized.contiguous().view(torch.int32).to(torch.int64)
bits &= 0xFFFFFFFF
ordered = bits ^ torch.where(
bits >> 31 != 0,
0xFFFFFFFF,
0x80000000,
)
index = torch.arange(score_stride, device="cuda") + 1
keys = (1 << 48) | (ordered << 16) | (0xFFFF - index)
return torch.where(live, keys, 0)
def assert_exact(expected_seq_lens: torch.Tensor) -> None:
write_mask, topk_mask = score_masks(expected_seq_lens)
write_mask = write_mask.unsqueeze(0).expand_as(deployed_score)
topk_mask = topk_mask.unsqueeze(0)
deployed = deployed_score
optimized = optimized_score
assert torch.equal(
deployed[write_mask].contiguous().view(torch.int32),
optimized[write_mask].contiguous().view(torch.int32),
)
assert torch.isnan(deployed[~write_mask]).all()
assert torch.isnan(optimized[~write_mask]).all()
deployed_keys = packed_keys(deployed, topk_mask)
optimized_keys = packed_keys(optimized, topk_mask)
assert torch.equal(deployed_keys, optimized_keys)
deployed_topk = torch.topk(deployed_keys, 16, dim=-1, sorted=True)
optimized_topk = torch.topk(optimized_keys, 16, dim=-1, sorted=True)
assert torch.equal(deployed_topk.values, optimized_topk.values)
assert torch.equal(deployed_topk.indices, optimized_topk.indices)
uniform = torch.full(
(num_reqs,),
max_block * BLOCK_SIZE,
device="cuda",
dtype=torch.int32,
)
one_active = torch.zeros(num_reqs, device="cuda", dtype=torch.int32)
one_active[0] = max(1, max_block // 2) * BLOCK_SIZE
patterns = [uniform, one_active]
non_aligned = torch.tensor(
[(max_block - 1) * BLOCK_SIZE + request % 3 + 1 for request in range(num_reqs)],
device="cuda",
dtype=torch.int32,
)
patterns.append(non_aligned)
seq_lens.copy_(uniform)
deployed_score.fill_(float("nan"))
optimized_score.fill_(float("nan"))
launch_deployed()
launch_optimized()
assert_exact(uniform)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
launch_optimized()
addresses = (
seq_lens.data_ptr(),
optimized_score.data_ptr(),
block_table.data_ptr(),
index_kv_cache.data_ptr(),
)
for pattern in patterns:
seq_lens.copy_(pattern)
deployed_score.fill_(float("nan"))
optimized_score.fill_(float("nan"))
launch_deployed()
graph.replay()
assert_exact(pattern)
assert addresses == (
seq_lens.data_ptr(),
optimized_score.data_ptr(),
block_table.data_ptr(),
index_kv_cache.data_ptr(),
)
@pytest.mark.parametrize(
("kv_cache_dtype", "expected_dtype"),
[
@@ -638,6 +1015,338 @@ def test_decode_index_topk_correctness(
_assert_topk_indices_equal_unordered(actual, expected)
@pytest.mark.parametrize(
(
"max_block",
"total_q",
"num_idx_heads",
"topk",
"is_gfx950",
"expected",
),
[
(0, 24, 1, 16, True, (2, False, False)),
(1, 1, 1, 16, True, (16, True, True)),
(512, 4, 1, 16, True, (16, True, True)),
(513, 24, 4, 16, True, (16, True, True)),
(2760, 5, 2, 16, True, (16, True, True)),
(8192, 64, 4, 16, True, (16, True, True)),
(8193, 4, 1, 16, True, (16, False, False)),
(8193, 16, 1, 16, True, (4, False, False)),
(65534, 64, 1, 16, True, (1, False, False)),
(513, 16, 1, 13, True, (4, False, False)),
(513, 16, 4, 16, False, (1, False, False)),
],
)
def test_amd_decode_topk_launch_policy(
max_block: int,
total_q: int,
num_idx_heads: int,
topk: int,
is_gfx950: bool,
expected: tuple[int, bool, bool],
):
"""All supported head and query layouts use adaptive selection."""
from vllm.models.minimax_m3.amd.ops.index_topk import (
_decode_topk_launch_policy,
)
assert (
_decode_topk_launch_policy(
max_block,
total_q,
num_idx_heads,
topk,
is_gfx950=is_gfx950,
)
== expected
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="MiniMax-M3 fused decode top-k is ROCm-only",
)
@pytest.mark.parametrize(
(
"num_topk_chunks",
"single_tile_guaranteed",
"adaptive_final_merge",
"max_blocks",
"topk",
"num_idx_heads",
),
[
(1, False, False, 65534, 16, 1),
(2, False, False, 8193, 16, 1),
(4, False, False, 8193, 16, 1),
(8, False, False, 8193, 16, 1),
(16, False, False, 8193, 16, 1),
(16, False, False, 65534, 16, 1),
(2, True, False, 513, 13, 1),
(4, False, True, 8192, 16, 2),
(16, True, True, 8192, 16, 4),
],
)
def test_amd_decode_fused_topk_total_order_and_replay(
num_topk_chunks: int,
single_tile_guaranteed: bool,
adaptive_final_merge: bool,
max_blocks: int,
topk: int,
num_idx_heads: int,
):
"""Atomic multi-chunk selection is ordered and resets graph state."""
from vllm.models.minimax_m3.amd.ops.index_topk import (
SPARSE_BLOCK_SIZE,
_decode_topk_fused_kernel,
)
set_random_seed(0)
block_size_t = 1 << (topk - 1).bit_length()
long_seq_lens_list = [
0,
1,
15 * SPARSE_BLOCK_SIZE,
16 * SPARSE_BLOCK_SIZE,
17 * SPARSE_BLOCK_SIZE,
255 * SPARSE_BLOCK_SIZE,
256 * SPARSE_BLOCK_SIZE,
257 * SPARSE_BLOCK_SIZE,
511 * SPARSE_BLOCK_SIZE,
512 * SPARSE_BLOCK_SIZE,
513 * SPARSE_BLOCK_SIZE,
max_blocks * SPARSE_BLOCK_SIZE,
]
short_seq_lens_list = [0, 1, 127, 128, 129, 0, 1, 128, 129, 0, 1, 129]
boundary_limit = 0
if num_topk_chunks == 16 and max_blocks >= 8192:
boundary_limit = 15
elif num_topk_chunks == 4 and adaptive_final_merge:
boundary_limit = 4
if boundary_limit:
boundary_blocks = [
(active_chunks - 1) * 512 + 1
for active_chunks in range(3, boundary_limit + 1)
]
assert [(num_blocks + 511) // 512 for num_blocks in boundary_blocks] == list(
range(3, boundary_limit + 1)
)
long_seq_lens_list.extend(
num_blocks * SPARSE_BLOCK_SIZE for num_blocks in boundary_blocks
)
short_seq_lens_list.extend(
num_blocks * SPARSE_BLOCK_SIZE for num_blocks in reversed(boundary_blocks)
)
batch = len(long_seq_lens_list)
score = torch.randn((num_idx_heads, batch, max_blocks), device="cuda")
score[:, :, 0] = float("nan")
score[:, :, 1] = float("inf")
score[:, :, 2] = float("-inf")
score[:, :, 3] = 0.0
score[:, :, 4] = -0.0
score[:, :, 5] = 7.0
score[:, :, 6] = -1e30
score[:, :, 256] = 7.0
score[:, :, 512] = 7.0
seq_lens = torch.empty(batch, device="cuda", dtype=torch.int32)
partial = torch.full(
(num_topk_chunks, num_idx_heads, batch, block_size_t),
-1,
device="cuda",
dtype=torch.int64,
)
counter = torch.zeros((num_idx_heads, batch), device="cuda", dtype=torch.int32)
actual = torch.full(
(num_idx_heads, batch, topk), -2, device="cuda", dtype=torch.int32
)
dummy_block_table = torch.zeros(
(batch, max_blocks), device="cuda", dtype=torch.int32
)
addresses = (partial.data_ptr(), counter.data_ptr(), actual.data_ptr())
long_seq_lens = torch.tensor(long_seq_lens_list, device="cuda", dtype=torch.int32)
short_seq_lens = torch.tensor(short_seq_lens_list, device="cuda", dtype=torch.int32)
def launch() -> None:
_decode_topk_fused_kernel[(batch, num_idx_heads, num_topk_chunks)](
score,
partial,
counter,
actual,
seq_lens,
dummy_block_table,
actual,
seq_lens,
1,
score.stride(0),
score.stride(1),
score.stride(2),
partial.stride(0),
partial.stride(1),
partial.stride(2),
partial.stride(3),
counter.stride(0),
counter.stride(1),
actual.stride(0),
actual.stride(1),
actual.stride(2),
dummy_block_table.stride(0),
actual.stride(1),
topk=topk,
block_size=SPARSE_BLOCK_SIZE,
pages_per_sparse_block=8,
block_page_stride=8,
NUM_TOPK_CHUNKS=num_topk_chunks,
BLOCK_SIZE_K=512,
BLOCK_SIZE_T=block_size_t,
EMIT_SPARSE_TABLE=False,
SINGLE_TILE_GUARANTEED=single_tile_guaranteed,
ADAPTIVE_FINAL_MERGE=adaptive_final_merge,
num_warps=4 if adaptive_final_merge else 8,
num_stages=2,
)
score_cpu = score.cpu()
def reference(seq_lens_list: list[int]) -> torch.Tensor:
expected = torch.full((num_idx_heads, batch, topk), -1, dtype=torch.int32)
for head in range(num_idx_heads):
for row, seq_len in enumerate(seq_lens_list):
num_blocks = (seq_len + SPARSE_BLOCK_SIZE - 1) // SPARSE_BLOCK_SIZE
values = score_cpu[head, row, :num_blocks].tolist()
values = [-1e30 if value != value else value for value in values]
selected = sorted(
range(num_blocks),
key=lambda index: (-values[index], index),
)[:topk]
expected[head, row, : len(selected)] = torch.tensor(selected)
return expected
seq_lens.copy_(long_seq_lens)
launch()
torch.accelerator.synchronize()
assert torch.equal(actual.cpu(), reference(long_seq_lens_list))
assert torch.count_nonzero(counter).item() == 0
seq_lens.copy_(short_seq_lens)
launch()
torch.accelerator.synchronize()
assert torch.equal(actual.cpu(), reference(short_seq_lens_list))
assert torch.count_nonzero(counter).item() == 0
replay_count = 8 if max_blocks == 65534 else 128
for replay in range(replay_count):
seq_lens.copy_(long_seq_lens if replay % 2 == 0 else short_seq_lens)
launch()
seq_lens.copy_(long_seq_lens)
launch()
torch.accelerator.synchronize()
assert torch.equal(actual.cpu(), reference(long_seq_lens_list))
assert torch.count_nonzero(counter).item() == 0
assert addresses == (partial.data_ptr(), counter.data_ptr(), actual.data_ptr())
if (num_topk_chunks, single_tile_guaranteed, adaptive_final_merge, max_blocks) in (
(4, False, False, 8193),
(16, False, False, 8193),
(4, False, True, 8192),
(16, True, True, 8192),
):
graph = torch.cuda.CUDAGraph()
torch.accelerator.synchronize()
with torch.cuda.graph(graph):
launch()
seq_lens.copy_(short_seq_lens)
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(actual.cpu(), reference(short_seq_lens_list))
assert torch.count_nonzero(counter).item() == 0
seq_lens.copy_(long_seq_lens)
graph.replay()
torch.accelerator.synchronize()
assert torch.equal(actual.cpu(), reference(long_seq_lens_list))
assert torch.count_nonzero(counter).item() == 0
assert addresses == (partial.data_ptr(), counter.data_ptr(), actual.data_ptr())
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="MiniMax-M3 fused decode top-k is ROCm-only",
)
@pytest.mark.parametrize(
("num_idx_heads", "decode_query_len", "max_decode_query_len"),
[(1, 1, 1), (2, 1, 4), (4, 4, 4), (4, 3, 8)],
)
def test_amd_decode_index_topk_end_to_end(
num_idx_heads: int,
decode_query_len: int,
max_decode_query_len: int,
):
"""ROCm BF16 score-to-selector parity crosses the atomic boundary."""
from vllm.models.minimax_m3.amd.ops.index_topk import (
minimax_m3_index_decode as amd_minimax_m3_index_decode,
)
torch.manual_seed(0)
topk = 16
head_dim = 16
seq_lens = torch.tensor([65537], device="cuda", dtype=torch.int32)
q_lens = torch.tensor([decode_query_len], device="cuda", dtype=torch.int32)
prefix_lens = seq_lens - q_lens
max_seq_len = int(seq_lens.max())
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
block_table = torch.randperm(
max_blocks, device="cuda", dtype=torch.int32
).unsqueeze(0)
idx_q = torch.randn(
decode_query_len,
num_idx_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
index_kv_cache = torch.randn(
max_blocks,
BLOCK_SIZE,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
completion_counter = torch.zeros(
(num_idx_heads, decode_query_len), device="cuda", dtype=torch.int32
)
actual = amd_minimax_m3_index_decode(
idx_q,
index_kv_cache,
block_table,
seq_lens,
max_seq_len=max_seq_len,
topk=topk,
init_blocks=0,
local_blocks=1,
num_kv_heads=num_idx_heads,
decode_query_len=decode_query_len,
max_decode_query_len=max_decode_query_len,
completion_counter=completion_counter,
)
expected = _reference_index_topk(
idx_q,
index_kv_cache,
block_table,
q_lens,
seq_lens,
prefix_lens,
topk,
0,
1,
)
torch.accelerator.synchronize()
_assert_topk_indices_equal_unordered(actual, expected)
assert torch.count_nonzero(completion_counter).item() == 0
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="fp8 e4m3 indexer cache is the SM100 (MSA) path.",
@@ -1250,6 +1959,137 @@ def test_aiter_decode_sparse_block_table_supports_spec_decode():
assert torch.equal(sparse_bt, expected_bt)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="MiniMax-M3 fused decode top-k is ROCm-only",
)
@pytest.mark.parametrize("block_page_stride", [8, 16])
@pytest.mark.parametrize(
"seq_lens_list",
[
[4097],
[65537],
[1, 129, 4097, 0],
[127, 128, 129, 256, 4097, 0],
[1, 127, 128, 129, 255, 256, 257, 4097, 0],
],
)
def test_amd_decode_fused_topk_emits_sparse_table(
block_page_stride: int, seq_lens_list: list[int]
):
"""The fused selector must match top-k followed by table construction."""
from vllm.models.minimax_m3.amd.ops.index_topk import (
minimax_m3_index_decode as amd_minimax_m3_index_decode,
)
from vllm.models.minimax_m3.amd.ops.sparse_pa import (
PAGES_PER_SPARSE_BLOCK,
minimax_m3_build_sparse_block_table_decode,
)
torch.manual_seed(0)
decode_query_len = 4
seq_lens = torch.tensor(seq_lens_list, device="cuda", dtype=torch.int32)
batch = seq_lens.numel()
total_q = batch * decode_query_len
max_seq_len = int(seq_lens.max())
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
index_block_table = torch.zeros(
(batch, max_blocks), device="cuda", dtype=torch.int32
)
for req_id, seq_len in enumerate(seq_lens.tolist()):
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
pages = torch.arange(max_blocks, device="cuda", dtype=torch.int32)
index_block_table[req_id, :num_blocks] = torch.roll(pages, req_id)[:num_blocks]
# The index cache and main-attention cache have independent physical page
# mappings in production. Disjoint values make using the wrong table fail.
attention_block_table = torch.full_like(index_block_table, -31337)
for req_id, seq_len in enumerate(seq_lens.tolist()):
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
first_page = 10_000 + req_id * max_blocks
attention_block_table[req_id, :num_blocks] = torch.arange(
first_page + num_blocks - 1,
first_page - 1,
-1,
device="cuda",
dtype=torch.int32,
)
assert not torch.equal(index_block_table, attention_block_table)
idx_q = torch.randn(total_q, 1, HEAD_DIM, device="cuda", dtype=torch.bfloat16)
index_kv_cache = torch.randn(
max_blocks,
BLOCK_SIZE,
HEAD_DIM,
device="cuda",
dtype=torch.bfloat16,
)
decode_kwargs = {
"max_seq_len": max_seq_len,
"topk": TOPK,
"init_blocks": 0,
"local_blocks": 1,
"num_kv_heads": 1,
"decode_query_len": decode_query_len,
"max_decode_query_len": decode_query_len,
}
completion_counter = torch.zeros((1, total_q), device="cuda", dtype=torch.int32)
expected_topk = amd_minimax_m3_index_decode(
idx_q,
index_kv_cache,
index_block_table,
seq_lens,
completion_counter=completion_counter,
**decode_kwargs,
)
expected_sparse_bt, expected_sparse_ctx = (
minimax_m3_build_sparse_block_table_decode(
expected_topk,
attention_block_table,
seq_lens,
decode_query_len,
block_page_stride,
)
)
topk_out = torch.full_like(expected_topk, -27182)
sparse_bt_out = torch.full(
(total_q, TOPK * PAGES_PER_SPARSE_BLOCK),
-27182,
device="cuda",
dtype=torch.int32,
)
sparse_ctx_out = torch.full((total_q,), -27182, device="cuda", dtype=torch.int32)
actual_topk = amd_minimax_m3_index_decode(
idx_q,
index_kv_cache,
index_block_table,
seq_lens,
out=topk_out,
attention_block_table=attention_block_table,
sparse_block_table_out=sparse_bt_out,
sparse_context_lens_out=sparse_ctx_out,
block_page_stride=block_page_stride,
completion_counter=completion_counter,
**decode_kwargs,
)
torch.accelerator.synchronize()
assert actual_topk.data_ptr() == topk_out.data_ptr()
assert torch.equal(actual_topk, expected_topk)
assert torch.equal(sparse_bt_out, expected_sparse_bt)
assert torch.equal(sparse_ctx_out, expected_sparse_ctx)
assert torch.count_nonzero(completion_counter).item() == 0
if seq_lens_list[-1] == 0:
padded_rows = slice(total_q - decode_query_len, total_q)
assert torch.all(actual_topk[:, padded_rows] == -1)
assert torch.count_nonzero(sparse_bt_out[padded_rows]) == 0
assert torch.count_nonzero(sparse_ctx_out[padded_rows]) == 0
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
@pytest.mark.parametrize(
"seq_lens_list",