[ROCm][DSv4.1] Paged MXFP4 sparse indexer on aiter's MQA-logits kernel (#58671)

Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Mehmet Cagri
2026-09-29 14:37:57 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Shanshan Shen
parent e05095c9e1
commit 63b9931b73
9 changed files with 2276 additions and 9 deletions
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm MXFP4 indexer reads the paged cache in place, so each prefill
chunk is split back into its requests' rows, which one launch finds through
the chunk's query_start_loc. A chunk is whole requests or one query slice of a
long request, and both have to come back with the right rows and the right
block-table rows.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm-only", allow_module_level=True)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
native_decode,
plan_gather_launches,
plan_prefill_chunks,
split_prefill_chunks,
)
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import rocm_mxfp4_consumer_rows
def _chunk(block_table, token_start, token_end, row_ends):
ends = torch.tensor(row_ends, dtype=torch.int32)
return DeepseekV32IndexerPrefillChunkMetadata(
block_table=block_table,
cu_seqlen_ks=torch.full_like(ends, 7),
cu_seqlen_ke=ends + 7,
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
token_to_seq=torch.zeros(1, dtype=torch.int32),
total_seq_lens=1,
token_start=token_start,
token_end=token_end,
num_reqs=block_table.shape[0],
)
# A decode (1 token), then prefills of 3, 3, 5 and 6 query tokens; the last
# one is sliced into two chunks.
QUERY_START_LOC = [0, 1, 4, 7, 12, 18]
SEQ_LENS = torch.tensor([40, 10, 12, 21, 64])
BLOCK_TABLE = torch.arange(20, dtype=torch.int32).view(5, 4)
CHUNKS = [
_chunk(BLOCK_TABLE[1:4], 1, 12, list(range(11))),
_chunk(BLOCK_TABLE[4:5], 12, 15, [29, 30, 30]),
_chunk(BLOCK_TABLE[4:5], 15, 18, [31, 31, 32]),
]
def _plans(min_gather_width, block=8):
return plan_prefill_chunks(
CHUNKS,
QUERY_START_LOC,
SEQ_LENS,
SEQ_LENS.int() // 2,
2,
block,
min_gather_width,
torch.tensor(QUERY_START_LOC, dtype=torch.int32),
)
def test_chunks_split_into_requests():
plans = _plans(20.0)
assert [p.requests for p in plans] == [
[(0, 3, 0), (3, 6, 1), (6, 11, 2)],
[(0, 3, 0)],
[(0, 3, 0)],
]
# every chunk launches once, with offsets local to the chunk
assert [p.query_start_loc.tolist() for p in plans] == [
[0, 3, 6, 11],
[0, 3],
[0, 3],
]
# the logits are as wide as the longest compressed context in the chunk
assert [p.width for p in plans] == [10, 32, 32]
assert [p.context_lens.tolist() for p in plans] == [[5, 6, 10], [32], [32]]
torch.testing.assert_close(plans[2].row_ends, torch.tensor([31, 31, 32]).int())
torch.testing.assert_close(plans[2].block_ends, torch.tensor([4, 4, 4]).int())
assert [p.use_gather for p in plans] == [False, True, True]
assert [p.first_request for p in plans] == [1, 4, 4]
# no candidate blocks, as in the model-neutral builder
for plan in _plans(None, block=0):
assert plan.block_ends is None and not plan.use_gather
def test_launches_follow_the_logits_budget(monkeypatch):
"""Each launch's fp32 logits fit in VLLM_SPARSE_INDEXER_MAX_LOGITS_MB. A dense
chunk's logits are [rows, longest context in the chunk]: K is read in place,
not gathered, so requests share a launch until their rows no longer fit, and
a request that does not fit alone is split on its query rows. A consumer's
logits are [rows, pool]."""
budget = 4 * 1000 * 30
contexts, queries = torch.tensor([1000, 600, 200]), torch.tensor([10, 10, 10])
assert split_prefill_chunks(contexts, queries, budget, 2) == [
(slice(2, 5), slice(0, 30))
]
# the third request would take the chunk past the budget
queries = torch.tensor([10, 10, 11])
assert split_prefill_chunks(contexts, queries, budget) == [
(slice(0, 2), slice(0, 20)),
(slice(2, 3), slice(0, 11)),
]
assert split_prefill_chunks(torch.tensor([1000]), torch.tensor([100]), budget) == [
(slice(0, 1), slice(lo, min(lo + 30, 100))) for lo in range(0, 100, 30)
]
# the default 512 MiB holds 128 rows of a 1M context, or 8192 of a 16K pool
assert split_prefill_chunks(
torch.tensor([1 << 20]), torch.tensor([200]), 512 << 20
) == [(slice(0, 1), slice(0, 128)), (slice(0, 1), slice(128, 200))]
monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512")
assert rocm_mxfp4_consumer_rows(16384) == 8192
def test_gather_launches_rejoin_a_requests_rows():
"""The consumers' logits are [rows, pool] whatever the context, so a
request's rows go back together across the chunks that cut them, and a
launch never mixes requests."""
# the first chunk is too narrow to gather, the last request's slices rejoin
(joined,) = plan_gather_launches(CHUNKS, _plans(20.0), 100)
assert (joined.token_start, joined.token_end) == (12, 18)
# every chunk gathers: the first chunk's three requests launch apart
launches = plan_gather_launches(CHUNKS, _plans(0.0), 100)
assert [(g.token_start, g.token_end) for g in launches] == [
(1, 4),
(4, 7),
(7, 12),
(12, 18),
]
assert plan_gather_launches(CHUNKS, _plans(None), 100) == []
def test_decode_launches_native_on_uniform_steps():
"""A decode step goes to the kernel as next_n-row sequences only when its
shape says so: each request has next_n rows, and cudagraph padding (query
length 0) comes only after them. A FULL graph captured on a uniform batch
then replays the same launch on a padded one."""
rows = torch.zeros(8, dtype=torch.int32)
row_block_table = torch.arange(32, dtype=torch.int32).view(8, 4)
context_lens = torch.tensor([9, 4, 7, 0, 5], dtype=torch.int32)
native = native_decode(rows, row_block_table, [2, 2, 2, 0], 2, context_lens)
assert native is not None and native.next_n == 2
assert native.context_lens.tolist() == [9, 4, 7, 0]
# every request's first flattened row, as a view
assert native.block_table.tolist() == row_block_table[::2].tolist()
assert native.block_table.data_ptr() == row_block_table.data_ptr()
assert native_decode(rows, row_block_table, [2, 0, 2, 2], 2, context_lens) is None
# a ragged step, rows padded past the requests, and no speculation
assert native_decode(rows[:5], row_block_table, [2, 1, 2], 2, context_lens) is None
assert native_decode(rows, row_block_table, [2, 2, 2], 2, context_lens) is None
assert native_decode(rows[:4], row_block_table, [1] * 4, 1, context_lens) is None