[ROCm][DSv4.1] Paged MXFP4 sparse indexer on aiter's MQA-logits kernel (#58671)

Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
Mehmet Cagri
2026-09-29 14:37:57 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Shanshan Shen
parent e05095c9e1
commit 63b9931b73
9 changed files with 2276 additions and 9 deletions
@@ -0,0 +1,599 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference.
The indexer K cache is a view into vLLM's block-major KV pool: its pages are
a block stride apart with other layers' pages in between, so the tests put
it there. The reference reads the keys back from a second cache the same
writer fills in natural order, so it shares none of the preshuffled
addressing it checks.
"""
import itertools
import types
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm-only", allow_module_level=True)
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.amd.rocm import (
rocm_mxfp4_indexer_k_store,
rocm_mxfp4_indexer_q_quant,
)
from vllm.platforms.rocm import get_cdna_version
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
RocmMxfp4IndexerMetadata,
native_decode,
plan_gather_launches,
plan_prefill_chunks,
)
from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
if get_cdna_version() != 4:
pytest.skip("CDNA4-only", allow_module_level=True)
HEADS, HEAD_DIM = 32, 128
WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32
# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled
# down so short test contexts still select.
CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32
MAX_LEN = 4096
DEVICE = "cuda"
_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
E2M1 = torch.cat([_E2M1, -_E2M1])
def _dequant(packed, e8m0):
"""[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32."""
table = E2M1.to(packed.device)
vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1)
scale = torch.pow(2.0, e8m0.float() - 127)
return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2)
@pytest.fixture(autouse=True)
def _workspace():
init_workspace_manager(torch.device(DEVICE))
yield
reset_workspace_manager()
class _Case:
"""Requests with their keys written into a block-major pool (ratio-2 and
ratio-1 pages side by side in every block) and into natural-order caches
the reference reads. The bf16 inputs are kept for the end-to-end test."""
def __init__(self, seq_lens, block):
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
k_norm_rope_mxfp4_cache,
)
self.seq_lens = seq_lens
num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3
perm = torch.randperm(num_blocks)
width = max(cdiv(n, block) for n in seq_lens)
self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32)
used = 0
for req, n in enumerate(seq_lens):
self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)]
used += cdiv(n, block)
self.block_table = self.block_table.to(DEVICE)
def align(n):
return cdiv(n, 576) * 576
page2, page1 = block // 2 * WIDTH, block * WIDTH
stride = align(page2) + align(page1) + 576
self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE)
self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool)
for i in range(num_blocks):
base = i * stride
self.page_bytes[base : base + page2] = True
base += align(page2)
self.page_bytes[base : base + page1] = True
self.cache = {
2: torch.as_strided(
self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0
),
1: torch.as_strided(
self.pool,
(num_blocks, block, WIDTH),
(stride, WIDTH, 1),
align(page2),
),
}
self.offsets = [0]
for n in seq_lens:
self.offsets.append(self.offsets[-1] + n)
# the same keys in natural order, one entry per page
self.natural = {
r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE)
for r in (1, 2)
}
self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE)
self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5
self.k_pre = {}
for req, n in enumerate(seq_lens):
if n == 0:
continue
k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
self.k_pre[req] = k_pre
pos = torch.arange(n, device=DEVICE)
for ratio, cache in self.cache.items():
comp = pos // ratio
boundary = (pos + 1) % ratio == 0
entries = cache.shape[1]
page = self.block_table[req, comp // entries].long()
slot = torch.where(boundary, page * entries + comp % entries, -1)
nat = torch.where(boundary, self.offsets[req] + comp, -1)
rocm_mxfp4_indexer_k_store(
k_pre,
pos,
cos_sin,
norm,
1e-6,
cache,
slot,
ratio,
True,
num_heads=HEADS,
)
k_norm_rope_mxfp4_cache(
k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio
)
def keys(self, ratio, req, n):
nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0]
return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :])
def _queries(num_rows):
q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8)
scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32))
q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE)
weights = torch.randn(num_rows, HEADS, device=DEVICE)
# the fused Q kernel returns the scales as one int32 per head
return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales)
def _scores(q, w, keys):
return (torch.relu(q @ keys.T) * w[:, None]).sum(0)
def _assert_topk(row, scores, k):
"""``row`` is a top-k of the entries of ``scores`` above -inf, up to the
fp32 reduce order."""
valid = scores > float("-inf")
want = min(k, int(valid.sum()))
sel = row[row >= 0].long()
assert sel.numel() == want and sel.unique().numel() == want
if want:
assert bool(valid[sel].all())
kth = scores[valid].topk(want).values[-1]
finite = scores[torch.isfinite(scores)]
tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0
assert bool((scores[sel] >= kth - tol).all())
def _block_maxima(scores):
end = scores.numel()
if end == 0:
return scores
padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf"))
padded[:end] = scores.cpu()
out = padded.view(-1, CAND_BLOCK).amax(1)
out[(end - 1) // CAND_BLOCK] = float("inf")
return out.to(scores.device)
def _forward_context(monkeypatch, metadata):
context = types.SimpleNamespace(
attn_metadata={"indexer": metadata},
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
monkeypatch.setattr(ops, "get_forward_context", lambda: context)
def _decode_metadata(case, rows, ratio, query_lens):
lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32)
lens = lens.to(DEVICE)
block_table = case.block_table[[req for req, _ in rows]].contiguous()
entries = case.cache[ratio].shape[1]
next_n = max(query_lens)
context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE)
native = native_decode(lens, block_table, query_lens, next_n, context_lens)
schedule = torch.empty(
ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n),
dtype=torch.int32,
device=DEVICE,
)
return RocmMxfp4IndexerMetadata(
seq_lens=None,
max_seq_len=max(case.seq_lens),
slot_mapping=None,
num_decodes=len(case.seq_lens),
num_decode_tokens=len(rows),
num_prefills=0,
num_prefill_tokens=0,
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
decode_row_lens=lens,
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
decode_native=native,
decode_schedule=ops.build_rocm_mxfp4_decode_schedule(
lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native
),
)
def _prefill_metadata(
case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather
):
chunks = []
for req_lo, req_hi, t0, t1 in chunk_bounds:
ends = torch.tensor(
[(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32
).to(DEVICE)
chunks.append(
DeepseekV32IndexerPrefillChunkMetadata(
block_table=case.block_table[req_lo:req_hi],
cu_seqlen_ks=torch.zeros_like(ends),
cu_seqlen_ke=ends,
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
total_seq_lens=1,
token_start=t0,
token_end=t1,
num_reqs=req_hi - req_lo,
)
)
seq_lens = torch.tensor(case.seq_lens)
context_lens = (seq_lens // ratio).int().to(DEVICE)
plans = plan_prefill_chunks(
chunks,
query_start_loc,
seq_lens,
context_lens,
ratio,
CAND_BLOCK,
0.0 if gather else None,
torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
)
return RocmMxfp4IndexerMetadata(
seq_lens=None,
max_seq_len=max(case.seq_lens),
slot_mapping=None,
num_decodes=0,
num_decode_tokens=0,
num_prefills=len(case.seq_lens),
num_prefill_tokens=len(rows),
prefill=types.SimpleNamespace(chunks=chunks),
prefill_plans=plans,
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
)
# The kernel prefers 128-token pages.
BLOCKS = pytest.mark.parametrize("block", [128, 64])
@BLOCKS
def test_k_store_stays_inside_its_pages(block):
"""The writer touches only its pages of the block-major pool. What they hold
is checked by the layer tests, which read them back through aiter's kernel."""
set_random_seed(0)
case = _Case([700, 333], block)
assert int(case.pool[~case.page_bytes].count_nonzero()) == 0
@pytest.mark.parametrize(
"rows, width, k",
# vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048
[(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)],
)
def test_topk_matches_torch_on_both_kernels(rows, width, k):
set_random_seed(0)
logits = torch.randn(rows, width, device=DEVICE)
lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE)
lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1
out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE)
ops._topk(logits, lengths, out, k)
for r in range(rows):
n = min(k, int(lengths[r]))
want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values
got = out[r]
assert torch.equal(got[got >= 0].sort().values, want.int())
assert int((got == -1).sum()) == k - n
def _run_layers(monkeypatch, case, rows, metadata):
"""Ratio-2 dense layer, the candidate source, and a consumer both ways;
each checked against the torch reference."""
num_rows = len(rows)
q, q_scale, weights, q_ref = _queries(num_rows)
hidden = torch.empty(num_rows, 1, device=DEVICE)
def topk_buffer():
return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE)
_forward_context(monkeypatch, metadata(2))
out = topk_buffer()
ops.rocm_mxfp4_sparse_attn_indexer(
hidden,
"indexer",
case.cache[2],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN // 2,
out,
compress_ratio=2,
)
for i, (req, pos) in enumerate(rows):
end = (pos + 1) // 2
keys = case.keys(2, req, end)
_assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK)
source = metadata(1)
_forward_context(monkeypatch, source)
pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE)
out = topk_buffer()
ops.rocm_mxfp4_sparse_attn_indexer(
hidden,
"indexer",
case.cache[1],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN,
out,
compress_ratio=1,
candidate_blocks=pool,
candidate_block_size=CAND_BLOCK,
candidate_write=True,
)
allowed = []
for i, (req, pos) in enumerate(rows):
scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1))
_assert_topk(out[i], scores, TOPK)
_assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS)
mask = torch.zeros_like(scores, dtype=torch.bool)
for block in pool[i][pool[i] >= 0].tolist():
mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True
allowed.append(torch.where(mask, scores, float("-inf")))
pa = ops._aiter()
build = pa.build_candidate_gather
builds = []
def counting_build(*args, **kwargs):
builds.append(1)
return build(*args, **kwargs)
monkeypatch.setattr(pa, "build_candidate_gather", counting_build)
for gather in (True, False):
consumer = metadata(1, gather)
consumer.decode_use_gather = gather
_forward_context(monkeypatch, consumer)
builds.clear()
# Two consumer layers of one step: the second reuses the first's pools.
for _ in range(2):
out = topk_buffer()
ops.rocm_mxfp4_sparse_mqa_indexer(
hidden,
"indexer",
case.cache[1],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN,
out,
1,
pool,
CAND_BLOCK,
POOL_BLOCKS * CAND_BLOCK,
)
for i in range(num_rows):
_assert_topk(out[i], allowed[i], TOPK)
if gather:
launches = len(consumer.gather_launches) + (consumer.decode is not None)
assert len(builds) == launches
@BLOCKS
@pytest.mark.parametrize(
"query_lens",
# Uniform steps launch the dense layers on next_n-row sequences, also with
# cudagraph padding (query length 0) after them; a ragged step keeps a row
# per token.
[[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]],
ids=["native6", "native2", "padded", "ragged"],
)
def test_decode_layers_match_reference(monkeypatch, block, query_lens):
set_random_seed(0)
next_n = max(query_lens)
case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block)
# a padding request has next_n rows of its own and no context
rows = [
(req, n - q + j if q else -1)
for req, (n, q) in enumerate(zip(case.seq_lens, query_lens))
for j in range(q or next_n)
]
_run_layers(
monkeypatch,
case,
rows,
lambda r, gather=False: _decode_metadata(case, rows, r, query_lens),
)
@pytest.mark.parametrize(
"seq_lens,new_tokens,chunks",
[
# one request past a cached prefix, then one sliced into two chunks
([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]),
# three requests in one chunk, the first two with equal rows
([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]),
# four requests with four different query lengths in one chunk
([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]),
],
ids=["sliced", "batched", "ragged"],
)
# The consumers rejoin a request's rows across chunks, then cut them at most
# gather_rows a launch.
@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"])
@BLOCKS
def test_prefill_layers_match_reference(
monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block
):
set_random_seed(0)
case = _Case(seq_lens, block)
rows = [
(req, n - q + i)
for req, (n, q) in enumerate(zip(seq_lens, new_tokens))
for i in range(q)
]
query_start_loc = [0, *itertools.accumulate(new_tokens)]
_run_layers(
monkeypatch,
case,
rows,
lambda r, gather=True: _prefill_metadata(
case, rows, r, chunks, query_start_loc, gather_rows, gather
),
)
def _rope_gptj(x, cos_sin, positions):
"""GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in
and out."""
x = x.bfloat16().float()
cos, sin = cos_sin[positions].chunk(2, dim=-1)
if x.dim() == 3:
cos, sin = cos[:, None], sin[:, None]
rot = x[..., -64:]
even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone()
rot[..., 0::2] = even * cos - odd * sin
rot[..., 1::2] = odd * cos + even * sin
return x.bfloat16()
def _mxfp4_round_trip(x):
"""MXFP4 round trip of x, rounded as the writers do: a scale of
2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to
even."""
blocks = x.float().unflatten(-1, (-1, 32))
amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126)
scale = torch.exp2(torch.ceil(torch.log2(amax / 6)))
y = (blocks / scale).clamp(-6, 6)
# Even codes first: argmin keeps the first of two equal distances.
grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device)
idx = (y.abs()[..., None] - grid).abs().argmin(-1)
return (grid[idx] * y.sign() * scale).flatten(-2)
def _torch_keys(case, ratio, req):
"""A request's compressed keys in torch: k_norm of each group's last
token, rotated at the group's first position, through MXFP4."""
k = case.k_pre[req][ratio - 1 :: ratio].float()
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm
pos = torch.arange(k.shape[0], device=DEVICE) * ratio
return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos))
def _topk_indices(q, weights, keys, ends, k):
"""Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end)."""
scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1)
cols = torch.arange(keys.shape[0], device=DEVICE)
return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices
def _recall(selected, reference):
"""Per row, the fraction of the reference top-k that ``selected`` picked."""
return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1)
@BLOCKS
@pytest.mark.parametrize("ratio", [2, 1])
@pytest.mark.parametrize("step", ["decode", "prefill"])
def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block):
"""End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the
paged pool, metadata from the planning functions the builder calls, and
the indexer op pick what the same MXFP4 math picks in torch."""
set_random_seed(0)
if step == "decode":
case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block)
rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)]
metadata = _decode_metadata(case, rows, ratio, [1] * len(rows))
else:
new_tokens = [300, 200, 250]
case = _Case([1500, 900, 2000], block)
rows = [
(req, n - q + i)
for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens))
for i in range(q)
]
metadata = _prefill_metadata(
case,
rows,
ratio,
[(0, len(new_tokens), 0, len(rows))],
[0, *itertools.accumulate(new_tokens)],
1 << 20,
gather=False,
)
num_rows = len(rows)
q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
w = torch.randn(num_rows, HEADS, device=DEVICE)
pos = torch.tensor([p for _, p in rows], device=DEVICE)
scales = (HEAD_DIM**-0.5, HEADS**-0.5)
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
pos, q_bf16, case.cos_sin, w, *scales
)
_forward_context(monkeypatch, metadata)
out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE)
ops.rocm_mxfp4_sparse_attn_indexer(
torch.empty(num_rows, 1, device=DEVICE),
"indexer",
case.cache[ratio],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN // ratio,
out,
compress_ratio=ratio,
)
ends = (pos + 1) // ratio
assert bool(((out >= 0) & (out < ends[:, None])).all())
assert all(row.unique().numel() == TOPK for row in out)
q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos))
w_ref = w * scales[0] * scales[1]
agree = []
for req in range(len(case.seq_lens)):
mine = torch.tensor([r == req for r, _ in rows], device=DEVICE)
ref = _topk_indices(
q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK
)
agree.append(_recall(out[mine], ref))
assert torch.cat(agree).mean().item() >= 0.99
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm MXFP4 indexer reads the paged cache in place, so each prefill
chunk is split back into its requests' rows, which one launch finds through
the chunk's query_start_loc. A chunk is whole requests or one query slice of a
long request, and both have to come back with the right rows and the right
block-table rows.
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm-only", allow_module_level=True)
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
native_decode,
plan_gather_launches,
plan_prefill_chunks,
split_prefill_chunks,
)
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import rocm_mxfp4_consumer_rows
def _chunk(block_table, token_start, token_end, row_ends):
ends = torch.tensor(row_ends, dtype=torch.int32)
return DeepseekV32IndexerPrefillChunkMetadata(
block_table=block_table,
cu_seqlen_ks=torch.full_like(ends, 7),
cu_seqlen_ke=ends + 7,
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
token_to_seq=torch.zeros(1, dtype=torch.int32),
total_seq_lens=1,
token_start=token_start,
token_end=token_end,
num_reqs=block_table.shape[0],
)
# A decode (1 token), then prefills of 3, 3, 5 and 6 query tokens; the last
# one is sliced into two chunks.
QUERY_START_LOC = [0, 1, 4, 7, 12, 18]
SEQ_LENS = torch.tensor([40, 10, 12, 21, 64])
BLOCK_TABLE = torch.arange(20, dtype=torch.int32).view(5, 4)
CHUNKS = [
_chunk(BLOCK_TABLE[1:4], 1, 12, list(range(11))),
_chunk(BLOCK_TABLE[4:5], 12, 15, [29, 30, 30]),
_chunk(BLOCK_TABLE[4:5], 15, 18, [31, 31, 32]),
]
def _plans(min_gather_width, block=8):
return plan_prefill_chunks(
CHUNKS,
QUERY_START_LOC,
SEQ_LENS,
SEQ_LENS.int() // 2,
2,
block,
min_gather_width,
torch.tensor(QUERY_START_LOC, dtype=torch.int32),
)
def test_chunks_split_into_requests():
plans = _plans(20.0)
assert [p.requests for p in plans] == [
[(0, 3, 0), (3, 6, 1), (6, 11, 2)],
[(0, 3, 0)],
[(0, 3, 0)],
]
# every chunk launches once, with offsets local to the chunk
assert [p.query_start_loc.tolist() for p in plans] == [
[0, 3, 6, 11],
[0, 3],
[0, 3],
]
# the logits are as wide as the longest compressed context in the chunk
assert [p.width for p in plans] == [10, 32, 32]
assert [p.context_lens.tolist() for p in plans] == [[5, 6, 10], [32], [32]]
torch.testing.assert_close(plans[2].row_ends, torch.tensor([31, 31, 32]).int())
torch.testing.assert_close(plans[2].block_ends, torch.tensor([4, 4, 4]).int())
assert [p.use_gather for p in plans] == [False, True, True]
assert [p.first_request for p in plans] == [1, 4, 4]
# no candidate blocks, as in the model-neutral builder
for plan in _plans(None, block=0):
assert plan.block_ends is None and not plan.use_gather
def test_launches_follow_the_logits_budget(monkeypatch):
"""Each launch's fp32 logits fit in VLLM_SPARSE_INDEXER_MAX_LOGITS_MB. A dense
chunk's logits are [rows, longest context in the chunk]: K is read in place,
not gathered, so requests share a launch until their rows no longer fit, and
a request that does not fit alone is split on its query rows. A consumer's
logits are [rows, pool]."""
budget = 4 * 1000 * 30
contexts, queries = torch.tensor([1000, 600, 200]), torch.tensor([10, 10, 10])
assert split_prefill_chunks(contexts, queries, budget, 2) == [
(slice(2, 5), slice(0, 30))
]
# the third request would take the chunk past the budget
queries = torch.tensor([10, 10, 11])
assert split_prefill_chunks(contexts, queries, budget) == [
(slice(0, 2), slice(0, 20)),
(slice(2, 3), slice(0, 11)),
]
assert split_prefill_chunks(torch.tensor([1000]), torch.tensor([100]), budget) == [
(slice(0, 1), slice(lo, min(lo + 30, 100))) for lo in range(0, 100, 30)
]
# the default 512 MiB holds 128 rows of a 1M context, or 8192 of a 16K pool
assert split_prefill_chunks(
torch.tensor([1 << 20]), torch.tensor([200]), 512 << 20
) == [(slice(0, 1), slice(0, 128)), (slice(0, 1), slice(128, 200))]
monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512")
assert rocm_mxfp4_consumer_rows(16384) == 8192
def test_gather_launches_rejoin_a_requests_rows():
"""The consumers' logits are [rows, pool] whatever the context, so a
request's rows go back together across the chunks that cut them, and a
launch never mixes requests."""
# the first chunk is too narrow to gather, the last request's slices rejoin
(joined,) = plan_gather_launches(CHUNKS, _plans(20.0), 100)
assert (joined.token_start, joined.token_end) == (12, 18)
# every chunk gathers: the first chunk's three requests launch apart
launches = plan_gather_launches(CHUNKS, _plans(0.0), 100)
assert [(g.token_start, g.token_end) for g in launches] == [
(1, 4),
(4, 7),
(7, 12),
(12, 18),
]
assert plan_gather_launches(CHUNKS, _plans(None), 100) == []
def test_decode_launches_native_on_uniform_steps():
"""A decode step goes to the kernel as next_n-row sequences only when its
shape says so: each request has next_n rows, and cudagraph padding (query
length 0) comes only after them. A FULL graph captured on a uniform batch
then replays the same launch on a padded one."""
rows = torch.zeros(8, dtype=torch.int32)
row_block_table = torch.arange(32, dtype=torch.int32).view(8, 4)
context_lens = torch.tensor([9, 4, 7, 0, 5], dtype=torch.int32)
native = native_decode(rows, row_block_table, [2, 2, 2, 0], 2, context_lens)
assert native is not None and native.next_n == 2
assert native.context_lens.tolist() == [9, 4, 7, 0]
# every request's first flattened row, as a view
assert native.block_table.tolist() == row_block_table[::2].tolist()
assert native.block_table.data_ptr() == row_block_table.data_ptr()
assert native_decode(rows, row_block_table, [2, 0, 2, 2], 2, context_lens) is None
# a ragged step, rows padded past the requests, and no speculation
assert native_decode(rows[:5], row_block_table, [2, 1, 2], 2, context_lens) is None
assert native_decode(rows, row_block_table, [2, 2, 2], 2, context_lens) is None
assert native_decode(rows[:4], row_block_table, [1] * 4, 1, context_lens) is None