[ROCm][DSv4.1] Paged MXFP4 sparse indexer on aiter's MQA-logits kernel (#58671)
Signed-off-by: Mehmet Cagri Kaymak <mehmet.kaymak@amd.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Shanshan Shen
parent
e05095c9e1
commit
63b9931b73
@@ -0,0 +1,599 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference.
|
||||
|
||||
The indexer K cache is a view into vLLM's block-major KV pool: its pages are
|
||||
a block stride apart with other layers' pages in between, so the tests put
|
||||
it there. The reference reads the keys back from a second cache the same
|
||||
writer fills in natural order, so it shares none of the preshuffled
|
||||
addressing it checks.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
rocm_mxfp4_indexer_k_store,
|
||||
rocm_mxfp4_indexer_q_quant,
|
||||
)
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerPrefillChunkMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
|
||||
RocmMxfp4IndexerMetadata,
|
||||
native_decode,
|
||||
plan_gather_launches,
|
||||
plan_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
if get_cdna_version() != 4:
|
||||
pytest.skip("CDNA4-only", allow_module_level=True)
|
||||
|
||||
HEADS, HEAD_DIM = 32, 128
|
||||
WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32
|
||||
# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled
|
||||
# down so short test contexts still select.
|
||||
CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32
|
||||
MAX_LEN = 4096
|
||||
DEVICE = "cuda"
|
||||
|
||||
_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
|
||||
E2M1 = torch.cat([_E2M1, -_E2M1])
|
||||
|
||||
|
||||
def _dequant(packed, e8m0):
|
||||
"""[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32."""
|
||||
table = E2M1.to(packed.device)
|
||||
vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1)
|
||||
scale = torch.pow(2.0, e8m0.float() - 127)
|
||||
return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _workspace():
|
||||
init_workspace_manager(torch.device(DEVICE))
|
||||
yield
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
class _Case:
|
||||
"""Requests with their keys written into a block-major pool (ratio-2 and
|
||||
ratio-1 pages side by side in every block) and into natural-order caches
|
||||
the reference reads. The bf16 inputs are kept for the end-to-end test."""
|
||||
|
||||
def __init__(self, seq_lens, block):
|
||||
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
|
||||
k_norm_rope_mxfp4_cache,
|
||||
)
|
||||
|
||||
self.seq_lens = seq_lens
|
||||
num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3
|
||||
perm = torch.randperm(num_blocks)
|
||||
width = max(cdiv(n, block) for n in seq_lens)
|
||||
self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32)
|
||||
used = 0
|
||||
for req, n in enumerate(seq_lens):
|
||||
self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)]
|
||||
used += cdiv(n, block)
|
||||
self.block_table = self.block_table.to(DEVICE)
|
||||
|
||||
def align(n):
|
||||
return cdiv(n, 576) * 576
|
||||
|
||||
page2, page1 = block // 2 * WIDTH, block * WIDTH
|
||||
stride = align(page2) + align(page1) + 576
|
||||
self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE)
|
||||
self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool)
|
||||
for i in range(num_blocks):
|
||||
base = i * stride
|
||||
self.page_bytes[base : base + page2] = True
|
||||
base += align(page2)
|
||||
self.page_bytes[base : base + page1] = True
|
||||
self.cache = {
|
||||
2: torch.as_strided(
|
||||
self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0
|
||||
),
|
||||
1: torch.as_strided(
|
||||
self.pool,
|
||||
(num_blocks, block, WIDTH),
|
||||
(stride, WIDTH, 1),
|
||||
align(page2),
|
||||
),
|
||||
}
|
||||
self.offsets = [0]
|
||||
for n in seq_lens:
|
||||
self.offsets.append(self.offsets[-1] + n)
|
||||
# the same keys in natural order, one entry per page
|
||||
self.natural = {
|
||||
r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE)
|
||||
for r in (1, 2)
|
||||
}
|
||||
self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE)
|
||||
self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5
|
||||
self.k_pre = {}
|
||||
for req, n in enumerate(seq_lens):
|
||||
if n == 0:
|
||||
continue
|
||||
k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
|
||||
self.k_pre[req] = k_pre
|
||||
pos = torch.arange(n, device=DEVICE)
|
||||
for ratio, cache in self.cache.items():
|
||||
comp = pos // ratio
|
||||
boundary = (pos + 1) % ratio == 0
|
||||
entries = cache.shape[1]
|
||||
page = self.block_table[req, comp // entries].long()
|
||||
slot = torch.where(boundary, page * entries + comp % entries, -1)
|
||||
nat = torch.where(boundary, self.offsets[req] + comp, -1)
|
||||
rocm_mxfp4_indexer_k_store(
|
||||
k_pre,
|
||||
pos,
|
||||
cos_sin,
|
||||
norm,
|
||||
1e-6,
|
||||
cache,
|
||||
slot,
|
||||
ratio,
|
||||
True,
|
||||
num_heads=HEADS,
|
||||
)
|
||||
k_norm_rope_mxfp4_cache(
|
||||
k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio
|
||||
)
|
||||
|
||||
def keys(self, ratio, req, n):
|
||||
nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0]
|
||||
return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :])
|
||||
|
||||
|
||||
def _queries(num_rows):
|
||||
q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8)
|
||||
scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32))
|
||||
q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE)
|
||||
weights = torch.randn(num_rows, HEADS, device=DEVICE)
|
||||
# the fused Q kernel returns the scales as one int32 per head
|
||||
return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales)
|
||||
|
||||
|
||||
def _scores(q, w, keys):
|
||||
return (torch.relu(q @ keys.T) * w[:, None]).sum(0)
|
||||
|
||||
|
||||
def _assert_topk(row, scores, k):
|
||||
"""``row`` is a top-k of the entries of ``scores`` above -inf, up to the
|
||||
fp32 reduce order."""
|
||||
valid = scores > float("-inf")
|
||||
want = min(k, int(valid.sum()))
|
||||
sel = row[row >= 0].long()
|
||||
assert sel.numel() == want and sel.unique().numel() == want
|
||||
if want:
|
||||
assert bool(valid[sel].all())
|
||||
kth = scores[valid].topk(want).values[-1]
|
||||
finite = scores[torch.isfinite(scores)]
|
||||
tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0
|
||||
assert bool((scores[sel] >= kth - tol).all())
|
||||
|
||||
|
||||
def _block_maxima(scores):
|
||||
end = scores.numel()
|
||||
if end == 0:
|
||||
return scores
|
||||
padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf"))
|
||||
padded[:end] = scores.cpu()
|
||||
out = padded.view(-1, CAND_BLOCK).amax(1)
|
||||
out[(end - 1) // CAND_BLOCK] = float("inf")
|
||||
return out.to(scores.device)
|
||||
|
||||
|
||||
def _forward_context(monkeypatch, metadata):
|
||||
context = types.SimpleNamespace(
|
||||
attn_metadata={"indexer": metadata},
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
monkeypatch.setattr(ops, "get_forward_context", lambda: context)
|
||||
|
||||
|
||||
def _decode_metadata(case, rows, ratio, query_lens):
|
||||
lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32)
|
||||
lens = lens.to(DEVICE)
|
||||
block_table = case.block_table[[req for req, _ in rows]].contiguous()
|
||||
entries = case.cache[ratio].shape[1]
|
||||
next_n = max(query_lens)
|
||||
context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE)
|
||||
native = native_decode(lens, block_table, query_lens, next_n, context_lens)
|
||||
schedule = torch.empty(
|
||||
ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n),
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
return RocmMxfp4IndexerMetadata(
|
||||
seq_lens=None,
|
||||
max_seq_len=max(case.seq_lens),
|
||||
slot_mapping=None,
|
||||
num_decodes=len(case.seq_lens),
|
||||
num_decode_tokens=len(rows),
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
|
||||
decode_row_lens=lens,
|
||||
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
|
||||
decode_native=native,
|
||||
decode_schedule=ops.build_rocm_mxfp4_decode_schedule(
|
||||
lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _prefill_metadata(
|
||||
case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather
|
||||
):
|
||||
chunks = []
|
||||
for req_lo, req_hi, t0, t1 in chunk_bounds:
|
||||
ends = torch.tensor(
|
||||
[(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32
|
||||
).to(DEVICE)
|
||||
chunks.append(
|
||||
DeepseekV32IndexerPrefillChunkMetadata(
|
||||
block_table=case.block_table[req_lo:req_hi],
|
||||
cu_seqlen_ks=torch.zeros_like(ends),
|
||||
cu_seqlen_ke=ends,
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
total_seq_lens=1,
|
||||
token_start=t0,
|
||||
token_end=t1,
|
||||
num_reqs=req_hi - req_lo,
|
||||
)
|
||||
)
|
||||
seq_lens = torch.tensor(case.seq_lens)
|
||||
context_lens = (seq_lens // ratio).int().to(DEVICE)
|
||||
plans = plan_prefill_chunks(
|
||||
chunks,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
context_lens,
|
||||
ratio,
|
||||
CAND_BLOCK,
|
||||
0.0 if gather else None,
|
||||
torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
return RocmMxfp4IndexerMetadata(
|
||||
seq_lens=None,
|
||||
max_seq_len=max(case.seq_lens),
|
||||
slot_mapping=None,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=len(case.seq_lens),
|
||||
num_prefill_tokens=len(rows),
|
||||
prefill=types.SimpleNamespace(chunks=chunks),
|
||||
prefill_plans=plans,
|
||||
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
|
||||
)
|
||||
|
||||
|
||||
# The kernel prefers 128-token pages.
|
||||
BLOCKS = pytest.mark.parametrize("block", [128, 64])
|
||||
|
||||
|
||||
@BLOCKS
|
||||
def test_k_store_stays_inside_its_pages(block):
|
||||
"""The writer touches only its pages of the block-major pool. What they hold
|
||||
is checked by the layer tests, which read them back through aiter's kernel."""
|
||||
set_random_seed(0)
|
||||
case = _Case([700, 333], block)
|
||||
assert int(case.pool[~case.page_bytes].count_nonzero()) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rows, width, k",
|
||||
# vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048
|
||||
[(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)],
|
||||
)
|
||||
def test_topk_matches_torch_on_both_kernels(rows, width, k):
|
||||
set_random_seed(0)
|
||||
logits = torch.randn(rows, width, device=DEVICE)
|
||||
lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE)
|
||||
lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1
|
||||
out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE)
|
||||
ops._topk(logits, lengths, out, k)
|
||||
for r in range(rows):
|
||||
n = min(k, int(lengths[r]))
|
||||
want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values
|
||||
got = out[r]
|
||||
assert torch.equal(got[got >= 0].sort().values, want.int())
|
||||
assert int((got == -1).sum()) == k - n
|
||||
|
||||
|
||||
def _run_layers(monkeypatch, case, rows, metadata):
|
||||
"""Ratio-2 dense layer, the candidate source, and a consumer both ways;
|
||||
each checked against the torch reference."""
|
||||
num_rows = len(rows)
|
||||
q, q_scale, weights, q_ref = _queries(num_rows)
|
||||
hidden = torch.empty(num_rows, 1, device=DEVICE)
|
||||
|
||||
def topk_buffer():
|
||||
return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
_forward_context(monkeypatch, metadata(2))
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[2],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN // 2,
|
||||
out,
|
||||
compress_ratio=2,
|
||||
)
|
||||
for i, (req, pos) in enumerate(rows):
|
||||
end = (pos + 1) // 2
|
||||
keys = case.keys(2, req, end)
|
||||
_assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK)
|
||||
|
||||
source = metadata(1)
|
||||
_forward_context(monkeypatch, source)
|
||||
pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE)
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[1],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN,
|
||||
out,
|
||||
compress_ratio=1,
|
||||
candidate_blocks=pool,
|
||||
candidate_block_size=CAND_BLOCK,
|
||||
candidate_write=True,
|
||||
)
|
||||
allowed = []
|
||||
for i, (req, pos) in enumerate(rows):
|
||||
scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1))
|
||||
_assert_topk(out[i], scores, TOPK)
|
||||
_assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS)
|
||||
mask = torch.zeros_like(scores, dtype=torch.bool)
|
||||
for block in pool[i][pool[i] >= 0].tolist():
|
||||
mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True
|
||||
allowed.append(torch.where(mask, scores, float("-inf")))
|
||||
|
||||
pa = ops._aiter()
|
||||
build = pa.build_candidate_gather
|
||||
builds = []
|
||||
|
||||
def counting_build(*args, **kwargs):
|
||||
builds.append(1)
|
||||
return build(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(pa, "build_candidate_gather", counting_build)
|
||||
for gather in (True, False):
|
||||
consumer = metadata(1, gather)
|
||||
consumer.decode_use_gather = gather
|
||||
_forward_context(monkeypatch, consumer)
|
||||
builds.clear()
|
||||
# Two consumer layers of one step: the second reuses the first's pools.
|
||||
for _ in range(2):
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_mqa_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[1],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN,
|
||||
out,
|
||||
1,
|
||||
pool,
|
||||
CAND_BLOCK,
|
||||
POOL_BLOCKS * CAND_BLOCK,
|
||||
)
|
||||
for i in range(num_rows):
|
||||
_assert_topk(out[i], allowed[i], TOPK)
|
||||
if gather:
|
||||
launches = len(consumer.gather_launches) + (consumer.decode is not None)
|
||||
assert len(builds) == launches
|
||||
|
||||
|
||||
@BLOCKS
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens",
|
||||
# Uniform steps launch the dense layers on next_n-row sequences, also with
|
||||
# cudagraph padding (query length 0) after them; a ragged step keeps a row
|
||||
# per token.
|
||||
[[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]],
|
||||
ids=["native6", "native2", "padded", "ragged"],
|
||||
)
|
||||
def test_decode_layers_match_reference(monkeypatch, block, query_lens):
|
||||
set_random_seed(0)
|
||||
next_n = max(query_lens)
|
||||
case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block)
|
||||
# a padding request has next_n rows of its own and no context
|
||||
rows = [
|
||||
(req, n - q + j if q else -1)
|
||||
for req, (n, q) in enumerate(zip(case.seq_lens, query_lens))
|
||||
for j in range(q or next_n)
|
||||
]
|
||||
_run_layers(
|
||||
monkeypatch,
|
||||
case,
|
||||
rows,
|
||||
lambda r, gather=False: _decode_metadata(case, rows, r, query_lens),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,new_tokens,chunks",
|
||||
[
|
||||
# one request past a cached prefix, then one sliced into two chunks
|
||||
([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]),
|
||||
# three requests in one chunk, the first two with equal rows
|
||||
([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]),
|
||||
# four requests with four different query lengths in one chunk
|
||||
([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]),
|
||||
],
|
||||
ids=["sliced", "batched", "ragged"],
|
||||
)
|
||||
# The consumers rejoin a request's rows across chunks, then cut them at most
|
||||
# gather_rows a launch.
|
||||
@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"])
|
||||
@BLOCKS
|
||||
def test_prefill_layers_match_reference(
|
||||
monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block
|
||||
):
|
||||
set_random_seed(0)
|
||||
case = _Case(seq_lens, block)
|
||||
rows = [
|
||||
(req, n - q + i)
|
||||
for req, (n, q) in enumerate(zip(seq_lens, new_tokens))
|
||||
for i in range(q)
|
||||
]
|
||||
query_start_loc = [0, *itertools.accumulate(new_tokens)]
|
||||
_run_layers(
|
||||
monkeypatch,
|
||||
case,
|
||||
rows,
|
||||
lambda r, gather=True: _prefill_metadata(
|
||||
case, rows, r, chunks, query_start_loc, gather_rows, gather
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _rope_gptj(x, cos_sin, positions):
|
||||
"""GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in
|
||||
and out."""
|
||||
x = x.bfloat16().float()
|
||||
cos, sin = cos_sin[positions].chunk(2, dim=-1)
|
||||
if x.dim() == 3:
|
||||
cos, sin = cos[:, None], sin[:, None]
|
||||
rot = x[..., -64:]
|
||||
even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone()
|
||||
rot[..., 0::2] = even * cos - odd * sin
|
||||
rot[..., 1::2] = odd * cos + even * sin
|
||||
return x.bfloat16()
|
||||
|
||||
|
||||
def _mxfp4_round_trip(x):
|
||||
"""MXFP4 round trip of x, rounded as the writers do: a scale of
|
||||
2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to
|
||||
even."""
|
||||
blocks = x.float().unflatten(-1, (-1, 32))
|
||||
amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126)
|
||||
scale = torch.exp2(torch.ceil(torch.log2(amax / 6)))
|
||||
y = (blocks / scale).clamp(-6, 6)
|
||||
# Even codes first: argmin keeps the first of two equal distances.
|
||||
grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device)
|
||||
idx = (y.abs()[..., None] - grid).abs().argmin(-1)
|
||||
return (grid[idx] * y.sign() * scale).flatten(-2)
|
||||
|
||||
|
||||
def _torch_keys(case, ratio, req):
|
||||
"""A request's compressed keys in torch: k_norm of each group's last
|
||||
token, rotated at the group's first position, through MXFP4."""
|
||||
k = case.k_pre[req][ratio - 1 :: ratio].float()
|
||||
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm
|
||||
pos = torch.arange(k.shape[0], device=DEVICE) * ratio
|
||||
return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos))
|
||||
|
||||
|
||||
def _topk_indices(q, weights, keys, ends, k):
|
||||
"""Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end)."""
|
||||
scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1)
|
||||
cols = torch.arange(keys.shape[0], device=DEVICE)
|
||||
return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices
|
||||
|
||||
|
||||
def _recall(selected, reference):
|
||||
"""Per row, the fraction of the reference top-k that ``selected`` picked."""
|
||||
return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1)
|
||||
|
||||
|
||||
@BLOCKS
|
||||
@pytest.mark.parametrize("ratio", [2, 1])
|
||||
@pytest.mark.parametrize("step", ["decode", "prefill"])
|
||||
def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block):
|
||||
"""End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the
|
||||
paged pool, metadata from the planning functions the builder calls, and
|
||||
the indexer op pick what the same MXFP4 math picks in torch."""
|
||||
set_random_seed(0)
|
||||
if step == "decode":
|
||||
case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block)
|
||||
rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)]
|
||||
metadata = _decode_metadata(case, rows, ratio, [1] * len(rows))
|
||||
else:
|
||||
new_tokens = [300, 200, 250]
|
||||
case = _Case([1500, 900, 2000], block)
|
||||
rows = [
|
||||
(req, n - q + i)
|
||||
for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens))
|
||||
for i in range(q)
|
||||
]
|
||||
metadata = _prefill_metadata(
|
||||
case,
|
||||
rows,
|
||||
ratio,
|
||||
[(0, len(new_tokens), 0, len(rows))],
|
||||
[0, *itertools.accumulate(new_tokens)],
|
||||
1 << 20,
|
||||
gather=False,
|
||||
)
|
||||
num_rows = len(rows)
|
||||
q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
|
||||
w = torch.randn(num_rows, HEADS, device=DEVICE)
|
||||
pos = torch.tensor([p for _, p in rows], device=DEVICE)
|
||||
scales = (HEAD_DIM**-0.5, HEADS**-0.5)
|
||||
|
||||
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
|
||||
pos, q_bf16, case.cos_sin, w, *scales
|
||||
)
|
||||
_forward_context(monkeypatch, metadata)
|
||||
out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE)
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
torch.empty(num_rows, 1, device=DEVICE),
|
||||
"indexer",
|
||||
case.cache[ratio],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN // ratio,
|
||||
out,
|
||||
compress_ratio=ratio,
|
||||
)
|
||||
ends = (pos + 1) // ratio
|
||||
assert bool(((out >= 0) & (out < ends[:, None])).all())
|
||||
assert all(row.unique().numel() == TOPK for row in out)
|
||||
|
||||
q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos))
|
||||
w_ref = w * scales[0] * scales[1]
|
||||
agree = []
|
||||
for req in range(len(case.seq_lens)):
|
||||
mine = torch.tensor([r == req for r, _ in rows], device=DEVICE)
|
||||
ref = _topk_indices(
|
||||
q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK
|
||||
)
|
||||
agree.append(_recall(out[mine], ref))
|
||||
assert torch.cat(agree).mean().item() >= 0.99
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm MXFP4 indexer reads the paged cache in place, so each prefill
|
||||
chunk is split back into its requests' rows, which one launch finds through
|
||||
the chunk's query_start_loc. A chunk is whole requests or one query slice of a
|
||||
long request, and both have to come back with the right rows and the right
|
||||
block-table rows.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerPrefillChunkMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
|
||||
native_decode,
|
||||
plan_gather_launches,
|
||||
plan_prefill_chunks,
|
||||
split_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import rocm_mxfp4_consumer_rows
|
||||
|
||||
|
||||
def _chunk(block_table, token_start, token_end, row_ends):
|
||||
ends = torch.tensor(row_ends, dtype=torch.int32)
|
||||
return DeepseekV32IndexerPrefillChunkMetadata(
|
||||
block_table=block_table,
|
||||
cu_seqlen_ks=torch.full_like(ends, 7),
|
||||
cu_seqlen_ke=ends + 7,
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32),
|
||||
total_seq_lens=1,
|
||||
token_start=token_start,
|
||||
token_end=token_end,
|
||||
num_reqs=block_table.shape[0],
|
||||
)
|
||||
|
||||
|
||||
# A decode (1 token), then prefills of 3, 3, 5 and 6 query tokens; the last
|
||||
# one is sliced into two chunks.
|
||||
QUERY_START_LOC = [0, 1, 4, 7, 12, 18]
|
||||
SEQ_LENS = torch.tensor([40, 10, 12, 21, 64])
|
||||
BLOCK_TABLE = torch.arange(20, dtype=torch.int32).view(5, 4)
|
||||
CHUNKS = [
|
||||
_chunk(BLOCK_TABLE[1:4], 1, 12, list(range(11))),
|
||||
_chunk(BLOCK_TABLE[4:5], 12, 15, [29, 30, 30]),
|
||||
_chunk(BLOCK_TABLE[4:5], 15, 18, [31, 31, 32]),
|
||||
]
|
||||
|
||||
|
||||
def _plans(min_gather_width, block=8):
|
||||
return plan_prefill_chunks(
|
||||
CHUNKS,
|
||||
QUERY_START_LOC,
|
||||
SEQ_LENS,
|
||||
SEQ_LENS.int() // 2,
|
||||
2,
|
||||
block,
|
||||
min_gather_width,
|
||||
torch.tensor(QUERY_START_LOC, dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
def test_chunks_split_into_requests():
|
||||
plans = _plans(20.0)
|
||||
|
||||
assert [p.requests for p in plans] == [
|
||||
[(0, 3, 0), (3, 6, 1), (6, 11, 2)],
|
||||
[(0, 3, 0)],
|
||||
[(0, 3, 0)],
|
||||
]
|
||||
# every chunk launches once, with offsets local to the chunk
|
||||
assert [p.query_start_loc.tolist() for p in plans] == [
|
||||
[0, 3, 6, 11],
|
||||
[0, 3],
|
||||
[0, 3],
|
||||
]
|
||||
# the logits are as wide as the longest compressed context in the chunk
|
||||
assert [p.width for p in plans] == [10, 32, 32]
|
||||
assert [p.context_lens.tolist() for p in plans] == [[5, 6, 10], [32], [32]]
|
||||
torch.testing.assert_close(plans[2].row_ends, torch.tensor([31, 31, 32]).int())
|
||||
torch.testing.assert_close(plans[2].block_ends, torch.tensor([4, 4, 4]).int())
|
||||
assert [p.use_gather for p in plans] == [False, True, True]
|
||||
assert [p.first_request for p in plans] == [1, 4, 4]
|
||||
# no candidate blocks, as in the model-neutral builder
|
||||
for plan in _plans(None, block=0):
|
||||
assert plan.block_ends is None and not plan.use_gather
|
||||
|
||||
|
||||
def test_launches_follow_the_logits_budget(monkeypatch):
|
||||
"""Each launch's fp32 logits fit in VLLM_SPARSE_INDEXER_MAX_LOGITS_MB. A dense
|
||||
chunk's logits are [rows, longest context in the chunk]: K is read in place,
|
||||
not gathered, so requests share a launch until their rows no longer fit, and
|
||||
a request that does not fit alone is split on its query rows. A consumer's
|
||||
logits are [rows, pool]."""
|
||||
budget = 4 * 1000 * 30
|
||||
contexts, queries = torch.tensor([1000, 600, 200]), torch.tensor([10, 10, 10])
|
||||
assert split_prefill_chunks(contexts, queries, budget, 2) == [
|
||||
(slice(2, 5), slice(0, 30))
|
||||
]
|
||||
# the third request would take the chunk past the budget
|
||||
queries = torch.tensor([10, 10, 11])
|
||||
assert split_prefill_chunks(contexts, queries, budget) == [
|
||||
(slice(0, 2), slice(0, 20)),
|
||||
(slice(2, 3), slice(0, 11)),
|
||||
]
|
||||
assert split_prefill_chunks(torch.tensor([1000]), torch.tensor([100]), budget) == [
|
||||
(slice(0, 1), slice(lo, min(lo + 30, 100))) for lo in range(0, 100, 30)
|
||||
]
|
||||
# the default 512 MiB holds 128 rows of a 1M context, or 8192 of a 16K pool
|
||||
assert split_prefill_chunks(
|
||||
torch.tensor([1 << 20]), torch.tensor([200]), 512 << 20
|
||||
) == [(slice(0, 1), slice(0, 128)), (slice(0, 1), slice(128, 200))]
|
||||
monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512")
|
||||
assert rocm_mxfp4_consumer_rows(16384) == 8192
|
||||
|
||||
|
||||
def test_gather_launches_rejoin_a_requests_rows():
|
||||
"""The consumers' logits are [rows, pool] whatever the context, so a
|
||||
request's rows go back together across the chunks that cut them, and a
|
||||
launch never mixes requests."""
|
||||
# the first chunk is too narrow to gather, the last request's slices rejoin
|
||||
(joined,) = plan_gather_launches(CHUNKS, _plans(20.0), 100)
|
||||
assert (joined.token_start, joined.token_end) == (12, 18)
|
||||
# every chunk gathers: the first chunk's three requests launch apart
|
||||
launches = plan_gather_launches(CHUNKS, _plans(0.0), 100)
|
||||
assert [(g.token_start, g.token_end) for g in launches] == [
|
||||
(1, 4),
|
||||
(4, 7),
|
||||
(7, 12),
|
||||
(12, 18),
|
||||
]
|
||||
assert plan_gather_launches(CHUNKS, _plans(None), 100) == []
|
||||
|
||||
|
||||
def test_decode_launches_native_on_uniform_steps():
|
||||
"""A decode step goes to the kernel as next_n-row sequences only when its
|
||||
shape says so: each request has next_n rows, and cudagraph padding (query
|
||||
length 0) comes only after them. A FULL graph captured on a uniform batch
|
||||
then replays the same launch on a padded one."""
|
||||
rows = torch.zeros(8, dtype=torch.int32)
|
||||
row_block_table = torch.arange(32, dtype=torch.int32).view(8, 4)
|
||||
context_lens = torch.tensor([9, 4, 7, 0, 5], dtype=torch.int32)
|
||||
|
||||
native = native_decode(rows, row_block_table, [2, 2, 2, 0], 2, context_lens)
|
||||
assert native is not None and native.next_n == 2
|
||||
assert native.context_lens.tolist() == [9, 4, 7, 0]
|
||||
# every request's first flattened row, as a view
|
||||
assert native.block_table.tolist() == row_block_table[::2].tolist()
|
||||
assert native.block_table.data_ptr() == row_block_table.data_ptr()
|
||||
|
||||
assert native_decode(rows, row_block_table, [2, 0, 2, 2], 2, context_lens) is None
|
||||
# a ragged step, rows padded past the requests, and no speculation
|
||||
assert native_decode(rows[:5], row_block_table, [2, 1, 2], 2, context_lens) is None
|
||||
assert native_decode(rows, row_block_table, [2, 2, 2], 2, context_lens) is None
|
||||
assert native_decode(rows[:4], row_block_table, [1] * 4, 1, context_lens) is None
|
||||
Reference in New Issue
Block a user