[ROCm][MiniMax-M3] Add AITER sparse paged attention (#47287)

Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Jun Kang Chow <junkangchow@gmail.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
Tan Pin Siang
2026-07-12 19:27:29 -07:00
committed by GitHub
co-authored by vllmellm Hongxia Yang Jun Kang Chow tjtanaa
parent e26264f3ef
commit ee5a89f4d7
15 changed files with 1554 additions and 128 deletions
@@ -820,6 +820,47 @@ def test_main_backend_layout_contract():
)
def test_aiter_sparse_pa_layout_contract(monkeypatch):
"""The shuffle-only AITER path retains separately contiguous K/V storage."""
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
monkeypatch.setattr(
sparse_attn_mod.rocm_aiter_ops,
"is_shuffle_kv_cache_enabled",
lambda: True,
)
nb, bs, h, d = 7, BLOCK_SIZE, 1, HEAD_DIM
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
assert logical == (nb, 2, bs, h, d)
assert order == (1, 0, 2, 3, 4)
physical_shape = tuple(logical[i] for i in order)
inv_order = [order.index(i) for i in range(len(order))]
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
logical_view = raw.permute(*inv_order)
key_cache, value_cache = logical_view.unbind(1)
assert key_cache.is_contiguous()
assert value_cache.is_contiguous()
def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch):
"""Do not pair AITER's separated cache layout with the Triton fallback."""
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
monkeypatch.setattr(
sparse_attn_mod.rocm_aiter_ops,
"is_shuffle_kv_cache_enabled",
lambda: True,
)
with pytest.raises(ValueError, match="num_kv_heads == 1"):
MiniMaxM3SparseBackend.get_kv_cache_shape(7, BLOCK_SIZE, 2, HEAD_DIM)
def test_main_backend_unknown_layout_raises(monkeypatch):
"""An unrecognized layout (injected past env-var validation) is rejected."""
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
@@ -171,10 +171,9 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
kv_cache_storage_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
kv_cache = torch.zeros(
num_blocks,
2,
block_size,
num_kv_heads,
HEAD_DIM,
block_size,
2 * HEAD_DIM,
dtype=kv_cache_storage_dtype,
device=device,
)
@@ -246,18 +245,21 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
torch.testing.assert_close(index_k, ik_ref, rtol=1e-2, atol=1e-2)
# ── Cache inserts. ──
# Main cache layout is [num_blocks, 2, block_size, num_kv_heads, head_dim]
# (the K/V axis sits *before* block_size); index cache is [nb, bs, head_dim].
# Main cache layout is [num_blocks, num_kv_heads, block_size, 2*head_dim];
# index cache is [num_blocks, block_size, head_dim].
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM) # v is raw (no norm/rope)
if kv_cache_dtype == "fp8":
expected_kv_cache = torch.zeros_like(kv_cache)
expected_k_cache, expected_v_cache = expected_kv_cache.transpose(1, 2).split(
HEAD_DIM, dim=-1
)
scale = torch.ones((), device=device)
ops.reshape_and_cache_flash(
k_out.view(num_tokens, num_kv_heads, HEAD_DIM),
v_out.view(num_tokens, num_kv_heads, HEAD_DIM),
expected_kv_cache[:, 0],
expected_kv_cache[:, 1],
expected_k_cache,
expected_v_cache,
slot_mapping,
kv_cache_dtype,
scale,
@@ -269,9 +271,11 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
s = slot_mapping[t].item()
b, pos = s // block_size, s % block_size
torch.testing.assert_close(
kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
kv_cache[b, :, pos, :HEAD_DIM], k_ref_h[t], rtol=1e-2, atol=1e-2
)
torch.testing.assert_close(
kv_cache[b, :, pos, HEAD_DIM:], v_ref_h[t], rtol=0, atol=0
)
torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
expected_index_cache[index_slot_mapping] = index_k
@@ -280,7 +284,121 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
)
# ── Test 3: fp8 (e4m3) index outputs ─────────────────────────────────────────
@pytest.mark.parametrize("num_tokens", [1, 64, 513])
@pytest.mark.parametrize("block_size", [16, 64])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
torch.manual_seed(2)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
num_heads, num_kv_heads, num_idx_heads = 16, 4, 4
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
iqsz, iksz = num_idx_heads * HEAD_DIM, HEAD_DIM
qkv = torch.randn(
num_tokens, qsz + 2 * kvsz + iqsz + iksz, dtype=dtype, device=device
)
qkv_orig = qkv.clone()
splits = [qsz, kvsz, kvsz, iqsz, iksz]
num_blocks = (num_tokens + block_size - 1) // block_size + 1
kv_cache_storage_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
kv_cache = torch.zeros(
num_blocks,
num_kv_heads,
block_size,
2 * HEAD_DIM,
dtype=kv_cache_storage_dtype,
device=device,
)
index_cache = torch.randn(
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
)
index_cache_orig = index_cache.clone()
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
q_out = torch.empty(num_tokens, qsz, dtype=dtype, device=device)
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
q_w,
k_w,
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
num_index_heads=num_idx_heads,
slot_mapping=slot_mapping,
kv_cache=kv_cache,
index_cache=index_cache,
block_size=block_size,
q_out=q_out,
kv_cache_dtype=kv_cache_dtype,
skip_index_branch=True,
)
_, k_out, v_out, index_q_out, index_k_out = qkv.split(splits, dim=-1)
q_in, k_in, v_in, index_q_in, index_k_in = qkv_orig.split(splits, dim=-1)
q_ref = norm_rope_ref(
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
).view(num_tokens, qsz)
k_ref = norm_rope_ref(
k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
k_w,
positions,
cos_sin,
eps,
).view(num_tokens, kvsz)
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
torch.testing.assert_close(index_q_out, index_q_in, rtol=0, atol=0)
torch.testing.assert_close(index_k_out, index_k_in, rtol=0, atol=0)
torch.testing.assert_close(index_cache, index_cache_orig, rtol=0, atol=0)
if kv_cache_dtype == "fp8":
expected_kv_cache = torch.zeros_like(kv_cache)
expected_k_cache, expected_v_cache = expected_kv_cache.transpose(1, 2).split(
HEAD_DIM, dim=-1
)
scale = torch.ones((), device=device)
ops.reshape_and_cache_flash(
k_out.view(num_tokens, num_kv_heads, HEAD_DIM),
v_out.view(num_tokens, num_kv_heads, HEAD_DIM),
expected_k_cache,
expected_v_cache,
slot_mapping,
kv_cache_dtype,
scale,
scale,
)
torch.testing.assert_close(kv_cache, expected_kv_cache, rtol=0, atol=0)
else:
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM)
for t in range(num_tokens):
s = slot_mapping[t].item()
b, pos = s // block_size, s % block_size
torch.testing.assert_close(
kv_cache[b, :, pos, :HEAD_DIM], k_ref_h[t], rtol=1e-2, atol=1e-2
)
torch.testing.assert_close(
kv_cache[b, :, pos, HEAD_DIM:], v_ref_h[t], rtol=0, atol=0
)
# ── Test 4: fp8 (e4m3) index outputs ─────────────────────────────────────────
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
@@ -324,10 +442,9 @@ def test_sparse_full_fp8_index(num_tokens, block_size):
qkv = qkv0.clone()
kv_cache = torch.zeros(
num_blocks,
2,
block_size,
num_kv_heads,
HEAD_DIM,
block_size,
2 * HEAD_DIM,
dtype=dtype,
device=device,
)
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
if not torch.cuda.is_available() or not current_platform.supports_fp8():
pytest.skip(
"MiniMax-M3 FP8 sparse attention scale tests require an FP8 GPU.",
allow_module_level=True,
)
if current_platform.is_rocm():
from vllm.models.minimax_m3.amd.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
else:
from vllm.models.minimax_m3.common.ops.sparse_attn import (
SPARSE_BLOCK_SIZE,
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
DEVICE = "cuda"
DTYPE = torch.bfloat16
HEAD_DIM = 128
NUM_KV_HEADS = 1
NUM_HEADS = 2
K_SCALE = 0.25
V_SCALE = 0.5
def _scale_tensors(mode: str, num_blocks: int):
if mode == "scalar":
k_scale = torch.tensor(K_SCALE, dtype=torch.float32, device=DEVICE)
v_scale = torch.tensor(V_SCALE, dtype=torch.float32, device=DEVICE)
else:
shape = (NUM_KV_HEADS, num_blocks * SPARSE_BLOCK_SIZE)
k_scale = torch.full(shape, K_SCALE, dtype=torch.float32, device=DEVICE)
v_scale = torch.full(shape, V_SCALE, dtype=torch.float32, device=DEVICE)
return k_scale, v_scale
def _make_kv_cache(num_blocks: int, seed: int):
torch.manual_seed(seed)
fp8_dtype = current_platform.fp8_dtype()
kv_ref = torch.randn(
num_blocks,
NUM_KV_HEADS,
SPARSE_BLOCK_SIZE,
2 * HEAD_DIM,
dtype=DTYPE,
device=DEVICE,
)
kv_fp8 = torch.empty_like(kv_ref, dtype=fp8_dtype)
kv_fp8[..., :HEAD_DIM] = (kv_ref[..., :HEAD_DIM].float() / K_SCALE).to(fp8_dtype)
kv_fp8[..., HEAD_DIM:] = (kv_ref[..., HEAD_DIM:].float() / V_SCALE).to(fp8_dtype)
kv_dequant = torch.empty_like(kv_ref)
kv_dequant[..., :HEAD_DIM] = (kv_fp8[..., :HEAD_DIM].float() * K_SCALE).to(DTYPE)
kv_dequant[..., HEAD_DIM:] = (kv_fp8[..., HEAD_DIM:].float() * V_SCALE).to(DTYPE)
return kv_fp8, kv_dequant
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
@torch.inference_mode()
def test_minimax_m3_sparse_prefill_fp8_kv_scales(scale_mode: str):
total_q = 17
num_blocks = 1
torch.manual_seed(0)
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=1)
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
block_table = torch.zeros(1, 1, dtype=torch.int32, device=DEVICE)
cu_seqlens = torch.tensor([0, total_q], dtype=torch.int32, device=DEVICE)
seq_lens = torch.tensor([total_q], dtype=torch.int32, device=DEVICE)
prefix_lens = torch.zeros(1, dtype=torch.int32, device=DEVICE)
got = torch.empty_like(q)
ref = torch.empty_like(q)
unscaled = torch.empty_like(q)
minimax_m3_sparse_attn(
q,
kv_fp8,
topk,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
total_q,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
got,
k_scale=k_scale,
v_scale=v_scale,
)
minimax_m3_sparse_attn(
q,
kv_dequant,
topk,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
total_q,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
ref,
)
minimax_m3_sparse_attn(
q,
kv_fp8,
topk,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
total_q,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
unscaled,
)
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
@torch.inference_mode()
def test_minimax_m3_sparse_decode_fp8_kv_scales(scale_mode: str):
total_q = 2
num_blocks = 1
torch.manual_seed(2)
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=3)
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
block_table = torch.zeros(total_q, 1, dtype=torch.int32, device=DEVICE)
seq_lens = torch.tensor([64, 128], dtype=torch.int32, device=DEVICE)
got = torch.empty_like(q)
ref = torch.empty_like(q)
unscaled = torch.empty_like(q)
minimax_m3_sparse_attn_decode(
q,
kv_fp8,
topk,
block_table,
seq_lens,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
got,
decode_query_len=1,
k_scale=k_scale,
v_scale=v_scale,
)
minimax_m3_sparse_attn_decode(
q,
kv_dequant,
topk,
block_table,
seq_lens,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
ref,
decode_query_len=1,
)
minimax_m3_sparse_attn_decode(
q,
kv_fp8,
topk,
block_table,
seq_lens,
NUM_KV_HEADS,
HEAD_DIM**-0.5,
unscaled,
decode_query_len=1,
)
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)