[ROCm][MiniMax-M3] Add AITER sparse paged attention (#47287)
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Jun Kang Chow <junkangchow@gmail.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
vllmellm
Hongxia Yang
Jun Kang Chow
tjtanaa
parent
e26264f3ef
commit
ee5a89f4d7
@@ -820,6 +820,47 @@ def test_main_backend_layout_contract():
|
||||
)
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_layout_contract(monkeypatch):
|
||||
"""The shuffle-only AITER path retains separately contiguous K/V storage."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
sparse_attn_mod.rocm_aiter_ops,
|
||||
"is_shuffle_kv_cache_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, 1, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
assert logical == (nb, 2, bs, h, d)
|
||||
assert order == (1, 0, 2, 3, 4)
|
||||
|
||||
physical_shape = tuple(logical[i] for i in order)
|
||||
inv_order = [order.index(i) for i in range(len(order))]
|
||||
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
|
||||
logical_view = raw.permute(*inv_order)
|
||||
key_cache, value_cache = logical_view.unbind(1)
|
||||
assert key_cache.is_contiguous()
|
||||
assert value_cache.is_contiguous()
|
||||
|
||||
|
||||
def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch):
|
||||
"""Do not pair AITER's separated cache layout with the Triton fallback."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
sparse_attn_mod.rocm_aiter_ops,
|
||||
"is_shuffle_kv_cache_enabled",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="num_kv_heads == 1"):
|
||||
MiniMaxM3SparseBackend.get_kv_cache_shape(7, BLOCK_SIZE, 2, HEAD_DIM)
|
||||
|
||||
|
||||
def test_main_backend_unknown_layout_raises(monkeypatch):
|
||||
"""An unrecognized layout (injected past env-var validation) is rejected."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
@@ -171,10 +171,9 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
|
||||
kv_cache_storage_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
2,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
block_size,
|
||||
2 * HEAD_DIM,
|
||||
dtype=kv_cache_storage_dtype,
|
||||
device=device,
|
||||
)
|
||||
@@ -246,18 +245,21 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
|
||||
torch.testing.assert_close(index_k, ik_ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# ── Cache inserts. ──
|
||||
# Main cache layout is [num_blocks, 2, block_size, num_kv_heads, head_dim]
|
||||
# (the K/V axis sits *before* block_size); index cache is [nb, bs, head_dim].
|
||||
# Main cache layout is [num_blocks, num_kv_heads, block_size, 2*head_dim];
|
||||
# index cache is [num_blocks, block_size, head_dim].
|
||||
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
|
||||
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM) # v is raw (no norm/rope)
|
||||
if kv_cache_dtype == "fp8":
|
||||
expected_kv_cache = torch.zeros_like(kv_cache)
|
||||
expected_k_cache, expected_v_cache = expected_kv_cache.transpose(1, 2).split(
|
||||
HEAD_DIM, dim=-1
|
||||
)
|
||||
scale = torch.ones((), device=device)
|
||||
ops.reshape_and_cache_flash(
|
||||
k_out.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
v_out.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
expected_kv_cache[:, 0],
|
||||
expected_kv_cache[:, 1],
|
||||
expected_k_cache,
|
||||
expected_v_cache,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
scale,
|
||||
@@ -269,9 +271,11 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
|
||||
s = slot_mapping[t].item()
|
||||
b, pos = s // block_size, s % block_size
|
||||
torch.testing.assert_close(
|
||||
kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
|
||||
kv_cache[b, :, pos, :HEAD_DIM], k_ref_h[t], rtol=1e-2, atol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
kv_cache[b, :, pos, HEAD_DIM:], v_ref_h[t], rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
|
||||
|
||||
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
|
||||
expected_index_cache[index_slot_mapping] = index_k
|
||||
@@ -280,7 +284,121 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
|
||||
)
|
||||
|
||||
|
||||
# ── Test 3: fp8 (e4m3) index outputs ─────────────────────────────────────────
|
||||
@pytest.mark.parametrize("num_tokens", [1, 64, 513])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
|
||||
torch.manual_seed(2)
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
|
||||
base, max_pos = 5_000_000.0, 4096
|
||||
num_heads, num_kv_heads, num_idx_heads = 16, 4, 4
|
||||
|
||||
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
|
||||
iqsz, iksz = num_idx_heads * HEAD_DIM, HEAD_DIM
|
||||
qkv = torch.randn(
|
||||
num_tokens, qsz + 2 * kvsz + iqsz + iksz, dtype=dtype, device=device
|
||||
)
|
||||
qkv_orig = qkv.clone()
|
||||
splits = [qsz, kvsz, kvsz, iqsz, iksz]
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
kv_cache_storage_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
num_kv_heads,
|
||||
block_size,
|
||||
2 * HEAD_DIM,
|
||||
dtype=kv_cache_storage_dtype,
|
||||
device=device,
|
||||
)
|
||||
index_cache = torch.randn(
|
||||
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
index_cache_orig = index_cache.clone()
|
||||
slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
q_out = torch.empty(num_tokens, qsz, dtype=dtype, device=device)
|
||||
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
q_w,
|
||||
k_w,
|
||||
cos_sin,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
ROTARY_DIM,
|
||||
eps,
|
||||
num_index_heads=num_idx_heads,
|
||||
slot_mapping=slot_mapping,
|
||||
kv_cache=kv_cache,
|
||||
index_cache=index_cache,
|
||||
block_size=block_size,
|
||||
q_out=q_out,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
skip_index_branch=True,
|
||||
)
|
||||
|
||||
_, k_out, v_out, index_q_out, index_k_out = qkv.split(splits, dim=-1)
|
||||
q_in, k_in, v_in, index_q_in, index_k_in = qkv_orig.split(splits, dim=-1)
|
||||
q_ref = norm_rope_ref(
|
||||
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = norm_rope_ref(
|
||||
k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
k_w,
|
||||
positions,
|
||||
cos_sin,
|
||||
eps,
|
||||
).view(num_tokens, kvsz)
|
||||
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(index_q_out, index_q_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(index_k_out, index_k_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(index_cache, index_cache_orig, rtol=0, atol=0)
|
||||
|
||||
if kv_cache_dtype == "fp8":
|
||||
expected_kv_cache = torch.zeros_like(kv_cache)
|
||||
expected_k_cache, expected_v_cache = expected_kv_cache.transpose(1, 2).split(
|
||||
HEAD_DIM, dim=-1
|
||||
)
|
||||
scale = torch.ones((), device=device)
|
||||
ops.reshape_and_cache_flash(
|
||||
k_out.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
v_out.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
expected_k_cache,
|
||||
expected_v_cache,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
scale,
|
||||
scale,
|
||||
)
|
||||
torch.testing.assert_close(kv_cache, expected_kv_cache, rtol=0, atol=0)
|
||||
else:
|
||||
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
|
||||
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM)
|
||||
for t in range(num_tokens):
|
||||
s = slot_mapping[t].item()
|
||||
b, pos = s // block_size, s % block_size
|
||||
torch.testing.assert_close(
|
||||
kv_cache[b, :, pos, :HEAD_DIM], k_ref_h[t], rtol=1e-2, atol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
kv_cache[b, :, pos, HEAD_DIM:], v_ref_h[t], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
# ── Test 4: fp8 (e4m3) index outputs ─────────────────────────────────────────
|
||||
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
|
||||
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
|
||||
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
|
||||
@@ -324,10 +442,9 @@ def test_sparse_full_fp8_index(num_tokens, block_size):
|
||||
qkv = qkv0.clone()
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
2,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
HEAD_DIM,
|
||||
block_size,
|
||||
2 * HEAD_DIM,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not torch.cuda.is_available() or not current_platform.supports_fp8():
|
||||
pytest.skip(
|
||||
"MiniMax-M3 FP8 sparse attention scale tests require an FP8 GPU.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.minimax_m3.amd.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
else:
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
SPARSE_BLOCK_SIZE,
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
|
||||
|
||||
DEVICE = "cuda"
|
||||
DTYPE = torch.bfloat16
|
||||
HEAD_DIM = 128
|
||||
NUM_KV_HEADS = 1
|
||||
NUM_HEADS = 2
|
||||
K_SCALE = 0.25
|
||||
V_SCALE = 0.5
|
||||
|
||||
|
||||
def _scale_tensors(mode: str, num_blocks: int):
|
||||
if mode == "scalar":
|
||||
k_scale = torch.tensor(K_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
v_scale = torch.tensor(V_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
else:
|
||||
shape = (NUM_KV_HEADS, num_blocks * SPARSE_BLOCK_SIZE)
|
||||
k_scale = torch.full(shape, K_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
v_scale = torch.full(shape, V_SCALE, dtype=torch.float32, device=DEVICE)
|
||||
return k_scale, v_scale
|
||||
|
||||
|
||||
def _make_kv_cache(num_blocks: int, seed: int):
|
||||
torch.manual_seed(seed)
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
kv_ref = torch.randn(
|
||||
num_blocks,
|
||||
NUM_KV_HEADS,
|
||||
SPARSE_BLOCK_SIZE,
|
||||
2 * HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
device=DEVICE,
|
||||
)
|
||||
kv_fp8 = torch.empty_like(kv_ref, dtype=fp8_dtype)
|
||||
kv_fp8[..., :HEAD_DIM] = (kv_ref[..., :HEAD_DIM].float() / K_SCALE).to(fp8_dtype)
|
||||
kv_fp8[..., HEAD_DIM:] = (kv_ref[..., HEAD_DIM:].float() / V_SCALE).to(fp8_dtype)
|
||||
|
||||
kv_dequant = torch.empty_like(kv_ref)
|
||||
kv_dequant[..., :HEAD_DIM] = (kv_fp8[..., :HEAD_DIM].float() * K_SCALE).to(DTYPE)
|
||||
kv_dequant[..., HEAD_DIM:] = (kv_fp8[..., HEAD_DIM:].float() * V_SCALE).to(DTYPE)
|
||||
return kv_fp8, kv_dequant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
|
||||
@torch.inference_mode()
|
||||
def test_minimax_m3_sparse_prefill_fp8_kv_scales(scale_mode: str):
|
||||
total_q = 17
|
||||
num_blocks = 1
|
||||
torch.manual_seed(0)
|
||||
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
|
||||
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=1)
|
||||
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
|
||||
|
||||
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
|
||||
block_table = torch.zeros(1, 1, dtype=torch.int32, device=DEVICE)
|
||||
cu_seqlens = torch.tensor([0, total_q], dtype=torch.int32, device=DEVICE)
|
||||
seq_lens = torch.tensor([total_q], dtype=torch.int32, device=DEVICE)
|
||||
prefix_lens = torch.zeros(1, dtype=torch.int32, device=DEVICE)
|
||||
got = torch.empty_like(q)
|
||||
ref = torch.empty_like(q)
|
||||
unscaled = torch.empty_like(q)
|
||||
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
total_q,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
got,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_dequant,
|
||||
topk,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
total_q,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
ref,
|
||||
)
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
total_q,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
unscaled,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
|
||||
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale_mode", ["scalar", "per_token_head"])
|
||||
@torch.inference_mode()
|
||||
def test_minimax_m3_sparse_decode_fp8_kv_scales(scale_mode: str):
|
||||
total_q = 2
|
||||
num_blocks = 1
|
||||
torch.manual_seed(2)
|
||||
q = torch.randn(total_q, NUM_HEADS, HEAD_DIM, dtype=DTYPE, device=DEVICE) * 0.1
|
||||
kv_fp8, kv_dequant = _make_kv_cache(num_blocks, seed=3)
|
||||
k_scale, v_scale = _scale_tensors(scale_mode, num_blocks)
|
||||
|
||||
topk = torch.zeros(NUM_KV_HEADS, total_q, 1, dtype=torch.int32, device=DEVICE)
|
||||
block_table = torch.zeros(total_q, 1, dtype=torch.int32, device=DEVICE)
|
||||
seq_lens = torch.tensor([64, 128], dtype=torch.int32, device=DEVICE)
|
||||
got = torch.empty_like(q)
|
||||
ref = torch.empty_like(q)
|
||||
unscaled = torch.empty_like(q)
|
||||
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
got,
|
||||
decode_query_len=1,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_dequant,
|
||||
topk,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
ref,
|
||||
decode_query_len=1,
|
||||
)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_fp8,
|
||||
topk,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
HEAD_DIM**-0.5,
|
||||
unscaled,
|
||||
decode_query_len=1,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(got, ref, rtol=2e-2, atol=2e-2)
|
||||
assert not torch.allclose(unscaled, ref, rtol=1e-1, atol=1e-1)
|
||||
Reference in New Issue
Block a user