[AMD][Minimax-M3][perf] Enable packed LBHNC AITER QK-norm fusion for MiniMax-M3 on ROCm (#54535)

Signed-off-by: Wei-Ting Liao <wei-ting.liao@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
weitliao
2026-09-21 16:38:30 +08:00
committed by GitHub
co-authored by Cursor Shanshan Shen
parent f05b88751e
commit b8cf275382
3 changed files with 636 additions and 91 deletions
+361 -10
View File
@@ -19,7 +19,10 @@ from vllm.models.minimax_m3.common.ops.sparse_attn import (
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
from vllm.models.minimax_m3.common.sparse_attention import MiniMaxM3SparseTritonImpl
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseTritonImpl,
minimax_m3_rebase_slots_to_page16,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.utils import record_kv_cache_layout
@@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheLayout,
MLAAttentionSpec,
compute_layer_kv_cache_shape_bytes,
get_kv_quant_mode,
)
if not (current_platform.is_cuda() or current_platform.is_rocm()):
@@ -1672,8 +1676,9 @@ def test_main_cache_layout_contract():
assert set(order) == set(range(len(order)))
def test_aiter_sparse_pa_layout_contract(monkeypatch):
"""The shuffle-only AITER path retains separately contiguous K/V storage."""
@pytest.mark.parametrize("layout", [KVCacheLayout.LHBNC, KVCacheLayout.LBHNC])
def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
"""Build the exact separate and packed page-16 views used by the model."""
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
@@ -1685,21 +1690,43 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch):
backend = sparse_attn_mod.MiniMaxM3SparseBackend
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
nb, h = 7, 1
nb, h = 3, 1
cache_dtype = current_platform.fp8_dtype()
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=h,
head_size=HEAD_DIM,
dtype=DTYPE,
dtype=cache_dtype,
kv_quant_mode=get_kv_quant_mode("fp8"),
num_head_slots=2,
state_content_bytes=h * HEAD_DIM * DTYPE.itemsize,
state_content_bytes=h * HEAD_DIM * cache_dtype.itemsize,
)
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
view = dense_kv_cache_views(raw, spec, nb, 1, KVCacheLayout.LHBNC)[0]
key_cache, value_cache = view.unbind(1)
assert key_cache.is_contiguous()
assert value_cache.is_contiguous()
kv_cache = dense_kv_cache_views(raw, spec, nb, 1, layout)[0].view(cache_dtype)
key_cache, value_cache = kv_cache.unbind(1)
x = 16 // cache_dtype.itemsize
pages_per_side = BLOCK_SIZE // 16
if key_cache.is_contiguous() and value_cache.is_contiguous():
k_src, v_src = key_cache, value_cache
block_pages, v_page_offset = pages_per_side, 0
else:
assert kv_cache.is_contiguous()
k_src = v_src = kv_cache
block_pages, v_page_offset = 2 * pages_per_side, pages_per_side
num_phys16 = nb * block_pages
k_view = k_src.view(num_phys16, h, HEAD_DIM // x, 16, x)
v_view = v_src.view(num_phys16, h, 16 // x, HEAD_DIM, x)[v_page_offset:]
assert k_view.is_contiguous()
assert v_view.is_contiguous()
if layout is KVCacheLayout.LHBNC:
assert k_view.shape == (24, 1, 8, 16, 16)
assert v_view.shape == (24, 1, 1, 128, 16)
else:
assert k_view.shape == (48, 1, 8, 16, 16)
assert v_view.shape == (40, 1, 1, 128, 16)
def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch):
@@ -1769,6 +1796,330 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
assert reason is None or "AITER sparse PA attend" not in reason
def test_aiter_consolidated_qknorm_enabled(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
scale = torch.tensor([0.25])
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, scale)
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e4m3", scale, scale)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8")
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, None)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", False)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: False)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
def _fake_aiter_qknorm_args(kv_cache_dtype: str = "auto") -> dict:
tensor = torch.empty(1)
return {
"qkv": tensor,
"q_norm_weight": tensor,
"k_norm_weight": tensor,
"cos_sin_cache": tensor,
"positions": tensor,
"num_heads": 16,
"num_kv_heads": 1,
"rotary_dim": 64,
"eps": 1e-6,
"slot_mapping": tensor,
"kv_cache_k": tensor,
"kv_cache_v": tensor,
"q_out": tensor,
"kv_cache_dtype": kv_cache_dtype,
}
def _install_fake_aiter_qknorm(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
"""Stub ``aiter.fused_qknorm_idxrqknorm`` so mapping tests run without AITER."""
import sys
import types
from typing import Any
calls: list[dict] = []
def fake_op(*args, **kwargs):
calls.append(kwargs)
existing = sys.modules.get("aiter")
if existing is not None:
monkeypatch.setattr(existing, "fused_qknorm_idxrqknorm", fake_op, raising=False)
else:
fake_mod: Any = types.ModuleType("aiter")
fake_mod.fused_qknorm_idxrqknorm = fake_op
monkeypatch.setitem(sys.modules, "aiter", fake_mod)
return calls
def test_aiter_consolidated_qknorm_packed_shuffle_unequal_spans(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
call_args = _fake_aiter_qknorm_args()
call_args["kv_cache_k"] = torch.empty(2, 1)
call_args["kv_cache_v"] = torch.empty(1, 1)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
assert len(calls) == 1
@pytest.mark.parametrize(
("kv_cache_dtype", "aiter_dtype", "passes_scales"),
[
("auto", "auto", False),
("fp8", "fp8_e4m3_static", True),
("fp8_e4m3", "fp8_e4m3_static", True),
],
)
def test_aiter_consolidated_qknorm_dtype_and_layout_mapping(
monkeypatch, kv_cache_dtype, aiter_dtype, passes_scales
):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
call_args = _fake_aiter_qknorm_args(kv_cache_dtype)
k_scale = torch.tensor([0.25])
v_scale = torch.tensor([0.5])
call_args.update(k_scale=k_scale, v_scale=v_scale)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
assert len(calls) == 1
kwargs = calls[0]
assert kwargs["kv_cache_dtype"] == aiter_dtype
assert kwargs["index_cache_dtype"] == "auto"
assert kwargs["block_size"] == 16
assert kwargs["asm_layout"] is True
if passes_scales:
assert kwargs["k_scale"] is k_scale
assert kwargs["v_scale"] is v_scale
else:
assert kwargs["k_scale"] is None
assert kwargs["v_scale"] is None
def test_aiter_consolidated_qknorm_fp8_index_dtype_mapping(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
fp8 = current_platform.fp8_dtype()
call_args = _fake_aiter_qknorm_args("fp8")
call_args.update(
k_scale=torch.tensor([0.25]),
v_scale=torch.tensor([0.5]),
index_cache=torch.empty(1, dtype=fp8),
index_q_out=torch.empty(1, dtype=fp8),
num_index_heads=1,
)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
assert calls[0]["index_cache_dtype"] == "fp8"
assert calls[0]["kv_cache_dtype"] == "fp8_e4m3_static"
@pytest.mark.parametrize("skip_index_branch", [False, True])
def test_aiter_consolidated_qknorm_full_and_skip_index_args(
monkeypatch, skip_index_branch
):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
call_args = _fake_aiter_qknorm_args()
if skip_index_branch:
call_args.update(skip_index_branch=True)
else:
tensor = torch.empty(1)
call_args.update(
index_q_norm_weight=tensor,
index_k_norm_weight=tensor,
num_index_heads=1,
index_cache=tensor,
index_q_out=tensor,
index_slot_mapping=tensor,
)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
kwargs = calls[0]
assert kwargs["skip_index_branch"] is skip_index_branch
if skip_index_branch:
assert kwargs["index_q_norm_weight"] is None
assert kwargs["index_k_norm_weight"] is None
assert kwargs["index_cache"] is None
assert kwargs["index_q_out"] is None
assert kwargs["index_slot_mapping"] is None
else:
assert kwargs["index_q_norm_weight"] is not None
assert kwargs["index_k_norm_weight"] is not None
assert kwargs["index_cache"] is not None
assert kwargs["index_q_out"] is not None
assert kwargs["index_slot_mapping"] is not None
def test_aiter_consolidated_qknorm_rejects_fp8_e5m2(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
scale = torch.tensor([0.25])
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale)
def test_aiter_sparse_pa_slot_rebase_preserves_page_offsets():
slots = torch.tensor([0, 15, 16, 127, 128, 129, -1])
actual = minimax_m3_rebase_slots_to_page16(slots, block_size=128)
expected = torch.tensor([0, 15, 16, 127, 256, 257, -1])
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="The consolidated packed-SHUFFLE AITER op is ROCm-only",
)
@pytest.mark.parametrize("skip_index_branch", [False, True])
@pytest.mark.parametrize("fp8_index", [False, True])
def test_aiter_consolidated_qknorm_real_packed_shuffle(skip_index_branch, fp8_index):
from vllm._aiter_ops import rocm_aiter_ops
torch.manual_seed(20260831)
num_tokens = 4
num_heads = 16
num_kv_heads = 1
num_index_heads = 1
rotary_dim = 64
head_dim = 128
logical_block_size = 128
num_blocks = 3
fp8_dtype = current_platform.fp8_dtype()
qkv = torch.randn(
num_tokens,
(num_heads + 2 * num_kv_heads + num_index_heads + 1) * head_dim,
dtype=torch.bfloat16,
device="cuda",
)
weights = [
torch.randn(head_dim, dtype=torch.bfloat16, device="cuda") for _ in range(4)
]
cos_sin_cache = torch.randn(16, rotary_dim, dtype=torch.bfloat16, device="cuda")
positions = torch.arange(num_tokens, dtype=torch.int64, device="cuda")
logical_slots = torch.tensor([0, 127, 128, -1], dtype=torch.int64, device="cuda")
slot_mapping = minimax_m3_rebase_slots_to_page16(logical_slots, logical_block_size)
index_slot_mapping = logical_slots.clone()
packed = torch.zeros(
num_blocks,
2,
logical_block_size,
head_dim,
dtype=fp8_dtype,
device="cuda",
)
x = 16 // packed.element_size()
pages_per_side = logical_block_size // 16
num_phys16 = num_blocks * 2 * pages_per_side
kv_cache_k = packed.view(num_phys16, num_kv_heads, head_dim // x, 16, x)
kv_cache_v = packed.view(num_phys16, num_kv_heads, 16 // x, head_dim, x)[
pages_per_side:
]
assert kv_cache_k.shape == (48, 1, 8, 16, 16)
assert kv_cache_v.shape == (40, 1, 1, 128, 16)
q_out = torch.zeros(
num_tokens, num_heads * head_dim, dtype=torch.bfloat16, device="cuda"
)
index_dtype = fp8_dtype if fp8_index else torch.bfloat16
if fp8_index:
index_q_out = torch.empty(
(num_tokens, num_index_heads * head_dim),
dtype=index_dtype,
device="cuda",
)
index_cache = torch.empty(
(num_blocks, logical_block_size, head_dim),
dtype=index_dtype,
device="cuda",
)
index_q_out.view(torch.uint8).fill_(7)
index_cache.view(torch.uint8).fill_(7)
else:
index_q_out = torch.full(
(num_tokens, num_index_heads * head_dim),
7,
dtype=torch.bfloat16,
device="cuda",
)
index_cache = torch.full(
(num_blocks, logical_block_size, head_dim),
7,
dtype=torch.bfloat16,
device="cuda",
)
scale = torch.tensor([0.02], dtype=torch.float32, device="cuda")
kwargs = {}
if not skip_index_branch:
kwargs = {
"index_q_norm_weight": weights[2],
"index_k_norm_weight": weights[3],
"index_cache": index_cache,
"index_q_out": index_q_out,
"index_slot_mapping": index_slot_mapping,
}
rocm_aiter_ops.fused_qknorm_idxrqknorm(
qkv,
weights[0],
weights[1],
cos_sin_cache,
positions,
num_heads,
num_kv_heads,
rotary_dim,
1e-6,
slot_mapping,
kv_cache_k,
kv_cache_v,
q_out,
"fp8",
scale,
scale,
num_index_heads=num_index_heads,
skip_index_branch=skip_index_branch,
**kwargs,
)
torch.accelerator.synchronize()
assert torch.isfinite(q_out).all()
assert torch.count_nonzero(q_out) > 0
assert torch.count_nonzero(packed) > 0
if fp8_index:
index_q_untouched = torch.all(index_q_out.view(torch.uint8) == 7)
index_cache_untouched = torch.all(index_cache.view(torch.uint8) == 7)
index_cache_t0 = torch.all(index_cache[0, 0].view(torch.uint8) == 7)
index_cache_t127 = torch.all(index_cache[0, 127].view(torch.uint8) == 7)
index_cache_t128 = torch.all(index_cache[1, 0].view(torch.uint8) == 7)
else:
index_q_untouched = torch.all(index_q_out == 7)
index_cache_untouched = torch.all(index_cache == 7)
index_cache_t0 = torch.all(index_cache[0, 0] == 7)
index_cache_t127 = torch.all(index_cache[0, 127] == 7)
index_cache_t128 = torch.all(index_cache[1, 0] == 7)
if skip_index_branch:
assert index_q_untouched
assert index_cache_untouched
else:
assert not index_q_untouched
assert not index_cache_t0
assert not index_cache_t127
assert not index_cache_t128
def test_indexer_cache_squeezes_to_contiguous_3d():
"""The indexer side cache is standardized 4D with H=1: under both layouts
the allocator's logical view stays contiguous and squeezes (as