[AMD][Minimax-M3][perf] Enable packed LBHNC AITER QK-norm fusion for MiniMax-M3 on ROCm (#54535)

Signed-off-by: Wei-Ting Liao <wei-ting.liao@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
weitliao
2026-09-21 16:38:30 +08:00
committed by GitHub
co-authored by Cursor Shanshan Shen
parent f05b88751e
commit b8cf275382
3 changed files with 636 additions and 91 deletions
+361 -10
View File
@@ -19,7 +19,10 @@ from vllm.models.minimax_m3.common.ops.sparse_attn import (
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
from vllm.models.minimax_m3.common.sparse_attention import MiniMaxM3SparseTritonImpl
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseTritonImpl,
minimax_m3_rebase_slots_to_page16,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.utils import record_kv_cache_layout
@@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheLayout,
MLAAttentionSpec,
compute_layer_kv_cache_shape_bytes,
get_kv_quant_mode,
)
if not (current_platform.is_cuda() or current_platform.is_rocm()):
@@ -1672,8 +1676,9 @@ def test_main_cache_layout_contract():
assert set(order) == set(range(len(order)))
def test_aiter_sparse_pa_layout_contract(monkeypatch):
"""The shuffle-only AITER path retains separately contiguous K/V storage."""
@pytest.mark.parametrize("layout", [KVCacheLayout.LHBNC, KVCacheLayout.LBHNC])
def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
"""Build the exact separate and packed page-16 views used by the model."""
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True)
@@ -1685,21 +1690,43 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch):
backend = sparse_attn_mod.MiniMaxM3SparseBackend
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
nb, h = 7, 1
nb, h = 3, 1
cache_dtype = current_platform.fp8_dtype()
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=h,
head_size=HEAD_DIM,
dtype=DTYPE,
dtype=cache_dtype,
kv_quant_mode=get_kv_quant_mode("fp8"),
num_head_slots=2,
state_content_bytes=h * HEAD_DIM * DTYPE.itemsize,
state_content_bytes=h * HEAD_DIM * cache_dtype.itemsize,
)
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
view = dense_kv_cache_views(raw, spec, nb, 1, KVCacheLayout.LHBNC)[0]
key_cache, value_cache = view.unbind(1)
assert key_cache.is_contiguous()
assert value_cache.is_contiguous()
kv_cache = dense_kv_cache_views(raw, spec, nb, 1, layout)[0].view(cache_dtype)
key_cache, value_cache = kv_cache.unbind(1)
x = 16 // cache_dtype.itemsize
pages_per_side = BLOCK_SIZE // 16
if key_cache.is_contiguous() and value_cache.is_contiguous():
k_src, v_src = key_cache, value_cache
block_pages, v_page_offset = pages_per_side, 0
else:
assert kv_cache.is_contiguous()
k_src = v_src = kv_cache
block_pages, v_page_offset = 2 * pages_per_side, pages_per_side
num_phys16 = nb * block_pages
k_view = k_src.view(num_phys16, h, HEAD_DIM // x, 16, x)
v_view = v_src.view(num_phys16, h, 16 // x, HEAD_DIM, x)[v_page_offset:]
assert k_view.is_contiguous()
assert v_view.is_contiguous()
if layout is KVCacheLayout.LHBNC:
assert k_view.shape == (24, 1, 8, 16, 16)
assert v_view.shape == (24, 1, 1, 128, 16)
else:
assert k_view.shape == (48, 1, 8, 16, 16)
assert v_view.shape == (40, 1, 1, 128, 16)
def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch):
@@ -1769,6 +1796,330 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
assert reason is None or "AITER sparse PA attend" not in reason
def test_aiter_consolidated_qknorm_enabled(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
scale = torch.tensor([0.25])
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, scale)
assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e4m3", scale, scale)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8")
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, None)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", False)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: False)
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto")
def _fake_aiter_qknorm_args(kv_cache_dtype: str = "auto") -> dict:
tensor = torch.empty(1)
return {
"qkv": tensor,
"q_norm_weight": tensor,
"k_norm_weight": tensor,
"cos_sin_cache": tensor,
"positions": tensor,
"num_heads": 16,
"num_kv_heads": 1,
"rotary_dim": 64,
"eps": 1e-6,
"slot_mapping": tensor,
"kv_cache_k": tensor,
"kv_cache_v": tensor,
"q_out": tensor,
"kv_cache_dtype": kv_cache_dtype,
}
def _install_fake_aiter_qknorm(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
"""Stub ``aiter.fused_qknorm_idxrqknorm`` so mapping tests run without AITER."""
import sys
import types
from typing import Any
calls: list[dict] = []
def fake_op(*args, **kwargs):
calls.append(kwargs)
existing = sys.modules.get("aiter")
if existing is not None:
monkeypatch.setattr(existing, "fused_qknorm_idxrqknorm", fake_op, raising=False)
else:
fake_mod: Any = types.ModuleType("aiter")
fake_mod.fused_qknorm_idxrqknorm = fake_op
monkeypatch.setitem(sys.modules, "aiter", fake_mod)
return calls
def test_aiter_consolidated_qknorm_packed_shuffle_unequal_spans(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
call_args = _fake_aiter_qknorm_args()
call_args["kv_cache_k"] = torch.empty(2, 1)
call_args["kv_cache_v"] = torch.empty(1, 1)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
assert len(calls) == 1
@pytest.mark.parametrize(
("kv_cache_dtype", "aiter_dtype", "passes_scales"),
[
("auto", "auto", False),
("fp8", "fp8_e4m3_static", True),
("fp8_e4m3", "fp8_e4m3_static", True),
],
)
def test_aiter_consolidated_qknorm_dtype_and_layout_mapping(
monkeypatch, kv_cache_dtype, aiter_dtype, passes_scales
):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
call_args = _fake_aiter_qknorm_args(kv_cache_dtype)
k_scale = torch.tensor([0.25])
v_scale = torch.tensor([0.5])
call_args.update(k_scale=k_scale, v_scale=v_scale)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
assert len(calls) == 1
kwargs = calls[0]
assert kwargs["kv_cache_dtype"] == aiter_dtype
assert kwargs["index_cache_dtype"] == "auto"
assert kwargs["block_size"] == 16
assert kwargs["asm_layout"] is True
if passes_scales:
assert kwargs["k_scale"] is k_scale
assert kwargs["v_scale"] is v_scale
else:
assert kwargs["k_scale"] is None
assert kwargs["v_scale"] is None
def test_aiter_consolidated_qknorm_fp8_index_dtype_mapping(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
fp8 = current_platform.fp8_dtype()
call_args = _fake_aiter_qknorm_args("fp8")
call_args.update(
k_scale=torch.tensor([0.25]),
v_scale=torch.tensor([0.5]),
index_cache=torch.empty(1, dtype=fp8),
index_q_out=torch.empty(1, dtype=fp8),
num_index_heads=1,
)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
assert calls[0]["index_cache_dtype"] == "fp8"
assert calls[0]["kv_cache_dtype"] == "fp8_e4m3_static"
@pytest.mark.parametrize("skip_index_branch", [False, True])
def test_aiter_consolidated_qknorm_full_and_skip_index_args(
monkeypatch, skip_index_branch
):
from vllm._aiter_ops import rocm_aiter_ops
calls = _install_fake_aiter_qknorm(monkeypatch)
call_args = _fake_aiter_qknorm_args()
if skip_index_branch:
call_args.update(skip_index_branch=True)
else:
tensor = torch.empty(1)
call_args.update(
index_q_norm_weight=tensor,
index_k_norm_weight=tensor,
num_index_heads=1,
index_cache=tensor,
index_q_out=tensor,
index_slot_mapping=tensor,
)
rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args)
kwargs = calls[0]
assert kwargs["skip_index_branch"] is skip_index_branch
if skip_index_branch:
assert kwargs["index_q_norm_weight"] is None
assert kwargs["index_k_norm_weight"] is None
assert kwargs["index_cache"] is None
assert kwargs["index_q_out"] is None
assert kwargs["index_slot_mapping"] is None
else:
assert kwargs["index_q_norm_weight"] is not None
assert kwargs["index_k_norm_weight"] is not None
assert kwargs["index_cache"] is not None
assert kwargs["index_q_out"] is not None
assert kwargs["index_slot_mapping"] is not None
def test_aiter_consolidated_qknorm_rejects_fp8_e5m2(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True)
scale = torch.tensor([0.25])
assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale)
def test_aiter_sparse_pa_slot_rebase_preserves_page_offsets():
slots = torch.tensor([0, 15, 16, 127, 128, 129, -1])
actual = minimax_m3_rebase_slots_to_page16(slots, block_size=128)
expected = torch.tensor([0, 15, 16, 127, 256, 257, -1])
assert torch.equal(actual, expected)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="The consolidated packed-SHUFFLE AITER op is ROCm-only",
)
@pytest.mark.parametrize("skip_index_branch", [False, True])
@pytest.mark.parametrize("fp8_index", [False, True])
def test_aiter_consolidated_qknorm_real_packed_shuffle(skip_index_branch, fp8_index):
from vllm._aiter_ops import rocm_aiter_ops
torch.manual_seed(20260831)
num_tokens = 4
num_heads = 16
num_kv_heads = 1
num_index_heads = 1
rotary_dim = 64
head_dim = 128
logical_block_size = 128
num_blocks = 3
fp8_dtype = current_platform.fp8_dtype()
qkv = torch.randn(
num_tokens,
(num_heads + 2 * num_kv_heads + num_index_heads + 1) * head_dim,
dtype=torch.bfloat16,
device="cuda",
)
weights = [
torch.randn(head_dim, dtype=torch.bfloat16, device="cuda") for _ in range(4)
]
cos_sin_cache = torch.randn(16, rotary_dim, dtype=torch.bfloat16, device="cuda")
positions = torch.arange(num_tokens, dtype=torch.int64, device="cuda")
logical_slots = torch.tensor([0, 127, 128, -1], dtype=torch.int64, device="cuda")
slot_mapping = minimax_m3_rebase_slots_to_page16(logical_slots, logical_block_size)
index_slot_mapping = logical_slots.clone()
packed = torch.zeros(
num_blocks,
2,
logical_block_size,
head_dim,
dtype=fp8_dtype,
device="cuda",
)
x = 16 // packed.element_size()
pages_per_side = logical_block_size // 16
num_phys16 = num_blocks * 2 * pages_per_side
kv_cache_k = packed.view(num_phys16, num_kv_heads, head_dim // x, 16, x)
kv_cache_v = packed.view(num_phys16, num_kv_heads, 16 // x, head_dim, x)[
pages_per_side:
]
assert kv_cache_k.shape == (48, 1, 8, 16, 16)
assert kv_cache_v.shape == (40, 1, 1, 128, 16)
q_out = torch.zeros(
num_tokens, num_heads * head_dim, dtype=torch.bfloat16, device="cuda"
)
index_dtype = fp8_dtype if fp8_index else torch.bfloat16
if fp8_index:
index_q_out = torch.empty(
(num_tokens, num_index_heads * head_dim),
dtype=index_dtype,
device="cuda",
)
index_cache = torch.empty(
(num_blocks, logical_block_size, head_dim),
dtype=index_dtype,
device="cuda",
)
index_q_out.view(torch.uint8).fill_(7)
index_cache.view(torch.uint8).fill_(7)
else:
index_q_out = torch.full(
(num_tokens, num_index_heads * head_dim),
7,
dtype=torch.bfloat16,
device="cuda",
)
index_cache = torch.full(
(num_blocks, logical_block_size, head_dim),
7,
dtype=torch.bfloat16,
device="cuda",
)
scale = torch.tensor([0.02], dtype=torch.float32, device="cuda")
kwargs = {}
if not skip_index_branch:
kwargs = {
"index_q_norm_weight": weights[2],
"index_k_norm_weight": weights[3],
"index_cache": index_cache,
"index_q_out": index_q_out,
"index_slot_mapping": index_slot_mapping,
}
rocm_aiter_ops.fused_qknorm_idxrqknorm(
qkv,
weights[0],
weights[1],
cos_sin_cache,
positions,
num_heads,
num_kv_heads,
rotary_dim,
1e-6,
slot_mapping,
kv_cache_k,
kv_cache_v,
q_out,
"fp8",
scale,
scale,
num_index_heads=num_index_heads,
skip_index_branch=skip_index_branch,
**kwargs,
)
torch.accelerator.synchronize()
assert torch.isfinite(q_out).all()
assert torch.count_nonzero(q_out) > 0
assert torch.count_nonzero(packed) > 0
if fp8_index:
index_q_untouched = torch.all(index_q_out.view(torch.uint8) == 7)
index_cache_untouched = torch.all(index_cache.view(torch.uint8) == 7)
index_cache_t0 = torch.all(index_cache[0, 0].view(torch.uint8) == 7)
index_cache_t127 = torch.all(index_cache[0, 127].view(torch.uint8) == 7)
index_cache_t128 = torch.all(index_cache[1, 0].view(torch.uint8) == 7)
else:
index_q_untouched = torch.all(index_q_out == 7)
index_cache_untouched = torch.all(index_cache == 7)
index_cache_t0 = torch.all(index_cache[0, 0] == 7)
index_cache_t127 = torch.all(index_cache[0, 127] == 7)
index_cache_t128 = torch.all(index_cache[1, 0] == 7)
if skip_index_branch:
assert index_q_untouched
assert index_cache_untouched
else:
assert not index_q_untouched
assert not index_cache_t0
assert not index_cache_t127
assert not index_cache_t128
def test_indexer_cache_squeezes_to_contiguous_3d():
"""The indexer side cache is standardized 4D with H=1: under both layouts
the allocator's logical view stays contiguous and squeezes (as
+112
View File
@@ -35,10 +35,22 @@ except ImportError:
# which is a host op, so we cache it once here.
FP8_DTYPE = current_platform.fp8_dtype()
_HIPB_MM_INITIALIZED_DEVICES: set[int] = set()
_FP8_E4M3_DTYPES = {
dtype
for dtype in (
getattr(torch, "float8_e4m3fn", None),
getattr(torch, "float8_e4m3fnuz", None),
)
if dtype is not None
}
KB = 1024
MB = 1024 * KB
def _is_fp8_e4m3_tensor(t: torch.Tensor | None) -> bool:
return t is not None and t.dtype in _FP8_E4M3_DTYPES
def _get_or_create_aiter_qr_rmsnorm_comm(
device_comm: Any,
) -> tuple[int, int, bool] | None:
@@ -2200,6 +2212,106 @@ class rocm_aiter_ops:
def is_shuffle_kv_cache_enabled(cls) -> bool:
return cls._SHUFFLE_KV_CACHE_ENABLED
@classmethod
@if_aiter_supported
def fused_qknorm_idxrqknorm_enabled(
cls,
kv_cache_dtype: str,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
) -> bool:
"""Whether MiniMax-M3 can use AITER's consolidated fused QK-norm.
Requires AITER to be installed and enabled. bf16 (``auto``) always
qualifies; fp8 e4m3 also needs K/V scales. Other cache dtypes fall
back to vLLM's fused kernel.
"""
if not cls._AITER_ENABLED:
return False
if kv_cache_dtype == "auto":
return True
if kv_cache_dtype in ("fp8", "fp8_e4m3"):
return k_scale is not None and v_scale is not None
return False
@classmethod
def fused_qknorm_idxrqknorm(
cls,
qkv: torch.Tensor,
q_norm_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads: int,
num_kv_heads: int,
rotary_dim: int,
eps: float,
slot_mapping: torch.Tensor,
kv_cache_k: torch.Tensor,
kv_cache_v: torch.Tensor,
q_out: torch.Tensor,
kv_cache_dtype: str,
k_scale: torch.Tensor | None = None,
v_scale: torch.Tensor | None = None,
index_q_norm_weight: torch.Tensor | None = None,
index_k_norm_weight: torch.Tensor | None = None,
num_index_heads: int = 0,
index_cache: torch.Tensor | None = None,
index_q_out: torch.Tensor | None = None,
index_slot_mapping: torch.Tensor | None = None,
skip_index_branch: bool = False,
) -> None:
"""Run consolidated MiniMax-M3 QK-norm fusion.
Callers must check ``fused_qknorm_idxrqknorm_enabled`` first. Runtime
failures from the AITER op are deliberately propagated.
"""
if kv_cache_dtype == "auto":
aiter_kv_cache_dtype = "auto"
aiter_k_scale = None
aiter_v_scale = None
elif kv_cache_dtype in ("fp8", "fp8_e4m3"):
aiter_kv_cache_dtype = "fp8_e4m3_static"
aiter_k_scale = k_scale
aiter_v_scale = v_scale
else:
raise ValueError(
"AITER fused QK-norm requires kv_cache_dtype 'auto', 'fp8', "
f"or 'fp8_e4m3', got {kv_cache_dtype!r}"
)
aiter_index_cache_dtype = "fp8" if _is_fp8_e4m3_tensor(index_cache) else "auto"
from aiter import fused_qknorm_idxrqknorm
fused_qknorm_idxrqknorm(
qkv,
q_norm_weight,
k_norm_weight,
cos_sin_cache,
positions,
num_heads,
num_kv_heads,
rotary_dim,
eps,
index_q_norm_weight=index_q_norm_weight,
index_k_norm_weight=index_k_norm_weight,
num_index_heads=num_index_heads,
slot_mapping=slot_mapping,
kv_cache_k=kv_cache_k,
kv_cache_v=kv_cache_v,
index_cache=index_cache,
block_size=16,
q_out=q_out,
index_q_out=index_q_out,
index_slot_mapping=index_slot_mapping,
kv_cache_dtype=aiter_kv_cache_dtype,
index_cache_dtype=aiter_index_cache_dtype,
k_scale=aiter_k_scale,
v_scale=aiter_v_scale,
asm_layout=True,
skip_index_branch=skip_index_branch,
)
@classmethod
@if_aiter_supported
def is_triton_unified_attn_enabled(cls) -> bool:
+163 -81
View File
@@ -890,6 +890,30 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
self._ensure_aiter_sparse_pa_kv_cache()
return self.kv_cache_k, self.kv_cache_v
def _get_aiter_sparse_pa_slot_mapping(
self,
slot_mapping: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
) -> torch.Tensor:
if value_cache.shape[0] == key_cache.shape[0]:
return slot_mapping
attn_metadata = get_forward_context().attn_metadata
page16_slot_mapping = None
if isinstance(attn_metadata, dict):
main_md = attn_metadata[self.layer_name]
assert isinstance(main_md, MiniMaxM3SparseMetadata)
page16_slot_mapping = main_md.page16_slot_mapping
if (
page16_slot_mapping is None
or page16_slot_mapping.shape != slot_mapping.shape
):
page16_slot_mapping = minimax_m3_rebase_slots_to_page16(
slot_mapping, self.kv_cache.shape[2]
)
return page16_slot_mapping
def _insert_aiter_sparse_pa_kv(
self,
k: torch.Tensor,
@@ -897,6 +921,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
index_k: torch.Tensor | None,
slot_mapping: torch.Tensor,
index_slot_mapping: torch.Tensor | None,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
) -> None:
if self.kv_cache.numel() == 0:
return
@@ -906,22 +932,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
minimax_m3_insert_index_cache,
)
key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache()
if value_cache.shape[0] != key_cache.shape[0]:
attn_metadata = get_forward_context().attn_metadata
page16_slot_mapping = None
if isinstance(attn_metadata, dict):
main_md = attn_metadata[self.layer_name]
assert isinstance(main_md, MiniMaxM3SparseMetadata)
page16_slot_mapping = main_md.page16_slot_mapping
if (
page16_slot_mapping is None
or page16_slot_mapping.shape != slot_mapping.shape
):
page16_slot_mapping = minimax_m3_rebase_slots_to_page16(
slot_mapping, self.kv_cache.shape[2]
)
slot_mapping = page16_slot_mapping
kv_cache_dtype = (
self.kv_cache_dtype
if is_quantized_kv_cache(self.kv_cache_dtype)
@@ -976,38 +986,78 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
main_slot_mapping = fwd_slot_mapping[self.layer_name]
q = qkv.new_empty((num_tokens, self.q_size))
key_cache = value_cache = page16_slot_mapping = None
k_scale = v_scale = None
use_fused_qknorm = False
if self.use_aiter_sparse_pa:
key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache()
page16_slot_mapping = self._get_aiter_sparse_pa_slot_mapping(
main_slot_mapping, key_cache, value_cache
)
k_scale = getattr(self, "_k_scale", None)
v_scale = getattr(self, "_v_scale", None)
use_fused_qknorm = rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled(
self.kv_cache_dtype, k_scale, v_scale
)
if self.skip_index_topk:
index_q = None
if self.use_aiter_sparse_pa:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
num_index_heads=self.num_idx_heads,
q_out=q,
skip_index_branch=True,
)
k_start = self.q_size
v_start = k_start + self.kv_size
k = qkv[:, k_start:v_start].view(
num_tokens, self.num_kv_heads, self.head_dim
)
v = qkv[:, v_start : v_start + self.kv_size].view(
num_tokens, self.num_kv_heads, self.head_dim
)
self._insert_aiter_sparse_pa_kv(
k,
v,
None,
main_slot_mapping,
None,
)
assert key_cache is not None
assert value_cache is not None
assert page16_slot_mapping is not None
if use_fused_qknorm:
rocm_aiter_ops.fused_qknorm_idxrqknorm(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
page16_slot_mapping,
key_cache,
value_cache,
q,
self.kv_cache_dtype,
k_scale,
v_scale,
num_index_heads=self.num_idx_heads,
skip_index_branch=True,
)
else:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
num_index_heads=self.num_idx_heads,
q_out=q,
skip_index_branch=True,
)
k_start = self.q_size
v_start = k_start + self.kv_size
k = qkv[:, k_start:v_start].view(
num_tokens, self.num_kv_heads, self.head_dim
)
v = qkv[:, v_start : v_start + self.kv_size].view(
num_tokens, self.num_kv_heads, self.head_dim
)
self._insert_aiter_sparse_pa_kv(
k,
v,
None,
page16_slot_mapping,
None,
key_cache,
value_cache,
)
else:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
@@ -1037,42 +1087,74 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
dtype=self.indexer.index_cache.dtype,
)
if self.use_aiter_sparse_pa:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
q_out=q,
index_q_out=index_q,
kv_cache_dtype=self.kv_cache_dtype,
)
k_start = self.q_size
v_start = k_start + self.kv_size
index_k_start = v_start + self.kv_size + self.index_q_size
k = qkv[:, k_start:v_start].view(
num_tokens, self.num_kv_heads, self.head_dim
)
v = qkv[:, v_start : v_start + self.kv_size].view(
num_tokens, self.num_kv_heads, self.head_dim
)
index_k = qkv[
:, index_k_start : index_k_start + self.idx_head_dim
].view(num_tokens, self.idx_head_dim)
self._insert_aiter_sparse_pa_kv(
k,
v,
index_k,
main_slot_mapping,
index_slot_mapping,
)
assert key_cache is not None
assert value_cache is not None
assert page16_slot_mapping is not None
index_cache = self.indexer.index_cache.kv_cache
if use_fused_qknorm:
rocm_aiter_ops.fused_qknorm_idxrqknorm(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
page16_slot_mapping,
key_cache,
value_cache,
q,
self.kv_cache_dtype,
k_scale,
v_scale,
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
index_cache,
index_q,
index_slot_mapping,
)
else:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
self.q_norm.weight,
self.k_norm.weight,
cos_sin_cache,
positions,
self.num_heads,
self.num_kv_heads,
rotary_dim,
eps,
self.index_q_norm.weight,
self.index_k_norm.weight,
self.num_idx_heads,
q_out=q,
index_q_out=index_q,
kv_cache_dtype=self.kv_cache_dtype,
)
k_start = self.q_size
v_start = k_start + self.kv_size
index_k_start = v_start + self.kv_size + self.index_q_size
k = qkv[:, k_start:v_start].view(
num_tokens, self.num_kv_heads, self.head_dim
)
v = qkv[:, v_start : v_start + self.kv_size].view(
num_tokens, self.num_kv_heads, self.head_dim
)
index_k = qkv[
:, index_k_start : index_k_start + self.idx_head_dim
].view(num_tokens, self.idx_head_dim)
self._insert_aiter_sparse_pa_kv(
k,
v,
index_k,
page16_slot_mapping,
index_slot_mapping,
key_cache,
value_cache,
)
else:
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,