From b8cf2753825d05d7f41eb4747480b78934953aac Mon Sep 17 00:00:00 2001 From: weitliao Date: Mon, 21 Sep 2026 01:38:30 -0700 Subject: [PATCH] [AMD][Minimax-M3][perf] Enable packed LBHNC AITER QK-norm fusion for MiniMax-M3 on ROCm (#54535) Signed-off-by: Wei-Ting Liao Co-authored-by: Cursor Co-authored-by: Shanshan Shen <467638484@qq.com> --- tests/kernels/attention/test_minimax_m3.py | 371 ++++++++++++++++++++- vllm/_aiter_ops.py | 112 +++++++ vllm/models/minimax_m3/amd/model.py | 244 +++++++++----- 3 files changed, 636 insertions(+), 91 deletions(-) diff --git a/tests/kernels/attention/test_minimax_m3.py b/tests/kernels/attention/test_minimax_m3.py index a7e6d499c1..9d3b1500e1 100644 --- a/tests/kernels/attention/test_minimax_m3.py +++ b/tests/kernels/attention/test_minimax_m3.py @@ -19,7 +19,10 @@ from vllm.models.minimax_m3.common.ops.sparse_attn import ( minimax_m3_sparse_attn, minimax_m3_sparse_attn_decode, ) -from vllm.models.minimax_m3.common.sparse_attention import MiniMaxM3SparseTritonImpl +from vllm.models.minimax_m3.common.sparse_attention import ( + MiniMaxM3SparseTritonImpl, + minimax_m3_rebase_slots_to_page16, +) from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.utils import record_kv_cache_layout @@ -28,6 +31,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheLayout, MLAAttentionSpec, compute_layer_kv_cache_shape_bytes, + get_kv_quant_mode, ) if not (current_platform.is_cuda() or current_platform.is_rocm()): @@ -1672,8 +1676,9 @@ def test_main_cache_layout_contract(): assert set(order) == set(range(len(order))) -def test_aiter_sparse_pa_layout_contract(monkeypatch): - """The shuffle-only AITER path retains separately contiguous K/V storage.""" +@pytest.mark.parametrize("layout", [KVCacheLayout.LHBNC, KVCacheLayout.LBHNC]) +def test_aiter_sparse_pa_layout_contract(monkeypatch, layout): + """Build the exact separate and packed page-16 views used by the model.""" import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod monkeypatch.setattr(sparse_attn_mod.rocm_aiter_ops, "is_enabled", lambda: True) @@ -1685,21 +1690,43 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch): backend = sparse_attn_mod.MiniMaxM3SparseBackend assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts() + assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts() - nb, h = 7, 1 + nb, h = 3, 1 + cache_dtype = current_platform.fp8_dtype() spec = FullAttentionSpec( block_size=BLOCK_SIZE, num_kv_heads=h, head_size=HEAD_DIM, - dtype=DTYPE, + dtype=cache_dtype, + kv_quant_mode=get_kv_quant_mode("fp8"), num_head_slots=2, - state_content_bytes=h * HEAD_DIM * DTYPE.itemsize, + state_content_bytes=h * HEAD_DIM * cache_dtype.itemsize, ) raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8) - view = dense_kv_cache_views(raw, spec, nb, 1, KVCacheLayout.LHBNC)[0] - key_cache, value_cache = view.unbind(1) - assert key_cache.is_contiguous() - assert value_cache.is_contiguous() + kv_cache = dense_kv_cache_views(raw, spec, nb, 1, layout)[0].view(cache_dtype) + key_cache, value_cache = kv_cache.unbind(1) + x = 16 // cache_dtype.itemsize + pages_per_side = BLOCK_SIZE // 16 + if key_cache.is_contiguous() and value_cache.is_contiguous(): + k_src, v_src = key_cache, value_cache + block_pages, v_page_offset = pages_per_side, 0 + else: + assert kv_cache.is_contiguous() + k_src = v_src = kv_cache + block_pages, v_page_offset = 2 * pages_per_side, pages_per_side + + num_phys16 = nb * block_pages + k_view = k_src.view(num_phys16, h, HEAD_DIM // x, 16, x) + v_view = v_src.view(num_phys16, h, 16 // x, HEAD_DIM, x)[v_page_offset:] + assert k_view.is_contiguous() + assert v_view.is_contiguous() + if layout is KVCacheLayout.LHBNC: + assert k_view.shape == (24, 1, 8, 16, 16) + assert v_view.shape == (24, 1, 1, 128, 16) + else: + assert k_view.shape == (48, 1, 8, 16, 16) + assert v_view.shape == (40, 1, 1, 128, 16) def test_aiter_sparse_pa_rejects_multiple_kv_heads(monkeypatch): @@ -1769,6 +1796,330 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch): assert reason is None or "AITER sparse PA attend" not in reason +def test_aiter_consolidated_qknorm_enabled(monkeypatch): + from vllm._aiter_ops import rocm_aiter_ops + + monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True) + monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True) + + scale = torch.tensor([0.25]) + assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto") + assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, scale) + assert rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e4m3", scale, scale) + assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8") + assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8", scale, None) + assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale) + + monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", False) + assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto") + + monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True) + monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: False) + assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("auto") + + +def _fake_aiter_qknorm_args(kv_cache_dtype: str = "auto") -> dict: + tensor = torch.empty(1) + return { + "qkv": tensor, + "q_norm_weight": tensor, + "k_norm_weight": tensor, + "cos_sin_cache": tensor, + "positions": tensor, + "num_heads": 16, + "num_kv_heads": 1, + "rotary_dim": 64, + "eps": 1e-6, + "slot_mapping": tensor, + "kv_cache_k": tensor, + "kv_cache_v": tensor, + "q_out": tensor, + "kv_cache_dtype": kv_cache_dtype, + } + + +def _install_fake_aiter_qknorm(monkeypatch: pytest.MonkeyPatch) -> list[dict]: + """Stub ``aiter.fused_qknorm_idxrqknorm`` so mapping tests run without AITER.""" + import sys + import types + from typing import Any + + calls: list[dict] = [] + + def fake_op(*args, **kwargs): + calls.append(kwargs) + + existing = sys.modules.get("aiter") + if existing is not None: + monkeypatch.setattr(existing, "fused_qknorm_idxrqknorm", fake_op, raising=False) + else: + fake_mod: Any = types.ModuleType("aiter") + fake_mod.fused_qknorm_idxrqknorm = fake_op + monkeypatch.setitem(sys.modules, "aiter", fake_mod) + return calls + + +def test_aiter_consolidated_qknorm_packed_shuffle_unequal_spans(monkeypatch): + from vllm._aiter_ops import rocm_aiter_ops + + calls = _install_fake_aiter_qknorm(monkeypatch) + call_args = _fake_aiter_qknorm_args() + call_args["kv_cache_k"] = torch.empty(2, 1) + call_args["kv_cache_v"] = torch.empty(1, 1) + + rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args) + assert len(calls) == 1 + + +@pytest.mark.parametrize( + ("kv_cache_dtype", "aiter_dtype", "passes_scales"), + [ + ("auto", "auto", False), + ("fp8", "fp8_e4m3_static", True), + ("fp8_e4m3", "fp8_e4m3_static", True), + ], +) +def test_aiter_consolidated_qknorm_dtype_and_layout_mapping( + monkeypatch, kv_cache_dtype, aiter_dtype, passes_scales +): + from vllm._aiter_ops import rocm_aiter_ops + + calls = _install_fake_aiter_qknorm(monkeypatch) + call_args = _fake_aiter_qknorm_args(kv_cache_dtype) + k_scale = torch.tensor([0.25]) + v_scale = torch.tensor([0.5]) + call_args.update(k_scale=k_scale, v_scale=v_scale) + + rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args) + assert len(calls) == 1 + kwargs = calls[0] + assert kwargs["kv_cache_dtype"] == aiter_dtype + assert kwargs["index_cache_dtype"] == "auto" + assert kwargs["block_size"] == 16 + assert kwargs["asm_layout"] is True + if passes_scales: + assert kwargs["k_scale"] is k_scale + assert kwargs["v_scale"] is v_scale + else: + assert kwargs["k_scale"] is None + assert kwargs["v_scale"] is None + + +def test_aiter_consolidated_qknorm_fp8_index_dtype_mapping(monkeypatch): + from vllm._aiter_ops import rocm_aiter_ops + + calls = _install_fake_aiter_qknorm(monkeypatch) + fp8 = current_platform.fp8_dtype() + call_args = _fake_aiter_qknorm_args("fp8") + call_args.update( + k_scale=torch.tensor([0.25]), + v_scale=torch.tensor([0.5]), + index_cache=torch.empty(1, dtype=fp8), + index_q_out=torch.empty(1, dtype=fp8), + num_index_heads=1, + ) + + rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args) + assert calls[0]["index_cache_dtype"] == "fp8" + assert calls[0]["kv_cache_dtype"] == "fp8_e4m3_static" + + +@pytest.mark.parametrize("skip_index_branch", [False, True]) +def test_aiter_consolidated_qknorm_full_and_skip_index_args( + monkeypatch, skip_index_branch +): + from vllm._aiter_ops import rocm_aiter_ops + + calls = _install_fake_aiter_qknorm(monkeypatch) + call_args = _fake_aiter_qknorm_args() + if skip_index_branch: + call_args.update(skip_index_branch=True) + else: + tensor = torch.empty(1) + call_args.update( + index_q_norm_weight=tensor, + index_k_norm_weight=tensor, + num_index_heads=1, + index_cache=tensor, + index_q_out=tensor, + index_slot_mapping=tensor, + ) + + rocm_aiter_ops.fused_qknorm_idxrqknorm(**call_args) + kwargs = calls[0] + assert kwargs["skip_index_branch"] is skip_index_branch + if skip_index_branch: + assert kwargs["index_q_norm_weight"] is None + assert kwargs["index_k_norm_weight"] is None + assert kwargs["index_cache"] is None + assert kwargs["index_q_out"] is None + assert kwargs["index_slot_mapping"] is None + else: + assert kwargs["index_q_norm_weight"] is not None + assert kwargs["index_k_norm_weight"] is not None + assert kwargs["index_cache"] is not None + assert kwargs["index_q_out"] is not None + assert kwargs["index_slot_mapping"] is not None + + +def test_aiter_consolidated_qknorm_rejects_fp8_e5m2(monkeypatch): + from vllm._aiter_ops import rocm_aiter_ops + + monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True) + monkeypatch.setattr(rocm_aiter_ops, "_AITER_ENABLED", True) + scale = torch.tensor([0.25]) + assert not rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled("fp8_e5m2", scale, scale) + + +def test_aiter_sparse_pa_slot_rebase_preserves_page_offsets(): + slots = torch.tensor([0, 15, 16, 127, 128, 129, -1]) + actual = minimax_m3_rebase_slots_to_page16(slots, block_size=128) + expected = torch.tensor([0, 15, 16, 127, 256, 257, -1]) + assert torch.equal(actual, expected) + + +@pytest.mark.skipif( + not current_platform.is_rocm(), + reason="The consolidated packed-SHUFFLE AITER op is ROCm-only", +) +@pytest.mark.parametrize("skip_index_branch", [False, True]) +@pytest.mark.parametrize("fp8_index", [False, True]) +def test_aiter_consolidated_qknorm_real_packed_shuffle(skip_index_branch, fp8_index): + from vllm._aiter_ops import rocm_aiter_ops + + torch.manual_seed(20260831) + num_tokens = 4 + num_heads = 16 + num_kv_heads = 1 + num_index_heads = 1 + rotary_dim = 64 + head_dim = 128 + logical_block_size = 128 + num_blocks = 3 + fp8_dtype = current_platform.fp8_dtype() + + qkv = torch.randn( + num_tokens, + (num_heads + 2 * num_kv_heads + num_index_heads + 1) * head_dim, + dtype=torch.bfloat16, + device="cuda", + ) + weights = [ + torch.randn(head_dim, dtype=torch.bfloat16, device="cuda") for _ in range(4) + ] + cos_sin_cache = torch.randn(16, rotary_dim, dtype=torch.bfloat16, device="cuda") + positions = torch.arange(num_tokens, dtype=torch.int64, device="cuda") + logical_slots = torch.tensor([0, 127, 128, -1], dtype=torch.int64, device="cuda") + slot_mapping = minimax_m3_rebase_slots_to_page16(logical_slots, logical_block_size) + index_slot_mapping = logical_slots.clone() + + packed = torch.zeros( + num_blocks, + 2, + logical_block_size, + head_dim, + dtype=fp8_dtype, + device="cuda", + ) + x = 16 // packed.element_size() + pages_per_side = logical_block_size // 16 + num_phys16 = num_blocks * 2 * pages_per_side + kv_cache_k = packed.view(num_phys16, num_kv_heads, head_dim // x, 16, x) + kv_cache_v = packed.view(num_phys16, num_kv_heads, 16 // x, head_dim, x)[ + pages_per_side: + ] + assert kv_cache_k.shape == (48, 1, 8, 16, 16) + assert kv_cache_v.shape == (40, 1, 1, 128, 16) + + q_out = torch.zeros( + num_tokens, num_heads * head_dim, dtype=torch.bfloat16, device="cuda" + ) + index_dtype = fp8_dtype if fp8_index else torch.bfloat16 + if fp8_index: + index_q_out = torch.empty( + (num_tokens, num_index_heads * head_dim), + dtype=index_dtype, + device="cuda", + ) + index_cache = torch.empty( + (num_blocks, logical_block_size, head_dim), + dtype=index_dtype, + device="cuda", + ) + index_q_out.view(torch.uint8).fill_(7) + index_cache.view(torch.uint8).fill_(7) + else: + index_q_out = torch.full( + (num_tokens, num_index_heads * head_dim), + 7, + dtype=torch.bfloat16, + device="cuda", + ) + index_cache = torch.full( + (num_blocks, logical_block_size, head_dim), + 7, + dtype=torch.bfloat16, + device="cuda", + ) + scale = torch.tensor([0.02], dtype=torch.float32, device="cuda") + + kwargs = {} + if not skip_index_branch: + kwargs = { + "index_q_norm_weight": weights[2], + "index_k_norm_weight": weights[3], + "index_cache": index_cache, + "index_q_out": index_q_out, + "index_slot_mapping": index_slot_mapping, + } + rocm_aiter_ops.fused_qknorm_idxrqknorm( + qkv, + weights[0], + weights[1], + cos_sin_cache, + positions, + num_heads, + num_kv_heads, + rotary_dim, + 1e-6, + slot_mapping, + kv_cache_k, + kv_cache_v, + q_out, + "fp8", + scale, + scale, + num_index_heads=num_index_heads, + skip_index_branch=skip_index_branch, + **kwargs, + ) + torch.accelerator.synchronize() + + assert torch.isfinite(q_out).all() + assert torch.count_nonzero(q_out) > 0 + assert torch.count_nonzero(packed) > 0 + if fp8_index: + index_q_untouched = torch.all(index_q_out.view(torch.uint8) == 7) + index_cache_untouched = torch.all(index_cache.view(torch.uint8) == 7) + index_cache_t0 = torch.all(index_cache[0, 0].view(torch.uint8) == 7) + index_cache_t127 = torch.all(index_cache[0, 127].view(torch.uint8) == 7) + index_cache_t128 = torch.all(index_cache[1, 0].view(torch.uint8) == 7) + else: + index_q_untouched = torch.all(index_q_out == 7) + index_cache_untouched = torch.all(index_cache == 7) + index_cache_t0 = torch.all(index_cache[0, 0] == 7) + index_cache_t127 = torch.all(index_cache[0, 127] == 7) + index_cache_t128 = torch.all(index_cache[1, 0] == 7) + if skip_index_branch: + assert index_q_untouched + assert index_cache_untouched + else: + assert not index_q_untouched + assert not index_cache_t0 + assert not index_cache_t127 + assert not index_cache_t128 + + def test_indexer_cache_squeezes_to_contiguous_3d(): """The indexer side cache is standardized 4D with H=1: under both layouts the allocator's logical view stays contiguous and squeezes (as diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index c514b7c0bf..1829d9f5e2 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -35,10 +35,22 @@ except ImportError: # which is a host op, so we cache it once here. FP8_DTYPE = current_platform.fp8_dtype() _HIPB_MM_INITIALIZED_DEVICES: set[int] = set() +_FP8_E4M3_DTYPES = { + dtype + for dtype in ( + getattr(torch, "float8_e4m3fn", None), + getattr(torch, "float8_e4m3fnuz", None), + ) + if dtype is not None +} KB = 1024 MB = 1024 * KB +def _is_fp8_e4m3_tensor(t: torch.Tensor | None) -> bool: + return t is not None and t.dtype in _FP8_E4M3_DTYPES + + def _get_or_create_aiter_qr_rmsnorm_comm( device_comm: Any, ) -> tuple[int, int, bool] | None: @@ -2200,6 +2212,106 @@ class rocm_aiter_ops: def is_shuffle_kv_cache_enabled(cls) -> bool: return cls._SHUFFLE_KV_CACHE_ENABLED + @classmethod + @if_aiter_supported + def fused_qknorm_idxrqknorm_enabled( + cls, + kv_cache_dtype: str, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, + ) -> bool: + """Whether MiniMax-M3 can use AITER's consolidated fused QK-norm. + + Requires AITER to be installed and enabled. bf16 (``auto``) always + qualifies; fp8 e4m3 also needs K/V scales. Other cache dtypes fall + back to vLLM's fused kernel. + """ + if not cls._AITER_ENABLED: + return False + if kv_cache_dtype == "auto": + return True + if kv_cache_dtype in ("fp8", "fp8_e4m3"): + return k_scale is not None and v_scale is not None + return False + + @classmethod + def fused_qknorm_idxrqknorm( + cls, + qkv: torch.Tensor, + q_norm_weight: torch.Tensor, + k_norm_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads: int, + num_kv_heads: int, + rotary_dim: int, + eps: float, + slot_mapping: torch.Tensor, + kv_cache_k: torch.Tensor, + kv_cache_v: torch.Tensor, + q_out: torch.Tensor, + kv_cache_dtype: str, + k_scale: torch.Tensor | None = None, + v_scale: torch.Tensor | None = None, + index_q_norm_weight: torch.Tensor | None = None, + index_k_norm_weight: torch.Tensor | None = None, + num_index_heads: int = 0, + index_cache: torch.Tensor | None = None, + index_q_out: torch.Tensor | None = None, + index_slot_mapping: torch.Tensor | None = None, + skip_index_branch: bool = False, + ) -> None: + """Run consolidated MiniMax-M3 QK-norm fusion. + + Callers must check ``fused_qknorm_idxrqknorm_enabled`` first. Runtime + failures from the AITER op are deliberately propagated. + """ + if kv_cache_dtype == "auto": + aiter_kv_cache_dtype = "auto" + aiter_k_scale = None + aiter_v_scale = None + elif kv_cache_dtype in ("fp8", "fp8_e4m3"): + aiter_kv_cache_dtype = "fp8_e4m3_static" + aiter_k_scale = k_scale + aiter_v_scale = v_scale + else: + raise ValueError( + "AITER fused QK-norm requires kv_cache_dtype 'auto', 'fp8', " + f"or 'fp8_e4m3', got {kv_cache_dtype!r}" + ) + aiter_index_cache_dtype = "fp8" if _is_fp8_e4m3_tensor(index_cache) else "auto" + + from aiter import fused_qknorm_idxrqknorm + + fused_qknorm_idxrqknorm( + qkv, + q_norm_weight, + k_norm_weight, + cos_sin_cache, + positions, + num_heads, + num_kv_heads, + rotary_dim, + eps, + index_q_norm_weight=index_q_norm_weight, + index_k_norm_weight=index_k_norm_weight, + num_index_heads=num_index_heads, + slot_mapping=slot_mapping, + kv_cache_k=kv_cache_k, + kv_cache_v=kv_cache_v, + index_cache=index_cache, + block_size=16, + q_out=q_out, + index_q_out=index_q_out, + index_slot_mapping=index_slot_mapping, + kv_cache_dtype=aiter_kv_cache_dtype, + index_cache_dtype=aiter_index_cache_dtype, + k_scale=aiter_k_scale, + v_scale=aiter_v_scale, + asm_layout=True, + skip_index_branch=skip_index_branch, + ) + @classmethod @if_aiter_supported def is_triton_unified_attn_enabled(cls) -> bool: diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index a0a58253d1..88651e944a 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -890,6 +890,30 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): self._ensure_aiter_sparse_pa_kv_cache() return self.kv_cache_k, self.kv_cache_v + def _get_aiter_sparse_pa_slot_mapping( + self, + slot_mapping: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + ) -> torch.Tensor: + if value_cache.shape[0] == key_cache.shape[0]: + return slot_mapping + + attn_metadata = get_forward_context().attn_metadata + page16_slot_mapping = None + if isinstance(attn_metadata, dict): + main_md = attn_metadata[self.layer_name] + assert isinstance(main_md, MiniMaxM3SparseMetadata) + page16_slot_mapping = main_md.page16_slot_mapping + if ( + page16_slot_mapping is None + or page16_slot_mapping.shape != slot_mapping.shape + ): + page16_slot_mapping = minimax_m3_rebase_slots_to_page16( + slot_mapping, self.kv_cache.shape[2] + ) + return page16_slot_mapping + def _insert_aiter_sparse_pa_kv( self, k: torch.Tensor, @@ -897,6 +921,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): index_k: torch.Tensor | None, slot_mapping: torch.Tensor, index_slot_mapping: torch.Tensor | None, + key_cache: torch.Tensor, + value_cache: torch.Tensor, ) -> None: if self.kv_cache.numel() == 0: return @@ -906,22 +932,6 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): minimax_m3_insert_index_cache, ) - key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache() - if value_cache.shape[0] != key_cache.shape[0]: - attn_metadata = get_forward_context().attn_metadata - page16_slot_mapping = None - if isinstance(attn_metadata, dict): - main_md = attn_metadata[self.layer_name] - assert isinstance(main_md, MiniMaxM3SparseMetadata) - page16_slot_mapping = main_md.page16_slot_mapping - if ( - page16_slot_mapping is None - or page16_slot_mapping.shape != slot_mapping.shape - ): - page16_slot_mapping = minimax_m3_rebase_slots_to_page16( - slot_mapping, self.kv_cache.shape[2] - ) - slot_mapping = page16_slot_mapping kv_cache_dtype = ( self.kv_cache_dtype if is_quantized_kv_cache(self.kv_cache_dtype) @@ -976,38 +986,78 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): main_slot_mapping = fwd_slot_mapping[self.layer_name] q = qkv.new_empty((num_tokens, self.q_size)) + key_cache = value_cache = page16_slot_mapping = None + k_scale = v_scale = None + use_fused_qknorm = False + if self.use_aiter_sparse_pa: + key_cache, value_cache = self.get_aiter_sparse_pa_kv_cache() + page16_slot_mapping = self._get_aiter_sparse_pa_slot_mapping( + main_slot_mapping, key_cache, value_cache + ) + k_scale = getattr(self, "_k_scale", None) + v_scale = getattr(self, "_v_scale", None) + use_fused_qknorm = rocm_aiter_ops.fused_qknorm_idxrqknorm_enabled( + self.kv_cache_dtype, k_scale, v_scale + ) if self.skip_index_topk: index_q = None if self.use_aiter_sparse_pa: - ops.fused_minimax_m3_qknorm_rope_kv_insert( - qkv, - self.q_norm.weight, - self.k_norm.weight, - cos_sin_cache, - positions, - self.num_heads, - self.num_kv_heads, - rotary_dim, - eps, - num_index_heads=self.num_idx_heads, - q_out=q, - skip_index_branch=True, - ) - k_start = self.q_size - v_start = k_start + self.kv_size - k = qkv[:, k_start:v_start].view( - num_tokens, self.num_kv_heads, self.head_dim - ) - v = qkv[:, v_start : v_start + self.kv_size].view( - num_tokens, self.num_kv_heads, self.head_dim - ) - self._insert_aiter_sparse_pa_kv( - k, - v, - None, - main_slot_mapping, - None, - ) + assert key_cache is not None + assert value_cache is not None + assert page16_slot_mapping is not None + if use_fused_qknorm: + rocm_aiter_ops.fused_qknorm_idxrqknorm( + qkv, + self.q_norm.weight, + self.k_norm.weight, + cos_sin_cache, + positions, + self.num_heads, + self.num_kv_heads, + rotary_dim, + eps, + page16_slot_mapping, + key_cache, + value_cache, + q, + self.kv_cache_dtype, + k_scale, + v_scale, + num_index_heads=self.num_idx_heads, + skip_index_branch=True, + ) + else: + ops.fused_minimax_m3_qknorm_rope_kv_insert( + qkv, + self.q_norm.weight, + self.k_norm.weight, + cos_sin_cache, + positions, + self.num_heads, + self.num_kv_heads, + rotary_dim, + eps, + num_index_heads=self.num_idx_heads, + q_out=q, + skip_index_branch=True, + ) + k_start = self.q_size + v_start = k_start + self.kv_size + k = qkv[:, k_start:v_start].view( + num_tokens, self.num_kv_heads, self.head_dim + ) + v = qkv[:, v_start : v_start + self.kv_size].view( + num_tokens, self.num_kv_heads, self.head_dim + ) + self._insert_aiter_sparse_pa_kv( + k, + v, + None, + page16_slot_mapping, + None, + key_cache, + value_cache, + ) else: ops.fused_minimax_m3_qknorm_rope_kv_insert( qkv, @@ -1037,42 +1087,74 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): dtype=self.indexer.index_cache.dtype, ) if self.use_aiter_sparse_pa: - ops.fused_minimax_m3_qknorm_rope_kv_insert( - qkv, - self.q_norm.weight, - self.k_norm.weight, - cos_sin_cache, - positions, - self.num_heads, - self.num_kv_heads, - rotary_dim, - eps, - self.index_q_norm.weight, - self.index_k_norm.weight, - self.num_idx_heads, - q_out=q, - index_q_out=index_q, - kv_cache_dtype=self.kv_cache_dtype, - ) - k_start = self.q_size - v_start = k_start + self.kv_size - index_k_start = v_start + self.kv_size + self.index_q_size - k = qkv[:, k_start:v_start].view( - num_tokens, self.num_kv_heads, self.head_dim - ) - v = qkv[:, v_start : v_start + self.kv_size].view( - num_tokens, self.num_kv_heads, self.head_dim - ) - index_k = qkv[ - :, index_k_start : index_k_start + self.idx_head_dim - ].view(num_tokens, self.idx_head_dim) - self._insert_aiter_sparse_pa_kv( - k, - v, - index_k, - main_slot_mapping, - index_slot_mapping, - ) + assert key_cache is not None + assert value_cache is not None + assert page16_slot_mapping is not None + index_cache = self.indexer.index_cache.kv_cache + if use_fused_qknorm: + rocm_aiter_ops.fused_qknorm_idxrqknorm( + qkv, + self.q_norm.weight, + self.k_norm.weight, + cos_sin_cache, + positions, + self.num_heads, + self.num_kv_heads, + rotary_dim, + eps, + page16_slot_mapping, + key_cache, + value_cache, + q, + self.kv_cache_dtype, + k_scale, + v_scale, + self.index_q_norm.weight, + self.index_k_norm.weight, + self.num_idx_heads, + index_cache, + index_q, + index_slot_mapping, + ) + else: + ops.fused_minimax_m3_qknorm_rope_kv_insert( + qkv, + self.q_norm.weight, + self.k_norm.weight, + cos_sin_cache, + positions, + self.num_heads, + self.num_kv_heads, + rotary_dim, + eps, + self.index_q_norm.weight, + self.index_k_norm.weight, + self.num_idx_heads, + q_out=q, + index_q_out=index_q, + kv_cache_dtype=self.kv_cache_dtype, + ) + k_start = self.q_size + v_start = k_start + self.kv_size + index_k_start = v_start + self.kv_size + self.index_q_size + k = qkv[:, k_start:v_start].view( + num_tokens, self.num_kv_heads, self.head_dim + ) + v = qkv[:, v_start : v_start + self.kv_size].view( + num_tokens, self.num_kv_heads, self.head_dim + ) + index_k = qkv[ + :, index_k_start : index_k_start + self.idx_head_dim + ].view(num_tokens, self.idx_head_dim) + self._insert_aiter_sparse_pa_kv( + k, + v, + index_k, + page16_slot_mapping, + index_slot_mapping, + key_cache, + value_cache, + ) else: ops.fused_minimax_m3_qknorm_rope_kv_insert( qkv,