diff --git a/tests/kernels/test_minimax_m3_amd_ops.py b/tests/kernels/test_minimax_m3_amd_ops.py index 997c619809..27bd74ce41 100644 --- a/tests/kernels/test_minimax_m3_amd_ops.py +++ b/tests/kernels/test_minimax_m3_amd_ops.py @@ -10,6 +10,7 @@ tests assert the two agree within tolerance: * Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch * Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul * Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math + * Strided K/V sparse-PA cache insert -> contiguous-copy insert The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T`` @@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: _mxfp8_e4m3_quantize_triton, dequant_mxfp8_to_bf16, ) +from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402 from vllm.models.minimax_m3.amd.ops import ( # noqa: E402 gemma_fused_add_rmsnorm, gemma_rmsnorm, @@ -608,3 +610,77 @@ def test_routed_experts_expert_map_delegates_to_kernel(): assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular + + +# --------------------------------------------------------------------------- # +# Strided K/V operand for the AITER sparse-PA cache insert +# --------------------------------------------------------------------------- # +def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16): + """K/V as the sparse-PA insert sees them: column slices of a fused qkv.""" + kv = num_kv_heads * head_dim + row = num_q_heads * head_dim + 2 * kv + 2 * head_dim + qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE) + k_start = num_q_heads * head_dim + k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim) + v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim) + return k, v + + +def _asm_kv_cache(pages, num_kv_heads, head_dim): + """Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``.""" + x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size() + key = torch.zeros( + (pages, num_kv_heads, head_dim // x, 16, x), + dtype=torch.bfloat16, + device=DEVICE, + ) + value = torch.zeros( + (pages, num_kv_heads, 16 // x, head_dim, x), + dtype=torch.bfloat16, + device=DEVICE, + ) + return key, value + + +@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024]) +@pytest.mark.parametrize("num_kv_heads", [1, 2]) +def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads): + """A strided K/V slice must insert bit-identically to a contiguous copy.""" + reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache + head_dim = 128 + pages = num_tokens // 16 + 4 + slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32) + + torch.manual_seed(num_tokens) + k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim) + assert num_tokens == 1 or not k.is_contiguous() + + caches = [] + for operand in (lambda t: t.contiguous(), _kv_insert_operand): + key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim) + reshape_and_cache( + operand(k), + operand(v), + key_cache, + value_cache, + slots, + kv_cache_dtype="auto", + asm_layout=True, + ) + caches.append((key_cache, value_cache)) + + (ref_k, ref_v), (got_k, got_v) = caches + assert torch.equal(ref_k, got_k) + assert torch.equal(ref_v, got_v) + + +def test_kv_insert_operand_copies_unsupported_layouts(): + """Only a row-strided slice passes through; anything else is copied.""" + k, _ = _fused_qkv_kv_slices(32, 2, 128) + assert _kv_insert_operand(k) is k + + inner_strided = k[:, :, ::2] + assert _kv_insert_operand(inner_strided).is_contiguous() + + flat = k.reshape(32, -1) + assert _kv_insert_operand(flat).is_contiguous() diff --git a/vllm/models/minimax_m3/amd/model.py b/vllm/models/minimax_m3/amd/model.py index 50cdb55183..a0a58253d1 100644 --- a/vllm/models/minimax_m3/amd/model.py +++ b/vllm/models/minimax_m3/amd/model.py @@ -579,6 +579,32 @@ class MiniMaxM3Attention(nn.Module): return output +# Widest row pitch aiter's reshape_and_cache can carry in its `int` stride args. +_MAX_KV_INSERT_ROW_STRIDE = 2**31 - 1 + + +def _kv_insert_operand(t: torch.Tensor) -> torch.Tensor: + """K or V operand for ``aiter.reshape_and_cache``, without a needless copy. + + The AITER sparse-PA insert takes K and V as column slices of the fused + ``[q | k | v | index_q | index_k]`` projection, so they are row-strided + views and never contiguous. ``reshape_and_cache`` does not need them to be: + it passes ``key.stride(0)`` to the kernel as the row pitch and reads + ``key[token_idx * key_stride + i]`` for ``i`` over ``num_heads * + head_size``, so the only requirement is that the trailing + ``(num_heads, head_size)`` block be contiguous within a row, which such a + slice always satisfies. Any other layout falls back to a copy. + """ + if ( + t.dim() == 3 + and t.stride(2) == 1 + and t.stride(1) == t.shape[2] + and t.stride(0) <= _MAX_KV_INSERT_ROW_STRIDE + ): + return t + return t.contiguous() + + class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): """Block-sparse attention layer with the lightning-indexer branch. @@ -902,8 +928,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase): else "auto" ) reshape_and_cache( - k.contiguous(), - v.contiguous(), + _kv_insert_operand(k), + _kv_insert_operand(v), key_cache, value_cache, slot_mapping,