[ROCm][Perf] Insert MiniMax-M3 sparse-PA K/V without a contiguous copy (#56849)

Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com>
This commit is contained in:
akii96
2026-09-17 09:44:54 -05:00
committed by GitHub
parent 9e1cd262cb
commit 438434b5b5
2 changed files with 104 additions and 2 deletions
+76
View File
@@ -10,6 +10,7 @@ tests assert the two agree within tolerance:
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
* Strided K/V sparse-PA cache insert -> contiguous-copy insert
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa:
_mxfp8_e4m3_quantize_triton,
dequant_mxfp8_to_bf16,
)
from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
gemma_fused_add_rmsnorm,
gemma_rmsnorm,
@@ -608,3 +610,77 @@ def test_routed_experts_expert_map_delegates_to_kernel():
assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask
assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map
assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular
# --------------------------------------------------------------------------- #
# Strided K/V operand for the AITER sparse-PA cache insert
# --------------------------------------------------------------------------- #
def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16):
"""K/V as the sparse-PA insert sees them: column slices of a fused qkv."""
kv = num_kv_heads * head_dim
row = num_q_heads * head_dim + 2 * kv + 2 * head_dim
qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE)
k_start = num_q_heads * head_dim
k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim)
v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim)
return k, v
def _asm_kv_cache(pages, num_kv_heads, head_dim):
"""Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``."""
x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size()
key = torch.zeros(
(pages, num_kv_heads, head_dim // x, 16, x),
dtype=torch.bfloat16,
device=DEVICE,
)
value = torch.zeros(
(pages, num_kv_heads, 16 // x, head_dim, x),
dtype=torch.bfloat16,
device=DEVICE,
)
return key, value
@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024])
@pytest.mark.parametrize("num_kv_heads", [1, 2])
def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads):
"""A strided K/V slice must insert bit-identically to a contiguous copy."""
reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache
head_dim = 128
pages = num_tokens // 16 + 4
slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32)
torch.manual_seed(num_tokens)
k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim)
assert num_tokens == 1 or not k.is_contiguous()
caches = []
for operand in (lambda t: t.contiguous(), _kv_insert_operand):
key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim)
reshape_and_cache(
operand(k),
operand(v),
key_cache,
value_cache,
slots,
kv_cache_dtype="auto",
asm_layout=True,
)
caches.append((key_cache, value_cache))
(ref_k, ref_v), (got_k, got_v) = caches
assert torch.equal(ref_k, got_k)
assert torch.equal(ref_v, got_v)
def test_kv_insert_operand_copies_unsupported_layouts():
"""Only a row-strided slice passes through; anything else is copied."""
k, _ = _fused_qkv_kv_slices(32, 2, 128)
assert _kv_insert_operand(k) is k
inner_strided = k[:, :, ::2]
assert _kv_insert_operand(inner_strided).is_contiguous()
flat = k.reshape(32, -1)
assert _kv_insert_operand(flat).is_contiguous()
+28 -2
View File
@@ -579,6 +579,32 @@ class MiniMaxM3Attention(nn.Module):
return output
# Widest row pitch aiter's reshape_and_cache can carry in its `int` stride args.
_MAX_KV_INSERT_ROW_STRIDE = 2**31 - 1
def _kv_insert_operand(t: torch.Tensor) -> torch.Tensor:
"""K or V operand for ``aiter.reshape_and_cache``, without a needless copy.
The AITER sparse-PA insert takes K and V as column slices of the fused
``[q | k | v | index_q | index_k]`` projection, so they are row-strided
views and never contiguous. ``reshape_and_cache`` does not need them to be:
it passes ``key.stride(0)`` to the kernel as the row pitch and reads
``key[token_idx * key_stride + i]`` for ``i`` over ``num_heads *
head_size``, so the only requirement is that the trailing
``(num_heads, head_size)`` block be contiguous within a row, which such a
slice always satisfies. Any other layout falls back to a copy.
"""
if (
t.dim() == 3
and t.stride(2) == 1
and t.stride(1) == t.shape[2]
and t.stride(0) <= _MAX_KV_INSERT_ROW_STRIDE
):
return t
return t.contiguous()
class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
"""Block-sparse attention layer with the lightning-indexer branch.
@@ -902,8 +928,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
else "auto"
)
reshape_and_cache(
k.contiguous(),
v.contiguous(),
_kv_insert_operand(k),
_kv_insert_operand(v),
key_cache,
value_cache,
slot_mapping,