[ROCm][Perf] Insert MiniMax-M3 sparse-PA K/V without a contiguous copy (#56849)
Signed-off-by: Aakif Nawaz <aakif.nawaz@amd.com>
This commit is contained in:
@@ -10,6 +10,7 @@ tests assert the two agree within tolerance:
|
||||
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
|
||||
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
|
||||
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
|
||||
* Strided K/V sparse-PA cache insert -> contiguous-copy insert
|
||||
|
||||
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
|
||||
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa:
|
||||
_mxfp8_e4m3_quantize_triton,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.model import _kv_insert_operand # noqa: E402
|
||||
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
|
||||
gemma_fused_add_rmsnorm,
|
||||
gemma_rmsnorm,
|
||||
@@ -608,3 +610,77 @@ def test_routed_experts_expert_map_delegates_to_kernel():
|
||||
assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask
|
||||
assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map
|
||||
assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Strided K/V operand for the AITER sparse-PA cache insert
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim, num_q_heads=16):
|
||||
"""K/V as the sparse-PA insert sees them: column slices of a fused qkv."""
|
||||
kv = num_kv_heads * head_dim
|
||||
row = num_q_heads * head_dim + 2 * kv + 2 * head_dim
|
||||
qkv = torch.randn((num_tokens, row), dtype=torch.bfloat16, device=DEVICE)
|
||||
k_start = num_q_heads * head_dim
|
||||
k = qkv[:, k_start : k_start + kv].view(num_tokens, num_kv_heads, head_dim)
|
||||
v = qkv[:, k_start + kv : k_start + 2 * kv].view(num_tokens, num_kv_heads, head_dim)
|
||||
return k, v
|
||||
|
||||
|
||||
def _asm_kv_cache(pages, num_kv_heads, head_dim):
|
||||
"""Page-16 K/V cache views matching ``_ensure_aiter_sparse_pa_kv_cache``."""
|
||||
x = 16 // torch.tensor([], dtype=torch.bfloat16).element_size()
|
||||
key = torch.zeros(
|
||||
(pages, num_kv_heads, head_dim // x, 16, x),
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
value = torch.zeros(
|
||||
(pages, num_kv_heads, 16 // x, head_dim, x),
|
||||
dtype=torch.bfloat16,
|
||||
device=DEVICE,
|
||||
)
|
||||
return key, value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 8, 129, 1024])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 2])
|
||||
def test_kv_insert_operand_matches_contiguous(num_tokens, num_kv_heads):
|
||||
"""A strided K/V slice must insert bit-identically to a contiguous copy."""
|
||||
reshape_and_cache = pytest.importorskip("aiter").reshape_and_cache
|
||||
head_dim = 128
|
||||
pages = num_tokens // 16 + 4
|
||||
slots = torch.randperm(pages * 16, device=DEVICE)[:num_tokens].to(torch.int32)
|
||||
|
||||
torch.manual_seed(num_tokens)
|
||||
k, v = _fused_qkv_kv_slices(num_tokens, num_kv_heads, head_dim)
|
||||
assert num_tokens == 1 or not k.is_contiguous()
|
||||
|
||||
caches = []
|
||||
for operand in (lambda t: t.contiguous(), _kv_insert_operand):
|
||||
key_cache, value_cache = _asm_kv_cache(pages, num_kv_heads, head_dim)
|
||||
reshape_and_cache(
|
||||
operand(k),
|
||||
operand(v),
|
||||
key_cache,
|
||||
value_cache,
|
||||
slots,
|
||||
kv_cache_dtype="auto",
|
||||
asm_layout=True,
|
||||
)
|
||||
caches.append((key_cache, value_cache))
|
||||
|
||||
(ref_k, ref_v), (got_k, got_v) = caches
|
||||
assert torch.equal(ref_k, got_k)
|
||||
assert torch.equal(ref_v, got_v)
|
||||
|
||||
|
||||
def test_kv_insert_operand_copies_unsupported_layouts():
|
||||
"""Only a row-strided slice passes through; anything else is copied."""
|
||||
k, _ = _fused_qkv_kv_slices(32, 2, 128)
|
||||
assert _kv_insert_operand(k) is k
|
||||
|
||||
inner_strided = k[:, :, ::2]
|
||||
assert _kv_insert_operand(inner_strided).is_contiguous()
|
||||
|
||||
flat = k.reshape(32, -1)
|
||||
assert _kv_insert_operand(flat).is_contiguous()
|
||||
|
||||
@@ -579,6 +579,32 @@ class MiniMaxM3Attention(nn.Module):
|
||||
return output
|
||||
|
||||
|
||||
# Widest row pitch aiter's reshape_and_cache can carry in its `int` stride args.
|
||||
_MAX_KV_INSERT_ROW_STRIDE = 2**31 - 1
|
||||
|
||||
|
||||
def _kv_insert_operand(t: torch.Tensor) -> torch.Tensor:
|
||||
"""K or V operand for ``aiter.reshape_and_cache``, without a needless copy.
|
||||
|
||||
The AITER sparse-PA insert takes K and V as column slices of the fused
|
||||
``[q | k | v | index_q | index_k]`` projection, so they are row-strided
|
||||
views and never contiguous. ``reshape_and_cache`` does not need them to be:
|
||||
it passes ``key.stride(0)`` to the kernel as the row pitch and reads
|
||||
``key[token_idx * key_stride + i]`` for ``i`` over ``num_heads *
|
||||
head_size``, so the only requirement is that the trailing
|
||||
``(num_heads, head_size)`` block be contiguous within a row, which such a
|
||||
slice always satisfies. Any other layout falls back to a copy.
|
||||
"""
|
||||
if (
|
||||
t.dim() == 3
|
||||
and t.stride(2) == 1
|
||||
and t.stride(1) == t.shape[2]
|
||||
and t.stride(0) <= _MAX_KV_INSERT_ROW_STRIDE
|
||||
):
|
||||
return t
|
||||
return t.contiguous()
|
||||
|
||||
|
||||
class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
"""Block-sparse attention layer with the lightning-indexer branch.
|
||||
|
||||
@@ -902,8 +928,8 @@ class MiniMaxM3SparseAttention(nn.Module, AttentionLayerBase):
|
||||
else "auto"
|
||||
)
|
||||
reshape_and_cache(
|
||||
k.contiguous(),
|
||||
v.contiguous(),
|
||||
_kv_insert_operand(k),
|
||||
_kv_insert_operand(v),
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
|
||||
Reference in New Issue
Block a user