Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
855 lines
31 KiB
Python
855 lines
31 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Correctness tests for MiniMax M3 sparse prefill attention kernels."""
|
|
|
|
import pytest
|
|
import torch
|
|
|
|
from vllm import _custom_ops as ops
|
|
from vllm.models.minimax_m3.common.indexer import (
|
|
MiniMaxM3IndexerBackend,
|
|
)
|
|
from vllm.models.minimax_m3.common.ops.index_topk import (
|
|
minimax_m3_index_decode,
|
|
minimax_m3_index_score,
|
|
minimax_m3_index_topk,
|
|
)
|
|
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
|
minimax_m3_sparse_attn,
|
|
minimax_m3_sparse_attn_decode,
|
|
)
|
|
from vllm.models.minimax_m3.common.sparse_attention import (
|
|
MiniMaxM3SparseBackend,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
|
from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
|
|
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
|
|
from vllm.v1.worker.utils import AttentionGroup
|
|
|
|
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
|
pytest.skip(
|
|
"MiniMax M3 attention kernels require CUDA or ROCm.",
|
|
allow_module_level=True,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def kv_layout(request):
|
|
"""Set the global KV cache layout for one test and restore it after."""
|
|
set_kv_cache_layout(request.param)
|
|
try:
|
|
yield request.param
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
|
|
|
|
def _stride_order_for(backend: type[MiniMaxM3SparseBackend], ndim: int) -> tuple:
|
|
"""Mirror the allocator's stride-order resolution (identity fallback)."""
|
|
try:
|
|
stride_order = backend.get_kv_cache_stride_order()
|
|
assert len(stride_order) == ndim
|
|
except (AttributeError, NotImplementedError):
|
|
stride_order = tuple(range(ndim))
|
|
return stride_order
|
|
|
|
|
|
def _allocate_main_kv_via_contract(
|
|
num_pages: int, device: torch.device | str = "cuda"
|
|
) -> torch.Tensor:
|
|
"""Build the main KV cache exactly as the production allocator does for the
|
|
currently active layout: allocate the physical (permuted) tensor, then
|
|
expose the inverse-permuted logical-NHD view the backend sees."""
|
|
logical_shape = MiniMaxM3SparseBackend.get_kv_cache_shape(
|
|
num_pages, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
|
)
|
|
stride_order = _stride_order_for(MiniMaxM3SparseBackend, len(logical_shape))
|
|
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
|
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
|
raw = torch.randn(physical_shape, device=device, dtype=DTYPE)
|
|
return raw.permute(*inv_order)
|
|
|
|
|
|
NUM_Q_HEADS = 32
|
|
NUM_KV_HEADS = 2
|
|
HEAD_DIM = 128
|
|
BLOCK_SIZE = 128
|
|
DTYPE = torch.bfloat16
|
|
SM_SCALE = HEAD_DIM**-0.5
|
|
TOPK = 16
|
|
|
|
|
|
# Index top-k kernels.
|
|
def _reference_index_topk(
|
|
idx_q: torch.Tensor,
|
|
index_kv_cache: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
q_lens: torch.Tensor,
|
|
seq_lens: torch.Tensor,
|
|
prefix_lens: torch.Tensor,
|
|
topk: int,
|
|
init_blocks: int,
|
|
local_blocks: int,
|
|
sm_scale: float,
|
|
) -> torch.Tensor:
|
|
total_q, num_idx_heads, _ = idx_q.shape
|
|
out = torch.full(
|
|
(num_idx_heads, total_q, topk), -1, device=idx_q.device, dtype=torch.int32
|
|
)
|
|
|
|
q_start = 0
|
|
for req_id, (q_len, seq_len, prefix_len) in enumerate(
|
|
zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
|
|
):
|
|
q_end = q_start + q_len
|
|
q = idx_q[q_start:q_end]
|
|
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
|
pages = block_table[req_id, :num_blocks]
|
|
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
|
|
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
|
|
|
|
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
|
|
k_pos = torch.arange(k.shape[0], device=idx_q.device)
|
|
score.masked_fill_(k_pos[None, :] > q_pos[:, None], -float("inf"))
|
|
score = score.reshape(num_idx_heads, q_len, num_blocks, BLOCK_SIZE)
|
|
score_tensor = score.max(dim=3).values
|
|
|
|
valid_blocks = (q_pos + BLOCK_SIZE) // BLOCK_SIZE
|
|
for local_q, num_valid_blocks in enumerate(valid_blocks.tolist()):
|
|
end = min(init_blocks, num_valid_blocks)
|
|
score_tensor[:, local_q, :end] = 1e30
|
|
start = max(0, num_valid_blocks - local_blocks)
|
|
score_tensor[:, local_q, start:num_valid_blocks] = 1e29
|
|
|
|
k = min(topk, num_valid_blocks)
|
|
topk_idx = score_tensor[:, local_q].topk(k, dim=1).indices
|
|
out[:, q_start + local_q, :k] = topk_idx
|
|
q_start = q_end
|
|
|
|
return out
|
|
|
|
|
|
def _assert_topk_indices_equal_unordered(
|
|
actual: torch.Tensor,
|
|
expected: torch.Tensor,
|
|
) -> None:
|
|
"""Compare selected sparse blocks without requiring a deterministic order."""
|
|
assert actual.shape == expected.shape
|
|
actual_flat = actual.cpu().reshape(-1, actual.shape[-1]).tolist()
|
|
expected_flat = expected.cpu().reshape(-1, expected.shape[-1]).tolist()
|
|
for actual_row, expected_row in zip(actual_flat, expected_flat):
|
|
assert set(actual_row) == set(expected_row)
|
|
|
|
|
|
def test_prefill_index_topk_correctness():
|
|
topk = 6
|
|
init_blocks = 0
|
|
local_blocks = 1
|
|
num_idx_heads = 2
|
|
head_dim = 16
|
|
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
|
|
prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
|
|
seq_lens = prefix_lens + q_lens
|
|
batch = q_lens.numel()
|
|
max_seq_len = seq_lens.max().item()
|
|
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
|
num_pages = batch * max_blocks
|
|
|
|
cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
|
|
cu_seqlens[1:] = q_lens.cumsum(0)
|
|
block_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
|
|
batch, max_blocks
|
|
)
|
|
idx_q = torch.ones(q_lens.sum().item(), num_idx_heads, head_dim, device="cuda")
|
|
index_kv_cache = torch.empty(num_pages, BLOCK_SIZE, head_dim, device="cuda")
|
|
for req_id in range(batch):
|
|
for block_id in range(max_blocks):
|
|
page = block_table[req_id, block_id]
|
|
index_kv_cache[page].fill_(block_id + 1)
|
|
|
|
score = minimax_m3_index_score(
|
|
idx_q,
|
|
index_kv_cache,
|
|
block_table,
|
|
cu_seqlens,
|
|
seq_lens,
|
|
prefix_lens,
|
|
max_query_len=q_lens.max().item(),
|
|
max_seq_len=max_seq_len,
|
|
num_kv_heads=num_idx_heads,
|
|
sm_scale=head_dim**-0.5,
|
|
)
|
|
actual = minimax_m3_index_topk(
|
|
score,
|
|
cu_seqlens,
|
|
prefix_lens,
|
|
max_query_len=q_lens.max().item(),
|
|
topk=topk,
|
|
init_blocks=init_blocks,
|
|
local_blocks=local_blocks,
|
|
)
|
|
expected = _reference_index_topk(
|
|
idx_q,
|
|
index_kv_cache,
|
|
block_table,
|
|
q_lens,
|
|
seq_lens,
|
|
prefix_lens,
|
|
topk,
|
|
init_blocks,
|
|
local_blocks,
|
|
head_dim**-0.5,
|
|
)
|
|
_assert_topk_indices_equal_unordered(actual, expected)
|
|
|
|
|
|
@pytest.mark.parametrize("decode_query_len", [1, 4])
|
|
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
|
|
def test_decode_index_topk_correctness(
|
|
decode_query_len: int,
|
|
num_padded_reqs: int,
|
|
):
|
|
topk = 6
|
|
init_blocks = 0
|
|
local_blocks = 1
|
|
num_idx_heads = 2
|
|
head_dim = 16
|
|
active_seq_lens = torch.tensor((7, 129, 1025), device="cuda", dtype=torch.int32)
|
|
q_lens = torch.full_like(active_seq_lens, decode_query_len)
|
|
prefix_lens = active_seq_lens - decode_query_len
|
|
active_batch = active_seq_lens.numel()
|
|
batch = active_batch + num_padded_reqs
|
|
seq_lens = torch.cat(
|
|
[
|
|
active_seq_lens,
|
|
torch.zeros(num_padded_reqs, device="cuda", dtype=torch.int32),
|
|
]
|
|
)
|
|
max_seq_len = active_seq_lens.max().item()
|
|
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
|
num_pages = active_batch * max_blocks
|
|
|
|
active_block_table = torch.randperm(
|
|
num_pages, device="cuda", dtype=torch.int32
|
|
).reshape(active_batch, max_blocks)
|
|
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
|
|
block_table[:active_batch] = active_block_table
|
|
idx_q = torch.randn(
|
|
batch * decode_query_len, num_idx_heads, head_dim, device="cuda"
|
|
)
|
|
index_kv_cache = torch.randn(num_pages, BLOCK_SIZE, head_dim, device="cuda")
|
|
|
|
actual = minimax_m3_index_decode(
|
|
idx_q,
|
|
index_kv_cache,
|
|
block_table,
|
|
seq_lens,
|
|
max_seq_len=max_seq_len,
|
|
topk=topk,
|
|
init_blocks=init_blocks,
|
|
local_blocks=local_blocks,
|
|
num_kv_heads=num_idx_heads,
|
|
sm_scale=head_dim**-0.5,
|
|
decode_query_len=decode_query_len,
|
|
)
|
|
expected = torch.full_like(actual, -1)
|
|
active_tokens = active_batch * decode_query_len
|
|
expected[:, :active_tokens] = _reference_index_topk(
|
|
idx_q[:active_tokens],
|
|
index_kv_cache,
|
|
block_table[:active_batch],
|
|
q_lens,
|
|
active_seq_lens,
|
|
prefix_lens,
|
|
topk,
|
|
init_blocks,
|
|
local_blocks,
|
|
head_dim**-0.5,
|
|
)
|
|
_assert_topk_indices_equal_unordered(actual, expected)
|
|
|
|
|
|
# Sparse attention kernels.
|
|
def _reference_sparse_attn(
|
|
q: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
topk_idx: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
q_lens: torch.Tensor,
|
|
seq_lens: torch.Tensor,
|
|
prefix_lens: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
out = torch.empty_like(q, dtype=torch.float32)
|
|
gqa_group_size = NUM_Q_HEADS // NUM_KV_HEADS
|
|
q_start = 0
|
|
for req_id, (q_len, seq_len, prefix_len) in enumerate(
|
|
zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
|
|
):
|
|
q_end = q_start + q_len
|
|
q_req = q[q_start:q_end]
|
|
positions = torch.arange(seq_len, device="cuda")
|
|
pages = block_table[req_id, positions // BLOCK_SIZE]
|
|
rows = positions % BLOCK_SIZE
|
|
k_req = kv_cache[pages, 0, rows]
|
|
v_req = kv_cache[pages, 1, rows].float()
|
|
|
|
q_pos = prefix_len + torch.arange(q_len, device="cuda")
|
|
key_blocks = positions // BLOCK_SIZE
|
|
causal_mask = positions.unsqueeze(0) <= q_pos.unsqueeze(1)
|
|
|
|
for kv_head in range(NUM_KV_HEADS):
|
|
selected = topk_idx[kv_head, q_start:q_end]
|
|
selected_mask = (key_blocks[None, :, None] == selected[:, None, :]).any(-1)
|
|
mask = causal_mask & selected_mask
|
|
head_start = kv_head * gqa_group_size
|
|
head_end = head_start + gqa_group_size
|
|
|
|
q_heads = q_req[:, head_start:head_end].transpose(0, 1)
|
|
k_head = k_req[:, kv_head].T.expand(gqa_group_size, -1, -1)
|
|
scores = torch.bmm(q_heads, k_head, out_dtype=torch.float32)
|
|
scores = scores.transpose(0, 1) * SM_SCALE
|
|
probs = torch.softmax(
|
|
scores.masked_fill(~mask[:, None, :], -float("inf")), -1
|
|
)
|
|
out[q_start:q_end, head_start:head_end] = torch.einsum(
|
|
"qhk,kd->qhd", probs, v_req[:, kv_head]
|
|
)
|
|
q_start += q_len
|
|
return out.to(q.dtype)
|
|
|
|
|
|
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
|
@pytest.mark.parametrize(
|
|
("q_lens", "kv_lens"),
|
|
[
|
|
((129, 257), (129, 257)),
|
|
((65, 129, 257), (129, 257, 385)),
|
|
],
|
|
)
|
|
def test_prefill_sparse_attention_correctness(
|
|
kv_layout: str,
|
|
q_lens: tuple[int, ...],
|
|
kv_lens: tuple[int, ...],
|
|
):
|
|
assert len(q_lens) == len(kv_lens)
|
|
assert all(kv_len >= q_len for q_len, kv_len in zip(q_lens, kv_lens))
|
|
|
|
# Build paged-KV metadata, including a non-identity page order.
|
|
batch = len(q_lens)
|
|
pages_per_req = [(kv_len + BLOCK_SIZE - 1) // BLOCK_SIZE for kv_len in kv_lens]
|
|
max_blocks = max(pages_per_req)
|
|
num_pages = sum(pages_per_req)
|
|
physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
|
|
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
|
|
base_page = 0
|
|
for req_id, num_req_pages in enumerate(pages_per_req):
|
|
block_table[req_id, :num_req_pages] = physical_pages[
|
|
base_page : base_page + num_req_pages
|
|
]
|
|
base_page += num_req_pages
|
|
|
|
q_lens_t = torch.tensor(q_lens, device="cuda", dtype=torch.int32)
|
|
seq_lens = torch.tensor(kv_lens, device="cuda", dtype=torch.int32)
|
|
prefix_lens = seq_lens - q_lens_t
|
|
cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
|
|
cu_seqlens[1:] = q_lens_t.cumsum(0)
|
|
total_q = sum(q_lens)
|
|
max_seqlen_q = max(q_lens)
|
|
|
|
q_shape = (total_q, NUM_Q_HEADS, HEAD_DIM)
|
|
q = torch.randn(q_shape, device="cuda", dtype=DTYPE)
|
|
# Allocate the main KV cache through the backend layout contract so the
|
|
# physical storage matches the active layout (contiguous NHD or strided
|
|
# HND), while the kernels and reference see the logical-NHD view.
|
|
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
|
|
|
# Build sparse block indices with the same contract as the real M3 indexer:
|
|
# one forced local block, then score-selected older causal blocks.
|
|
topk_shape = (NUM_KV_HEADS, total_q, TOPK)
|
|
topk_idx = torch.full(topk_shape, -1, device="cuda", dtype=torch.int32)
|
|
q_start = 0
|
|
for q_len, prefix_len in zip(q_lens_t.tolist(), prefix_lens.tolist()):
|
|
for local_q in range(q_len):
|
|
current_block = (prefix_len + local_q) // BLOCK_SIZE
|
|
older_blocks = torch.randperm(
|
|
current_block, device="cuda", dtype=torch.int32
|
|
)
|
|
selected = torch.cat(
|
|
[
|
|
torch.tensor([current_block], device="cuda", dtype=torch.int32),
|
|
older_blocks[: TOPK - 1],
|
|
]
|
|
)
|
|
topk_idx[:, q_start + local_q, : selected.numel()] = selected
|
|
q_start += q_len
|
|
|
|
actual = torch.empty_like(q)
|
|
minimax_m3_sparse_attn(
|
|
q,
|
|
kv_cache,
|
|
topk_idx,
|
|
block_table,
|
|
cu_seqlens,
|
|
seq_lens,
|
|
prefix_lens,
|
|
max_seqlen_q,
|
|
NUM_KV_HEADS,
|
|
SM_SCALE,
|
|
actual,
|
|
)
|
|
|
|
expected = _reference_sparse_attn(
|
|
q,
|
|
kv_cache,
|
|
topk_idx,
|
|
block_table,
|
|
q_lens_t,
|
|
seq_lens,
|
|
prefix_lens,
|
|
)
|
|
torch.accelerator.synchronize()
|
|
|
|
error = (actual.float() - expected.float()).abs()
|
|
assert error.mean().item() < 2.5e-4
|
|
assert error.max().item() < 1.7e-2
|
|
|
|
|
|
def test_main_backend_layout_contract():
|
|
"""The main sparse backend exposes the logical-NHD shape and the
|
|
flash_attn-style stride order for each layout."""
|
|
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
|
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
|
assert logical == (nb, 2, bs, h, d)
|
|
# The old HND-ordered shape is no longer the logical shape.
|
|
assert logical != (nb, 2, h, bs, d)
|
|
|
|
try:
|
|
set_kv_cache_layout("HND")
|
|
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4)
|
|
set_kv_cache_layout("NHD")
|
|
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4)
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
|
|
for layout in ("NHD", "HND"):
|
|
try:
|
|
set_kv_cache_layout(layout)
|
|
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
# Valid permutation: no duplicates, covers every axis.
|
|
assert set(order) == set(range(len(order)))
|
|
|
|
# M3 has no cross-layer KV blocks.
|
|
with pytest.raises(NotImplementedError):
|
|
MiniMaxM3SparseBackend.get_kv_cache_stride_order(
|
|
include_num_layers_dimension=True
|
|
)
|
|
|
|
|
|
def test_main_backend_unknown_layout_raises(monkeypatch):
|
|
"""An unrecognized layout (injected past env-var validation) is rejected."""
|
|
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
|
|
|
monkeypatch.setattr(sparse_attn_mod, "get_kv_cache_layout", lambda: "BOGUS")
|
|
with pytest.raises(ValueError, match="Unknown cache layout format"):
|
|
MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
|
|
|
|
|
def test_indexer_backend_stride_order_is_identity():
|
|
"""The 3-dim indexer cache must not inherit the parent's 5-element stride
|
|
order; it overrides to the 3-element identity so the allocator keeps the
|
|
contiguous layout."""
|
|
assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
|
|
|
|
# Cross-layer (per-layer-stacked) KV blocks are not supported.
|
|
with pytest.raises(NotImplementedError):
|
|
MiniMaxM3IndexerBackend.get_kv_cache_stride_order(
|
|
include_num_layers_dimension=True
|
|
)
|
|
|
|
# The stride order matches the 3-dim indexer shape rank.
|
|
indexer_shape = MiniMaxM3IndexerBackend.get_kv_cache_shape(
|
|
5, BLOCK_SIZE, 1, HEAD_DIM
|
|
)
|
|
assert len(indexer_shape) == 3
|
|
assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
|
|
|
|
|
|
def test_hnd_allocation_is_byte_identical_to_transpose():
|
|
"""Under HND the backend-visible logical view is byte-identical to the
|
|
pre-change allocate-HND-then-transpose(2, 3) workaround."""
|
|
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
|
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
|
try:
|
|
set_kv_cache_layout("HND")
|
|
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
|
|
physical_shape = tuple(logical[i] for i in stride_order)
|
|
# The physical (permuted) shape equals the old hardcoded HND shape.
|
|
assert physical_shape == (nb, 2, h, bs, d)
|
|
|
|
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
|
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
|
|
view = raw.permute(*inv_order)
|
|
expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3)
|
|
|
|
assert view.shape == expected.shape
|
|
assert view.stride() == expected.stride()
|
|
assert view.storage_offset() == expected.storage_offset()
|
|
|
|
# Negative: the identity (wrong) stride order under HND does not reproduce
|
|
# the transpose view.
|
|
wrong_view = raw.view(logical)
|
|
assert wrong_view.stride() != expected.stride()
|
|
|
|
|
|
def test_main_cache_is_block_first_and_unpadded():
|
|
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
|
is used for the main GQA cache: its spec is unpadded and the physical
|
|
layout keeps num_blocks as the first dimension under both layouts."""
|
|
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
|
|
|
spec = FullAttentionSpec(
|
|
block_size=BLOCK_SIZE,
|
|
num_kv_heads=NUM_KV_HEADS,
|
|
head_size=HEAD_DIM,
|
|
head_size_v=HEAD_DIM,
|
|
dtype=DTYPE,
|
|
)
|
|
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
|
assert spec.page_size_padded is None
|
|
|
|
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(
|
|
4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
|
)
|
|
for layout in ("NHD", "HND"):
|
|
try:
|
|
set_kv_cache_layout(layout)
|
|
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
inv_order = [order.index(i) for i in range(len(order))]
|
|
# Physical first dim is num_blocks (block-first); required by the
|
|
# padded-strided branch's block-first assumption if it were ever taken.
|
|
assert inv_order[0] == 0
|
|
assert logical[order[0]] == logical[0]
|
|
|
|
|
|
def _build_decode_inputs(
|
|
seq_lens_list: tuple[int, ...],
|
|
decode_query_len: int = 1,
|
|
num_padded_reqs: int = 0,
|
|
):
|
|
"""Shared decode setup: uniform query tokens per request, a non-identity
|
|
block table, and topk indices selecting the current block plus older causal
|
|
blocks for each query token."""
|
|
active_batch = len(seq_lens_list)
|
|
batch = active_batch + num_padded_reqs
|
|
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
|
|
max_blocks = max(pages_per_req)
|
|
num_pages = sum(pages_per_req)
|
|
physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
|
|
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
|
|
base_page = 0
|
|
for req_id, num_req_pages in enumerate(pages_per_req):
|
|
block_table[req_id, :num_req_pages] = physical_pages[
|
|
base_page : base_page + num_req_pages
|
|
]
|
|
base_page += num_req_pages
|
|
|
|
seq_lens = torch.tensor(
|
|
(*seq_lens_list, *([0] * num_padded_reqs)),
|
|
device="cuda",
|
|
dtype=torch.int32,
|
|
)
|
|
q = torch.randn(
|
|
batch * decode_query_len, NUM_Q_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE
|
|
)
|
|
|
|
topk_idx = torch.full(
|
|
(NUM_KV_HEADS, batch * decode_query_len, TOPK),
|
|
-1,
|
|
device="cuda",
|
|
dtype=torch.int32,
|
|
)
|
|
token_id = 0
|
|
for req_id, seq_len in enumerate(seq_lens_list):
|
|
for local_q in range(decode_query_len):
|
|
query_pos = seq_len - decode_query_len + local_q
|
|
current_block = query_pos // BLOCK_SIZE
|
|
older_blocks = torch.randperm(
|
|
current_block, device="cuda", dtype=torch.int32
|
|
)
|
|
selected = torch.cat(
|
|
[
|
|
torch.tensor([current_block], device="cuda", dtype=torch.int32),
|
|
older_blocks[: TOPK - 1],
|
|
]
|
|
)
|
|
topk_idx[:, token_id, : selected.numel()] = selected
|
|
token_id += 1
|
|
|
|
return q, block_table, seq_lens, topk_idx, num_pages
|
|
|
|
|
|
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
|
@pytest.mark.parametrize(
|
|
"seq_lens_list",
|
|
[(130, 257), (129, 200, 384)],
|
|
)
|
|
@pytest.mark.parametrize("decode_query_len", [1, 4])
|
|
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
|
|
def test_decode_sparse_attention_correctness(
|
|
kv_layout: str,
|
|
seq_lens_list: tuple[int, ...],
|
|
decode_query_len: int,
|
|
num_padded_reqs: int,
|
|
):
|
|
"""Decode (split-K) parity under both layouts: this is the only coverage of
|
|
the decode-site cache feed, and the strided HND case fails if the kernel
|
|
ignores the cache strides."""
|
|
torch.manual_seed(0)
|
|
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
|
seq_lens_list, decode_query_len, num_padded_reqs
|
|
)
|
|
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
|
|
|
actual = torch.empty_like(q)
|
|
minimax_m3_sparse_attn_decode(
|
|
q,
|
|
kv_cache,
|
|
topk_idx,
|
|
block_table,
|
|
seq_lens,
|
|
NUM_KV_HEADS,
|
|
SM_SCALE,
|
|
actual,
|
|
decode_query_len,
|
|
)
|
|
|
|
# Reuse the prefill reference: decode is a uniform query chunk ending at
|
|
# seq_len - 1 for each request.
|
|
active_batch = len(seq_lens_list)
|
|
active_tokens = active_batch * decode_query_len
|
|
q_lens_t = torch.full(
|
|
(len(seq_lens_list),), decode_query_len, device="cuda", dtype=torch.int32
|
|
)
|
|
active_seq_lens = seq_lens[:active_batch]
|
|
prefix_lens = active_seq_lens - q_lens_t
|
|
expected = _reference_sparse_attn(
|
|
q[:active_tokens],
|
|
kv_cache,
|
|
topk_idx[:, :active_tokens],
|
|
block_table[:active_batch],
|
|
q_lens_t,
|
|
active_seq_lens,
|
|
prefix_lens,
|
|
)
|
|
torch.accelerator.synchronize()
|
|
|
|
error = (actual[:active_tokens].float() - expected.float()).abs()
|
|
assert error.mean().item() < 2.5e-4
|
|
assert error.max().item() < 1.7e-2
|
|
|
|
|
|
def test_decode_wrong_layout_breaks_parity():
|
|
"""Negative (AC-3/AC-5): consuming the physical HND buffer as if it were
|
|
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
|
reorders the K/V content, so the decode output no longer matches the
|
|
reference computed on the correct logical view. The mislabeled tensor keeps
|
|
the same shape as the correct view, so the kernel stays in bounds."""
|
|
torch.manual_seed(0)
|
|
seq_lens_list = (130, 257)
|
|
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
|
|
|
# Physical HND storage [blocks, 2, heads, block, dim].
|
|
phys = torch.randn(
|
|
(num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE
|
|
)
|
|
# Correct logical-NHD view (strided) vs. the same bytes mislabeled as a
|
|
# contiguous-NHD cache — same shape, different content mapping.
|
|
correct = phys.permute(0, 1, 3, 2, 4)
|
|
wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM)
|
|
|
|
q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32)
|
|
prefix_lens = seq_lens - q_lens_t
|
|
expected = _reference_sparse_attn(
|
|
q, correct, topk_idx, block_table, q_lens_t, seq_lens, prefix_lens
|
|
)
|
|
|
|
actual = torch.empty_like(q)
|
|
minimax_m3_sparse_attn_decode(
|
|
q, wrong, topk_idx, block_table, seq_lens, NUM_KV_HEADS, SM_SCALE, actual, 1
|
|
)
|
|
torch.accelerator.synchronize()
|
|
assert (actual.float() - expected.float()).abs().max().item() > 1.7e-2
|
|
|
|
|
|
def _make_attn_group(backend, spec):
|
|
return AttentionGroup(
|
|
backend=backend,
|
|
layer_names=["main"],
|
|
kv_cache_spec=spec,
|
|
kv_cache_group_id=0,
|
|
)
|
|
|
|
|
|
def test_main_cache_byte_identical_through_production_allocator():
|
|
"""AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
|
|
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
|
same shape, stride, and storage offset as the pre-change
|
|
allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates
|
|
through the same path to its 3-dim shape."""
|
|
nb = 4
|
|
spec = FullAttentionSpec(
|
|
block_size=BLOCK_SIZE,
|
|
num_kv_heads=NUM_KV_HEADS,
|
|
head_size=HEAD_DIM,
|
|
head_size_v=HEAD_DIM,
|
|
dtype=DTYPE,
|
|
)
|
|
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
|
group = _make_attn_group(MiniMaxM3SparseBackend, spec)
|
|
try:
|
|
set_kv_cache_layout("HND")
|
|
kv_caches = _reshape_kv_cache([group], {"main": raw}, "auto", [BLOCK_SIZE], {})
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
view = kv_caches["main"]
|
|
|
|
oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM))
|
|
oracle = oracle.transpose(2, 3)
|
|
assert tuple(view.shape) == tuple(oracle.shape)
|
|
assert view.stride() == oracle.stride()
|
|
assert view.storage_offset() == oracle.storage_offset()
|
|
|
|
# Indexer cache allocates through the same path under both layouts.
|
|
ispec = MLAAttentionSpec(
|
|
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
|
)
|
|
for layout in ("NHD", "HND"):
|
|
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
|
igroup = AttentionGroup(
|
|
backend=MiniMaxM3IndexerBackend,
|
|
layer_names=["idx"],
|
|
kv_cache_spec=ispec,
|
|
kv_cache_group_id=0,
|
|
)
|
|
try:
|
|
set_kv_cache_layout(layout)
|
|
iout = _reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
assert tuple(iout["idx"].shape) == (nb, BLOCK_SIZE, HEAD_DIM)
|
|
|
|
|
|
def test_indexer_inherited_stride_order_trips_allocator_assert():
|
|
"""AC-4 negative: without the indexer override, the inherited 5-element
|
|
stride order trips the allocator's `len(stride_order) == len(shape)` assert
|
|
for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
|
|
allocator's `(AttributeError, NotImplementedError)` fallback."""
|
|
|
|
class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
|
|
# Simulate inheriting the parent's 5-element stride order.
|
|
get_kv_cache_stride_order = staticmethod(
|
|
MiniMaxM3SparseBackend.get_kv_cache_stride_order
|
|
)
|
|
|
|
nb = 4
|
|
ispec = MLAAttentionSpec(
|
|
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
|
)
|
|
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
|
igroup = AttentionGroup(
|
|
backend=_BrokenIndexerBackend,
|
|
layer_names=["idx"],
|
|
kv_cache_spec=ispec,
|
|
kv_cache_group_id=0,
|
|
)
|
|
try:
|
|
set_kv_cache_layout("HND")
|
|
with pytest.raises(AssertionError):
|
|
_reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
|
|
|
|
def test_padded_main_cache_is_flagged():
|
|
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
|
contiguous-view branch (`page_size_padded is None`). A spec that sets
|
|
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
|
|
|
|
def _require_unpadded_block_first(spec, stride_order):
|
|
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
|
assert spec.page_size_padded is None, (
|
|
"main GQA cache must be unpadded to use the contiguous-view "
|
|
"allocator branch"
|
|
)
|
|
assert inv_order[0] == 0, "main GQA cache must remain block-first"
|
|
|
|
try:
|
|
set_kv_cache_layout("HND")
|
|
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
|
finally:
|
|
set_kv_cache_layout(None)
|
|
|
|
good = FullAttentionSpec(
|
|
block_size=BLOCK_SIZE,
|
|
num_kv_heads=NUM_KV_HEADS,
|
|
head_size=HEAD_DIM,
|
|
head_size_v=HEAD_DIM,
|
|
dtype=DTYPE,
|
|
)
|
|
_require_unpadded_block_first(good, stride_order) # passes
|
|
|
|
padded = FullAttentionSpec(
|
|
block_size=BLOCK_SIZE,
|
|
num_kv_heads=NUM_KV_HEADS,
|
|
head_size=HEAD_DIM,
|
|
head_size_v=HEAD_DIM,
|
|
dtype=DTYPE,
|
|
page_size_padded=good.page_size_bytes + 128,
|
|
)
|
|
with pytest.raises(AssertionError):
|
|
_require_unpadded_block_first(padded, stride_order)
|
|
|
|
|
|
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
|
def test_reshape_and_cache_flash_write_persists(kv_layout: str):
|
|
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
|
`self.kv_cache.unbind(1)` directly. Writing through those views must persist
|
|
into the bound storage (read back through an independent logical view) under
|
|
both layouts — a `.contiguous()` copy of the unbind slice would leave the
|
|
bound storage unchanged."""
|
|
torch.manual_seed(0)
|
|
num_pages = 4
|
|
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
|
with torch.no_grad():
|
|
kv_cache.zero_()
|
|
|
|
# Exactly the production write-site code under test.
|
|
key_cache, value_cache = kv_cache.unbind(1)
|
|
|
|
num_tokens = 12
|
|
slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[
|
|
:num_tokens
|
|
].to(torch.int64)
|
|
key = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
|
|
value = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
|
|
scale = torch.ones((), device="cuda")
|
|
ops.reshape_and_cache_flash(
|
|
key, value, key_cache, value_cache, slot_mapping, "auto", scale, scale
|
|
)
|
|
torch.accelerator.synchronize()
|
|
|
|
# Read back through the independent logical view; proves the writes landed
|
|
# in the engine-bound storage, not a detached copy.
|
|
for t in range(num_tokens):
|
|
slot = int(slot_mapping[t].item())
|
|
blk, intra = divmod(slot, BLOCK_SIZE)
|
|
torch.testing.assert_close(kv_cache[blk, 0, intra], key[t])
|
|
torch.testing.assert_close(kv_cache[blk, 1, intra], value[t])
|