[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
+854
View File
@@ -0,0 +1,854 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for MiniMax M3 sparse prefill attention kernels."""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.models.minimax_m3.common.indexer import (
MiniMaxM3IndexerBackend,
)
from vllm.models.minimax_m3.common.ops.index_topk import (
minimax_m3_index_decode,
minimax_m3_index_score,
minimax_m3_index_topk,
)
from vllm.models.minimax_m3.common.ops.sparse_attn import (
minimax_m3_sparse_attn,
minimax_m3_sparse_attn_decode,
)
from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import set_kv_cache_layout
from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
from vllm.v1.worker.utils import AttentionGroup
if not (current_platform.is_cuda() or current_platform.is_rocm()):
pytest.skip(
"MiniMax M3 attention kernels require CUDA or ROCm.",
allow_module_level=True,
)
@pytest.fixture
def kv_layout(request):
"""Set the global KV cache layout for one test and restore it after."""
set_kv_cache_layout(request.param)
try:
yield request.param
finally:
set_kv_cache_layout(None)
def _stride_order_for(backend: type[MiniMaxM3SparseBackend], ndim: int) -> tuple:
"""Mirror the allocator's stride-order resolution (identity fallback)."""
try:
stride_order = backend.get_kv_cache_stride_order()
assert len(stride_order) == ndim
except (AttributeError, NotImplementedError):
stride_order = tuple(range(ndim))
return stride_order
def _allocate_main_kv_via_contract(
num_pages: int, device: torch.device | str = "cuda"
) -> torch.Tensor:
"""Build the main KV cache exactly as the production allocator does for the
currently active layout: allocate the physical (permuted) tensor, then
expose the inverse-permuted logical-NHD view the backend sees."""
logical_shape = MiniMaxM3SparseBackend.get_kv_cache_shape(
num_pages, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
)
stride_order = _stride_order_for(MiniMaxM3SparseBackend, len(logical_shape))
physical_shape = tuple(logical_shape[i] for i in stride_order)
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
raw = torch.randn(physical_shape, device=device, dtype=DTYPE)
return raw.permute(*inv_order)
NUM_Q_HEADS = 32
NUM_KV_HEADS = 2
HEAD_DIM = 128
BLOCK_SIZE = 128
DTYPE = torch.bfloat16
SM_SCALE = HEAD_DIM**-0.5
TOPK = 16
# Index top-k kernels.
def _reference_index_topk(
idx_q: torch.Tensor,
index_kv_cache: torch.Tensor,
block_table: torch.Tensor,
q_lens: torch.Tensor,
seq_lens: torch.Tensor,
prefix_lens: torch.Tensor,
topk: int,
init_blocks: int,
local_blocks: int,
sm_scale: float,
) -> torch.Tensor:
total_q, num_idx_heads, _ = idx_q.shape
out = torch.full(
(num_idx_heads, total_q, topk), -1, device=idx_q.device, dtype=torch.int32
)
q_start = 0
for req_id, (q_len, seq_len, prefix_len) in enumerate(
zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
):
q_end = q_start + q_len
q = idx_q[q_start:q_end]
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
pages = block_table[req_id, :num_blocks]
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
k_pos = torch.arange(k.shape[0], device=idx_q.device)
score.masked_fill_(k_pos[None, :] > q_pos[:, None], -float("inf"))
score = score.reshape(num_idx_heads, q_len, num_blocks, BLOCK_SIZE)
score_tensor = score.max(dim=3).values
valid_blocks = (q_pos + BLOCK_SIZE) // BLOCK_SIZE
for local_q, num_valid_blocks in enumerate(valid_blocks.tolist()):
end = min(init_blocks, num_valid_blocks)
score_tensor[:, local_q, :end] = 1e30
start = max(0, num_valid_blocks - local_blocks)
score_tensor[:, local_q, start:num_valid_blocks] = 1e29
k = min(topk, num_valid_blocks)
topk_idx = score_tensor[:, local_q].topk(k, dim=1).indices
out[:, q_start + local_q, :k] = topk_idx
q_start = q_end
return out
def _assert_topk_indices_equal_unordered(
actual: torch.Tensor,
expected: torch.Tensor,
) -> None:
"""Compare selected sparse blocks without requiring a deterministic order."""
assert actual.shape == expected.shape
actual_flat = actual.cpu().reshape(-1, actual.shape[-1]).tolist()
expected_flat = expected.cpu().reshape(-1, expected.shape[-1]).tolist()
for actual_row, expected_row in zip(actual_flat, expected_flat):
assert set(actual_row) == set(expected_row)
def test_prefill_index_topk_correctness():
topk = 6
init_blocks = 0
local_blocks = 1
num_idx_heads = 2
head_dim = 16
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
seq_lens = prefix_lens + q_lens
batch = q_lens.numel()
max_seq_len = seq_lens.max().item()
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
num_pages = batch * max_blocks
cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
cu_seqlens[1:] = q_lens.cumsum(0)
block_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
batch, max_blocks
)
idx_q = torch.ones(q_lens.sum().item(), num_idx_heads, head_dim, device="cuda")
index_kv_cache = torch.empty(num_pages, BLOCK_SIZE, head_dim, device="cuda")
for req_id in range(batch):
for block_id in range(max_blocks):
page = block_table[req_id, block_id]
index_kv_cache[page].fill_(block_id + 1)
score = minimax_m3_index_score(
idx_q,
index_kv_cache,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
max_query_len=q_lens.max().item(),
max_seq_len=max_seq_len,
num_kv_heads=num_idx_heads,
sm_scale=head_dim**-0.5,
)
actual = minimax_m3_index_topk(
score,
cu_seqlens,
prefix_lens,
max_query_len=q_lens.max().item(),
topk=topk,
init_blocks=init_blocks,
local_blocks=local_blocks,
)
expected = _reference_index_topk(
idx_q,
index_kv_cache,
block_table,
q_lens,
seq_lens,
prefix_lens,
topk,
init_blocks,
local_blocks,
head_dim**-0.5,
)
_assert_topk_indices_equal_unordered(actual, expected)
@pytest.mark.parametrize("decode_query_len", [1, 4])
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
def test_decode_index_topk_correctness(
decode_query_len: int,
num_padded_reqs: int,
):
topk = 6
init_blocks = 0
local_blocks = 1
num_idx_heads = 2
head_dim = 16
active_seq_lens = torch.tensor((7, 129, 1025), device="cuda", dtype=torch.int32)
q_lens = torch.full_like(active_seq_lens, decode_query_len)
prefix_lens = active_seq_lens - decode_query_len
active_batch = active_seq_lens.numel()
batch = active_batch + num_padded_reqs
seq_lens = torch.cat(
[
active_seq_lens,
torch.zeros(num_padded_reqs, device="cuda", dtype=torch.int32),
]
)
max_seq_len = active_seq_lens.max().item()
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
num_pages = active_batch * max_blocks
active_block_table = torch.randperm(
num_pages, device="cuda", dtype=torch.int32
).reshape(active_batch, max_blocks)
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
block_table[:active_batch] = active_block_table
idx_q = torch.randn(
batch * decode_query_len, num_idx_heads, head_dim, device="cuda"
)
index_kv_cache = torch.randn(num_pages, BLOCK_SIZE, head_dim, device="cuda")
actual = minimax_m3_index_decode(
idx_q,
index_kv_cache,
block_table,
seq_lens,
max_seq_len=max_seq_len,
topk=topk,
init_blocks=init_blocks,
local_blocks=local_blocks,
num_kv_heads=num_idx_heads,
sm_scale=head_dim**-0.5,
decode_query_len=decode_query_len,
)
expected = torch.full_like(actual, -1)
active_tokens = active_batch * decode_query_len
expected[:, :active_tokens] = _reference_index_topk(
idx_q[:active_tokens],
index_kv_cache,
block_table[:active_batch],
q_lens,
active_seq_lens,
prefix_lens,
topk,
init_blocks,
local_blocks,
head_dim**-0.5,
)
_assert_topk_indices_equal_unordered(actual, expected)
# Sparse attention kernels.
def _reference_sparse_attn(
q: torch.Tensor,
kv_cache: torch.Tensor,
topk_idx: torch.Tensor,
block_table: torch.Tensor,
q_lens: torch.Tensor,
seq_lens: torch.Tensor,
prefix_lens: torch.Tensor,
) -> torch.Tensor:
out = torch.empty_like(q, dtype=torch.float32)
gqa_group_size = NUM_Q_HEADS // NUM_KV_HEADS
q_start = 0
for req_id, (q_len, seq_len, prefix_len) in enumerate(
zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
):
q_end = q_start + q_len
q_req = q[q_start:q_end]
positions = torch.arange(seq_len, device="cuda")
pages = block_table[req_id, positions // BLOCK_SIZE]
rows = positions % BLOCK_SIZE
k_req = kv_cache[pages, 0, rows]
v_req = kv_cache[pages, 1, rows].float()
q_pos = prefix_len + torch.arange(q_len, device="cuda")
key_blocks = positions // BLOCK_SIZE
causal_mask = positions.unsqueeze(0) <= q_pos.unsqueeze(1)
for kv_head in range(NUM_KV_HEADS):
selected = topk_idx[kv_head, q_start:q_end]
selected_mask = (key_blocks[None, :, None] == selected[:, None, :]).any(-1)
mask = causal_mask & selected_mask
head_start = kv_head * gqa_group_size
head_end = head_start + gqa_group_size
q_heads = q_req[:, head_start:head_end].transpose(0, 1)
k_head = k_req[:, kv_head].T.expand(gqa_group_size, -1, -1)
scores = torch.bmm(q_heads, k_head, out_dtype=torch.float32)
scores = scores.transpose(0, 1) * SM_SCALE
probs = torch.softmax(
scores.masked_fill(~mask[:, None, :], -float("inf")), -1
)
out[q_start:q_end, head_start:head_end] = torch.einsum(
"qhk,kd->qhd", probs, v_req[:, kv_head]
)
q_start += q_len
return out.to(q.dtype)
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
@pytest.mark.parametrize(
("q_lens", "kv_lens"),
[
((129, 257), (129, 257)),
((65, 129, 257), (129, 257, 385)),
],
)
def test_prefill_sparse_attention_correctness(
kv_layout: str,
q_lens: tuple[int, ...],
kv_lens: tuple[int, ...],
):
assert len(q_lens) == len(kv_lens)
assert all(kv_len >= q_len for q_len, kv_len in zip(q_lens, kv_lens))
# Build paged-KV metadata, including a non-identity page order.
batch = len(q_lens)
pages_per_req = [(kv_len + BLOCK_SIZE - 1) // BLOCK_SIZE for kv_len in kv_lens]
max_blocks = max(pages_per_req)
num_pages = sum(pages_per_req)
physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
base_page = 0
for req_id, num_req_pages in enumerate(pages_per_req):
block_table[req_id, :num_req_pages] = physical_pages[
base_page : base_page + num_req_pages
]
base_page += num_req_pages
q_lens_t = torch.tensor(q_lens, device="cuda", dtype=torch.int32)
seq_lens = torch.tensor(kv_lens, device="cuda", dtype=torch.int32)
prefix_lens = seq_lens - q_lens_t
cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
cu_seqlens[1:] = q_lens_t.cumsum(0)
total_q = sum(q_lens)
max_seqlen_q = max(q_lens)
q_shape = (total_q, NUM_Q_HEADS, HEAD_DIM)
q = torch.randn(q_shape, device="cuda", dtype=DTYPE)
# Allocate the main KV cache through the backend layout contract so the
# physical storage matches the active layout (contiguous NHD or strided
# HND), while the kernels and reference see the logical-NHD view.
kv_cache = _allocate_main_kv_via_contract(num_pages)
# Build sparse block indices with the same contract as the real M3 indexer:
# one forced local block, then score-selected older causal blocks.
topk_shape = (NUM_KV_HEADS, total_q, TOPK)
topk_idx = torch.full(topk_shape, -1, device="cuda", dtype=torch.int32)
q_start = 0
for q_len, prefix_len in zip(q_lens_t.tolist(), prefix_lens.tolist()):
for local_q in range(q_len):
current_block = (prefix_len + local_q) // BLOCK_SIZE
older_blocks = torch.randperm(
current_block, device="cuda", dtype=torch.int32
)
selected = torch.cat(
[
torch.tensor([current_block], device="cuda", dtype=torch.int32),
older_blocks[: TOPK - 1],
]
)
topk_idx[:, q_start + local_q, : selected.numel()] = selected
q_start += q_len
actual = torch.empty_like(q)
minimax_m3_sparse_attn(
q,
kv_cache,
topk_idx,
block_table,
cu_seqlens,
seq_lens,
prefix_lens,
max_seqlen_q,
NUM_KV_HEADS,
SM_SCALE,
actual,
)
expected = _reference_sparse_attn(
q,
kv_cache,
topk_idx,
block_table,
q_lens_t,
seq_lens,
prefix_lens,
)
torch.accelerator.synchronize()
error = (actual.float() - expected.float()).abs()
assert error.mean().item() < 2.5e-4
assert error.max().item() < 1.7e-2
def test_main_backend_layout_contract():
"""The main sparse backend exposes the logical-NHD shape and the
flash_attn-style stride order for each layout."""
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
assert logical == (nb, 2, bs, h, d)
# The old HND-ordered shape is no longer the logical shape.
assert logical != (nb, 2, h, bs, d)
try:
set_kv_cache_layout("HND")
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4)
set_kv_cache_layout("NHD")
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4)
finally:
set_kv_cache_layout(None)
for layout in ("NHD", "HND"):
try:
set_kv_cache_layout(layout)
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
finally:
set_kv_cache_layout(None)
# Valid permutation: no duplicates, covers every axis.
assert set(order) == set(range(len(order)))
# M3 has no cross-layer KV blocks.
with pytest.raises(NotImplementedError):
MiniMaxM3SparseBackend.get_kv_cache_stride_order(
include_num_layers_dimension=True
)
def test_main_backend_unknown_layout_raises(monkeypatch):
"""An unrecognized layout (injected past env-var validation) is rejected."""
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
monkeypatch.setattr(sparse_attn_mod, "get_kv_cache_layout", lambda: "BOGUS")
with pytest.raises(ValueError, match="Unknown cache layout format"):
MiniMaxM3SparseBackend.get_kv_cache_stride_order()
def test_indexer_backend_stride_order_is_identity():
"""The 3-dim indexer cache must not inherit the parent's 5-element stride
order; it overrides to the 3-element identity so the allocator keeps the
contiguous layout."""
assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
# Cross-layer (per-layer-stacked) KV blocks are not supported.
with pytest.raises(NotImplementedError):
MiniMaxM3IndexerBackend.get_kv_cache_stride_order(
include_num_layers_dimension=True
)
# The stride order matches the 3-dim indexer shape rank.
indexer_shape = MiniMaxM3IndexerBackend.get_kv_cache_shape(
5, BLOCK_SIZE, 1, HEAD_DIM
)
assert len(indexer_shape) == 3
assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
def test_hnd_allocation_is_byte_identical_to_transpose():
"""Under HND the backend-visible logical view is byte-identical to the
pre-change allocate-HND-then-transpose(2, 3) workaround."""
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
try:
set_kv_cache_layout("HND")
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
finally:
set_kv_cache_layout(None)
physical_shape = tuple(logical[i] for i in stride_order)
# The physical (permuted) shape equals the old hardcoded HND shape.
assert physical_shape == (nb, 2, h, bs, d)
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
view = raw.permute(*inv_order)
expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3)
assert view.shape == expected.shape
assert view.stride() == expected.stride()
assert view.storage_offset() == expected.storage_offset()
# Negative: the identity (wrong) stride order under HND does not reproduce
# the transpose view.
wrong_view = raw.view(logical)
assert wrong_view.stride() != expected.stride()
def test_main_cache_is_block_first_and_unpadded():
"""The allocator's contiguous-view branch (not the padded-strided branch)
is used for the main GQA cache: its spec is unpadded and the physical
layout keeps num_blocks as the first dimension under both layouts."""
from vllm.v1.kv_cache_interface import FullAttentionSpec
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_DIM,
head_size_v=HEAD_DIM,
dtype=DTYPE,
)
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
assert spec.page_size_padded is None
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(
4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
)
for layout in ("NHD", "HND"):
try:
set_kv_cache_layout(layout)
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
finally:
set_kv_cache_layout(None)
inv_order = [order.index(i) for i in range(len(order))]
# Physical first dim is num_blocks (block-first); required by the
# padded-strided branch's block-first assumption if it were ever taken.
assert inv_order[0] == 0
assert logical[order[0]] == logical[0]
def _build_decode_inputs(
seq_lens_list: tuple[int, ...],
decode_query_len: int = 1,
num_padded_reqs: int = 0,
):
"""Shared decode setup: uniform query tokens per request, a non-identity
block table, and topk indices selecting the current block plus older causal
blocks for each query token."""
active_batch = len(seq_lens_list)
batch = active_batch + num_padded_reqs
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
max_blocks = max(pages_per_req)
num_pages = sum(pages_per_req)
physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
base_page = 0
for req_id, num_req_pages in enumerate(pages_per_req):
block_table[req_id, :num_req_pages] = physical_pages[
base_page : base_page + num_req_pages
]
base_page += num_req_pages
seq_lens = torch.tensor(
(*seq_lens_list, *([0] * num_padded_reqs)),
device="cuda",
dtype=torch.int32,
)
q = torch.randn(
batch * decode_query_len, NUM_Q_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE
)
topk_idx = torch.full(
(NUM_KV_HEADS, batch * decode_query_len, TOPK),
-1,
device="cuda",
dtype=torch.int32,
)
token_id = 0
for req_id, seq_len in enumerate(seq_lens_list):
for local_q in range(decode_query_len):
query_pos = seq_len - decode_query_len + local_q
current_block = query_pos // BLOCK_SIZE
older_blocks = torch.randperm(
current_block, device="cuda", dtype=torch.int32
)
selected = torch.cat(
[
torch.tensor([current_block], device="cuda", dtype=torch.int32),
older_blocks[: TOPK - 1],
]
)
topk_idx[:, token_id, : selected.numel()] = selected
token_id += 1
return q, block_table, seq_lens, topk_idx, num_pages
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
@pytest.mark.parametrize(
"seq_lens_list",
[(130, 257), (129, 200, 384)],
)
@pytest.mark.parametrize("decode_query_len", [1, 4])
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
def test_decode_sparse_attention_correctness(
kv_layout: str,
seq_lens_list: tuple[int, ...],
decode_query_len: int,
num_padded_reqs: int,
):
"""Decode (split-K) parity under both layouts: this is the only coverage of
the decode-site cache feed, and the strided HND case fails if the kernel
ignores the cache strides."""
torch.manual_seed(0)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
seq_lens_list, decode_query_len, num_padded_reqs
)
kv_cache = _allocate_main_kv_via_contract(num_pages)
actual = torch.empty_like(q)
minimax_m3_sparse_attn_decode(
q,
kv_cache,
topk_idx,
block_table,
seq_lens,
NUM_KV_HEADS,
SM_SCALE,
actual,
decode_query_len,
)
# Reuse the prefill reference: decode is a uniform query chunk ending at
# seq_len - 1 for each request.
active_batch = len(seq_lens_list)
active_tokens = active_batch * decode_query_len
q_lens_t = torch.full(
(len(seq_lens_list),), decode_query_len, device="cuda", dtype=torch.int32
)
active_seq_lens = seq_lens[:active_batch]
prefix_lens = active_seq_lens - q_lens_t
expected = _reference_sparse_attn(
q[:active_tokens],
kv_cache,
topk_idx[:, :active_tokens],
block_table[:active_batch],
q_lens_t,
active_seq_lens,
prefix_lens,
)
torch.accelerator.synchronize()
error = (actual[:active_tokens].float() - expected.float()).abs()
assert error.mean().item() < 2.5e-4
assert error.max().item() < 1.7e-2
def test_decode_wrong_layout_breaks_parity():
"""Negative (AC-3/AC-5): consuming the physical HND buffer as if it were
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
reorders the K/V content, so the decode output no longer matches the
reference computed on the correct logical view. The mislabeled tensor keeps
the same shape as the correct view, so the kernel stays in bounds."""
torch.manual_seed(0)
seq_lens_list = (130, 257)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
# Physical HND storage [blocks, 2, heads, block, dim].
phys = torch.randn(
(num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE
)
# Correct logical-NHD view (strided) vs. the same bytes mislabeled as a
# contiguous-NHD cache — same shape, different content mapping.
correct = phys.permute(0, 1, 3, 2, 4)
wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM)
q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32)
prefix_lens = seq_lens - q_lens_t
expected = _reference_sparse_attn(
q, correct, topk_idx, block_table, q_lens_t, seq_lens, prefix_lens
)
actual = torch.empty_like(q)
minimax_m3_sparse_attn_decode(
q, wrong, topk_idx, block_table, seq_lens, NUM_KV_HEADS, SM_SCALE, actual, 1
)
torch.accelerator.synchronize()
assert (actual.float() - expected.float()).abs().max().item() > 1.7e-2
def _make_attn_group(backend, spec):
return AttentionGroup(
backend=backend,
layer_names=["main"],
kv_cache_spec=spec,
kv_cache_group_id=0,
)
def test_main_cache_byte_identical_through_production_allocator():
"""AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
`FullAttentionSpec` under HND and assert the backend-visible view has the
same shape, stride, and storage offset as the pre-change
allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates
through the same path to its 3-dim shape."""
nb = 4
spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_DIM,
head_size_v=HEAD_DIM,
dtype=DTYPE,
)
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
group = _make_attn_group(MiniMaxM3SparseBackend, spec)
try:
set_kv_cache_layout("HND")
kv_caches = _reshape_kv_cache([group], {"main": raw}, "auto", [BLOCK_SIZE], {})
finally:
set_kv_cache_layout(None)
view = kv_caches["main"]
oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM))
oracle = oracle.transpose(2, 3)
assert tuple(view.shape) == tuple(oracle.shape)
assert view.stride() == oracle.stride()
assert view.storage_offset() == oracle.storage_offset()
# Indexer cache allocates through the same path under both layouts.
ispec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
)
for layout in ("NHD", "HND"):
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
igroup = AttentionGroup(
backend=MiniMaxM3IndexerBackend,
layer_names=["idx"],
kv_cache_spec=ispec,
kv_cache_group_id=0,
)
try:
set_kv_cache_layout(layout)
iout = _reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
finally:
set_kv_cache_layout(None)
assert tuple(iout["idx"].shape) == (nb, BLOCK_SIZE, HEAD_DIM)
def test_indexer_inherited_stride_order_trips_allocator_assert():
"""AC-4 negative: without the indexer override, the inherited 5-element
stride order trips the allocator's `len(stride_order) == len(shape)` assert
for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
allocator's `(AttributeError, NotImplementedError)` fallback."""
class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
# Simulate inheriting the parent's 5-element stride order.
get_kv_cache_stride_order = staticmethod(
MiniMaxM3SparseBackend.get_kv_cache_stride_order
)
nb = 4
ispec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
)
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
igroup = AttentionGroup(
backend=_BrokenIndexerBackend,
layer_names=["idx"],
kv_cache_spec=ispec,
kv_cache_group_id=0,
)
try:
set_kv_cache_layout("HND")
with pytest.raises(AssertionError):
_reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
finally:
set_kv_cache_layout(None)
def test_padded_main_cache_is_flagged():
"""AC-2.1 negative: the M3 main cache relies on the allocator's
contiguous-view branch (`page_size_padded is None`). A spec that sets
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
def _require_unpadded_block_first(spec, stride_order):
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
assert spec.page_size_padded is None, (
"main GQA cache must be unpadded to use the contiguous-view "
"allocator branch"
)
assert inv_order[0] == 0, "main GQA cache must remain block-first"
try:
set_kv_cache_layout("HND")
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
finally:
set_kv_cache_layout(None)
good = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_DIM,
head_size_v=HEAD_DIM,
dtype=DTYPE,
)
_require_unpadded_block_first(good, stride_order) # passes
padded = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=NUM_KV_HEADS,
head_size=HEAD_DIM,
head_size_v=HEAD_DIM,
dtype=DTYPE,
page_size_padded=good.page_size_bytes + 128,
)
with pytest.raises(AssertionError):
_require_unpadded_block_first(padded, stride_order)
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
def test_reshape_and_cache_flash_write_persists(kv_layout: str):
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
`self.kv_cache.unbind(1)` directly. Writing through those views must persist
into the bound storage (read back through an independent logical view) under
both layouts — a `.contiguous()` copy of the unbind slice would leave the
bound storage unchanged."""
torch.manual_seed(0)
num_pages = 4
kv_cache = _allocate_main_kv_via_contract(num_pages)
with torch.no_grad():
kv_cache.zero_()
# Exactly the production write-site code under test.
key_cache, value_cache = kv_cache.unbind(1)
num_tokens = 12
slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[
:num_tokens
].to(torch.int64)
key = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
value = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
scale = torch.ones((), device="cuda")
ops.reshape_and_cache_flash(
key, value, key_cache, value_cache, slot_mapping, "auto", scale, scale
)
torch.accelerator.synchronize()
# Read back through the independent logical view; proves the writes landed
# in the engine-bound storage, not a detached copy.
for t in range(num_tokens):
slot = int(slot_mapping[t].item())
blk, intra = divmod(slot, BLOCK_SIZE)
torch.testing.assert_close(kv_cache[blk, 0, intra], key[t])
torch.testing.assert_close(kv_cache[blk, 1, intra], value[t])
@@ -0,0 +1,109 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the manual AllReduce + GemmaRMSNorm fusion used by MiniMax M3.
``fused_allreduce_gemma_rms_norm`` must match the unfused model path, i.e.
``GemmaRMSNorm(all_reduce(partial), residual)``, both on the flashinfer fast
path (TP>1 with flashinfer + NVSwitch) and on the eager fallback (TP==1, or when
flashinfer is unavailable / the GPU has no NVSwitch).
"""
import pytest
import torch
from torch.multiprocessing import spawn
from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
fused_allreduce_gemma_rms_norm,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.torch_utils import set_random_seed
@ensure_current_vllm_config()
def _worker_fused_ar_norm(
local_rank,
world_size,
port,
num_tokens,
hidden_size,
dtype,
seed,
eps,
):
"""Per-rank worker: compare the fused helper vs all_reduce + GemmaRMSNorm."""
device = torch.device(f"cuda:{local_rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(
world_size, 1, local_rank, port, local_rank=local_rank
)
# Norm weights are identical across ranks (replicated GemmaRMSNorm).
set_random_seed(seed)
norm = GemmaRMSNorm(hidden_size, eps=eps).cuda().to(dtype)
with torch.no_grad():
norm.weight.normal_(mean=0.0, std=0.1)
# Residual is shared across ranks; the partial o_proj output differs per rank
# (each rank holds a partial sum that all_reduce combines).
torch.manual_seed(seed + 7)
residual = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
torch.manual_seed(seed + 1000 + local_rank)
partial = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
# Reference: the unfused model path.
reduced = tensor_model_parallel_all_reduce(partial.clone())
ref_out, ref_res = norm(reduced, residual.clone())
# Fused helper (flashinfer fast path when available, else fallback).
out, res = fused_allreduce_gemma_rms_norm(partial.clone(), residual.clone(), norm)
torch.accelerator.synchronize()
torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=2e-2)
torch.testing.assert_close(res, ref_res, atol=2e-2, rtol=2e-2)
cleanup_dist_env_and_memory()
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="CUDA required",
)
# world_size=1 exercises the TP==1 identity branch on a single GPU; >1 exercises
# the all_reduce + GemmaRMSNorm equivalence (flashinfer kernel or fallback).
@pytest.mark.parametrize("world_size", [1, 2, 4])
@pytest.mark.parametrize("num_tokens", [1, 128, 333])
@pytest.mark.parametrize("hidden_size", [2048, 4096])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.parametrize("seed", [42])
def test_fused_allreduce_gemma_rms_norm(
world_size,
num_tokens,
hidden_size,
dtype,
eps,
seed,
):
num_gpus = current_platform.device_count()
if num_gpus < world_size:
pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
port = str(get_open_port())
spawn(
_worker_fused_ar_norm,
args=(
world_size,
port,
num_tokens,
hidden_size,
dtype,
seed,
eps,
),
nprocs=world_size,
join=True,
)
+21 -15
View File
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for fp32_router_gemm kernel: activation×weight→fp32, H=3072, E=256.
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
Supported (hidden_size, num_experts) pairs:
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
Correctness baseline: torch.matmul in float64.
"""
@@ -10,8 +13,8 @@ import torch
from vllm._custom_ops import fp32_router_gemm
NUM_EXPERTS = 256
HIDDEN_DIM = 3072
# (hidden_size, num_experts)
SHAPES = [(3072, 256), (6144, 128)]
# Absolute tolerance for fp32 kernel vs float64 reference
ATOL_FP32 = 2e-4
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
@@ -30,49 +33,52 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 8, 16, 32])
def test_fp32_activation(num_tokens: int):
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""fp32 activation → fp32 output should match reference closely."""
_requires_sm90()
torch.manual_seed(42)
device = torch.device("cuda")
mat_a = torch.randn(num_tokens, HIDDEN_DIM, dtype=torch.float32, device=device)
mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
mat_a = torch.randn(num_tokens, hidden_dim, dtype=torch.float32, device=device)
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
out = fp32_router_gemm(mat_a, mat_b)
ref = _ref(mat_a, mat_b)
assert out.shape == (num_tokens, NUM_EXPERTS)
assert out.shape == (num_tokens, num_experts)
assert out.dtype == torch.float32
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 8, 16, 32])
def test_bf16_activation(num_tokens: int):
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""bf16 activation → fp32 output should match reference within bf16 error."""
_requires_sm90()
torch.manual_seed(42)
device = torch.device("cuda")
mat_a_bf16 = torch.randn(
num_tokens, HIDDEN_DIM, dtype=torch.bfloat16, device=device
num_tokens, hidden_dim, dtype=torch.bfloat16, device=device
)
mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
out = fp32_router_gemm(mat_a_bf16, mat_b)
ref = _ref(mat_a_bf16, mat_b).to(device)
assert out.shape == (num_tokens, NUM_EXPERTS)
assert out.shape == (num_tokens, num_experts)
assert out.dtype == torch.float32
torch.testing.assert_close(out, ref, atol=ATOL_BF16, rtol=0)
def test_output_shape_and_dtype():
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
def test_output_shape_and_dtype(hidden_dim: int, num_experts: int):
"""Basic shape and dtype checks."""
_requires_sm90()
device = torch.device("cuda")
mat_a = torch.randn(4, HIDDEN_DIM, dtype=torch.float32, device=device)
mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
mat_a = torch.randn(4, hidden_dim, dtype=torch.float32, device=device)
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
out = fp32_router_gemm(mat_a, mat_b)
assert out.shape == (4, NUM_EXPERTS)
assert out.shape == (4, num_experts)
assert out.dtype == torch.float32
assert out.device.type == "cuda"
@@ -0,0 +1,244 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for the horizontally-fused MiniMax-M3 attention pre-processing
kernel:
fused_minimax_m3_qknorm_rope_kv_insert
- q / k / index_q / index_k: Gemma RMSNorm + partial NeoX RoPE (in place)
- sparse (insert) mode: scatter k/v into the paged bf16 KV cache and the
index key into the index cache by its own slot mapping.
Reference: PyTorch Gemma RMSNorm with the same dtype materialization boundary
as the unfused path, followed by vLLM CUDA rotary_embedding-style NeoX RoPE.
"""
import pytest
import torch
import vllm._custom_ops as ops
HEAD_DIM = 128
ROTARY_DIM = 64
def _op_available() -> bool:
return hasattr(torch.ops._C, "fused_minimax_m3_qknorm_rope_kv_insert")
pytestmark = pytest.mark.skipif(
not torch.cuda.is_available() or not _op_available(),
reason="CUDA not available or fused MiniMax-M3 op not built in",
)
def make_cos_sin_cache(max_pos, rotary_dim, base, dtype, device):
inv_freq = 1.0 / (
base
** (
torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
/ rotary_dim
)
)
t = torch.arange(max_pos, dtype=torch.float32, device=device)
freqs = torch.einsum("i,j->ij", t, inv_freq) # [max_pos, rotary_dim/2]
cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1) # [max_pos, rotary_dim]
return cache.to(dtype)
def gemma_rmsnorm(x, weight, eps):
"""x: [..., 128]; weight: [128]. Returns original dtype."""
xf = x.float()
var = xf.pow(2).mean(dim=-1, keepdim=True)
out = xf * torch.rsqrt(var + eps)
out = out * (1.0 + weight.float())
return out.to(x.dtype)
def apply_rope_neox_partial(x, positions, cos_sin_cache, rotary_dim):
"""NeoX-style RoPE on the leading rotary_dim dims; rest pass through.
x: [num_tokens, num_heads, head_dim]
cos_sin_cache: [max_pos, rotary_dim] (cos||sin), read as float (matches the
kernel, which loads the bf16 cache and converts to fp32).
"""
half = rotary_dim // 2
cs = cos_sin_cache[positions].float() # [num_tokens, rotary_dim]
cos = cs[..., :half].unsqueeze(1) # [nt, 1, half]
sin = cs[..., half:].unsqueeze(1)
rot = x[..., :rotary_dim].float()
x1 = rot[..., :half]
x2 = rot[..., half:]
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
out = x.clone()
out[..., :half] = o1
out[..., half:rotary_dim] = o2
return out.to(x.dtype)
def norm_rope_ref(x, weight, positions, cos_sin_cache, eps):
"""[nt, nheads, 128] -> Gemma norm + neox partial rope."""
normed = gemma_rmsnorm(x, weight, eps)
roped = apply_rope_neox_partial(normed, positions, cos_sin_cache, ROTARY_DIM)
return roped
# ── Test 1: dense mode (norm+rope only, no index, no insert) ─────────────────
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
@pytest.mark.parametrize("num_heads,num_kv_heads", [(8, 2), (16, 4), (64, 4)])
def test_dense_norm_rope(num_tokens, num_heads, num_kv_heads):
torch.manual_seed(0)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
qkv = torch.randn(num_tokens, qsz + 2 * kvsz, dtype=dtype, device=device)
qkv_orig = qkv.clone()
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv, q_w, k_w, cos_sin, positions, num_heads, num_kv_heads, ROTARY_DIM, eps
)
q_out, k_out, v_out = qkv.split([qsz, kvsz, kvsz], dim=-1)
q_in, k_in, v_in = qkv_orig.split([qsz, kvsz, kvsz], dim=-1)
q_ref = norm_rope_ref(
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
).view(num_tokens, qsz)
k_ref = norm_rope_ref(
k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
k_w,
positions,
cos_sin,
eps,
).view(num_tokens, kvsz)
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
# V is untouched.
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
# ── Test 2: sparse mode (full: index branch + cache inserts) ─────────────────
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
@pytest.mark.parametrize("block_size", [16, 64])
def test_sparse_full(num_tokens, block_size):
torch.manual_seed(1)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
num_heads, num_kv_heads, num_idx_heads = 16, 4, 4
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
iq_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
ik_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
iqsz, iksz = num_idx_heads * HEAD_DIM, HEAD_DIM
# Single fused tensor packing [q | k | v | index_q | index_k].
qkv = torch.randn(
num_tokens, qsz + 2 * kvsz + iqsz + iksz, dtype=dtype, device=device
)
qkv_orig = qkv.clone()
splits = [qsz, kvsz, kvsz, iqsz, iksz]
num_blocks = (num_tokens + block_size - 1) // block_size + 1
kv_cache = torch.zeros(
num_blocks, 2, block_size, num_kv_heads, HEAD_DIM, dtype=dtype, device=device
)
index_cache = torch.zeros(
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
)
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
index_slot_mapping = torch.roll(slot_mapping, shifts=1)
# Contiguous gather targets: the kernel writes the normed/roped q and
# index_q here (de-interleaved from the packed qkv); k/v/index_k stay in
# place inside qkv and are scatter-inserted into the caches.
q_out = torch.empty(num_tokens, qsz, dtype=dtype, device=device)
index_q = torch.empty(num_tokens, iqsz, dtype=dtype, device=device)
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
q_w,
k_w,
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
iq_w,
ik_w,
num_idx_heads,
slot_mapping,
index_slot_mapping,
kv_cache,
index_cache,
block_size,
q_out,
index_q,
)
# ── norm+rope parity. q/index_q land in their gather buffers; k/index_k are
# rewritten in place inside qkv. ──
_, k_out, _, _, index_k = qkv.split(splits, dim=-1)
q_in, k_in, v_in, iq_orig, ik_orig = qkv_orig.split(splits, dim=-1)
q_ref = norm_rope_ref(
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
).view(num_tokens, qsz)
k_ref = norm_rope_ref(
k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
k_w,
positions,
cos_sin,
eps,
).view(num_tokens, kvsz)
iq_ref = norm_rope_ref(
iq_orig.view(num_tokens, num_idx_heads, HEAD_DIM),
iq_w,
positions,
cos_sin,
eps,
).view(num_tokens, num_idx_heads * HEAD_DIM)
ik_ref = norm_rope_ref(
ik_orig.view(num_tokens, 1, HEAD_DIM), ik_w, positions, cos_sin, eps
).view(num_tokens, HEAD_DIM)
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(index_q, iq_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(index_k, ik_ref, rtol=1e-2, atol=1e-2)
# ── Cache inserts. ──
# Main cache layout is [num_blocks, 2, block_size, num_kv_heads, head_dim]
# (the K/V axis sits *before* block_size); index cache is [nb, bs, head_dim].
idx_flat = index_cache.view(num_blocks * block_size, HEAD_DIM)
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM) # v is raw (no norm/rope)
for t in range(num_tokens):
s = slot_mapping[t].item()
b, pos = s // block_size, s % block_size
torch.testing.assert_close(
kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
)
torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
index_s = index_slot_mapping[t].item()
torch.testing.assert_close(idx_flat[index_s], ik_ref[t], rtol=1e-2, atol=1e-2)
+337
View File
@@ -0,0 +1,337 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reference-vs-optimized unit tests for the MiniMax-M3 AMD/ROCm fused kernels.
Each optimized kernel added for the ROCm port has a slow PyTorch reference; the
tests assert the two agree within tolerance:
* Gemma RMSNorm (plain + fused-add-residual) -> fp32 PyTorch normalize
* SwiGLU-OAI (split layout) -> fp32 PyTorch elementwise
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
makes the shape ``[K//32, N]`` and Triton raises before producing output, so any
regression there fails these tests loudly.
Hardware scope: the whole module is ROCm-only (these are the AMD path; NVIDIA
uses the FlashInfer kernels). The norm/activation/quant kernels run on any ROCm
arch; the native MXFP8 ``dot_scaled`` linear/MoE tests are additionally gated to
CDNA4 gfx95x (``@requires_gfx950``) since gfx942 uses the BF16 emulation path.
Run: pytest tests/kernels/test_minimax_m3_amd_ops.py -v
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("MiniMax-M3 AMD fused ops require ROCm.", allow_module_level=True)
if not torch.cuda.is_available():
pytest.skip("Requires a GPU.", allow_module_level=True)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: E402
_mxfp8_e4m3_quantize_torch,
_mxfp8_e4m3_quantize_triton,
dequant_mxfp8_to_bf16,
)
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
gemma_fused_add_rmsnorm,
gemma_rmsnorm,
swiglu_oai_split,
)
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
DEVICE = "cuda"
EPS = 1e-6
def _gcn_arch() -> str:
try:
return torch.cuda.get_device_properties(0).gcnArchName
except Exception: # pragma: no cover - no device / non-AMD
return ""
# The pure-Triton norm/activation/quant kernels run on any ROCm arch (CDNA3
# gfx942 and CDNA4 gfx950). The native MXFP8 ``dot_scaled`` GEMMs (linear + MoE)
# use CDNA4 hardware microscaling and are gated to gfx95x in the source
# (``RocmDotScaledMxfp8LinearKernel.is_supported``; the MoE oracle routes gfx942
# to the BF16 emulation path instead) — so those tests are gfx950-only.
requires_gfx950 = pytest.mark.skipif(
"gfx95" not in _gcn_arch(),
reason="native MXFP8 dot_scaled is a CDNA4 (gfx95x) feature; "
"gfx942 uses the BF16 emulation path instead.",
)
def _relerr(a: torch.Tensor, b: torch.Tensor) -> float:
a = a.float()
b = b.float()
return ((a - b).norm() / (b.norm() + 1e-8)).item()
# --------------------------------------------------------------------------- #
# Gemma RMSNorm
# --------------------------------------------------------------------------- #
def _ref_gemma_rmsnorm(x, w, eps, residual=None):
orig_dtype = x.dtype
xf = x.float()
res_out = None
if residual is not None:
xf = xf + residual.float()
res_out = xf.to(orig_dtype)
xf = xf * torch.rsqrt(xf.pow(2).mean(dim=-1, keepdim=True) + eps)
xf = xf * (1.0 + w.float())
out = xf.to(orig_dtype)
return out if residual is None else (out, res_out)
@pytest.mark.parametrize("shape", [(1, 4096), (37, 6144), (128, 2048)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("seed", [0, 1234])
@torch.inference_mode()
def test_gemma_rmsnorm(shape, dtype, seed):
torch.manual_seed(seed)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
got = gemma_rmsnorm(x, w, EPS)
ref = _ref_gemma_rmsnorm(x, w, EPS)
assert got.shape == x.shape
assert _relerr(got, ref) < 5e-3
@pytest.mark.parametrize("shape", [(1, 6144), (64, 4096)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_gemma_fused_add_rmsnorm(shape, dtype):
torch.manual_seed(0)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
res = torch.randn(*shape, device=DEVICE, dtype=dtype)
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
got_out, got_res = gemma_fused_add_rmsnorm(x, res, w, EPS)
ref_out, ref_res = _ref_gemma_rmsnorm(x, w, EPS, residual=res)
assert _relerr(got_out, ref_out) < 5e-3
# residual_out is the pre-norm sum (x + res): bit-for-bit identical cast.
assert torch.equal(got_res, ref_res)
@torch.inference_mode()
def test_gemma_rmsnorm_per_head_strided():
"""q_norm/k_norm normalize a non-contiguous ``qkv.split`` slice over head_dim."""
torch.manual_seed(0)
T, H, D, kv = 7, 48, 128, 8
total = (H + 2 * kv) * D
qkv = torch.randn(T, total, device=DEVICE, dtype=torch.bfloat16)
q = qkv[..., : H * D] # non-contiguous view (row stride == total)
q_by_head = q.view(T, H, D)
assert not q_by_head.is_contiguous()
w = torch.randn(D, device=DEVICE, dtype=torch.bfloat16) * 0.1
got = gemma_rmsnorm(q_by_head, w, EPS)
ref = _ref_gemma_rmsnorm(q_by_head, w, EPS)
assert got.shape == q_by_head.shape
assert _relerr(got, ref) < 5e-3
def test_num_warps_monotonic():
assert _num_warps(128) <= _num_warps(2048) <= _num_warps(8192)
# --------------------------------------------------------------------------- #
# SwiGLU-OAI (split layout)
# --------------------------------------------------------------------------- #
def _ref_swiglu(gate_up, alpha, beta, limit):
d = gate_up.shape[-1] // 2
gate = gate_up[..., :d].float()
up = gate_up[..., d:].float()
if limit is not None:
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
return (gate * torch.sigmoid(alpha * gate) * (up + beta)).to(gate_up.dtype)
@pytest.mark.parametrize("m,inter", [(1, 768), (64, 1536), (128, 1024)])
@pytest.mark.parametrize("limit", [7.0, None])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_swiglu_oai_split(m, inter, limit, dtype):
torch.manual_seed(0)
gate_up = torch.randn(m, 2 * inter, device=DEVICE, dtype=dtype)
got = swiglu_oai_split(gate_up, alpha=1.702, beta=1.0, limit=limit)
ref = _ref_swiglu(gate_up, 1.702, 1.0, limit)
assert got.shape == (m, inter)
assert _relerr(got, ref) < 5e-3
# --------------------------------------------------------------------------- #
# Fused MXFP8 activation quant (Triton vs torch reference)
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("shape", [(64, 4096), (1, 6144), (333, 2048)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_mxfp8_quant_triton_matches_torch(shape, dtype):
torch.manual_seed(0)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
# E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
# most a 1-step difference at exact powers of two.
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
# Dequantized values agree to fp8 granularity.
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
deq_k = dequant_mxfp8_to_bf16(xq_k, s_k)
assert _relerr(deq_k, deq_t) < 1e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 linear (dot_scaled) vs dequant-to-bf16 matmul
# --------------------------------------------------------------------------- #
@requires_gfx950
@pytest.mark.parametrize("m,n,k", [(64, 256, 128), (37, 512, 256), (1, 6144, 4096)])
@torch.inference_mode()
def test_mxfp8_native_linear(m, n, k):
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
_mxfp8_dot_scaled_linear,
)
torch.manual_seed(0)
w_bf16 = torch.randn(n, k, device=DEVICE, dtype=torch.bfloat16) * 0.1
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
x = torch.randn(m, k, device=DEVICE, dtype=torch.bfloat16) * 0.5
got = _mxfp8_dot_scaled_linear(x, w_fp8, w_scale)
# Reference: consume the SAME quantized weights (isolates activation-quant
# noise) -> dequant to bf16, plain matmul.
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
ref = torch.nn.functional.linear(x, w_deq).to(x.dtype)
assert got.shape == (m, n)
# Only the activation is re-quantized inside the kernel -> small MX noise.
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 MoE (dot_scaled grouped GEMM) vs dequant-to-bf16 MoE math
# --------------------------------------------------------------------------- #
def _ref_moe(x, w13, w2, topk_weights, topk_ids, alpha, beta, limit):
T, H = x.shape
inter = w2.shape[-1]
top_k = topk_ids.shape[1]
out = torch.zeros(T, H, device=x.device, dtype=torch.float32)
for t in range(T):
for j in range(top_k):
e = int(topk_ids[t, j].item())
g1 = x[t].float() @ w13[e].float().T # [2I]
gate = g1[:inter]
up = g1[inter:]
if limit is not None:
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
act = gate * torch.sigmoid(alpha * gate) * (up + beta)
g2 = act @ w2[e].float().T # [H]
out[t] += topk_weights[t, j].float() * g2
return out.to(x.dtype)
@requires_gfx950
@pytest.mark.parametrize(
"T,H,inter,E,top_k", [(8, 256, 512, 8, 2), (1, 512, 256, 16, 4)]
)
@torch.inference_mode()
def test_mxfp8_native_moe(T, H, inter, E, top_k):
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
fused_moe_mxfp8_native,
)
torch.manual_seed(0)
alpha, beta, limit = 1.702, 1.0, 7.0
w13_bf16 = torch.randn(E, 2 * inter, H, device=DEVICE, dtype=torch.bfloat16) * 0.1
w2_bf16 = torch.randn(E, H, inter, device=DEVICE, dtype=torch.bfloat16) * 0.1
w13_fp8, w13_scale = _mxfp8_e4m3_quantize_torch(
w13_bf16, is_sf_swizzled_layout=False
)
w2_fp8, w2_scale = _mxfp8_e4m3_quantize_torch(w2_bf16, is_sf_swizzled_layout=False)
x = torch.randn(T, H, device=DEVICE, dtype=torch.bfloat16) * 0.5
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
topk_weights = topk_weights.to(torch.float32)
topk_ids = topk_ids.to(torch.int32)
got = fused_moe_mxfp8_native(
x,
w13_fp8,
w13_scale,
w2_fp8,
w2_scale,
topk_weights,
topk_ids,
alpha=alpha,
beta=beta,
limit=limit,
global_num_experts=E,
expert_map=None,
)
# Reference consumes the dequantized weights (same bits the kernel reads).
w13_deq = dequant_mxfp8_to_bf16(w13_fp8, w13_scale)
w2_deq = dequant_mxfp8_to_bf16(w2_fp8, w2_scale)
ref = _ref_moe(x, w13_deq, w2_deq, topk_weights, topk_ids, alpha, beta, limit)
assert got.shape == (T, H)
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# MXFP8 linear emulation: BF16-at-load (default) vs per-step dequant + switch
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("shape", [(512, 2048), (1, 6144)])
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("dequant_at_load", [True, False])
@torch.inference_mode()
def test_mxfp8_linear_emulation_bf16_at_load(
shape, act_dtype, dequant_at_load, monkeypatch
):
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
same result; the dtype-match (BF16/FP16 activations) must also hold."""
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
EmulationMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
monkeypatch.setenv(
"VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "1" if dequant_at_load else "0"
)
N, K = shape
torch.manual_seed(0)
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16)
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
assert w_scale.shape == (N, K // 32)
# Reference: dequant once, plain linear in the activation dtype.
w_ref = dequant_mxfp8_to_bf16(w_fp8, w_scale).to(act_dtype)
x = torch.randn(7, K, device=DEVICE, dtype=act_dtype)
out_ref = torch.nn.functional.linear(x, w_ref)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig())
kernel.process_weights_after_loading(layer)
if dequant_at_load:
# weights converted to BF16 at load (>= 2-byte)
assert layer.weight.element_size() >= 2
else:
# opt-out: weights stay 1-byte MXFP8, dequant happens per-step
assert layer.weight.element_size() == 1
out = kernel.apply_weights(layer, x)
assert out.dtype == act_dtype # dtype-match preserved (no tl.dot/F.linear crash)
assert _relerr(out.float(), out_ref.float()) < 2e-2
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
These exercise the vendored processor directly (no checkpoint / GPU needed), so
they validate the long-side resize spec and the resulting prompt-token counts
deterministically.
"""
import pytest
import torch
from vllm.transformers_utils.processors.minimax_m3 import (
IMAGE_MAX_TOTAL_PIXELS,
MIN_SHORT_SIDE_PIXEL,
VIDEO_MAX_TOTAL_PIXELS,
MiniMaxM3VLImageProcessor,
MiniMaxM3VLVideoProcessor,
smart_resize,
)
# Long sides are multiples of patch_size*merge_size (28) so the rounding is
# exact and the expected token counts are unambiguous.
LONG_SIDES = [252, 504, 1008]
MERGE2 = 2**2 # merge_size ** 2
def _image_tokens(grid_thw) -> int:
g = list(grid_thw)
return int(g[0] * g[1] * g[2]) // MERGE2
# --------------------------------------------------------------------------- #
# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
# --------------------------------------------------------------------------- #
def test_smart_resize_long_side_shrink():
# (a) long side exceeds the cap -> shrink so the long side equals the cap.
h, w = smart_resize(
2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
)
assert max(h, w) == 1008
assert (h, w) == (1008, 504) # aspect ratio preserved
def test_smart_resize_short_side_enlarge():
# (b) long side within the cap but short side below the floor -> enlarge so
# the short side reaches min_short_side_pixel.
h, w = smart_resize(
200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
)
assert min(h, w) == MIN_SHORT_SIDE_PIXEL # 112
def test_smart_resize_total_pixels_raises():
# (c) still over the area cap after resizing -> raise instead of inferring.
with pytest.raises(ValueError, match="max_total_pixels"):
smart_resize(
5000,
5000,
factor=28,
max_long_side_pixel=4000,
max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
)
def test_smart_resize_backward_compatible_area_bound():
# Without max_long_side_pixel the original Qwen-style area bound is used.
assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
# --------------------------------------------------------------------------- #
# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
# --------------------------------------------------------------------------- #
def test_image_tokens_increase_with_max_long_side_pixel():
proc = MiniMaxM3VLImageProcessor()
counts = []
for long_side in LONG_SIDES:
patches = proc.get_number_of_image_patches(
2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
)
counts.append(patches // MERGE2)
assert counts == [81, 324, 1296]
assert counts[0] < counts[1] < counts[2]
def test_image_processor_defaults_match_spec():
proc = MiniMaxM3VLImageProcessor()
assert proc.max_long_side_pixel is None # opt-in
assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
def test_image_preprocess_pipeline_monotonic():
proc = MiniMaxM3VLImageProcessor()
image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
counts = []
for long_side in LONG_SIDES:
out = proc.preprocess(
[image],
do_resize=True,
max_long_side_pixel=long_side,
return_tensors="pt",
)
counts.append(_image_tokens(out["image_grid_thw"][0]))
assert counts == [81, 324, 1296]
# --------------------------------------------------------------------------- #
# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
# --------------------------------------------------------------------------- #
def test_video_tokens_increase_with_max_long_side_pixel():
proc = MiniMaxM3VLVideoProcessor()
assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
counts = []
for long_side in LONG_SIDES:
out = proc.preprocess(
videos=[video],
do_resize=True,
max_long_side_pixel=long_side,
return_tensors="pt",
)
counts.append(_image_tokens(out["video_grid_thw"][0]))
assert counts[0] < counts[1] < counts[2]
def test_video_volumetric_cap_raises():
proc = MiniMaxM3VLVideoProcessor()
# 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
with pytest.raises(ValueError, match="max_total_pixels"):
proc.preprocess(
videos=[video],
do_resize=True,
max_long_side_pixel=1008,
return_tensors="pt",
)
+15
View File
@@ -420,6 +420,11 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"MiniMaxAI/MiniMax-M2",
trust_remote_code=True,
),
"MiniMaxM3SparseForCausalLM": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
trust_remote_code=True,
is_available_online=False,
),
"Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"),
"MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
"MistralLarge3ForCausalLM": _HfExamplesInfo(
@@ -1099,6 +1104,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"MiniMaxAI/MiniMax-VL-01",
trust_remote_code=True,
),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
trust_remote_code=True,
is_available_online=False,
),
"Mistral3ForConditionalGeneration": _HfExamplesInfo(
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
@@ -1601,6 +1611,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="XiaomiMiMo/MiMo-V2.5-Omni",
is_available_online=False,
),
"MiniMaxM3MTP": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
trust_remote_code=True,
is_available_online=False,
),
"NemotronHMTPModel": _HfExamplesInfo(
"nvidia/Nemotron-Super-Placeholder",
speculative_model="nvidia/Nemotron-Super-Placeholder",
@@ -0,0 +1,320 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import string
from collections.abc import Sequence
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.reasoning import ReasoningParserManager
from vllm.reasoning.minimax_m3_reasoning_parser import MiniMaxM3ReasoningParser
pytestmark = pytest.mark.skip_global_cleanup
class MiniMaxM3Tokenizer:
"""Small tokenizer with MiniMax M3 reasoning tags as special tokens."""
special_tokens = ("<mm:think>", "</mm:think>")
def __init__(self):
self._token_to_id: dict[str, int] = {}
self._id_to_token: dict[int, str] = {}
for token in self.special_tokens:
self._add_token(token)
for char in string.printable:
self._add_token(char)
def _add_token(self, token: str) -> int:
token_id = self._token_to_id.get(token)
if token_id is None:
token_id = len(self._token_to_id) + 1
self._token_to_id[token] = token_id
self._id_to_token[token_id] = token
return token_id
def get_vocab(self) -> dict[str, int]:
return dict(self._token_to_id)
def encode(
self,
text: str,
truncation: bool | None = None,
max_length: int | None = None,
add_special_tokens: bool = True,
) -> list[int]:
return [self._add_token(token) for token in self.tokenize(text)]
def decode(
self, ids: Sequence[int] | int, skip_special_tokens: bool = False
) -> str:
if isinstance(ids, int):
ids = [ids]
return "".join(self._id_to_token[token_id] for token_id in ids)
def tokenize(self, text: str) -> list[str]:
tokens: list[str] = []
pos = 0
while pos < len(text):
for special_token in self.special_tokens:
if text.startswith(special_token, pos):
tokens.append(special_token)
pos += len(special_token)
break
else:
tokens.append(text[pos])
pos += 1
return tokens
def convert_ids_to_tokens(
self,
ids: Sequence[int],
skip_special_tokens: bool = False,
) -> list[str]:
return [self._id_to_token[token_id] for token_id in ids]
def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
if isinstance(tokens, str):
return self._add_token(tokens)
return [self._add_token(token) for token in tokens]
def convert_tokens_to_string(self, tokens: list[str]) -> str:
return "".join(tokens)
def make_parser(
chat_template_kwargs: dict[str, str] | None = None,
) -> tuple[MiniMaxM3ReasoningParser, MiniMaxM3Tokenizer]:
tokenizer = MiniMaxM3Tokenizer()
return (
MiniMaxM3ReasoningParser(tokenizer, chat_template_kwargs=chat_template_kwargs),
tokenizer,
)
def run_streaming(
parser: MiniMaxM3ReasoningParser,
tokenizer: MiniMaxM3Tokenizer,
chunks: list[str],
) -> tuple[str | None, str | None, list[bool]]:
previous_text = ""
previous_token_ids: list[int] = []
reasoning_parts: list[str] = []
content_parts: list[str] = []
reasoning_end_states: list[bool] = []
for chunk in chunks:
delta_token_ids = tokenizer.encode(chunk, add_special_tokens=False)
current_text = previous_text + chunk
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_reasoning_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
)
reasoning_end_states.append(
parser.is_reasoning_end_streaming(current_token_ids, delta_token_ids)
)
if delta is not None:
if delta.reasoning is not None:
reasoning_parts.append(delta.reasoning)
if delta.content is not None:
content_parts.append(delta.content)
previous_text = current_text
previous_token_ids = current_token_ids
return (
"".join(reasoning_parts) or None,
"".join(content_parts) or None,
reasoning_end_states,
)
def test_parser_registration():
parser_cls = ReasoningParserManager.get_reasoning_parser("minimax_m3")
assert parser_cls is MiniMaxM3ReasoningParser
def test_nonstreaming_extracts_explicit_reasoning_block():
parser, _ = make_parser()
request = ChatCompletionRequest(messages=[], model="test-model")
reasoning, content = parser.extract_reasoning(
"<mm:think>plan</mm:think>answer", request
)
assert reasoning == "plan"
assert content == "answer"
def test_nonstreaming_without_start_tag_is_content():
parser, _ = make_parser()
request = ChatCompletionRequest(messages=[], model="test-model")
reasoning, content = parser.extract_reasoning("plain answer", request)
assert reasoning is None
assert content == "plain answer"
def test_nonstreaming_drops_leading_end_tag():
parser, _ = make_parser()
request = ChatCompletionRequest(messages=[], model="test-model")
reasoning, content = parser.extract_reasoning("</mm:think>answer", request)
assert reasoning is None
assert content == "answer"
def test_nonstreaming_non_leading_end_tag_is_content():
parser, _ = make_parser()
request = ChatCompletionRequest(messages=[], model="test-model")
reasoning, content = parser.extract_reasoning("XXX</mm:think>YYY", request)
assert reasoning is None
assert content == "XXX</mm:think>YYY"
def test_nonstreaming_enabled_mode_starts_in_reasoning():
parser, _ = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
request = ChatCompletionRequest(messages=[], model="test-model")
reasoning, content = parser.extract_reasoning("plan</mm:think>answer", request)
assert reasoning == "plan"
assert content == "answer"
def test_nonstreaming_open_reasoning_block():
parser, _ = make_parser()
request = ChatCompletionRequest(messages=[], model="test-model")
reasoning, content = parser.extract_reasoning("<mm:think>still thinking", request)
assert reasoning == "still thinking"
assert content is None
def test_streaming_reasoning_tags_are_not_returned():
parser, tokenizer = make_parser()
reasoning, content, end_states = run_streaming(
parser,
tokenizer,
["<mm:think>", "plan", "</mm:think>", "answer"],
)
assert reasoning == "plan"
assert content == "answer"
assert end_states == [False, False, True, True]
def test_streaming_boundary_can_emit_reasoning_and_content():
parser, tokenizer = make_parser()
reasoning, content, end_states = run_streaming(
parser,
tokenizer,
["<mm:think>plan</mm:think>answer"],
)
assert reasoning == "plan"
assert content == "answer"
assert end_states == [True]
def test_streaming_drops_leading_end_tag():
parser, tokenizer = make_parser()
reasoning, content, end_states = run_streaming(
parser,
tokenizer,
["</mm:think>", "answer"],
)
assert reasoning is None
assert content == "answer"
assert end_states == [True, True]
def test_streaming_non_leading_end_tag_is_content():
parser, tokenizer = make_parser()
reasoning, content, end_states = run_streaming(
parser,
tokenizer,
["XXX</mm:think>YYY"],
)
assert reasoning is None
assert content == "XXX</mm:think>YYY"
assert end_states == [True]
def test_streaming_enabled_mode_starts_in_reasoning():
parser, tokenizer = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
reasoning, content, end_states = run_streaming(
parser,
tokenizer,
["plan", "</mm:think>", "answer"],
)
assert reasoning == "plan"
assert content == "answer"
assert end_states == [False, True, True]
def test_streaming_plain_content_ends_reasoning_phase():
parser, tokenizer = make_parser()
reasoning, content, end_states = run_streaming(
parser,
tokenizer,
["plain ", "answer"],
)
assert reasoning is None
assert content == "plain answer"
assert end_states == [True, True]
def test_token_id_helpers():
parser, tokenizer = make_parser()
output_ids = tokenizer.encode(
"<mm:think>abc</mm:think>def", add_special_tokens=False
)
open_reasoning_ids = tokenizer.encode("<mm:think>abc", add_special_tokens=False)
content_ids = tokenizer.encode("plain", add_special_tokens=False)
assert parser.is_reasoning_end(output_ids)
assert not parser.is_reasoning_end(open_reasoning_ids)
assert not parser.is_reasoning_end(content_ids)
assert tokenizer.decode(parser.extract_content_ids(output_ids)) == "def"
assert parser.extract_content_ids(open_reasoning_ids) == []
assert parser.extract_content_ids(content_ids) == content_ids
assert parser.count_reasoning_tokens(output_ids) == len(tokenizer.encode("abc"))
def test_token_id_helpers_enabled_mode():
parser, tokenizer = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
output_ids = tokenizer.encode("abc</mm:think>def", add_special_tokens=False)
open_reasoning_ids = tokenizer.encode("abc", add_special_tokens=False)
assert parser.is_reasoning_end(output_ids)
assert not parser.is_reasoning_end(open_reasoning_ids)
assert tokenizer.decode(parser.extract_content_ids(output_ids)) == "def"
assert parser.extract_content_ids(open_reasoning_ids) == []
assert parser.count_reasoning_tokens(output_ids) == len(tokenizer.encode("abc"))
assert parser.count_reasoning_tokens(open_reasoning_ids) == len(
tokenizer.encode("abc")
)
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
NS = "]<]minimax[>["
EOS_ID = 99
class FakeTokenizer:
"""Minimal fake tokenizer for unit tests."""
def __init__(self):
self.model_tokenizer = True
self.vocab: dict[str, int] = {}
def get_vocab(self) -> dict[str, int]:
return self.vocab
def sample_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="create_order",
parameters={
"type": "object",
"properties": {
"user_id": {"type": "integer"},
"urgent": {"type": "boolean"},
"note": {"type": "string"},
"shipping": {
"type": "object",
"properties": {
"city": {"type": "string"},
"zip": {"type": "integer"},
},
},
"items": {
"type": "array",
"items": {
"type": "object",
"properties": {
"sku": {"type": "string"},
"qty": {"type": "integer"},
},
},
},
"metadata": {
"type": "object",
"additionalProperties": {"type": "string"},
},
"duplicate_demo": {"type": "object"},
},
},
),
)
]
@pytest.fixture
def parser() -> MinimaxM3ToolParser:
return MinimaxM3ToolParser(FakeTokenizer(), tools=sample_tools())
def build_order_call() -> str:
return (
f"{NS}<tool_call>\n"
f'{NS}<invoke name="create_order">'
f"{NS}<user_id>42{NS}</user_id>"
f"{NS}<urgent>true{NS}</urgent>"
f"{NS}<note>Please leave at front desk.{NS}</note>"
f"{NS}<shipping>"
f"{NS}<city>Singapore{NS}</city>"
f"{NS}<zip>018956{NS}</zip>"
f"{NS}</shipping>"
f"{NS}<items>"
f"{NS}<item>{NS}<sku>book-001{NS}</sku>{NS}<qty>2{NS}</qty>{NS}</item>"
f"{NS}<item>{NS}<sku>pen-007{NS}</sku>{NS}<qty>5{NS}</qty>{NS}</item>"
f"{NS}</items>"
f"{NS}<metadata>"
f"{NS}<source>mobile{NS}</source>"
f"{NS}<campaign>may-launch{NS}</campaign>"
f"{NS}</metadata>"
f"{NS}<duplicate_demo>"
f"{NS}<tag>a{NS}</tag>"
f"{NS}<tag>b{NS}</tag>"
f"{NS}</duplicate_demo>"
f"{NS}</invoke>\n"
f"{NS}</tool_call>"
)
def build_order_invocation(user_id: int) -> str:
return (
f'{NS}<invoke name="create_order">'
f"{NS}<user_id>{user_id}{NS}</user_id>"
f"{NS}</invoke>"
)
def build_multiple_order_call() -> str:
return (
f"{NS}<tool_call>\n"
f"{build_order_invocation(1)}\n"
f"{build_order_invocation(2)}\n"
f"{NS}</tool_call>"
)
def _feed(
parser: MinimaxM3ToolParser, chunks: list[str | tuple[str, list[int]]]
) -> list[DeltaMessage]:
previous = ""
results: list[DeltaMessage] = []
for chunk in chunks:
if isinstance(chunk, tuple):
delta, delta_ids = chunk
else:
delta = chunk
delta_ids = []
current = previous + delta
result = parser.extract_tool_calls_streaming(
previous_text=previous,
current_text=current,
delta_text=delta,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=delta_ids,
request=None,
)
if result is not None:
results.append(result)
previous = current
return results
def _collect_content(results: list[DeltaMessage]) -> str:
return "".join(result.content for result in results if result.content)
def _collect_tool_calls(results: list[DeltaMessage]) -> dict[int, dict[str, Any]]:
tool_calls: dict[int, dict[str, Any]] = {}
for result in results:
for tool_call in result.tool_calls or []:
tool_calls.setdefault(
tool_call.index,
{"id": None, "name": "", "arguments": ""},
)
if tool_call.id:
tool_calls[tool_call.index]["id"] = tool_call.id
if tool_call.function:
if tool_call.function.name:
tool_calls[tool_call.index]["name"] += tool_call.function.name
if tool_call.function.arguments:
tool_calls[tool_call.index]["arguments"] += (
tool_call.function.arguments
)
return tool_calls
def test_minimax_m3_parser_registered():
assert ToolParserManager.get_tool_parser("minimax_m3") is MinimaxM3ToolParser
def test_non_streaming_nested_tool_call(parser):
result = parser.extract_tool_calls(
"I will create it.\n" + build_order_call(),
request=None,
)
assert result.tools_called
assert result.content == "I will create it.\n"
assert len(result.tool_calls) == 1
tool_call = result.tool_calls[0]
assert tool_call.function.name == "create_order"
assert json.loads(tool_call.function.arguments) == {
"user_id": 42,
"urgent": True,
"note": "Please leave at front desk.",
"shipping": {"city": "Singapore", "zip": 18956},
"items": [
{"sku": "book-001", "qty": 2},
{"sku": "pen-007", "qty": 5},
],
"metadata": {
"source": "mobile",
"campaign": "may-launch",
},
"duplicate_demo": {"tag": ["a", "b"]},
}
def test_non_streaming_without_tool_call_keeps_content(parser):
result = parser.extract_tool_calls("plain response", request=None)
assert not result.tools_called
assert result.tool_calls == []
assert result.content == "plain response"
def test_non_streaming_multiple_tool_calls(parser):
result = parser.extract_tool_calls(build_multiple_order_call(), request=None)
assert result.tools_called
assert result.content is None
assert [tool_call.function.name for tool_call in result.tool_calls] == [
"create_order",
"create_order",
]
assert [
json.loads(tool_call.function.arguments)["user_id"]
for tool_call in result.tool_calls
] == [1, 2]
def test_streaming_without_tool_call_emits_text(parser):
results = _feed(parser, ["plain ", "response"])
assert _collect_content(results) == "plain response"
assert _collect_tool_calls(results) == {}
def test_streaming_nested_tool_call(parser):
tool_call_text = build_order_call()
results = _feed(
parser,
[
"I will create it.\n",
tool_call_text[:5],
tool_call_text[5:17],
tool_call_text[17:120],
tool_call_text[120:],
("", [EOS_ID]),
],
)
assert _collect_content(results) == "I will create it.\n"
tool_calls = _collect_tool_calls(results)
assert len(tool_calls) == 1
assert tool_calls[0]["name"] == "create_order"
assert tool_calls[0]["id"] is not None
assert json.loads(tool_calls[0]["arguments"]) == json.loads(
parser.streamed_args_for_tool[0]
)
assert json.loads(parser.prev_tool_call_arr[0]["arguments"])["items"][1]["qty"] == 5
assert results[-1].content is None