[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -0,0 +1,854 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for MiniMax M3 sparse prefill attention kernels."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.models.minimax_m3.common.indexer import (
|
||||
MiniMaxM3IndexerBackend,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.ops.index_topk import (
|
||||
minimax_m3_index_decode,
|
||||
minimax_m3_index_score,
|
||||
minimax_m3_index_topk,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.ops.sparse_attn import (
|
||||
minimax_m3_sparse_attn,
|
||||
minimax_m3_sparse_attn_decode,
|
||||
)
|
||||
from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import set_kv_cache_layout
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
pytest.skip(
|
||||
"MiniMax M3 attention kernels require CUDA or ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def kv_layout(request):
|
||||
"""Set the global KV cache layout for one test and restore it after."""
|
||||
set_kv_cache_layout(request.param)
|
||||
try:
|
||||
yield request.param
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
|
||||
def _stride_order_for(backend: type[MiniMaxM3SparseBackend], ndim: int) -> tuple:
|
||||
"""Mirror the allocator's stride-order resolution (identity fallback)."""
|
||||
try:
|
||||
stride_order = backend.get_kv_cache_stride_order()
|
||||
assert len(stride_order) == ndim
|
||||
except (AttributeError, NotImplementedError):
|
||||
stride_order = tuple(range(ndim))
|
||||
return stride_order
|
||||
|
||||
|
||||
def _allocate_main_kv_via_contract(
|
||||
num_pages: int, device: torch.device | str = "cuda"
|
||||
) -> torch.Tensor:
|
||||
"""Build the main KV cache exactly as the production allocator does for the
|
||||
currently active layout: allocate the physical (permuted) tensor, then
|
||||
expose the inverse-permuted logical-NHD view the backend sees."""
|
||||
logical_shape = MiniMaxM3SparseBackend.get_kv_cache_shape(
|
||||
num_pages, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
)
|
||||
stride_order = _stride_order_for(MiniMaxM3SparseBackend, len(logical_shape))
|
||||
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
raw = torch.randn(physical_shape, device=device, dtype=DTYPE)
|
||||
return raw.permute(*inv_order)
|
||||
|
||||
|
||||
NUM_Q_HEADS = 32
|
||||
NUM_KV_HEADS = 2
|
||||
HEAD_DIM = 128
|
||||
BLOCK_SIZE = 128
|
||||
DTYPE = torch.bfloat16
|
||||
SM_SCALE = HEAD_DIM**-0.5
|
||||
TOPK = 16
|
||||
|
||||
|
||||
# Index top-k kernels.
|
||||
def _reference_index_topk(
|
||||
idx_q: torch.Tensor,
|
||||
index_kv_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
q_lens: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
prefix_lens: torch.Tensor,
|
||||
topk: int,
|
||||
init_blocks: int,
|
||||
local_blocks: int,
|
||||
sm_scale: float,
|
||||
) -> torch.Tensor:
|
||||
total_q, num_idx_heads, _ = idx_q.shape
|
||||
out = torch.full(
|
||||
(num_idx_heads, total_q, topk), -1, device=idx_q.device, dtype=torch.int32
|
||||
)
|
||||
|
||||
q_start = 0
|
||||
for req_id, (q_len, seq_len, prefix_len) in enumerate(
|
||||
zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
|
||||
):
|
||||
q_end = q_start + q_len
|
||||
q = idx_q[q_start:q_end]
|
||||
num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
pages = block_table[req_id, :num_blocks]
|
||||
k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
|
||||
score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
|
||||
|
||||
q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
|
||||
k_pos = torch.arange(k.shape[0], device=idx_q.device)
|
||||
score.masked_fill_(k_pos[None, :] > q_pos[:, None], -float("inf"))
|
||||
score = score.reshape(num_idx_heads, q_len, num_blocks, BLOCK_SIZE)
|
||||
score_tensor = score.max(dim=3).values
|
||||
|
||||
valid_blocks = (q_pos + BLOCK_SIZE) // BLOCK_SIZE
|
||||
for local_q, num_valid_blocks in enumerate(valid_blocks.tolist()):
|
||||
end = min(init_blocks, num_valid_blocks)
|
||||
score_tensor[:, local_q, :end] = 1e30
|
||||
start = max(0, num_valid_blocks - local_blocks)
|
||||
score_tensor[:, local_q, start:num_valid_blocks] = 1e29
|
||||
|
||||
k = min(topk, num_valid_blocks)
|
||||
topk_idx = score_tensor[:, local_q].topk(k, dim=1).indices
|
||||
out[:, q_start + local_q, :k] = topk_idx
|
||||
q_start = q_end
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def _assert_topk_indices_equal_unordered(
|
||||
actual: torch.Tensor,
|
||||
expected: torch.Tensor,
|
||||
) -> None:
|
||||
"""Compare selected sparse blocks without requiring a deterministic order."""
|
||||
assert actual.shape == expected.shape
|
||||
actual_flat = actual.cpu().reshape(-1, actual.shape[-1]).tolist()
|
||||
expected_flat = expected.cpu().reshape(-1, expected.shape[-1]).tolist()
|
||||
for actual_row, expected_row in zip(actual_flat, expected_flat):
|
||||
assert set(actual_row) == set(expected_row)
|
||||
|
||||
|
||||
def test_prefill_index_topk_correctness():
|
||||
topk = 6
|
||||
init_blocks = 0
|
||||
local_blocks = 1
|
||||
num_idx_heads = 2
|
||||
head_dim = 16
|
||||
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
|
||||
prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
|
||||
seq_lens = prefix_lens + q_lens
|
||||
batch = q_lens.numel()
|
||||
max_seq_len = seq_lens.max().item()
|
||||
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
num_pages = batch * max_blocks
|
||||
|
||||
cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
|
||||
cu_seqlens[1:] = q_lens.cumsum(0)
|
||||
block_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
|
||||
batch, max_blocks
|
||||
)
|
||||
idx_q = torch.ones(q_lens.sum().item(), num_idx_heads, head_dim, device="cuda")
|
||||
index_kv_cache = torch.empty(num_pages, BLOCK_SIZE, head_dim, device="cuda")
|
||||
for req_id in range(batch):
|
||||
for block_id in range(max_blocks):
|
||||
page = block_table[req_id, block_id]
|
||||
index_kv_cache[page].fill_(block_id + 1)
|
||||
|
||||
score = minimax_m3_index_score(
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
max_query_len=q_lens.max().item(),
|
||||
max_seq_len=max_seq_len,
|
||||
num_kv_heads=num_idx_heads,
|
||||
sm_scale=head_dim**-0.5,
|
||||
)
|
||||
actual = minimax_m3_index_topk(
|
||||
score,
|
||||
cu_seqlens,
|
||||
prefix_lens,
|
||||
max_query_len=q_lens.max().item(),
|
||||
topk=topk,
|
||||
init_blocks=init_blocks,
|
||||
local_blocks=local_blocks,
|
||||
)
|
||||
expected = _reference_index_topk(
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
block_table,
|
||||
q_lens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
topk,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
head_dim**-0.5,
|
||||
)
|
||||
_assert_topk_indices_equal_unordered(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_query_len", [1, 4])
|
||||
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
|
||||
def test_decode_index_topk_correctness(
|
||||
decode_query_len: int,
|
||||
num_padded_reqs: int,
|
||||
):
|
||||
topk = 6
|
||||
init_blocks = 0
|
||||
local_blocks = 1
|
||||
num_idx_heads = 2
|
||||
head_dim = 16
|
||||
active_seq_lens = torch.tensor((7, 129, 1025), device="cuda", dtype=torch.int32)
|
||||
q_lens = torch.full_like(active_seq_lens, decode_query_len)
|
||||
prefix_lens = active_seq_lens - decode_query_len
|
||||
active_batch = active_seq_lens.numel()
|
||||
batch = active_batch + num_padded_reqs
|
||||
seq_lens = torch.cat(
|
||||
[
|
||||
active_seq_lens,
|
||||
torch.zeros(num_padded_reqs, device="cuda", dtype=torch.int32),
|
||||
]
|
||||
)
|
||||
max_seq_len = active_seq_lens.max().item()
|
||||
max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
|
||||
num_pages = active_batch * max_blocks
|
||||
|
||||
active_block_table = torch.randperm(
|
||||
num_pages, device="cuda", dtype=torch.int32
|
||||
).reshape(active_batch, max_blocks)
|
||||
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
|
||||
block_table[:active_batch] = active_block_table
|
||||
idx_q = torch.randn(
|
||||
batch * decode_query_len, num_idx_heads, head_dim, device="cuda"
|
||||
)
|
||||
index_kv_cache = torch.randn(num_pages, BLOCK_SIZE, head_dim, device="cuda")
|
||||
|
||||
actual = minimax_m3_index_decode(
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
block_table,
|
||||
seq_lens,
|
||||
max_seq_len=max_seq_len,
|
||||
topk=topk,
|
||||
init_blocks=init_blocks,
|
||||
local_blocks=local_blocks,
|
||||
num_kv_heads=num_idx_heads,
|
||||
sm_scale=head_dim**-0.5,
|
||||
decode_query_len=decode_query_len,
|
||||
)
|
||||
expected = torch.full_like(actual, -1)
|
||||
active_tokens = active_batch * decode_query_len
|
||||
expected[:, :active_tokens] = _reference_index_topk(
|
||||
idx_q[:active_tokens],
|
||||
index_kv_cache,
|
||||
block_table[:active_batch],
|
||||
q_lens,
|
||||
active_seq_lens,
|
||||
prefix_lens,
|
||||
topk,
|
||||
init_blocks,
|
||||
local_blocks,
|
||||
head_dim**-0.5,
|
||||
)
|
||||
_assert_topk_indices_equal_unordered(actual, expected)
|
||||
|
||||
|
||||
# Sparse attention kernels.
|
||||
def _reference_sparse_attn(
|
||||
q: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
topk_idx: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
q_lens: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
prefix_lens: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
out = torch.empty_like(q, dtype=torch.float32)
|
||||
gqa_group_size = NUM_Q_HEADS // NUM_KV_HEADS
|
||||
q_start = 0
|
||||
for req_id, (q_len, seq_len, prefix_len) in enumerate(
|
||||
zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
|
||||
):
|
||||
q_end = q_start + q_len
|
||||
q_req = q[q_start:q_end]
|
||||
positions = torch.arange(seq_len, device="cuda")
|
||||
pages = block_table[req_id, positions // BLOCK_SIZE]
|
||||
rows = positions % BLOCK_SIZE
|
||||
k_req = kv_cache[pages, 0, rows]
|
||||
v_req = kv_cache[pages, 1, rows].float()
|
||||
|
||||
q_pos = prefix_len + torch.arange(q_len, device="cuda")
|
||||
key_blocks = positions // BLOCK_SIZE
|
||||
causal_mask = positions.unsqueeze(0) <= q_pos.unsqueeze(1)
|
||||
|
||||
for kv_head in range(NUM_KV_HEADS):
|
||||
selected = topk_idx[kv_head, q_start:q_end]
|
||||
selected_mask = (key_blocks[None, :, None] == selected[:, None, :]).any(-1)
|
||||
mask = causal_mask & selected_mask
|
||||
head_start = kv_head * gqa_group_size
|
||||
head_end = head_start + gqa_group_size
|
||||
|
||||
q_heads = q_req[:, head_start:head_end].transpose(0, 1)
|
||||
k_head = k_req[:, kv_head].T.expand(gqa_group_size, -1, -1)
|
||||
scores = torch.bmm(q_heads, k_head, out_dtype=torch.float32)
|
||||
scores = scores.transpose(0, 1) * SM_SCALE
|
||||
probs = torch.softmax(
|
||||
scores.masked_fill(~mask[:, None, :], -float("inf")), -1
|
||||
)
|
||||
out[q_start:q_end, head_start:head_end] = torch.einsum(
|
||||
"qhk,kd->qhd", probs, v_req[:, kv_head]
|
||||
)
|
||||
q_start += q_len
|
||||
return out.to(q.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
@pytest.mark.parametrize(
|
||||
("q_lens", "kv_lens"),
|
||||
[
|
||||
((129, 257), (129, 257)),
|
||||
((65, 129, 257), (129, 257, 385)),
|
||||
],
|
||||
)
|
||||
def test_prefill_sparse_attention_correctness(
|
||||
kv_layout: str,
|
||||
q_lens: tuple[int, ...],
|
||||
kv_lens: tuple[int, ...],
|
||||
):
|
||||
assert len(q_lens) == len(kv_lens)
|
||||
assert all(kv_len >= q_len for q_len, kv_len in zip(q_lens, kv_lens))
|
||||
|
||||
# Build paged-KV metadata, including a non-identity page order.
|
||||
batch = len(q_lens)
|
||||
pages_per_req = [(kv_len + BLOCK_SIZE - 1) // BLOCK_SIZE for kv_len in kv_lens]
|
||||
max_blocks = max(pages_per_req)
|
||||
num_pages = sum(pages_per_req)
|
||||
physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
|
||||
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
|
||||
base_page = 0
|
||||
for req_id, num_req_pages in enumerate(pages_per_req):
|
||||
block_table[req_id, :num_req_pages] = physical_pages[
|
||||
base_page : base_page + num_req_pages
|
||||
]
|
||||
base_page += num_req_pages
|
||||
|
||||
q_lens_t = torch.tensor(q_lens, device="cuda", dtype=torch.int32)
|
||||
seq_lens = torch.tensor(kv_lens, device="cuda", dtype=torch.int32)
|
||||
prefix_lens = seq_lens - q_lens_t
|
||||
cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
|
||||
cu_seqlens[1:] = q_lens_t.cumsum(0)
|
||||
total_q = sum(q_lens)
|
||||
max_seqlen_q = max(q_lens)
|
||||
|
||||
q_shape = (total_q, NUM_Q_HEADS, HEAD_DIM)
|
||||
q = torch.randn(q_shape, device="cuda", dtype=DTYPE)
|
||||
# Allocate the main KV cache through the backend layout contract so the
|
||||
# physical storage matches the active layout (contiguous NHD or strided
|
||||
# HND), while the kernels and reference see the logical-NHD view.
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
|
||||
# Build sparse block indices with the same contract as the real M3 indexer:
|
||||
# one forced local block, then score-selected older causal blocks.
|
||||
topk_shape = (NUM_KV_HEADS, total_q, TOPK)
|
||||
topk_idx = torch.full(topk_shape, -1, device="cuda", dtype=torch.int32)
|
||||
q_start = 0
|
||||
for q_len, prefix_len in zip(q_lens_t.tolist(), prefix_lens.tolist()):
|
||||
for local_q in range(q_len):
|
||||
current_block = (prefix_len + local_q) // BLOCK_SIZE
|
||||
older_blocks = torch.randperm(
|
||||
current_block, device="cuda", dtype=torch.int32
|
||||
)
|
||||
selected = torch.cat(
|
||||
[
|
||||
torch.tensor([current_block], device="cuda", dtype=torch.int32),
|
||||
older_blocks[: TOPK - 1],
|
||||
]
|
||||
)
|
||||
topk_idx[:, q_start + local_q, : selected.numel()] = selected
|
||||
q_start += q_len
|
||||
|
||||
actual = torch.empty_like(q)
|
||||
minimax_m3_sparse_attn(
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
block_table,
|
||||
cu_seqlens,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
max_seqlen_q,
|
||||
NUM_KV_HEADS,
|
||||
SM_SCALE,
|
||||
actual,
|
||||
)
|
||||
|
||||
expected = _reference_sparse_attn(
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
block_table,
|
||||
q_lens_t,
|
||||
seq_lens,
|
||||
prefix_lens,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
error = (actual.float() - expected.float()).abs()
|
||||
assert error.mean().item() < 2.5e-4
|
||||
assert error.max().item() < 1.7e-2
|
||||
|
||||
|
||||
def test_main_backend_layout_contract():
|
||||
"""The main sparse backend exposes the logical-NHD shape and the
|
||||
flash_attn-style stride order for each layout."""
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
assert logical == (nb, 2, bs, h, d)
|
||||
# The old HND-ordered shape is no longer the logical shape.
|
||||
assert logical != (nb, 2, h, bs, d)
|
||||
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4)
|
||||
set_kv_cache_layout("NHD")
|
||||
assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4)
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
for layout in ("NHD", "HND"):
|
||||
try:
|
||||
set_kv_cache_layout(layout)
|
||||
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
# Valid permutation: no duplicates, covers every axis.
|
||||
assert set(order) == set(range(len(order)))
|
||||
|
||||
# M3 has no cross-layer KV blocks.
|
||||
with pytest.raises(NotImplementedError):
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=True
|
||||
)
|
||||
|
||||
|
||||
def test_main_backend_unknown_layout_raises(monkeypatch):
|
||||
"""An unrecognized layout (injected past env-var validation) is rejected."""
|
||||
import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
|
||||
|
||||
monkeypatch.setattr(sparse_attn_mod, "get_kv_cache_layout", lambda: "BOGUS")
|
||||
with pytest.raises(ValueError, match="Unknown cache layout format"):
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
|
||||
|
||||
def test_indexer_backend_stride_order_is_identity():
|
||||
"""The 3-dim indexer cache must not inherit the parent's 5-element stride
|
||||
order; it overrides to the 3-element identity so the allocator keeps the
|
||||
contiguous layout."""
|
||||
assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
|
||||
|
||||
# Cross-layer (per-layer-stacked) KV blocks are not supported.
|
||||
with pytest.raises(NotImplementedError):
|
||||
MiniMaxM3IndexerBackend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=True
|
||||
)
|
||||
|
||||
# The stride order matches the 3-dim indexer shape rank.
|
||||
indexer_shape = MiniMaxM3IndexerBackend.get_kv_cache_shape(
|
||||
5, BLOCK_SIZE, 1, HEAD_DIM
|
||||
)
|
||||
assert len(indexer_shape) == 3
|
||||
assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
|
||||
|
||||
|
||||
def test_hnd_allocation_is_byte_identical_to_transpose():
|
||||
"""Under HND the backend-visible logical view is byte-identical to the
|
||||
pre-change allocate-HND-then-transpose(2, 3) workaround."""
|
||||
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
physical_shape = tuple(logical[i] for i in stride_order)
|
||||
# The physical (permuted) shape equals the old hardcoded HND shape.
|
||||
assert physical_shape == (nb, 2, h, bs, d)
|
||||
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
|
||||
view = raw.permute(*inv_order)
|
||||
expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3)
|
||||
|
||||
assert view.shape == expected.shape
|
||||
assert view.stride() == expected.stride()
|
||||
assert view.storage_offset() == expected.storage_offset()
|
||||
|
||||
# Negative: the identity (wrong) stride order under HND does not reproduce
|
||||
# the transpose view.
|
||||
wrong_view = raw.view(logical)
|
||||
assert wrong_view.stride() != expected.stride()
|
||||
|
||||
|
||||
def test_main_cache_is_block_first_and_unpadded():
|
||||
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
||||
is used for the main GQA cache: its spec is unpadded and the physical
|
||||
layout keeps num_blocks as the first dimension under both layouts."""
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
||||
assert spec.page_size_padded is None
|
||||
|
||||
logical = MiniMaxM3SparseBackend.get_kv_cache_shape(
|
||||
4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
)
|
||||
for layout in ("NHD", "HND"):
|
||||
try:
|
||||
set_kv_cache_layout(layout)
|
||||
order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
inv_order = [order.index(i) for i in range(len(order))]
|
||||
# Physical first dim is num_blocks (block-first); required by the
|
||||
# padded-strided branch's block-first assumption if it were ever taken.
|
||||
assert inv_order[0] == 0
|
||||
assert logical[order[0]] == logical[0]
|
||||
|
||||
|
||||
def _build_decode_inputs(
|
||||
seq_lens_list: tuple[int, ...],
|
||||
decode_query_len: int = 1,
|
||||
num_padded_reqs: int = 0,
|
||||
):
|
||||
"""Shared decode setup: uniform query tokens per request, a non-identity
|
||||
block table, and topk indices selecting the current block plus older causal
|
||||
blocks for each query token."""
|
||||
active_batch = len(seq_lens_list)
|
||||
batch = active_batch + num_padded_reqs
|
||||
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
|
||||
max_blocks = max(pages_per_req)
|
||||
num_pages = sum(pages_per_req)
|
||||
physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
|
||||
block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
|
||||
base_page = 0
|
||||
for req_id, num_req_pages in enumerate(pages_per_req):
|
||||
block_table[req_id, :num_req_pages] = physical_pages[
|
||||
base_page : base_page + num_req_pages
|
||||
]
|
||||
base_page += num_req_pages
|
||||
|
||||
seq_lens = torch.tensor(
|
||||
(*seq_lens_list, *([0] * num_padded_reqs)),
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
q = torch.randn(
|
||||
batch * decode_query_len, NUM_Q_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE
|
||||
)
|
||||
|
||||
topk_idx = torch.full(
|
||||
(NUM_KV_HEADS, batch * decode_query_len, TOPK),
|
||||
-1,
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
token_id = 0
|
||||
for req_id, seq_len in enumerate(seq_lens_list):
|
||||
for local_q in range(decode_query_len):
|
||||
query_pos = seq_len - decode_query_len + local_q
|
||||
current_block = query_pos // BLOCK_SIZE
|
||||
older_blocks = torch.randperm(
|
||||
current_block, device="cuda", dtype=torch.int32
|
||||
)
|
||||
selected = torch.cat(
|
||||
[
|
||||
torch.tensor([current_block], device="cuda", dtype=torch.int32),
|
||||
older_blocks[: TOPK - 1],
|
||||
]
|
||||
)
|
||||
topk_idx[:, token_id, : selected.numel()] = selected
|
||||
token_id += 1
|
||||
|
||||
return q, block_table, seq_lens, topk_idx, num_pages
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens_list",
|
||||
[(130, 257), (129, 200, 384)],
|
||||
)
|
||||
@pytest.mark.parametrize("decode_query_len", [1, 4])
|
||||
@pytest.mark.parametrize("num_padded_reqs", [0, 2])
|
||||
def test_decode_sparse_attention_correctness(
|
||||
kv_layout: str,
|
||||
seq_lens_list: tuple[int, ...],
|
||||
decode_query_len: int,
|
||||
num_padded_reqs: int,
|
||||
):
|
||||
"""Decode (split-K) parity under both layouts: this is the only coverage of
|
||||
the decode-site cache feed, and the strided HND case fails if the kernel
|
||||
ignores the cache strides."""
|
||||
torch.manual_seed(0)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
||||
seq_lens_list, decode_query_len, num_padded_reqs
|
||||
)
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
|
||||
actual = torch.empty_like(q)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
SM_SCALE,
|
||||
actual,
|
||||
decode_query_len,
|
||||
)
|
||||
|
||||
# Reuse the prefill reference: decode is a uniform query chunk ending at
|
||||
# seq_len - 1 for each request.
|
||||
active_batch = len(seq_lens_list)
|
||||
active_tokens = active_batch * decode_query_len
|
||||
q_lens_t = torch.full(
|
||||
(len(seq_lens_list),), decode_query_len, device="cuda", dtype=torch.int32
|
||||
)
|
||||
active_seq_lens = seq_lens[:active_batch]
|
||||
prefix_lens = active_seq_lens - q_lens_t
|
||||
expected = _reference_sparse_attn(
|
||||
q[:active_tokens],
|
||||
kv_cache,
|
||||
topk_idx[:, :active_tokens],
|
||||
block_table[:active_batch],
|
||||
q_lens_t,
|
||||
active_seq_lens,
|
||||
prefix_lens,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
error = (actual[:active_tokens].float() - expected.float()).abs()
|
||||
assert error.mean().item() < 2.5e-4
|
||||
assert error.max().item() < 1.7e-2
|
||||
|
||||
|
||||
def test_decode_wrong_layout_breaks_parity():
|
||||
"""Negative (AC-3/AC-5): consuming the physical HND buffer as if it were
|
||||
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
||||
reorders the K/V content, so the decode output no longer matches the
|
||||
reference computed on the correct logical view. The mislabeled tensor keeps
|
||||
the same shape as the correct view, so the kernel stays in bounds."""
|
||||
torch.manual_seed(0)
|
||||
seq_lens_list = (130, 257)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
||||
|
||||
# Physical HND storage [blocks, 2, heads, block, dim].
|
||||
phys = torch.randn(
|
||||
(num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE
|
||||
)
|
||||
# Correct logical-NHD view (strided) vs. the same bytes mislabeled as a
|
||||
# contiguous-NHD cache — same shape, different content mapping.
|
||||
correct = phys.permute(0, 1, 3, 2, 4)
|
||||
wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM)
|
||||
|
||||
q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32)
|
||||
prefix_lens = seq_lens - q_lens_t
|
||||
expected = _reference_sparse_attn(
|
||||
q, correct, topk_idx, block_table, q_lens_t, seq_lens, prefix_lens
|
||||
)
|
||||
|
||||
actual = torch.empty_like(q)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q, wrong, topk_idx, block_table, seq_lens, NUM_KV_HEADS, SM_SCALE, actual, 1
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
assert (actual.float() - expected.float()).abs().max().item() > 1.7e-2
|
||||
|
||||
|
||||
def _make_attn_group(backend, spec):
|
||||
return AttentionGroup(
|
||||
backend=backend,
|
||||
layer_names=["main"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
|
||||
|
||||
def test_main_cache_byte_identical_through_production_allocator():
|
||||
"""AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
|
||||
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
||||
same shape, stride, and storage offset as the pre-change
|
||||
allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates
|
||||
through the same path to its 3-dim shape."""
|
||||
nb = 4
|
||||
spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||
group = _make_attn_group(MiniMaxM3SparseBackend, spec)
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
kv_caches = _reshape_kv_cache([group], {"main": raw}, "auto", [BLOCK_SIZE], {})
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
view = kv_caches["main"]
|
||||
|
||||
oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM))
|
||||
oracle = oracle.transpose(2, 3)
|
||||
assert tuple(view.shape) == tuple(oracle.shape)
|
||||
assert view.stride() == oracle.stride()
|
||||
assert view.storage_offset() == oracle.storage_offset()
|
||||
|
||||
# Indexer cache allocates through the same path under both layouts.
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
)
|
||||
for layout in ("NHD", "HND"):
|
||||
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
||||
igroup = AttentionGroup(
|
||||
backend=MiniMaxM3IndexerBackend,
|
||||
layer_names=["idx"],
|
||||
kv_cache_spec=ispec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
try:
|
||||
set_kv_cache_layout(layout)
|
||||
iout = _reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
assert tuple(iout["idx"].shape) == (nb, BLOCK_SIZE, HEAD_DIM)
|
||||
|
||||
|
||||
def test_indexer_inherited_stride_order_trips_allocator_assert():
|
||||
"""AC-4 negative: without the indexer override, the inherited 5-element
|
||||
stride order trips the allocator's `len(stride_order) == len(shape)` assert
|
||||
for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
|
||||
allocator's `(AttributeError, NotImplementedError)` fallback."""
|
||||
|
||||
class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
|
||||
# Simulate inheriting the parent's 5-element stride order.
|
||||
get_kv_cache_stride_order = staticmethod(
|
||||
MiniMaxM3SparseBackend.get_kv_cache_stride_order
|
||||
)
|
||||
|
||||
nb = 4
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
)
|
||||
iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
|
||||
igroup = AttentionGroup(
|
||||
backend=_BrokenIndexerBackend,
|
||||
layer_names=["idx"],
|
||||
kv_cache_spec=ispec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
with pytest.raises(AssertionError):
|
||||
_reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
|
||||
def test_padded_main_cache_is_flagged():
|
||||
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
||||
contiguous-view branch (`page_size_padded is None`). A spec that sets
|
||||
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
|
||||
|
||||
def _require_unpadded_block_first(spec, stride_order):
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
assert spec.page_size_padded is None, (
|
||||
"main GQA cache must be unpadded to use the contiguous-view "
|
||||
"allocator branch"
|
||||
)
|
||||
assert inv_order[0] == 0, "main GQA cache must remain block-first"
|
||||
|
||||
try:
|
||||
set_kv_cache_layout("HND")
|
||||
stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
|
||||
finally:
|
||||
set_kv_cache_layout(None)
|
||||
|
||||
good = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
)
|
||||
_require_unpadded_block_first(good, stride_order) # passes
|
||||
|
||||
padded = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=NUM_KV_HEADS,
|
||||
head_size=HEAD_DIM,
|
||||
head_size_v=HEAD_DIM,
|
||||
dtype=DTYPE,
|
||||
page_size_padded=good.page_size_bytes + 128,
|
||||
)
|
||||
with pytest.raises(AssertionError):
|
||||
_require_unpadded_block_first(padded, stride_order)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
def test_reshape_and_cache_flash_write_persists(kv_layout: str):
|
||||
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
||||
`self.kv_cache.unbind(1)` directly. Writing through those views must persist
|
||||
into the bound storage (read back through an independent logical view) under
|
||||
both layouts — a `.contiguous()` copy of the unbind slice would leave the
|
||||
bound storage unchanged."""
|
||||
torch.manual_seed(0)
|
||||
num_pages = 4
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages)
|
||||
with torch.no_grad():
|
||||
kv_cache.zero_()
|
||||
|
||||
# Exactly the production write-site code under test.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
|
||||
num_tokens = 12
|
||||
slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[
|
||||
:num_tokens
|
||||
].to(torch.int64)
|
||||
key = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
|
||||
value = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
|
||||
scale = torch.ones((), device="cuda")
|
||||
ops.reshape_and_cache_flash(
|
||||
key, value, key_cache, value_cache, slot_mapping, "auto", scale, scale
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# Read back through the independent logical view; proves the writes landed
|
||||
# in the engine-bound storage, not a detached copy.
|
||||
for t in range(num_tokens):
|
||||
slot = int(slot_mapping[t].item())
|
||||
blk, intra = divmod(slot, BLOCK_SIZE)
|
||||
torch.testing.assert_close(kv_cache[blk, 0, intra], key[t])
|
||||
torch.testing.assert_close(kv_cache[blk, 1, intra], value[t])
|
||||
@@ -0,0 +1,109 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the manual AllReduce + GemmaRMSNorm fusion used by MiniMax M3.
|
||||
|
||||
``fused_allreduce_gemma_rms_norm`` must match the unfused model path, i.e.
|
||||
``GemmaRMSNorm(all_reduce(partial), residual)``, both on the flashinfer fast
|
||||
path (TP>1 with flashinfer + NVSwitch) and on the eager fallback (TP==1, or when
|
||||
flashinfer is unavailable / the GPU has no NVSwitch).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch.multiprocessing import spawn
|
||||
|
||||
from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
|
||||
fused_allreduce_gemma_rms_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
|
||||
@ensure_current_vllm_config()
|
||||
def _worker_fused_ar_norm(
|
||||
local_rank,
|
||||
world_size,
|
||||
port,
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype,
|
||||
seed,
|
||||
eps,
|
||||
):
|
||||
"""Per-rank worker: compare the fused helper vs all_reduce + GemmaRMSNorm."""
|
||||
device = torch.device(f"cuda:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(
|
||||
world_size, 1, local_rank, port, local_rank=local_rank
|
||||
)
|
||||
|
||||
# Norm weights are identical across ranks (replicated GemmaRMSNorm).
|
||||
set_random_seed(seed)
|
||||
norm = GemmaRMSNorm(hidden_size, eps=eps).cuda().to(dtype)
|
||||
with torch.no_grad():
|
||||
norm.weight.normal_(mean=0.0, std=0.1)
|
||||
|
||||
# Residual is shared across ranks; the partial o_proj output differs per rank
|
||||
# (each rank holds a partial sum that all_reduce combines).
|
||||
torch.manual_seed(seed + 7)
|
||||
residual = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
torch.manual_seed(seed + 1000 + local_rank)
|
||||
partial = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
|
||||
# Reference: the unfused model path.
|
||||
reduced = tensor_model_parallel_all_reduce(partial.clone())
|
||||
ref_out, ref_res = norm(reduced, residual.clone())
|
||||
|
||||
# Fused helper (flashinfer fast path when available, else fallback).
|
||||
out, res = fused_allreduce_gemma_rms_norm(partial.clone(), residual.clone(), norm)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(res, ref_res, atol=2e-2, rtol=2e-2)
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="CUDA required",
|
||||
)
|
||||
# world_size=1 exercises the TP==1 identity branch on a single GPU; >1 exercises
|
||||
# the all_reduce + GemmaRMSNorm equivalence (flashinfer kernel or fallback).
|
||||
@pytest.mark.parametrize("world_size", [1, 2, 4])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128, 333])
|
||||
@pytest.mark.parametrize("hidden_size", [2048, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@pytest.mark.parametrize("seed", [42])
|
||||
def test_fused_allreduce_gemma_rms_norm(
|
||||
world_size,
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype,
|
||||
eps,
|
||||
seed,
|
||||
):
|
||||
num_gpus = current_platform.device_count()
|
||||
if num_gpus < world_size:
|
||||
pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
|
||||
port = str(get_open_port())
|
||||
spawn(
|
||||
_worker_fused_ar_norm,
|
||||
args=(
|
||||
world_size,
|
||||
port,
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype,
|
||||
seed,
|
||||
eps,
|
||||
),
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
)
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for fp32_router_gemm kernel: activation×weight→fp32, H=3072, E=256.
|
||||
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
|
||||
|
||||
Supported (hidden_size, num_experts) pairs:
|
||||
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
|
||||
Correctness baseline: torch.matmul in float64.
|
||||
"""
|
||||
@@ -10,8 +13,8 @@ import torch
|
||||
|
||||
from vllm._custom_ops import fp32_router_gemm
|
||||
|
||||
NUM_EXPERTS = 256
|
||||
HIDDEN_DIM = 3072
|
||||
# (hidden_size, num_experts)
|
||||
SHAPES = [(3072, 256), (6144, 128)]
|
||||
# Absolute tolerance for fp32 kernel vs float64 reference
|
||||
ATOL_FP32 = 2e-4
|
||||
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
|
||||
@@ -30,49 +33,52 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
|
||||
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 8, 16, 32])
|
||||
def test_fp32_activation(num_tokens: int):
|
||||
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
"""fp32 activation → fp32 output should match reference closely."""
|
||||
_requires_sm90()
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
mat_a = torch.randn(num_tokens, HIDDEN_DIM, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
|
||||
mat_a = torch.randn(num_tokens, hidden_dim, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
|
||||
out = fp32_router_gemm(mat_a, mat_b)
|
||||
ref = _ref(mat_a, mat_b)
|
||||
|
||||
assert out.shape == (num_tokens, NUM_EXPERTS)
|
||||
assert out.shape == (num_tokens, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 2, 4, 8, 16, 32])
|
||||
def test_bf16_activation(num_tokens: int):
|
||||
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
"""bf16 activation → fp32 output should match reference within bf16 error."""
|
||||
_requires_sm90()
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
mat_a_bf16 = torch.randn(
|
||||
num_tokens, HIDDEN_DIM, dtype=torch.bfloat16, device=device
|
||||
num_tokens, hidden_dim, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
|
||||
out = fp32_router_gemm(mat_a_bf16, mat_b)
|
||||
ref = _ref(mat_a_bf16, mat_b).to(device)
|
||||
|
||||
assert out.shape == (num_tokens, NUM_EXPERTS)
|
||||
assert out.shape == (num_tokens, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
torch.testing.assert_close(out, ref, atol=ATOL_BF16, rtol=0)
|
||||
|
||||
|
||||
def test_output_shape_and_dtype():
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
def test_output_shape_and_dtype(hidden_dim: int, num_experts: int):
|
||||
"""Basic shape and dtype checks."""
|
||||
_requires_sm90()
|
||||
device = torch.device("cuda")
|
||||
mat_a = torch.randn(4, HIDDEN_DIM, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
|
||||
mat_a = torch.randn(4, hidden_dim, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
out = fp32_router_gemm(mat_a, mat_b)
|
||||
assert out.shape == (4, NUM_EXPERTS)
|
||||
assert out.shape == (4, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
assert out.device.type == "cuda"
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for the horizontally-fused MiniMax-M3 attention pre-processing
|
||||
kernel:
|
||||
|
||||
fused_minimax_m3_qknorm_rope_kv_insert
|
||||
- q / k / index_q / index_k: Gemma RMSNorm + partial NeoX RoPE (in place)
|
||||
- sparse (insert) mode: scatter k/v into the paged bf16 KV cache and the
|
||||
index key into the index cache by its own slot mapping.
|
||||
|
||||
Reference: PyTorch Gemma RMSNorm with the same dtype materialization boundary
|
||||
as the unfused path, followed by vLLM CUDA rotary_embedding-style NeoX RoPE.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
HEAD_DIM = 128
|
||||
ROTARY_DIM = 64
|
||||
|
||||
|
||||
def _op_available() -> bool:
|
||||
return hasattr(torch.ops._C, "fused_minimax_m3_qknorm_rope_kv_insert")
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or not _op_available(),
|
||||
reason="CUDA not available or fused MiniMax-M3 op not built in",
|
||||
)
|
||||
|
||||
|
||||
def make_cos_sin_cache(max_pos, rotary_dim, base, dtype, device):
|
||||
inv_freq = 1.0 / (
|
||||
base
|
||||
** (
|
||||
torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
|
||||
/ rotary_dim
|
||||
)
|
||||
)
|
||||
t = torch.arange(max_pos, dtype=torch.float32, device=device)
|
||||
freqs = torch.einsum("i,j->ij", t, inv_freq) # [max_pos, rotary_dim/2]
|
||||
cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1) # [max_pos, rotary_dim]
|
||||
return cache.to(dtype)
|
||||
|
||||
|
||||
def gemma_rmsnorm(x, weight, eps):
|
||||
"""x: [..., 128]; weight: [128]. Returns original dtype."""
|
||||
xf = x.float()
|
||||
var = xf.pow(2).mean(dim=-1, keepdim=True)
|
||||
out = xf * torch.rsqrt(var + eps)
|
||||
out = out * (1.0 + weight.float())
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
def apply_rope_neox_partial(x, positions, cos_sin_cache, rotary_dim):
|
||||
"""NeoX-style RoPE on the leading rotary_dim dims; rest pass through.
|
||||
|
||||
x: [num_tokens, num_heads, head_dim]
|
||||
cos_sin_cache: [max_pos, rotary_dim] (cos||sin), read as float (matches the
|
||||
kernel, which loads the bf16 cache and converts to fp32).
|
||||
"""
|
||||
half = rotary_dim // 2
|
||||
cs = cos_sin_cache[positions].float() # [num_tokens, rotary_dim]
|
||||
cos = cs[..., :half].unsqueeze(1) # [nt, 1, half]
|
||||
sin = cs[..., half:].unsqueeze(1)
|
||||
|
||||
rot = x[..., :rotary_dim].float()
|
||||
x1 = rot[..., :half]
|
||||
x2 = rot[..., half:]
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
out = x.clone()
|
||||
out[..., :half] = o1
|
||||
out[..., half:rotary_dim] = o2
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
def norm_rope_ref(x, weight, positions, cos_sin_cache, eps):
|
||||
"""[nt, nheads, 128] -> Gemma norm + neox partial rope."""
|
||||
normed = gemma_rmsnorm(x, weight, eps)
|
||||
roped = apply_rope_neox_partial(normed, positions, cos_sin_cache, ROTARY_DIM)
|
||||
return roped
|
||||
|
||||
|
||||
# ── Test 1: dense mode (norm+rope only, no index, no insert) ─────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
|
||||
@pytest.mark.parametrize("num_heads,num_kv_heads", [(8, 2), (16, 4), (64, 4)])
|
||||
def test_dense_norm_rope(num_tokens, num_heads, num_kv_heads):
|
||||
torch.manual_seed(0)
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
|
||||
base, max_pos = 5_000_000.0, 4096
|
||||
|
||||
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
|
||||
qkv = torch.randn(num_tokens, qsz + 2 * kvsz, dtype=dtype, device=device)
|
||||
qkv_orig = qkv.clone()
|
||||
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv, q_w, k_w, cos_sin, positions, num_heads, num_kv_heads, ROTARY_DIM, eps
|
||||
)
|
||||
q_out, k_out, v_out = qkv.split([qsz, kvsz, kvsz], dim=-1)
|
||||
|
||||
q_in, k_in, v_in = qkv_orig.split([qsz, kvsz, kvsz], dim=-1)
|
||||
q_ref = norm_rope_ref(
|
||||
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = norm_rope_ref(
|
||||
k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
k_w,
|
||||
positions,
|
||||
cos_sin,
|
||||
eps,
|
||||
).view(num_tokens, kvsz)
|
||||
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
# V is untouched.
|
||||
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
|
||||
|
||||
|
||||
# ── Test 2: sparse mode (full: index branch + cache inserts) ─────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_sparse_full(num_tokens, block_size):
|
||||
torch.manual_seed(1)
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
|
||||
base, max_pos = 5_000_000.0, 4096
|
||||
num_heads, num_kv_heads, num_idx_heads = 16, 4, 4
|
||||
|
||||
q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
iq_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
ik_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
|
||||
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
|
||||
iqsz, iksz = num_idx_heads * HEAD_DIM, HEAD_DIM
|
||||
# Single fused tensor packing [q | k | v | index_q | index_k].
|
||||
qkv = torch.randn(
|
||||
num_tokens, qsz + 2 * kvsz + iqsz + iksz, dtype=dtype, device=device
|
||||
)
|
||||
qkv_orig = qkv.clone()
|
||||
splits = [qsz, kvsz, kvsz, iqsz, iksz]
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks, 2, block_size, num_kv_heads, HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
index_cache = torch.zeros(
|
||||
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
index_slot_mapping = torch.roll(slot_mapping, shifts=1)
|
||||
|
||||
# Contiguous gather targets: the kernel writes the normed/roped q and
|
||||
# index_q here (de-interleaved from the packed qkv); k/v/index_k stay in
|
||||
# place inside qkv and are scatter-inserted into the caches.
|
||||
q_out = torch.empty(num_tokens, qsz, dtype=dtype, device=device)
|
||||
index_q = torch.empty(num_tokens, iqsz, dtype=dtype, device=device)
|
||||
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
q_w,
|
||||
k_w,
|
||||
cos_sin,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
ROTARY_DIM,
|
||||
eps,
|
||||
iq_w,
|
||||
ik_w,
|
||||
num_idx_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
kv_cache,
|
||||
index_cache,
|
||||
block_size,
|
||||
q_out,
|
||||
index_q,
|
||||
)
|
||||
|
||||
# ── norm+rope parity. q/index_q land in their gather buffers; k/index_k are
|
||||
# rewritten in place inside qkv. ──
|
||||
_, k_out, _, _, index_k = qkv.split(splits, dim=-1)
|
||||
q_in, k_in, v_in, iq_orig, ik_orig = qkv_orig.split(splits, dim=-1)
|
||||
q_ref = norm_rope_ref(
|
||||
q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = norm_rope_ref(
|
||||
k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
|
||||
k_w,
|
||||
positions,
|
||||
cos_sin,
|
||||
eps,
|
||||
).view(num_tokens, kvsz)
|
||||
iq_ref = norm_rope_ref(
|
||||
iq_orig.view(num_tokens, num_idx_heads, HEAD_DIM),
|
||||
iq_w,
|
||||
positions,
|
||||
cos_sin,
|
||||
eps,
|
||||
).view(num_tokens, num_idx_heads * HEAD_DIM)
|
||||
ik_ref = norm_rope_ref(
|
||||
ik_orig.view(num_tokens, 1, HEAD_DIM), ik_w, positions, cos_sin, eps
|
||||
).view(num_tokens, HEAD_DIM)
|
||||
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(index_q, iq_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(index_k, ik_ref, rtol=1e-2, atol=1e-2)
|
||||
|
||||
# ── Cache inserts. ──
|
||||
# Main cache layout is [num_blocks, 2, block_size, num_kv_heads, head_dim]
|
||||
# (the K/V axis sits *before* block_size); index cache is [nb, bs, head_dim].
|
||||
idx_flat = index_cache.view(num_blocks * block_size, HEAD_DIM)
|
||||
k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
|
||||
v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM) # v is raw (no norm/rope)
|
||||
for t in range(num_tokens):
|
||||
s = slot_mapping[t].item()
|
||||
b, pos = s // block_size, s % block_size
|
||||
torch.testing.assert_close(
|
||||
kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
|
||||
)
|
||||
torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
|
||||
index_s = index_slot_mapping[t].item()
|
||||
torch.testing.assert_close(idx_flat[index_s], ik_ref[t], rtol=1e-2, atol=1e-2)
|
||||
@@ -0,0 +1,337 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reference-vs-optimized unit tests for the MiniMax-M3 AMD/ROCm fused kernels.
|
||||
|
||||
Each optimized kernel added for the ROCm port has a slow PyTorch reference; the
|
||||
tests assert the two agree within tolerance:
|
||||
|
||||
* Gemma RMSNorm (plain + fused-add-residual) -> fp32 PyTorch normalize
|
||||
* SwiGLU-OAI (split layout) -> fp32 PyTorch elementwise
|
||||
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
|
||||
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
|
||||
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
|
||||
|
||||
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
|
||||
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
|
||||
makes the shape ``[K//32, N]`` and Triton raises before producing output, so any
|
||||
regression there fails these tests loudly.
|
||||
|
||||
Hardware scope: the whole module is ROCm-only (these are the AMD path; NVIDIA
|
||||
uses the FlashInfer kernels). The norm/activation/quant kernels run on any ROCm
|
||||
arch; the native MXFP8 ``dot_scaled`` linear/MoE tests are additionally gated to
|
||||
CDNA4 gfx95x (``@requires_gfx950``) since gfx942 uses the BF16 emulation path.
|
||||
|
||||
Run: pytest tests/kernels/test_minimax_m3_amd_ops.py -v
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("MiniMax-M3 AMD fused ops require ROCm.", allow_module_level=True)
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("Requires a GPU.", allow_module_level=True)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: E402
|
||||
_mxfp8_e4m3_quantize_torch,
|
||||
_mxfp8_e4m3_quantize_triton,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
|
||||
gemma_fused_add_rmsnorm,
|
||||
gemma_rmsnorm,
|
||||
swiglu_oai_split,
|
||||
)
|
||||
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
|
||||
|
||||
DEVICE = "cuda"
|
||||
EPS = 1e-6
|
||||
|
||||
|
||||
def _gcn_arch() -> str:
|
||||
try:
|
||||
return torch.cuda.get_device_properties(0).gcnArchName
|
||||
except Exception: # pragma: no cover - no device / non-AMD
|
||||
return ""
|
||||
|
||||
|
||||
# The pure-Triton norm/activation/quant kernels run on any ROCm arch (CDNA3
|
||||
# gfx942 and CDNA4 gfx950). The native MXFP8 ``dot_scaled`` GEMMs (linear + MoE)
|
||||
# use CDNA4 hardware microscaling and are gated to gfx95x in the source
|
||||
# (``RocmDotScaledMxfp8LinearKernel.is_supported``; the MoE oracle routes gfx942
|
||||
# to the BF16 emulation path instead) — so those tests are gfx950-only.
|
||||
requires_gfx950 = pytest.mark.skipif(
|
||||
"gfx95" not in _gcn_arch(),
|
||||
reason="native MXFP8 dot_scaled is a CDNA4 (gfx95x) feature; "
|
||||
"gfx942 uses the BF16 emulation path instead.",
|
||||
)
|
||||
|
||||
|
||||
def _relerr(a: torch.Tensor, b: torch.Tensor) -> float:
|
||||
a = a.float()
|
||||
b = b.float()
|
||||
return ((a - b).norm() / (b.norm() + 1e-8)).item()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Gemma RMSNorm
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_gemma_rmsnorm(x, w, eps, residual=None):
|
||||
orig_dtype = x.dtype
|
||||
xf = x.float()
|
||||
res_out = None
|
||||
if residual is not None:
|
||||
xf = xf + residual.float()
|
||||
res_out = xf.to(orig_dtype)
|
||||
xf = xf * torch.rsqrt(xf.pow(2).mean(dim=-1, keepdim=True) + eps)
|
||||
xf = xf * (1.0 + w.float())
|
||||
out = xf.to(orig_dtype)
|
||||
return out if residual is None else (out, res_out)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 4096), (37, 6144), (128, 2048)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("seed", [0, 1234])
|
||||
@torch.inference_mode()
|
||||
def test_gemma_rmsnorm(shape, dtype, seed):
|
||||
torch.manual_seed(seed)
|
||||
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
|
||||
got = gemma_rmsnorm(x, w, EPS)
|
||||
ref = _ref_gemma_rmsnorm(x, w, EPS)
|
||||
assert got.shape == x.shape
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shape", [(1, 6144), (64, 4096)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_gemma_fused_add_rmsnorm(shape, dtype):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
res = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
|
||||
got_out, got_res = gemma_fused_add_rmsnorm(x, res, w, EPS)
|
||||
ref_out, ref_res = _ref_gemma_rmsnorm(x, w, EPS, residual=res)
|
||||
assert _relerr(got_out, ref_out) < 5e-3
|
||||
# residual_out is the pre-norm sum (x + res): bit-for-bit identical cast.
|
||||
assert torch.equal(got_res, ref_res)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gemma_rmsnorm_per_head_strided():
|
||||
"""q_norm/k_norm normalize a non-contiguous ``qkv.split`` slice over head_dim."""
|
||||
torch.manual_seed(0)
|
||||
T, H, D, kv = 7, 48, 128, 8
|
||||
total = (H + 2 * kv) * D
|
||||
qkv = torch.randn(T, total, device=DEVICE, dtype=torch.bfloat16)
|
||||
q = qkv[..., : H * D] # non-contiguous view (row stride == total)
|
||||
q_by_head = q.view(T, H, D)
|
||||
assert not q_by_head.is_contiguous()
|
||||
w = torch.randn(D, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
got = gemma_rmsnorm(q_by_head, w, EPS)
|
||||
ref = _ref_gemma_rmsnorm(q_by_head, w, EPS)
|
||||
assert got.shape == q_by_head.shape
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
def test_num_warps_monotonic():
|
||||
assert _num_warps(128) <= _num_warps(2048) <= _num_warps(8192)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# SwiGLU-OAI (split layout)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_swiglu(gate_up, alpha, beta, limit):
|
||||
d = gate_up.shape[-1] // 2
|
||||
gate = gate_up[..., :d].float()
|
||||
up = gate_up[..., d:].float()
|
||||
if limit is not None:
|
||||
gate = gate.clamp(max=limit)
|
||||
up = up.clamp(min=-limit, max=limit)
|
||||
return (gate * torch.sigmoid(alpha * gate) * (up + beta)).to(gate_up.dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,inter", [(1, 768), (64, 1536), (128, 1024)])
|
||||
@pytest.mark.parametrize("limit", [7.0, None])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_swiglu_oai_split(m, inter, limit, dtype):
|
||||
torch.manual_seed(0)
|
||||
gate_up = torch.randn(m, 2 * inter, device=DEVICE, dtype=dtype)
|
||||
got = swiglu_oai_split(gate_up, alpha=1.702, beta=1.0, limit=limit)
|
||||
ref = _ref_swiglu(gate_up, 1.702, 1.0, limit)
|
||||
assert got.shape == (m, inter)
|
||||
assert _relerr(got, ref) < 5e-3
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fused MXFP8 activation quant (Triton vs torch reference)
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("shape", [(64, 4096), (1, 6144), (333, 2048)])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_quant_triton_matches_torch(shape, dtype):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
|
||||
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
|
||||
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
|
||||
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
|
||||
# E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
|
||||
# most a 1-step difference at exact powers of two.
|
||||
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
|
||||
# Dequantized values agree to fp8 granularity.
|
||||
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
|
||||
deq_k = dequant_mxfp8_to_bf16(xq_k, s_k)
|
||||
assert _relerr(deq_k, deq_t) < 1e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 linear (dot_scaled) vs dequant-to-bf16 matmul
|
||||
# --------------------------------------------------------------------------- #
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize("m,n,k", [(64, 256, 128), (37, 512, 256), (1, 6144, 4096)])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_native_linear(m, n, k):
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
_mxfp8_dot_scaled_linear,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
w_bf16 = torch.randn(n, k, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
x = torch.randn(m, k, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
|
||||
got = _mxfp8_dot_scaled_linear(x, w_fp8, w_scale)
|
||||
# Reference: consume the SAME quantized weights (isolates activation-quant
|
||||
# noise) -> dequant to bf16, plain matmul.
|
||||
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
|
||||
ref = torch.nn.functional.linear(x, w_deq).to(x.dtype)
|
||||
assert got.shape == (m, n)
|
||||
# Only the activation is re-quantized inside the kernel -> small MX noise.
|
||||
assert _relerr(got, ref) < 5e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Native MXFP8 MoE (dot_scaled grouped GEMM) vs dequant-to-bf16 MoE math
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _ref_moe(x, w13, w2, topk_weights, topk_ids, alpha, beta, limit):
|
||||
T, H = x.shape
|
||||
inter = w2.shape[-1]
|
||||
top_k = topk_ids.shape[1]
|
||||
out = torch.zeros(T, H, device=x.device, dtype=torch.float32)
|
||||
for t in range(T):
|
||||
for j in range(top_k):
|
||||
e = int(topk_ids[t, j].item())
|
||||
g1 = x[t].float() @ w13[e].float().T # [2I]
|
||||
gate = g1[:inter]
|
||||
up = g1[inter:]
|
||||
if limit is not None:
|
||||
gate = gate.clamp(max=limit)
|
||||
up = up.clamp(min=-limit, max=limit)
|
||||
act = gate * torch.sigmoid(alpha * gate) * (up + beta)
|
||||
g2 = act @ w2[e].float().T # [H]
|
||||
out[t] += topk_weights[t, j].float() * g2
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@pytest.mark.parametrize(
|
||||
"T,H,inter,E,top_k", [(8, 256, 512, 8, 2), (1, 512, 256, 16, 4)]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_native_moe(T, H, inter, E, top_k):
|
||||
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
|
||||
fused_moe_mxfp8_native,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
alpha, beta, limit = 1.702, 1.0, 7.0
|
||||
w13_bf16 = torch.randn(E, 2 * inter, H, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w2_bf16 = torch.randn(E, H, inter, device=DEVICE, dtype=torch.bfloat16) * 0.1
|
||||
w13_fp8, w13_scale = _mxfp8_e4m3_quantize_torch(
|
||||
w13_bf16, is_sf_swizzled_layout=False
|
||||
)
|
||||
w2_fp8, w2_scale = _mxfp8_e4m3_quantize_torch(w2_bf16, is_sf_swizzled_layout=False)
|
||||
|
||||
x = torch.randn(T, H, device=DEVICE, dtype=torch.bfloat16) * 0.5
|
||||
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
|
||||
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
got = fused_moe_mxfp8_native(
|
||||
x,
|
||||
w13_fp8,
|
||||
w13_scale,
|
||||
w2_fp8,
|
||||
w2_scale,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
alpha=alpha,
|
||||
beta=beta,
|
||||
limit=limit,
|
||||
global_num_experts=E,
|
||||
expert_map=None,
|
||||
)
|
||||
# Reference consumes the dequantized weights (same bits the kernel reads).
|
||||
w13_deq = dequant_mxfp8_to_bf16(w13_fp8, w13_scale)
|
||||
w2_deq = dequant_mxfp8_to_bf16(w2_fp8, w2_scale)
|
||||
ref = _ref_moe(x, w13_deq, w2_deq, topk_weights, topk_ids, alpha, beta, limit)
|
||||
assert got.shape == (T, H)
|
||||
assert _relerr(got, ref) < 5e-2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# MXFP8 linear emulation: BF16-at-load (default) vs per-step dequant + switch
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("shape", [(512, 2048), (1, 6144)])
|
||||
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("dequant_at_load", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_mxfp8_linear_emulation_bf16_at_load(
|
||||
shape, act_dtype, dequant_at_load, monkeypatch
|
||||
):
|
||||
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
|
||||
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
|
||||
same result; the dtype-match (BF16/FP16 activations) must also hold."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
EmulationMxfp8LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
|
||||
Mxfp8LinearLayerConfig,
|
||||
)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "1" if dequant_at_load else "0"
|
||||
)
|
||||
N, K = shape
|
||||
torch.manual_seed(0)
|
||||
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16)
|
||||
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
|
||||
assert w_scale.shape == (N, K // 32)
|
||||
|
||||
# Reference: dequant once, plain linear in the activation dtype.
|
||||
w_ref = dequant_mxfp8_to_bf16(w_fp8, w_scale).to(act_dtype)
|
||||
x = torch.randn(7, K, device=DEVICE, dtype=act_dtype)
|
||||
out_ref = torch.nn.functional.linear(x, w_ref)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
|
||||
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
|
||||
|
||||
kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
if dequant_at_load:
|
||||
# weights converted to BF16 at load (>= 2-byte)
|
||||
assert layer.weight.element_size() >= 2
|
||||
else:
|
||||
# opt-out: weights stay 1-byte MXFP8, dequant happens per-step
|
||||
assert layer.weight.element_size() == 1
|
||||
|
||||
out = kernel.apply_weights(layer, x)
|
||||
assert out.dtype == act_dtype # dtype-match preserved (no tl.dot/F.linear crash)
|
||||
assert _relerr(out.float(), out_ref.float()) < 2e-2
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
|
||||
|
||||
These exercise the vendored processor directly (no checkpoint / GPU needed), so
|
||||
they validate the long-side resize spec and the resulting prompt-token counts
|
||||
deterministically.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.transformers_utils.processors.minimax_m3 import (
|
||||
IMAGE_MAX_TOTAL_PIXELS,
|
||||
MIN_SHORT_SIDE_PIXEL,
|
||||
VIDEO_MAX_TOTAL_PIXELS,
|
||||
MiniMaxM3VLImageProcessor,
|
||||
MiniMaxM3VLVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
# Long sides are multiples of patch_size*merge_size (28) so the rounding is
|
||||
# exact and the expected token counts are unambiguous.
|
||||
LONG_SIDES = [252, 504, 1008]
|
||||
MERGE2 = 2**2 # merge_size ** 2
|
||||
|
||||
|
||||
def _image_tokens(grid_thw) -> int:
|
||||
g = list(grid_thw)
|
||||
return int(g[0] * g[1] * g[2]) // MERGE2
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_smart_resize_long_side_shrink():
|
||||
# (a) long side exceeds the cap -> shrink so the long side equals the cap.
|
||||
h, w = smart_resize(
|
||||
2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
|
||||
)
|
||||
assert max(h, w) == 1008
|
||||
assert (h, w) == (1008, 504) # aspect ratio preserved
|
||||
|
||||
|
||||
def test_smart_resize_short_side_enlarge():
|
||||
# (b) long side within the cap but short side below the floor -> enlarge so
|
||||
# the short side reaches min_short_side_pixel.
|
||||
h, w = smart_resize(
|
||||
200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
|
||||
)
|
||||
assert min(h, w) == MIN_SHORT_SIDE_PIXEL # 112
|
||||
|
||||
|
||||
def test_smart_resize_total_pixels_raises():
|
||||
# (c) still over the area cap after resizing -> raise instead of inferring.
|
||||
with pytest.raises(ValueError, match="max_total_pixels"):
|
||||
smart_resize(
|
||||
5000,
|
||||
5000,
|
||||
factor=28,
|
||||
max_long_side_pixel=4000,
|
||||
max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
|
||||
)
|
||||
|
||||
|
||||
def test_smart_resize_backward_compatible_area_bound():
|
||||
# Without max_long_side_pixel the original Qwen-style area bound is used.
|
||||
assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_image_tokens_increase_with_max_long_side_pixel():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
patches = proc.get_number_of_image_patches(
|
||||
2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
|
||||
)
|
||||
counts.append(patches // MERGE2)
|
||||
|
||||
assert counts == [81, 324, 1296]
|
||||
assert counts[0] < counts[1] < counts[2]
|
||||
|
||||
|
||||
def test_image_processor_defaults_match_spec():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
assert proc.max_long_side_pixel is None # opt-in
|
||||
assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
|
||||
assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
|
||||
|
||||
|
||||
def test_image_preprocess_pipeline_monotonic():
|
||||
proc = MiniMaxM3VLImageProcessor()
|
||||
image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
out = proc.preprocess(
|
||||
[image],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=long_side,
|
||||
return_tensors="pt",
|
||||
)
|
||||
counts.append(_image_tokens(out["image_grid_thw"][0]))
|
||||
assert counts == [81, 324, 1296]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_video_tokens_increase_with_max_long_side_pixel():
|
||||
proc = MiniMaxM3VLVideoProcessor()
|
||||
assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
|
||||
video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
|
||||
counts = []
|
||||
for long_side in LONG_SIDES:
|
||||
out = proc.preprocess(
|
||||
videos=[video],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=long_side,
|
||||
return_tensors="pt",
|
||||
)
|
||||
counts.append(_image_tokens(out["video_grid_thw"][0]))
|
||||
assert counts[0] < counts[1] < counts[2]
|
||||
|
||||
|
||||
def test_video_volumetric_cap_raises():
|
||||
proc = MiniMaxM3VLVideoProcessor()
|
||||
# 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
|
||||
video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
|
||||
with pytest.raises(ValueError, match="max_total_pixels"):
|
||||
proc.preprocess(
|
||||
videos=[video],
|
||||
do_resize=True,
|
||||
max_long_side_pixel=1008,
|
||||
return_tensors="pt",
|
||||
)
|
||||
@@ -420,6 +420,11 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"MiniMaxAI/MiniMax-M2",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"MiniMaxM3SparseForCausalLM": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"),
|
||||
"MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
|
||||
"MistralLarge3ForCausalLM": _HfExamplesInfo(
|
||||
@@ -1099,6 +1104,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"MiniMaxAI/MiniMax-VL-01",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Mistral3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
|
||||
extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
|
||||
@@ -1601,6 +1611,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="XiaomiMiMo/MiMo-V2.5-Omni",
|
||||
is_available_online=False,
|
||||
),
|
||||
"MiniMaxM3MTP": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"NemotronHMTPModel": _HfExamplesInfo(
|
||||
"nvidia/Nemotron-Super-Placeholder",
|
||||
speculative_model="nvidia/Nemotron-Super-Placeholder",
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import string
|
||||
from collections.abc import Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.reasoning.minimax_m3_reasoning_parser import MiniMaxM3ReasoningParser
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class MiniMaxM3Tokenizer:
|
||||
"""Small tokenizer with MiniMax M3 reasoning tags as special tokens."""
|
||||
|
||||
special_tokens = ("<mm:think>", "</mm:think>")
|
||||
|
||||
def __init__(self):
|
||||
self._token_to_id: dict[str, int] = {}
|
||||
self._id_to_token: dict[int, str] = {}
|
||||
for token in self.special_tokens:
|
||||
self._add_token(token)
|
||||
for char in string.printable:
|
||||
self._add_token(char)
|
||||
|
||||
def _add_token(self, token: str) -> int:
|
||||
token_id = self._token_to_id.get(token)
|
||||
if token_id is None:
|
||||
token_id = len(self._token_to_id) + 1
|
||||
self._token_to_id[token] = token_id
|
||||
self._id_to_token[token_id] = token
|
||||
return token_id
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return dict(self._token_to_id)
|
||||
|
||||
def encode(
|
||||
self,
|
||||
text: str,
|
||||
truncation: bool | None = None,
|
||||
max_length: int | None = None,
|
||||
add_special_tokens: bool = True,
|
||||
) -> list[int]:
|
||||
return [self._add_token(token) for token in self.tokenize(text)]
|
||||
|
||||
def decode(
|
||||
self, ids: Sequence[int] | int, skip_special_tokens: bool = False
|
||||
) -> str:
|
||||
if isinstance(ids, int):
|
||||
ids = [ids]
|
||||
return "".join(self._id_to_token[token_id] for token_id in ids)
|
||||
|
||||
def tokenize(self, text: str) -> list[str]:
|
||||
tokens: list[str] = []
|
||||
pos = 0
|
||||
while pos < len(text):
|
||||
for special_token in self.special_tokens:
|
||||
if text.startswith(special_token, pos):
|
||||
tokens.append(special_token)
|
||||
pos += len(special_token)
|
||||
break
|
||||
else:
|
||||
tokens.append(text[pos])
|
||||
pos += 1
|
||||
return tokens
|
||||
|
||||
def convert_ids_to_tokens(
|
||||
self,
|
||||
ids: Sequence[int],
|
||||
skip_special_tokens: bool = False,
|
||||
) -> list[str]:
|
||||
return [self._id_to_token[token_id] for token_id in ids]
|
||||
|
||||
def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
|
||||
if isinstance(tokens, str):
|
||||
return self._add_token(tokens)
|
||||
return [self._add_token(token) for token in tokens]
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
return "".join(tokens)
|
||||
|
||||
|
||||
def make_parser(
|
||||
chat_template_kwargs: dict[str, str] | None = None,
|
||||
) -> tuple[MiniMaxM3ReasoningParser, MiniMaxM3Tokenizer]:
|
||||
tokenizer = MiniMaxM3Tokenizer()
|
||||
return (
|
||||
MiniMaxM3ReasoningParser(tokenizer, chat_template_kwargs=chat_template_kwargs),
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
|
||||
def run_streaming(
|
||||
parser: MiniMaxM3ReasoningParser,
|
||||
tokenizer: MiniMaxM3Tokenizer,
|
||||
chunks: list[str],
|
||||
) -> tuple[str | None, str | None, list[bool]]:
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
reasoning_end_states: list[bool] = []
|
||||
|
||||
for chunk in chunks:
|
||||
delta_token_ids = tokenizer.encode(chunk, add_special_tokens=False)
|
||||
current_text = previous_text + chunk
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
delta = parser.extract_reasoning_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
)
|
||||
reasoning_end_states.append(
|
||||
parser.is_reasoning_end_streaming(current_token_ids, delta_token_ids)
|
||||
)
|
||||
|
||||
if delta is not None:
|
||||
if delta.reasoning is not None:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content is not None:
|
||||
content_parts.append(delta.content)
|
||||
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
|
||||
return (
|
||||
"".join(reasoning_parts) or None,
|
||||
"".join(content_parts) or None,
|
||||
reasoning_end_states,
|
||||
)
|
||||
|
||||
|
||||
def test_parser_registration():
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("minimax_m3")
|
||||
|
||||
assert parser_cls is MiniMaxM3ReasoningParser
|
||||
|
||||
|
||||
def test_nonstreaming_extracts_explicit_reasoning_block():
|
||||
parser, _ = make_parser()
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<mm:think>plan</mm:think>answer", request
|
||||
)
|
||||
|
||||
assert reasoning == "plan"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_nonstreaming_without_start_tag_is_content():
|
||||
parser, _ = make_parser()
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
reasoning, content = parser.extract_reasoning("plain answer", request)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "plain answer"
|
||||
|
||||
|
||||
def test_nonstreaming_drops_leading_end_tag():
|
||||
parser, _ = make_parser()
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
reasoning, content = parser.extract_reasoning("</mm:think>answer", request)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_nonstreaming_non_leading_end_tag_is_content():
|
||||
parser, _ = make_parser()
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
reasoning, content = parser.extract_reasoning("XXX</mm:think>YYY", request)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "XXX</mm:think>YYY"
|
||||
|
||||
|
||||
def test_nonstreaming_enabled_mode_starts_in_reasoning():
|
||||
parser, _ = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
reasoning, content = parser.extract_reasoning("plan</mm:think>answer", request)
|
||||
|
||||
assert reasoning == "plan"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_nonstreaming_open_reasoning_block():
|
||||
parser, _ = make_parser()
|
||||
request = ChatCompletionRequest(messages=[], model="test-model")
|
||||
|
||||
reasoning, content = parser.extract_reasoning("<mm:think>still thinking", request)
|
||||
|
||||
assert reasoning == "still thinking"
|
||||
assert content is None
|
||||
|
||||
|
||||
def test_streaming_reasoning_tags_are_not_returned():
|
||||
parser, tokenizer = make_parser()
|
||||
|
||||
reasoning, content, end_states = run_streaming(
|
||||
parser,
|
||||
tokenizer,
|
||||
["<mm:think>", "plan", "</mm:think>", "answer"],
|
||||
)
|
||||
|
||||
assert reasoning == "plan"
|
||||
assert content == "answer"
|
||||
assert end_states == [False, False, True, True]
|
||||
|
||||
|
||||
def test_streaming_boundary_can_emit_reasoning_and_content():
|
||||
parser, tokenizer = make_parser()
|
||||
|
||||
reasoning, content, end_states = run_streaming(
|
||||
parser,
|
||||
tokenizer,
|
||||
["<mm:think>plan</mm:think>answer"],
|
||||
)
|
||||
|
||||
assert reasoning == "plan"
|
||||
assert content == "answer"
|
||||
assert end_states == [True]
|
||||
|
||||
|
||||
def test_streaming_drops_leading_end_tag():
|
||||
parser, tokenizer = make_parser()
|
||||
|
||||
reasoning, content, end_states = run_streaming(
|
||||
parser,
|
||||
tokenizer,
|
||||
["</mm:think>", "answer"],
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "answer"
|
||||
assert end_states == [True, True]
|
||||
|
||||
|
||||
def test_streaming_non_leading_end_tag_is_content():
|
||||
parser, tokenizer = make_parser()
|
||||
|
||||
reasoning, content, end_states = run_streaming(
|
||||
parser,
|
||||
tokenizer,
|
||||
["XXX</mm:think>YYY"],
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "XXX</mm:think>YYY"
|
||||
assert end_states == [True]
|
||||
|
||||
|
||||
def test_streaming_enabled_mode_starts_in_reasoning():
|
||||
parser, tokenizer = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
|
||||
|
||||
reasoning, content, end_states = run_streaming(
|
||||
parser,
|
||||
tokenizer,
|
||||
["plan", "</mm:think>", "answer"],
|
||||
)
|
||||
|
||||
assert reasoning == "plan"
|
||||
assert content == "answer"
|
||||
assert end_states == [False, True, True]
|
||||
|
||||
|
||||
def test_streaming_plain_content_ends_reasoning_phase():
|
||||
parser, tokenizer = make_parser()
|
||||
|
||||
reasoning, content, end_states = run_streaming(
|
||||
parser,
|
||||
tokenizer,
|
||||
["plain ", "answer"],
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "plain answer"
|
||||
assert end_states == [True, True]
|
||||
|
||||
|
||||
def test_token_id_helpers():
|
||||
parser, tokenizer = make_parser()
|
||||
output_ids = tokenizer.encode(
|
||||
"<mm:think>abc</mm:think>def", add_special_tokens=False
|
||||
)
|
||||
open_reasoning_ids = tokenizer.encode("<mm:think>abc", add_special_tokens=False)
|
||||
content_ids = tokenizer.encode("plain", add_special_tokens=False)
|
||||
|
||||
assert parser.is_reasoning_end(output_ids)
|
||||
assert not parser.is_reasoning_end(open_reasoning_ids)
|
||||
assert not parser.is_reasoning_end(content_ids)
|
||||
assert tokenizer.decode(parser.extract_content_ids(output_ids)) == "def"
|
||||
assert parser.extract_content_ids(open_reasoning_ids) == []
|
||||
assert parser.extract_content_ids(content_ids) == content_ids
|
||||
assert parser.count_reasoning_tokens(output_ids) == len(tokenizer.encode("abc"))
|
||||
|
||||
|
||||
def test_token_id_helpers_enabled_mode():
|
||||
parser, tokenizer = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
|
||||
output_ids = tokenizer.encode("abc</mm:think>def", add_special_tokens=False)
|
||||
open_reasoning_ids = tokenizer.encode("abc", add_special_tokens=False)
|
||||
|
||||
assert parser.is_reasoning_end(output_ids)
|
||||
assert not parser.is_reasoning_end(open_reasoning_ids)
|
||||
assert tokenizer.decode(parser.extract_content_ids(output_ids)) == "def"
|
||||
assert parser.extract_content_ids(open_reasoning_ids) == []
|
||||
assert parser.count_reasoning_tokens(output_ids) == len(tokenizer.encode("abc"))
|
||||
assert parser.count_reasoning_tokens(open_reasoning_ids) == len(
|
||||
tokenizer.encode("abc")
|
||||
)
|
||||
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
NS = "]<]minimax[>["
|
||||
EOS_ID = 99
|
||||
|
||||
|
||||
class FakeTokenizer:
|
||||
"""Minimal fake tokenizer for unit tests."""
|
||||
|
||||
def __init__(self):
|
||||
self.model_tokenizer = True
|
||||
self.vocab: dict[str, int] = {}
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return self.vocab
|
||||
|
||||
|
||||
def sample_tools() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="create_order",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "integer"},
|
||||
"urgent": {"type": "boolean"},
|
||||
"note": {"type": "string"},
|
||||
"shipping": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"zip": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sku": {"type": "string"},
|
||||
"qty": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"},
|
||||
},
|
||||
"duplicate_demo": {"type": "object"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser() -> MinimaxM3ToolParser:
|
||||
return MinimaxM3ToolParser(FakeTokenizer(), tools=sample_tools())
|
||||
|
||||
|
||||
def build_order_call() -> str:
|
||||
return (
|
||||
f"{NS}<tool_call>\n"
|
||||
f'{NS}<invoke name="create_order">'
|
||||
f"{NS}<user_id>42{NS}</user_id>"
|
||||
f"{NS}<urgent>true{NS}</urgent>"
|
||||
f"{NS}<note>Please leave at front desk.{NS}</note>"
|
||||
f"{NS}<shipping>"
|
||||
f"{NS}<city>Singapore{NS}</city>"
|
||||
f"{NS}<zip>018956{NS}</zip>"
|
||||
f"{NS}</shipping>"
|
||||
f"{NS}<items>"
|
||||
f"{NS}<item>{NS}<sku>book-001{NS}</sku>{NS}<qty>2{NS}</qty>{NS}</item>"
|
||||
f"{NS}<item>{NS}<sku>pen-007{NS}</sku>{NS}<qty>5{NS}</qty>{NS}</item>"
|
||||
f"{NS}</items>"
|
||||
f"{NS}<metadata>"
|
||||
f"{NS}<source>mobile{NS}</source>"
|
||||
f"{NS}<campaign>may-launch{NS}</campaign>"
|
||||
f"{NS}</metadata>"
|
||||
f"{NS}<duplicate_demo>"
|
||||
f"{NS}<tag>a{NS}</tag>"
|
||||
f"{NS}<tag>b{NS}</tag>"
|
||||
f"{NS}</duplicate_demo>"
|
||||
f"{NS}</invoke>\n"
|
||||
f"{NS}</tool_call>"
|
||||
)
|
||||
|
||||
|
||||
def build_order_invocation(user_id: int) -> str:
|
||||
return (
|
||||
f'{NS}<invoke name="create_order">'
|
||||
f"{NS}<user_id>{user_id}{NS}</user_id>"
|
||||
f"{NS}</invoke>"
|
||||
)
|
||||
|
||||
|
||||
def build_multiple_order_call() -> str:
|
||||
return (
|
||||
f"{NS}<tool_call>\n"
|
||||
f"{build_order_invocation(1)}\n"
|
||||
f"{build_order_invocation(2)}\n"
|
||||
f"{NS}</tool_call>"
|
||||
)
|
||||
|
||||
|
||||
def _feed(
|
||||
parser: MinimaxM3ToolParser, chunks: list[str | tuple[str, list[int]]]
|
||||
) -> list[DeltaMessage]:
|
||||
previous = ""
|
||||
results: list[DeltaMessage] = []
|
||||
for chunk in chunks:
|
||||
if isinstance(chunk, tuple):
|
||||
delta, delta_ids = chunk
|
||||
else:
|
||||
delta = chunk
|
||||
delta_ids = []
|
||||
|
||||
current = previous + delta
|
||||
result = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous,
|
||||
current_text=current,
|
||||
delta_text=delta,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=delta_ids,
|
||||
request=None,
|
||||
)
|
||||
if result is not None:
|
||||
results.append(result)
|
||||
previous = current
|
||||
return results
|
||||
|
||||
|
||||
def _collect_content(results: list[DeltaMessage]) -> str:
|
||||
return "".join(result.content for result in results if result.content)
|
||||
|
||||
|
||||
def _collect_tool_calls(results: list[DeltaMessage]) -> dict[int, dict[str, Any]]:
|
||||
tool_calls: dict[int, dict[str, Any]] = {}
|
||||
for result in results:
|
||||
for tool_call in result.tool_calls or []:
|
||||
tool_calls.setdefault(
|
||||
tool_call.index,
|
||||
{"id": None, "name": "", "arguments": ""},
|
||||
)
|
||||
if tool_call.id:
|
||||
tool_calls[tool_call.index]["id"] = tool_call.id
|
||||
if tool_call.function:
|
||||
if tool_call.function.name:
|
||||
tool_calls[tool_call.index]["name"] += tool_call.function.name
|
||||
if tool_call.function.arguments:
|
||||
tool_calls[tool_call.index]["arguments"] += (
|
||||
tool_call.function.arguments
|
||||
)
|
||||
return tool_calls
|
||||
|
||||
|
||||
def test_minimax_m3_parser_registered():
|
||||
assert ToolParserManager.get_tool_parser("minimax_m3") is MinimaxM3ToolParser
|
||||
|
||||
|
||||
def test_non_streaming_nested_tool_call(parser):
|
||||
result = parser.extract_tool_calls(
|
||||
"I will create it.\n" + build_order_call(),
|
||||
request=None,
|
||||
)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content == "I will create it.\n"
|
||||
assert len(result.tool_calls) == 1
|
||||
tool_call = result.tool_calls[0]
|
||||
assert tool_call.function.name == "create_order"
|
||||
assert json.loads(tool_call.function.arguments) == {
|
||||
"user_id": 42,
|
||||
"urgent": True,
|
||||
"note": "Please leave at front desk.",
|
||||
"shipping": {"city": "Singapore", "zip": 18956},
|
||||
"items": [
|
||||
{"sku": "book-001", "qty": 2},
|
||||
{"sku": "pen-007", "qty": 5},
|
||||
],
|
||||
"metadata": {
|
||||
"source": "mobile",
|
||||
"campaign": "may-launch",
|
||||
},
|
||||
"duplicate_demo": {"tag": ["a", "b"]},
|
||||
}
|
||||
|
||||
|
||||
def test_non_streaming_without_tool_call_keeps_content(parser):
|
||||
result = parser.extract_tool_calls("plain response", request=None)
|
||||
|
||||
assert not result.tools_called
|
||||
assert result.tool_calls == []
|
||||
assert result.content == "plain response"
|
||||
|
||||
|
||||
def test_non_streaming_multiple_tool_calls(parser):
|
||||
result = parser.extract_tool_calls(build_multiple_order_call(), request=None)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content is None
|
||||
assert [tool_call.function.name for tool_call in result.tool_calls] == [
|
||||
"create_order",
|
||||
"create_order",
|
||||
]
|
||||
assert [
|
||||
json.loads(tool_call.function.arguments)["user_id"]
|
||||
for tool_call in result.tool_calls
|
||||
] == [1, 2]
|
||||
|
||||
|
||||
def test_streaming_without_tool_call_emits_text(parser):
|
||||
results = _feed(parser, ["plain ", "response"])
|
||||
|
||||
assert _collect_content(results) == "plain response"
|
||||
assert _collect_tool_calls(results) == {}
|
||||
|
||||
|
||||
def test_streaming_nested_tool_call(parser):
|
||||
tool_call_text = build_order_call()
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"I will create it.\n",
|
||||
tool_call_text[:5],
|
||||
tool_call_text[5:17],
|
||||
tool_call_text[17:120],
|
||||
tool_call_text[120:],
|
||||
("", [EOS_ID]),
|
||||
],
|
||||
)
|
||||
|
||||
assert _collect_content(results) == "I will create it.\n"
|
||||
tool_calls = _collect_tool_calls(results)
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["name"] == "create_order"
|
||||
assert tool_calls[0]["id"] is not None
|
||||
assert json.loads(tool_calls[0]["arguments"]) == json.loads(
|
||||
parser.streamed_args_for_tool[0]
|
||||
)
|
||||
assert json.loads(parser.prev_tool_call_arr[0]["arguments"])["items"][1]["qty"] == 5
|
||||
assert results[-1].content is None
|
||||
Reference in New Issue
Block a user