[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
+337
View File
@@ -0,0 +1,337 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reference-vs-optimized unit tests for the MiniMax-M3 AMD/ROCm fused kernels.
Each optimized kernel added for the ROCm port has a slow PyTorch reference; the
tests assert the two agree within tolerance:
* Gemma RMSNorm (plain + fused-add-residual) -> fp32 PyTorch normalize
* SwiGLU-OAI (split layout) -> fp32 PyTorch elementwise
* Fused MXFP8 activation quant (Triton) -> _mxfp8_e4m3_quantize_torch
* Native MXFP8 linear (dot_scaled) -> dequant-to-bf16 @ matmul
* Native MXFP8 MoE (dot_scaled grouped GEMM) -> dequant-to-bf16 MoE math
The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
makes the shape ``[K//32, N]`` and Triton raises before producing output, so any
regression there fails these tests loudly.
Hardware scope: the whole module is ROCm-only (these are the AMD path; NVIDIA
uses the FlashInfer kernels). The norm/activation/quant kernels run on any ROCm
arch; the native MXFP8 ``dot_scaled`` linear/MoE tests are additionally gated to
CDNA4 gfx95x (``@requires_gfx950``) since gfx942 uses the BF16 emulation path.
Run: pytest tests/kernels/test_minimax_m3_amd_ops.py -v
"""
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("MiniMax-M3 AMD fused ops require ROCm.", allow_module_level=True)
if not torch.cuda.is_available():
pytest.skip("Requires a GPU.", allow_module_level=True)
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( # noqa: E402
_mxfp8_e4m3_quantize_torch,
_mxfp8_e4m3_quantize_triton,
dequant_mxfp8_to_bf16,
)
from vllm.models.minimax_m3.amd.ops import ( # noqa: E402
gemma_fused_add_rmsnorm,
gemma_rmsnorm,
swiglu_oai_split,
)
from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps # noqa: E402
DEVICE = "cuda"
EPS = 1e-6
def _gcn_arch() -> str:
try:
return torch.cuda.get_device_properties(0).gcnArchName
except Exception: # pragma: no cover - no device / non-AMD
return ""
# The pure-Triton norm/activation/quant kernels run on any ROCm arch (CDNA3
# gfx942 and CDNA4 gfx950). The native MXFP8 ``dot_scaled`` GEMMs (linear + MoE)
# use CDNA4 hardware microscaling and are gated to gfx95x in the source
# (``RocmDotScaledMxfp8LinearKernel.is_supported``; the MoE oracle routes gfx942
# to the BF16 emulation path instead) — so those tests are gfx950-only.
requires_gfx950 = pytest.mark.skipif(
"gfx95" not in _gcn_arch(),
reason="native MXFP8 dot_scaled is a CDNA4 (gfx95x) feature; "
"gfx942 uses the BF16 emulation path instead.",
)
def _relerr(a: torch.Tensor, b: torch.Tensor) -> float:
a = a.float()
b = b.float()
return ((a - b).norm() / (b.norm() + 1e-8)).item()
# --------------------------------------------------------------------------- #
# Gemma RMSNorm
# --------------------------------------------------------------------------- #
def _ref_gemma_rmsnorm(x, w, eps, residual=None):
orig_dtype = x.dtype
xf = x.float()
res_out = None
if residual is not None:
xf = xf + residual.float()
res_out = xf.to(orig_dtype)
xf = xf * torch.rsqrt(xf.pow(2).mean(dim=-1, keepdim=True) + eps)
xf = xf * (1.0 + w.float())
out = xf.to(orig_dtype)
return out if residual is None else (out, res_out)
@pytest.mark.parametrize("shape", [(1, 4096), (37, 6144), (128, 2048)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("seed", [0, 1234])
@torch.inference_mode()
def test_gemma_rmsnorm(shape, dtype, seed):
torch.manual_seed(seed)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
got = gemma_rmsnorm(x, w, EPS)
ref = _ref_gemma_rmsnorm(x, w, EPS)
assert got.shape == x.shape
assert _relerr(got, ref) < 5e-3
@pytest.mark.parametrize("shape", [(1, 6144), (64, 4096)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_gemma_fused_add_rmsnorm(shape, dtype):
torch.manual_seed(0)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
res = torch.randn(*shape, device=DEVICE, dtype=dtype)
w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
got_out, got_res = gemma_fused_add_rmsnorm(x, res, w, EPS)
ref_out, ref_res = _ref_gemma_rmsnorm(x, w, EPS, residual=res)
assert _relerr(got_out, ref_out) < 5e-3
# residual_out is the pre-norm sum (x + res): bit-for-bit identical cast.
assert torch.equal(got_res, ref_res)
@torch.inference_mode()
def test_gemma_rmsnorm_per_head_strided():
"""q_norm/k_norm normalize a non-contiguous ``qkv.split`` slice over head_dim."""
torch.manual_seed(0)
T, H, D, kv = 7, 48, 128, 8
total = (H + 2 * kv) * D
qkv = torch.randn(T, total, device=DEVICE, dtype=torch.bfloat16)
q = qkv[..., : H * D] # non-contiguous view (row stride == total)
q_by_head = q.view(T, H, D)
assert not q_by_head.is_contiguous()
w = torch.randn(D, device=DEVICE, dtype=torch.bfloat16) * 0.1
got = gemma_rmsnorm(q_by_head, w, EPS)
ref = _ref_gemma_rmsnorm(q_by_head, w, EPS)
assert got.shape == q_by_head.shape
assert _relerr(got, ref) < 5e-3
def test_num_warps_monotonic():
assert _num_warps(128) <= _num_warps(2048) <= _num_warps(8192)
# --------------------------------------------------------------------------- #
# SwiGLU-OAI (split layout)
# --------------------------------------------------------------------------- #
def _ref_swiglu(gate_up, alpha, beta, limit):
d = gate_up.shape[-1] // 2
gate = gate_up[..., :d].float()
up = gate_up[..., d:].float()
if limit is not None:
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
return (gate * torch.sigmoid(alpha * gate) * (up + beta)).to(gate_up.dtype)
@pytest.mark.parametrize("m,inter", [(1, 768), (64, 1536), (128, 1024)])
@pytest.mark.parametrize("limit", [7.0, None])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_swiglu_oai_split(m, inter, limit, dtype):
torch.manual_seed(0)
gate_up = torch.randn(m, 2 * inter, device=DEVICE, dtype=dtype)
got = swiglu_oai_split(gate_up, alpha=1.702, beta=1.0, limit=limit)
ref = _ref_swiglu(gate_up, 1.702, 1.0, limit)
assert got.shape == (m, inter)
assert _relerr(got, ref) < 5e-3
# --------------------------------------------------------------------------- #
# Fused MXFP8 activation quant (Triton vs torch reference)
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("shape", [(64, 4096), (1, 6144), (333, 2048)])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@torch.inference_mode()
def test_mxfp8_quant_triton_matches_torch(shape, dtype):
torch.manual_seed(0)
x = torch.randn(*shape, device=DEVICE, dtype=dtype)
xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
# E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
# most a 1-step difference at exact powers of two.
assert (s_k.int() - s_t.int()).abs().max().item() <= 1
# Dequantized values agree to fp8 granularity.
deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
deq_k = dequant_mxfp8_to_bf16(xq_k, s_k)
assert _relerr(deq_k, deq_t) < 1e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 linear (dot_scaled) vs dequant-to-bf16 matmul
# --------------------------------------------------------------------------- #
@requires_gfx950
@pytest.mark.parametrize("m,n,k", [(64, 256, 128), (37, 512, 256), (1, 6144, 4096)])
@torch.inference_mode()
def test_mxfp8_native_linear(m, n, k):
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
_mxfp8_dot_scaled_linear,
)
torch.manual_seed(0)
w_bf16 = torch.randn(n, k, device=DEVICE, dtype=torch.bfloat16) * 0.1
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
x = torch.randn(m, k, device=DEVICE, dtype=torch.bfloat16) * 0.5
got = _mxfp8_dot_scaled_linear(x, w_fp8, w_scale)
# Reference: consume the SAME quantized weights (isolates activation-quant
# noise) -> dequant to bf16, plain matmul.
w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
ref = torch.nn.functional.linear(x, w_deq).to(x.dtype)
assert got.shape == (m, n)
# Only the activation is re-quantized inside the kernel -> small MX noise.
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# Native MXFP8 MoE (dot_scaled grouped GEMM) vs dequant-to-bf16 MoE math
# --------------------------------------------------------------------------- #
def _ref_moe(x, w13, w2, topk_weights, topk_ids, alpha, beta, limit):
T, H = x.shape
inter = w2.shape[-1]
top_k = topk_ids.shape[1]
out = torch.zeros(T, H, device=x.device, dtype=torch.float32)
for t in range(T):
for j in range(top_k):
e = int(topk_ids[t, j].item())
g1 = x[t].float() @ w13[e].float().T # [2I]
gate = g1[:inter]
up = g1[inter:]
if limit is not None:
gate = gate.clamp(max=limit)
up = up.clamp(min=-limit, max=limit)
act = gate * torch.sigmoid(alpha * gate) * (up + beta)
g2 = act @ w2[e].float().T # [H]
out[t] += topk_weights[t, j].float() * g2
return out.to(x.dtype)
@requires_gfx950
@pytest.mark.parametrize(
"T,H,inter,E,top_k", [(8, 256, 512, 8, 2), (1, 512, 256, 16, 4)]
)
@torch.inference_mode()
def test_mxfp8_native_moe(T, H, inter, E, top_k):
from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
fused_moe_mxfp8_native,
)
torch.manual_seed(0)
alpha, beta, limit = 1.702, 1.0, 7.0
w13_bf16 = torch.randn(E, 2 * inter, H, device=DEVICE, dtype=torch.bfloat16) * 0.1
w2_bf16 = torch.randn(E, H, inter, device=DEVICE, dtype=torch.bfloat16) * 0.1
w13_fp8, w13_scale = _mxfp8_e4m3_quantize_torch(
w13_bf16, is_sf_swizzled_layout=False
)
w2_fp8, w2_scale = _mxfp8_e4m3_quantize_torch(w2_bf16, is_sf_swizzled_layout=False)
x = torch.randn(T, H, device=DEVICE, dtype=torch.bfloat16) * 0.5
logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
topk_weights = topk_weights.to(torch.float32)
topk_ids = topk_ids.to(torch.int32)
got = fused_moe_mxfp8_native(
x,
w13_fp8,
w13_scale,
w2_fp8,
w2_scale,
topk_weights,
topk_ids,
alpha=alpha,
beta=beta,
limit=limit,
global_num_experts=E,
expert_map=None,
)
# Reference consumes the dequantized weights (same bits the kernel reads).
w13_deq = dequant_mxfp8_to_bf16(w13_fp8, w13_scale)
w2_deq = dequant_mxfp8_to_bf16(w2_fp8, w2_scale)
ref = _ref_moe(x, w13_deq, w2_deq, topk_weights, topk_ids, alpha, beta, limit)
assert got.shape == (T, H)
assert _relerr(got, ref) < 5e-2
# --------------------------------------------------------------------------- #
# MXFP8 linear emulation: BF16-at-load (default) vs per-step dequant + switch
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("shape", [(512, 2048), (1, 6144)])
@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("dequant_at_load", [True, False])
@torch.inference_mode()
def test_mxfp8_linear_emulation_bf16_at_load(
shape, act_dtype, dequant_at_load, monkeypatch
):
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
same result; the dtype-match (BF16/FP16 activations) must also hold."""
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
EmulationMxfp8LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
Mxfp8LinearLayerConfig,
)
monkeypatch.setenv(
"VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "1" if dequant_at_load else "0"
)
N, K = shape
torch.manual_seed(0)
w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16)
w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
assert w_scale.shape == (N, K // 32)
# Reference: dequant once, plain linear in the activation dtype.
w_ref = dequant_mxfp8_to_bf16(w_fp8, w_scale).to(act_dtype)
x = torch.randn(7, K, device=DEVICE, dtype=act_dtype)
out_ref = torch.nn.functional.linear(x, w_ref)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig())
kernel.process_weights_after_loading(layer)
if dequant_at_load:
# weights converted to BF16 at load (>= 2-byte)
assert layer.weight.element_size() >= 2
else:
# opt-out: weights stay 1-byte MXFP8, dequant happens per-step
assert layer.weight.element_size() == 1
out = kernel.apply_weights(layer, x)
assert out.dtype == act_dtype # dtype-match preserved (no tl.dot/F.linear crash)
assert _relerr(out.float(), out_ref.float()) < 2e-2