[Bugfix][Model] Fix MiniMax-M3 NVFP4 inference correctness (#48929)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: Pavani Majety <pmajety@nvidia.com>
This commit is contained in:
Gabriel Wu
2026-08-05 07:25:46 -07:00
committed by GitHub
co-authored by Pavani Majety
parent cd930c8e78
commit 08b8613b7b
2 changed files with 100 additions and 31 deletions
+69
View File
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
@@ -52,6 +53,74 @@ MNK_FACTORS = [
]
@pytest.mark.parametrize(
"activation",
[MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUOAI_UNINTERLEAVE],
)
def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
from flashinfer.fused_moe.core import ActivationType
moe_config = FusedMoEConfig(
num_experts=2,
experts_per_token=1,
hidden_dim=128,
intermediate_size=128,
num_local_experts=2,
num_logical_experts=2,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.TopK,
)
quant_config = FusedMoEQuantConfig.make(
gemm1_alpha=1.702,
gemm1_beta=1.0,
gemm1_clamp_limit=7.0,
)
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
call_args = {}
def fake_flashinfer_cutlass_fused_moe(**kwargs):
call_args.update(kwargs)
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.experts."
"flashinfer_cutlass_moe.flashinfer_cutlass_fused_moe",
fake_flashinfer_cutlass_fused_moe,
)
experts.apply(
output=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
hidden_states=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
w1=torch.empty((2, 256, 128), device="cuda", dtype=torch.bfloat16),
w2=torch.empty((2, 128, 128), device="cuda", dtype=torch.bfloat16),
topk_weights=torch.ones((1, 1), device="cuda", dtype=torch.float32),
topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
activation=activation,
global_num_experts=2,
expert_map=None,
a1q_scale=None,
a2_scale=None,
workspace13=None,
workspace2=None,
expert_tokens_meta=None,
apply_router_weight_on_input=False,
)
assert experts._supports_activation(activation)
assert call_args["activation_type"] == ActivationType.Swiglu
for name, value in (
("swiglu_alpha", 1.702),
("swiglu_beta", 1.0),
("swiglu_limit", 7.0),
):
torch.testing.assert_close(
call_args[name],
torch.full((2,), value, device="cuda", dtype=torch.float32),
)
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
@pytest.mark.parametrize("e", [40, 64, 256])
@pytest.mark.parametrize("topk", [1, 6, 8])
@@ -13,6 +13,9 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
activation_to_flashinfer_type,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
@@ -93,29 +96,30 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
# - pass per-block weight scales to the kernel
# - skip input activation quantization (kernel applies scaling)
self.use_deepseek_fp8_block_scale = quant_config.is_block_quantized
self.gemm1_clamp_limit: torch.Tensor | None = None
if quant_config.gemm1_clamp_limit is not None:
self.gemm1_clamp_limit = torch.tensor(
[quant_config.gemm1_clamp_limit] * self.num_experts,
def _per_expert(value: float | None) -> torch.Tensor | None:
if value is None:
return None
return torch.full(
(self.num_experts,),
float(value),
dtype=torch.float32,
device=self.device,
)
self.gemm1_clamp_limit = _per_expert(quant_config.gemm1_clamp_limit)
self.gemm1_alpha = _per_expert(quant_config.gemm1_alpha)
self.gemm1_beta = _per_expert(quant_config.gemm1_beta)
if quant_config.weight_quant_dtype == "mxfp4":
# This value is used specifically for gpt-oss,
# Need to revisit this for other models
self.gemm1_alpha = torch.tensor(
[1.702] * self.num_experts, dtype=torch.float32, device=self.device
)
self.gemm1_beta = torch.tensor(
[1.0] * self.num_experts, dtype=torch.float32, device=self.device
)
if self.gemm1_alpha is None:
self.gemm1_alpha = _per_expert(1.702)
if self.gemm1_beta is None:
self.gemm1_beta = _per_expert(1.0)
if self.gemm1_clamp_limit is None:
self.gemm1_clamp_limit = torch.tensor(
[7.0] * self.num_experts,
dtype=torch.float32,
device=self.device,
)
self.gemm1_clamp_limit = _per_expert(7.0)
if quant_config.quant_dtype == "mxfp8":
self.fake_input_scale = torch.ones(
self.num_experts,
@@ -190,6 +194,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
MoEActivation.GELU_TANH,
MoEActivation.RELU2_NO_MUL,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
]
@staticmethod
@@ -263,18 +268,6 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool | None,
):
from flashinfer.fused_moe.core import ActivationType
activation_str_to_value_map = {
MoEActivation.SILU: ActivationType.Swiglu, # This is the default
MoEActivation.GELU_TANH: ActivationType.Geglu,
MoEActivation.SWIGLUOAI: ActivationType.Swiglu, # gpt-oss alias
MoEActivation.RELU2_NO_MUL: ActivationType.Relu2,
}
assert activation in activation_str_to_value_map, (
f"{activation=} missing from {activation_str_to_value_map.keys()=}"
)
quant_scales = None
fc1_expert_weights = None
fc2_expert_weights = None
@@ -282,9 +275,16 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
fc2_expert_biases = None
swiglu_alpha = None
swiglu_beta = None
swiglu_limit = (
self.gemm1_clamp_limit if activation == MoEActivation.SILU else None
)
swiglu_limit = None
if activation == MoEActivation.SILU:
swiglu_limit = self.gemm1_clamp_limit
elif activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
):
swiglu_alpha = self.gemm1_alpha
swiglu_beta = self.gemm1_beta
swiglu_limit = self.gemm1_clamp_limit
use_mxfp8_act_scaling = False
use_w4_group_scaling = False
# Select quantization metadata based on FP8 format/path
@@ -392,7 +392,7 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
tp_rank=self.tp_rank,
ep_size=self.ep_size,
ep_rank=self.ep_rank,
activation_type=activation_str_to_value_map[activation],
activation_type=activation_to_flashinfer_type(activation),
# Informs FlashInfer to use the block-scale decoding path when True
use_deepseek_fp8_block_scale=self.use_deepseek_fp8_block_scale,
use_mxfp8_act_scaling=use_mxfp8_act_scaling,