[Bugfix][Model] Fix MiniMax-M3 NVFP4 inference correctness (#48929)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: Pavani Majety <pmajety@nvidia.com>
This commit is contained in:
co-authored by
Pavani Majety
parent
cd930c8e78
commit
08b8613b7b
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
@@ -52,6 +53,74 @@ MNK_FACTORS = [
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"activation",
|
||||
[MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUOAI_UNINTERLEAVE],
|
||||
)
|
||||
def test_flashinfer_swigluoai_params_are_forwarded(activation, monkeypatch):
|
||||
from flashinfer.fused_moe.core import ActivationType
|
||||
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=2,
|
||||
experts_per_token=1,
|
||||
hidden_dim=128,
|
||||
intermediate_size=128,
|
||||
num_local_experts=2,
|
||||
num_logical_experts=2,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
)
|
||||
quant_config = FusedMoEQuantConfig.make(
|
||||
gemm1_alpha=1.702,
|
||||
gemm1_beta=1.0,
|
||||
gemm1_clamp_limit=7.0,
|
||||
)
|
||||
experts = FlashInferExperts(moe_config=moe_config, quant_config=quant_config)
|
||||
|
||||
call_args = {}
|
||||
|
||||
def fake_flashinfer_cutlass_fused_moe(**kwargs):
|
||||
call_args.update(kwargs)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.experts."
|
||||
"flashinfer_cutlass_moe.flashinfer_cutlass_fused_moe",
|
||||
fake_flashinfer_cutlass_fused_moe,
|
||||
)
|
||||
experts.apply(
|
||||
output=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
|
||||
hidden_states=torch.empty((1, 128), device="cuda", dtype=torch.bfloat16),
|
||||
w1=torch.empty((2, 256, 128), device="cuda", dtype=torch.bfloat16),
|
||||
w2=torch.empty((2, 128, 128), device="cuda", dtype=torch.bfloat16),
|
||||
topk_weights=torch.ones((1, 1), device="cuda", dtype=torch.float32),
|
||||
topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
|
||||
activation=activation,
|
||||
global_num_experts=2,
|
||||
expert_map=None,
|
||||
a1q_scale=None,
|
||||
a2_scale=None,
|
||||
workspace13=None,
|
||||
workspace2=None,
|
||||
expert_tokens_meta=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
assert experts._supports_activation(activation)
|
||||
assert call_args["activation_type"] == ActivationType.Swiglu
|
||||
for name, value in (
|
||||
("swiglu_alpha", 1.702),
|
||||
("swiglu_beta", 1.0),
|
||||
("swiglu_limit", 7.0),
|
||||
):
|
||||
torch.testing.assert_close(
|
||||
call_args[name],
|
||||
torch.full((2,), value, device="cuda", dtype=torch.float32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k", MNK_FACTORS)
|
||||
@pytest.mark.parametrize("e", [40, 64, 256])
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8])
|
||||
|
||||
Reference in New Issue
Block a user