[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fnmatch import fnmatch
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, cast
import torch
from torch.nn.parameter import Parameter
import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config import get_current_vllm_config
from vllm.logger import init_logger
@@ -27,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
make_fp8_moe_quant_config,
@@ -1720,6 +1722,22 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
return "modelopt_mxfp8"
return None
@classmethod
def from_config(cls, config: dict[str, Any]) -> "ModelOptMxFp8Config":
# MiniMax-style checkpoints tag `quant_method: "mxfp8"` + `ignored_layers`
# (same on-disk format as ModelOpt MXFP8); normalize to the ModelOpt
# schema and reuse the shared parser.
if "quantization" not in config and not config.get("quant_algo"):
config = {
"quant_method": "modelopt",
"quantization": {
"quant_algo": "MXFP8",
"kv_cache_quant_algo": config.get("kv_cache_quant_algo"),
"exclude_modules": config.get("ignored_layers", []) or [],
},
}
return cast("ModelOptMxFp8Config", super().from_config(config))
@classmethod
def _from_config(
cls,
@@ -1823,6 +1841,12 @@ class ModelOptMxFp8LinearMethod(LinearMethodBase):
layer.register_parameter("weight_scale", weight_scale)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# Idempotent: the emulation kernel may dequant the weight to BF16 at load
# time (>=2-byte). If already converted, there is nothing left to do --
# avoid re-running the MXFP8-only validation/conversion below.
if layer.weight.element_size() >= 2:
return
# Validate weight tensor
if layer.weight.ndim != 2:
raise ValueError(
@@ -2065,6 +2089,44 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
torch.stack(w2_scale_shuffled).contiguous(),
)
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
"""One-time MXFP8->BF16 weight dequant for the emulation path.
On devices without a native MXFP8 MoE kernel (e.g. gfx942 / MI300),
``Mxfp8EmulationTritonExperts`` otherwise dequantizes every expert
weight to BF16 on *every* forward step -- the dominant cost (conc1
~1.3 tok/s). Doing the dequant once here and replacing the MXFP8
parameters with BF16 makes the MoE run exactly like a plain BF16
checkpoint (full precision, no per-step dequant); SwiGLU-OAI is still
applied by the experts' ``activation()`` override. The MXFP8 weights
are freed by ``replace_parameter`` (BF16 is 2x their size; the small
E8M0 scale tensors are left in place, unused).
"""
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
)
target_dtype = getattr(layer, "orig_dtype", torch.bfloat16)
num_experts = layer.w13_weight.shape[0]
# dequant_mxfp8_to_bf16 handles arbitrary leading dims (*x.shape[:-1]),
# so dequant the whole [E, N, K] weight in one vectorized call.
w13_bf16 = dequant_mxfp8_to_bf16(layer.w13_weight, layer.w13_weight_scale).to(
target_dtype
)
w2_bf16 = dequant_mxfp8_to_bf16(layer.w2_weight, layer.w2_weight_scale).to(
target_dtype
)
replace_parameter(layer, "w13_weight", w13_bf16)
replace_parameter(layer, "w2_weight", w2_bf16)
logger.info_once(
"MXFP8->BF16 load-time dequant complete (%d experts/layer); MoE "
"now runs in BF16 with no per-step dequant.",
num_experts,
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
# TODO(bnell): why is this required only for mxfp8?
if getattr(layer, "_already_called_process_weights_after_loading", False):
@@ -2102,6 +2164,17 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
routing_tables=layer._expert_routing_tables(),
)
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
# experts would dequant MXFP8->BF16 every forward step. Convert the
# weights to BF16 once, here, so the MoE runs like a BF16 checkpoint.
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the 1-byte
# MXFP8 weights and dequant per-step (~half the memory, much slower).
if (
self.mxfp8_backend == Fp8MoeBackend.EMULATION
and envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD
):
self._dequant_mxfp8_weights_to_bf16(layer)
def maybe_make_prepare_finalize(
self,
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
@@ -2131,6 +2204,9 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
a1_scale=None,
a2_scale=None,
block_shape=self.weight_block_size,
swiglu_limit=getattr(layer, "swiglu_limit", None),
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
gemm1_beta=getattr(layer, "swiglu_beta", None),
)
def apply_monolithic(