[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -2,11 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fnmatch import fnmatch
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
import vllm.envs as envs
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
@@ -27,6 +28,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
make_fp8_moe_quant_config,
|
||||
@@ -1720,6 +1722,22 @@ class ModelOptMxFp8Config(ModelOptQuantConfigBase):
|
||||
return "modelopt_mxfp8"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, config: dict[str, Any]) -> "ModelOptMxFp8Config":
|
||||
# MiniMax-style checkpoints tag `quant_method: "mxfp8"` + `ignored_layers`
|
||||
# (same on-disk format as ModelOpt MXFP8); normalize to the ModelOpt
|
||||
# schema and reuse the shared parser.
|
||||
if "quantization" not in config and not config.get("quant_algo"):
|
||||
config = {
|
||||
"quant_method": "modelopt",
|
||||
"quantization": {
|
||||
"quant_algo": "MXFP8",
|
||||
"kv_cache_quant_algo": config.get("kv_cache_quant_algo"),
|
||||
"exclude_modules": config.get("ignored_layers", []) or [],
|
||||
},
|
||||
}
|
||||
return cast("ModelOptMxFp8Config", super().from_config(config))
|
||||
|
||||
@classmethod
|
||||
def _from_config(
|
||||
cls,
|
||||
@@ -1823,6 +1841,12 @@ class ModelOptMxFp8LinearMethod(LinearMethodBase):
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Idempotent: the emulation kernel may dequant the weight to BF16 at load
|
||||
# time (>=2-byte). If already converted, there is nothing left to do --
|
||||
# avoid re-running the MXFP8-only validation/conversion below.
|
||||
if layer.weight.element_size() >= 2:
|
||||
return
|
||||
|
||||
# Validate weight tensor
|
||||
if layer.weight.ndim != 2:
|
||||
raise ValueError(
|
||||
@@ -2065,6 +2089,44 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
torch.stack(w2_scale_shuffled).contiguous(),
|
||||
)
|
||||
|
||||
def _dequant_mxfp8_weights_to_bf16(self, layer: RoutedExperts) -> None:
|
||||
"""One-time MXFP8->BF16 weight dequant for the emulation path.
|
||||
|
||||
On devices without a native MXFP8 MoE kernel (e.g. gfx942 / MI300),
|
||||
``Mxfp8EmulationTritonExperts`` otherwise dequantizes every expert
|
||||
weight to BF16 on *every* forward step -- the dominant cost (conc1
|
||||
~1.3 tok/s). Doing the dequant once here and replacing the MXFP8
|
||||
parameters with BF16 makes the MoE run exactly like a plain BF16
|
||||
checkpoint (full precision, no per-step dequant); SwiGLU-OAI is still
|
||||
applied by the experts' ``activation()`` override. The MXFP8 weights
|
||||
are freed by ``replace_parameter`` (BF16 is 2x their size; the small
|
||||
E8M0 scale tensors are left in place, unused).
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
|
||||
target_dtype = getattr(layer, "orig_dtype", torch.bfloat16)
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
|
||||
# dequant_mxfp8_to_bf16 handles arbitrary leading dims (*x.shape[:-1]),
|
||||
# so dequant the whole [E, N, K] weight in one vectorized call.
|
||||
w13_bf16 = dequant_mxfp8_to_bf16(layer.w13_weight, layer.w13_weight_scale).to(
|
||||
target_dtype
|
||||
)
|
||||
w2_bf16 = dequant_mxfp8_to_bf16(layer.w2_weight, layer.w2_weight_scale).to(
|
||||
target_dtype
|
||||
)
|
||||
|
||||
replace_parameter(layer, "w13_weight", w13_bf16)
|
||||
replace_parameter(layer, "w2_weight", w2_bf16)
|
||||
|
||||
logger.info_once(
|
||||
"MXFP8->BF16 load-time dequant complete (%d experts/layer); MoE "
|
||||
"now runs in BF16 with no per-step dequant.",
|
||||
num_experts,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# TODO(bnell): why is this required only for mxfp8?
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
@@ -2102,6 +2164,17 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
# No native MXFP8 MoE kernel on this device (e.g. gfx942): the emulation
|
||||
# experts would dequant MXFP8->BF16 every forward step. Convert the
|
||||
# weights to BF16 once, here, so the MoE runs like a BF16 checkpoint.
|
||||
# Opt out (VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0) to keep the 1-byte
|
||||
# MXFP8 weights and dequant per-step (~half the memory, much slower).
|
||||
if (
|
||||
self.mxfp8_backend == Fp8MoeBackend.EMULATION
|
||||
and envs.VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD
|
||||
):
|
||||
self._dequant_mxfp8_weights_to_bf16(layer)
|
||||
|
||||
def maybe_make_prepare_finalize(
|
||||
self,
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
@@ -2131,6 +2204,9 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=self.weight_block_size,
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
)
|
||||
|
||||
def apply_monolithic(
|
||||
|
||||
Reference in New Issue
Block a user