[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -17,7 +17,12 @@ class MoEActivation(Enum):
|
||||
GELU = "gelu"
|
||||
GELU_TANH = "gelu_tanh"
|
||||
RELU2 = "relu2"
|
||||
# SWIGLUOAI expects gate/up *interleaved* in w13 ([gate0, up0, gate1, ...]),
|
||||
# as in gpt-oss checkpoints. SWIGLUOAI_UNINTERLEAVE has identical math but
|
||||
# expects the *packed* layout ([all gates; all ups]), as produced by a
|
||||
# MergedColumnParallelLinear gate_up_proj (e.g. MiniMax-M3).
|
||||
SWIGLUOAI = "swigluoai"
|
||||
SWIGLUOAI_UNINTERLEAVE = "swigluoai_uninterleave"
|
||||
SWIGLUSTEP = "swiglustep"
|
||||
|
||||
# Non-gated activations (no mul with gate) expect input of shape [..., d]
|
||||
@@ -73,6 +78,7 @@ _CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.GELU: "gelu_and_mul",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "silu_and_mul_with_clamp",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
|
||||
MoEActivation.RELU2: "relu2",
|
||||
MoEActivation.SILU_NO_MUL: "silu_and_mul",
|
||||
@@ -105,8 +111,17 @@ def apply_moe_activation(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
*,
|
||||
clamp_limit: float | None = None,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function."""
|
||||
"""Apply MoE activation function.
|
||||
|
||||
``clamp_limit``/``alpha``/``beta`` (from the quant config) drive the clamped
|
||||
SwiGLU kernels: ``SILU`` + ``clamp_limit`` and ``SWIGLUOAI_UNINTERLEAVE`` both
|
||||
map to ``silu_and_mul_with_clamp``. Other activations ignore them.
|
||||
"""
|
||||
assert input.dim() == 2, "Input must be 2D"
|
||||
assert output.dim() == 2, "Output must be 2D"
|
||||
if activation.is_gated:
|
||||
@@ -122,13 +137,21 @@ def apply_moe_activation(
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == MoEActivation.SILU:
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
if clamp_limit is not None:
|
||||
# Fused silu(clamp(gate)) * clamp(up); equivalent to swiglu_limit_func.
|
||||
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, 1.0, 0.0)
|
||||
else:
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU:
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU_TANH:
|
||||
torch.ops._C.gelu_tanh_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
# SwiGLU-OAI on packed w13 (gate = first half, up = second half).
|
||||
assert clamp_limit is not None, "SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
|
||||
torch.ops._C.silu_and_mul_with_clamp(output, input, clamp_limit, alpha, beta)
|
||||
elif activation == MoEActivation.SWIGLUSTEP:
|
||||
from vllm.model_executor.layers.activation import swiglustep_and_mul_triton
|
||||
|
||||
|
||||
Reference in New Issue
Block a user