[Model] Add MiniMax M3 support (#45381)

Signed-off-by: youkaichao <youkaichao@gmail.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
youkaichao
2026-06-16 01:01:25 +08:00
committed by GitHub
co-authored by OpenAI Codex Isotr0py Thien Tran Bugen Zhao Jee Jee Li Roger Wang functionstackx Yongye Zhu Jee Jee Li
parent a3195fab7b
commit 0a1c5034f5
108 changed files with 14734 additions and 311 deletions
+22 -5
View File
@@ -158,17 +158,28 @@ class SiluAndMulWithClamp(CustomOp):
Computes:
gate = clamp(x[..., :d], max=swiglu_limit)
up = clamp(x[..., d:], min=-swiglu_limit, max=swiglu_limit)
out = silu(gate) * up
where d = x.shape[-1] // 2.
out = gate * sigmoid(alpha * gate) * (up + beta)
where d = x.shape[-1] // 2. The defaults alpha=1.0, beta=0.0 reduce this to
``silu(gate) * up``; SwiGLU-OAI style models pass alpha (sigmoid scale) and
beta=1.0 (up bias).
Shapes:
x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
return: (num_tokens, d) or (batch_size, seq_len, d)
"""
def __init__(self, swiglu_limit: float, *, compile_native: bool = True):
def __init__(
self,
swiglu_limit: float,
alpha: float = 1.0,
beta: float = 0.0,
*,
compile_native: bool = True,
):
super().__init__(compile_native=compile_native)
self.swiglu_limit = float(swiglu_limit)
self.alpha = float(alpha)
self.beta = float(beta)
if current_platform.is_rocm() or current_platform.is_xpu():
self._forward_method = self.forward_native
elif current_platform.is_cuda_alike():
@@ -180,18 +191,24 @@ class SiluAndMulWithClamp(CustomOp):
d = x.shape[-1] // 2
gate = torch.clamp(x[..., :d], max=self.swiglu_limit)
up = torch.clamp(x[..., d:], min=-self.swiglu_limit, max=self.swiglu_limit)
return F.silu(gate) * up
return gate * torch.sigmoid(self.alpha * gate) * (up + self.beta)
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
output_shape = x.shape[:-1] + (d,)
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
self.op(out, x, self.swiglu_limit)
self.op(out, x, self.swiglu_limit, self.alpha, self.beta)
return out
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
return self.forward_native(x)
def extra_repr(self) -> str:
return (
f"swiglu_limit={self.swiglu_limit!r}, "
f"alpha={self.alpha!r}, beta={self.beta!r}"
)
# --8<-- [start:mul_and_silu]
@CustomOp.register("mul_and_silu")