[Model] Add MiniMax M3 support (#45381)
Signed-off-by: youkaichao <youkaichao@gmail.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
Isotr0py
Thien Tran
Bugen Zhao
Jee Jee Li
Roger Wang
functionstackx
Yongye Zhu
Jee Jee Li
parent
a3195fab7b
commit
0a1c5034f5
@@ -158,17 +158,28 @@ class SiluAndMulWithClamp(CustomOp):
|
||||
Computes:
|
||||
gate = clamp(x[..., :d], max=swiglu_limit)
|
||||
up = clamp(x[..., d:], min=-swiglu_limit, max=swiglu_limit)
|
||||
out = silu(gate) * up
|
||||
where d = x.shape[-1] // 2.
|
||||
out = gate * sigmoid(alpha * gate) * (up + beta)
|
||||
where d = x.shape[-1] // 2. The defaults alpha=1.0, beta=0.0 reduce this to
|
||||
``silu(gate) * up``; SwiGLU-OAI style models pass alpha (sigmoid scale) and
|
||||
beta=1.0 (up bias).
|
||||
|
||||
Shapes:
|
||||
x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
|
||||
return: (num_tokens, d) or (batch_size, seq_len, d)
|
||||
"""
|
||||
|
||||
def __init__(self, swiglu_limit: float, *, compile_native: bool = True):
|
||||
def __init__(
|
||||
self,
|
||||
swiglu_limit: float,
|
||||
alpha: float = 1.0,
|
||||
beta: float = 0.0,
|
||||
*,
|
||||
compile_native: bool = True,
|
||||
):
|
||||
super().__init__(compile_native=compile_native)
|
||||
self.swiglu_limit = float(swiglu_limit)
|
||||
self.alpha = float(alpha)
|
||||
self.beta = float(beta)
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
self._forward_method = self.forward_native
|
||||
elif current_platform.is_cuda_alike():
|
||||
@@ -180,18 +191,24 @@ class SiluAndMulWithClamp(CustomOp):
|
||||
d = x.shape[-1] // 2
|
||||
gate = torch.clamp(x[..., :d], max=self.swiglu_limit)
|
||||
up = torch.clamp(x[..., d:], min=-self.swiglu_limit, max=self.swiglu_limit)
|
||||
return F.silu(gate) * up
|
||||
return gate * torch.sigmoid(self.alpha * gate) * (up + self.beta)
|
||||
|
||||
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
output_shape = x.shape[:-1] + (d,)
|
||||
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
|
||||
self.op(out, x, self.swiglu_limit)
|
||||
self.op(out, x, self.swiglu_limit, self.alpha, self.beta)
|
||||
return out
|
||||
|
||||
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.forward_native(x)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return (
|
||||
f"swiglu_limit={self.swiglu_limit!r}, "
|
||||
f"alpha={self.alpha!r}, beta={self.beta!r}"
|
||||
)
|
||||
|
||||
|
||||
# --8<-- [start:mul_and_silu]
|
||||
@CustomOp.register("mul_and_silu")
|
||||
|
||||
Reference in New Issue
Block a user