[Model] Fix Gemma4 MoE activation mismatch (#41574)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
eaec7be446
commit
6bb924bbf3
@@ -15,6 +15,7 @@ class MoEActivation(Enum):
|
||||
# and produce output of shape [..., d]
|
||||
SILU = "silu"
|
||||
GELU = "gelu"
|
||||
GELU_TANH = "gelu_tanh"
|
||||
RELU2 = "relu2"
|
||||
SWIGLUOAI = "swigluoai"
|
||||
SWIGLUSTEP = "swiglustep"
|
||||
@@ -24,6 +25,7 @@ class MoEActivation(Enum):
|
||||
# NOTE: Non-gated activations require the "_no_mul" suffix to be present.
|
||||
SILU_NO_MUL = "silu_no_mul"
|
||||
GELU_NO_MUL = "gelu_no_mul"
|
||||
GELU_TANH_NO_MUL = "gelu_tanh_no_mul"
|
||||
RELU2_NO_MUL = "relu2_no_mul"
|
||||
|
||||
@property
|
||||
@@ -53,6 +55,7 @@ class MoEActivation(Enum):
|
||||
@classmethod
|
||||
def from_str(cls, s: str) -> "MoEActivation":
|
||||
"""Parse from string for backward compatibility."""
|
||||
s = _STR_ALIASES.get(s, s)
|
||||
for member in cls:
|
||||
if member.value == s:
|
||||
return member
|
||||
@@ -61,20 +64,27 @@ class MoEActivation(Enum):
|
||||
|
||||
|
||||
# Module-level lookup tables used by MoEActivation functions.
|
||||
_STR_ALIASES: dict[str, str] = {
|
||||
"gelu_pytorch_tanh": "gelu_tanh",
|
||||
}
|
||||
|
||||
_CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.SILU: "silu_and_mul",
|
||||
MoEActivation.GELU: "gelu_and_mul",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
|
||||
MoEActivation.RELU2: "relu2",
|
||||
MoEActivation.SILU_NO_MUL: "silu_and_mul",
|
||||
MoEActivation.GELU_NO_MUL: "gelu_and_mul",
|
||||
MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_and_mul",
|
||||
MoEActivation.RELU2_NO_MUL: "relu2",
|
||||
}
|
||||
|
||||
_WITHOUT_MUL: dict[MoEActivation, MoEActivation] = {
|
||||
MoEActivation.SILU: MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU: MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH: MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2: MoEActivation.RELU2_NO_MUL,
|
||||
}
|
||||
|
||||
@@ -115,6 +125,8 @@ def apply_moe_activation(
|
||||
torch.ops._C.silu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU:
|
||||
torch.ops._C.gelu_and_mul(output, input)
|
||||
elif activation == MoEActivation.GELU_TANH:
|
||||
torch.ops._C.gelu_tanh_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
elif activation == MoEActivation.SWIGLUSTEP:
|
||||
@@ -127,6 +139,8 @@ def apply_moe_activation(
|
||||
output.copy_(F.silu(input))
|
||||
elif activation == MoEActivation.GELU_NO_MUL:
|
||||
output.copy_(F.gelu(input))
|
||||
elif activation == MoEActivation.GELU_TANH_NO_MUL:
|
||||
output.copy_(F.gelu(input, approximate="tanh"))
|
||||
elif activation == MoEActivation.RELU2_NO_MUL:
|
||||
F.relu(input, inplace=True)
|
||||
torch.square(input, out=output)
|
||||
|
||||
@@ -786,9 +786,11 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
|
||||
|
||||
@@ -152,10 +152,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
|
||||
|
||||
@@ -613,10 +613,12 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
|
||||
|
||||
@@ -1941,10 +1941,12 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SWIGLUSTEP,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
|
||||
|
||||
@@ -360,7 +360,7 @@ class Gemma4MoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
custom_routing_function=routing_function,
|
||||
activation="gelu",
|
||||
activation="gelu_tanh",
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor, router_logits: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user