[Model] Fix Gemma4 MoE activation mismatch (#41574)

Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com>
Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
Luciano Martins
2026-05-05 04:34:11 +00:00
committed by GitHub
co-authored by Luciano Martins
parent eaec7be446
commit 6bb924bbf3
6 changed files with 23 additions and 1 deletions
@@ -15,6 +15,7 @@ class MoEActivation(Enum):
# and produce output of shape [..., d]
SILU = "silu"
GELU = "gelu"
GELU_TANH = "gelu_tanh"
RELU2 = "relu2"
SWIGLUOAI = "swigluoai"
SWIGLUSTEP = "swiglustep"
@@ -24,6 +25,7 @@ class MoEActivation(Enum):
# NOTE: Non-gated activations require the "_no_mul" suffix to be present.
SILU_NO_MUL = "silu_no_mul"
GELU_NO_MUL = "gelu_no_mul"
GELU_TANH_NO_MUL = "gelu_tanh_no_mul"
RELU2_NO_MUL = "relu2_no_mul"
@property
@@ -53,6 +55,7 @@ class MoEActivation(Enum):
@classmethod
def from_str(cls, s: str) -> "MoEActivation":
"""Parse from string for backward compatibility."""
s = _STR_ALIASES.get(s, s)
for member in cls:
if member.value == s:
return member
@@ -61,20 +64,27 @@ class MoEActivation(Enum):
# Module-level lookup tables used by MoEActivation functions.
_STR_ALIASES: dict[str, str] = {
"gelu_pytorch_tanh": "gelu_tanh",
}
_CUSTOM_OP_NAMES: dict[MoEActivation, str] = {
MoEActivation.SILU: "silu_and_mul",
MoEActivation.GELU: "gelu_and_mul",
MoEActivation.GELU_TANH: "gelu_tanh_and_mul",
MoEActivation.SWIGLUOAI: "swigluoai_and_mul",
MoEActivation.SWIGLUSTEP: "swiglustep_and_mul",
MoEActivation.RELU2: "relu2",
MoEActivation.SILU_NO_MUL: "silu_and_mul",
MoEActivation.GELU_NO_MUL: "gelu_and_mul",
MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_and_mul",
MoEActivation.RELU2_NO_MUL: "relu2",
}
_WITHOUT_MUL: dict[MoEActivation, MoEActivation] = {
MoEActivation.SILU: MoEActivation.SILU_NO_MUL,
MoEActivation.GELU: MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH: MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2: MoEActivation.RELU2_NO_MUL,
}
@@ -115,6 +125,8 @@ def apply_moe_activation(
torch.ops._C.silu_and_mul(output, input)
elif activation == MoEActivation.GELU:
torch.ops._C.gelu_and_mul(output, input)
elif activation == MoEActivation.GELU_TANH:
torch.ops._C.gelu_tanh_and_mul(output, input)
elif activation == MoEActivation.SWIGLUOAI:
torch.ops._C.swigluoai_and_mul(output, input)
elif activation == MoEActivation.SWIGLUSTEP:
@@ -127,6 +139,8 @@ def apply_moe_activation(
output.copy_(F.silu(input))
elif activation == MoEActivation.GELU_NO_MUL:
output.copy_(F.gelu(input))
elif activation == MoEActivation.GELU_TANH_NO_MUL:
output.copy_(F.gelu(input, approximate="tanh"))
elif activation == MoEActivation.RELU2_NO_MUL:
F.relu(input, inplace=True)
torch.square(input, out=output)
@@ -786,9 +786,11 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
@@ -152,10 +152,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
@@ -613,10 +613,12 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular):
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
@@ -1941,10 +1941,12 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
return activation in [
MoEActivation.SILU,
MoEActivation.GELU,
MoEActivation.GELU_TANH,
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUSTEP,
MoEActivation.SILU_NO_MUL,
MoEActivation.GELU_NO_MUL,
MoEActivation.GELU_TANH_NO_MUL,
MoEActivation.RELU2_NO_MUL,
]
+1 -1
View File
@@ -360,7 +360,7 @@ class Gemma4MoE(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.experts",
custom_routing_function=routing_function,
activation="gelu",
activation="gelu_tanh",
)
def forward(self, x: torch.Tensor, router_logits: torch.Tensor) -> torch.Tensor: