diff --git a/vllm/model_executor/layers/fused_moe/activation.py b/vllm/model_executor/layers/fused_moe/activation.py index 3112b3054f..b2e67e6220 100644 --- a/vllm/model_executor/layers/fused_moe/activation.py +++ b/vllm/model_executor/layers/fused_moe/activation.py @@ -15,6 +15,7 @@ class MoEActivation(Enum): # and produce output of shape [..., d] SILU = "silu" GELU = "gelu" + GELU_TANH = "gelu_tanh" RELU2 = "relu2" SWIGLUOAI = "swigluoai" SWIGLUSTEP = "swiglustep" @@ -24,6 +25,7 @@ class MoEActivation(Enum): # NOTE: Non-gated activations require the "_no_mul" suffix to be present. SILU_NO_MUL = "silu_no_mul" GELU_NO_MUL = "gelu_no_mul" + GELU_TANH_NO_MUL = "gelu_tanh_no_mul" RELU2_NO_MUL = "relu2_no_mul" @property @@ -53,6 +55,7 @@ class MoEActivation(Enum): @classmethod def from_str(cls, s: str) -> "MoEActivation": """Parse from string for backward compatibility.""" + s = _STR_ALIASES.get(s, s) for member in cls: if member.value == s: return member @@ -61,20 +64,27 @@ class MoEActivation(Enum): # Module-level lookup tables used by MoEActivation functions. +_STR_ALIASES: dict[str, str] = { + "gelu_pytorch_tanh": "gelu_tanh", +} + _CUSTOM_OP_NAMES: dict[MoEActivation, str] = { MoEActivation.SILU: "silu_and_mul", MoEActivation.GELU: "gelu_and_mul", + MoEActivation.GELU_TANH: "gelu_tanh_and_mul", MoEActivation.SWIGLUOAI: "swigluoai_and_mul", MoEActivation.SWIGLUSTEP: "swiglustep_and_mul", MoEActivation.RELU2: "relu2", MoEActivation.SILU_NO_MUL: "silu_and_mul", MoEActivation.GELU_NO_MUL: "gelu_and_mul", + MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_and_mul", MoEActivation.RELU2_NO_MUL: "relu2", } _WITHOUT_MUL: dict[MoEActivation, MoEActivation] = { MoEActivation.SILU: MoEActivation.SILU_NO_MUL, MoEActivation.GELU: MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH: MoEActivation.GELU_TANH_NO_MUL, MoEActivation.RELU2: MoEActivation.RELU2_NO_MUL, } @@ -115,6 +125,8 @@ def apply_moe_activation( torch.ops._C.silu_and_mul(output, input) elif activation == MoEActivation.GELU: torch.ops._C.gelu_and_mul(output, input) + elif activation == MoEActivation.GELU_TANH: + torch.ops._C.gelu_tanh_and_mul(output, input) elif activation == MoEActivation.SWIGLUOAI: torch.ops._C.swigluoai_and_mul(output, input) elif activation == MoEActivation.SWIGLUSTEP: @@ -127,6 +139,8 @@ def apply_moe_activation( output.copy_(F.silu(input)) elif activation == MoEActivation.GELU_NO_MUL: output.copy_(F.gelu(input)) + elif activation == MoEActivation.GELU_TANH_NO_MUL: + output.copy_(F.gelu(input, approximate="tanh")) elif activation == MoEActivation.RELU2_NO_MUL: F.relu(input, inplace=True) torch.square(input, out=output) diff --git a/vllm/model_executor/layers/fused_moe/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/fused_batched_moe.py index bd54cd636b..8638a11466 100644 --- a/vllm/model_executor/layers/fused_moe/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_batched_moe.py @@ -786,9 +786,11 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular): return activation in [ MoEActivation.SILU, MoEActivation.GELU, + MoEActivation.GELU_TANH, MoEActivation.SWIGLUOAI, MoEActivation.SILU_NO_MUL, MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH_NO_MUL, MoEActivation.RELU2_NO_MUL, ] diff --git a/vllm/model_executor/layers/fused_moe/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/fused_humming_moe.py index 64d12f9558..5876f1c87e 100644 --- a/vllm/model_executor/layers/fused_moe/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_humming_moe.py @@ -152,10 +152,12 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): return activation in [ MoEActivation.SILU, MoEActivation.GELU, + MoEActivation.GELU_TANH, MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUSTEP, MoEActivation.SILU_NO_MUL, MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH_NO_MUL, MoEActivation.RELU2_NO_MUL, ] diff --git a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py b/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py index ebd3301970..3487ac1766 100644 --- a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py @@ -613,10 +613,12 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular): return activation in [ MoEActivation.SILU, MoEActivation.GELU, + MoEActivation.GELU_TANH, MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUSTEP, MoEActivation.SILU_NO_MUL, MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH_NO_MUL, MoEActivation.RELU2_NO_MUL, ] diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 7e7bcc7099..1a65593425 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -1941,10 +1941,12 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular): return activation in [ MoEActivation.SILU, MoEActivation.GELU, + MoEActivation.GELU_TANH, MoEActivation.SWIGLUOAI, MoEActivation.SWIGLUSTEP, MoEActivation.SILU_NO_MUL, MoEActivation.GELU_NO_MUL, + MoEActivation.GELU_TANH_NO_MUL, MoEActivation.RELU2_NO_MUL, ] diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 4c8edf2a01..31f2d6a28d 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -360,7 +360,7 @@ class Gemma4MoE(nn.Module): quant_config=quant_config, prefix=f"{prefix}.experts", custom_routing_function=routing_function, - activation="gelu", + activation="gelu_tanh", ) def forward(self, x: torch.Tensor, router_logits: torch.Tensor) -> torch.Tensor: