[Models][Gemma3/Gemma4] Support hidden_act variants in gated MLP (#40588)

Signed-off-by: Hiroaki Mikami <hiroaki8270+github@gmail.com>
This commit is contained in:
Hiroaki Mikami
2026-05-08 11:29:11 -07:00
committed by GitHub
parent 4140faa4a5
commit 90f145aaf7
4 changed files with 67 additions and 16 deletions
+3
View File
@@ -712,6 +712,7 @@ _ACTIVATION_REGISTRY = LazyDict(
"relu": lambda: nn.ReLU(),
"relu2": lambda: ReLUSquaredActivation(),
"silu": lambda: nn.SiLU(),
"swish": lambda: nn.SiLU(),
"quick_gelu": lambda: QuickGELU(),
"tanh": lambda: nn.Tanh(),
"sigmoid": lambda: nn.Sigmoid(),
@@ -751,7 +752,9 @@ def get_act_fn(act_fn_name: str) -> nn.Module:
_ACTIVATION_AND_MUL_REGISTRY: LazyDict[nn.Module] = LazyDict(
{
"gelu": lambda: GeluAndMul(),
"gelu_pytorch_tanh": lambda: GeluAndMul(approximate="tanh"),
"silu": lambda: SiluAndMul(),
"swish": lambda: SiluAndMul(),
"geglu": lambda: GeluAndMul(),
"swigluoai": lambda: SwigluOAIAndMul(),
}
+2 -8
View File
@@ -26,7 +26,7 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import GeluAndMul
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import (
Attention,
EncoderOnlyAttention,
@@ -88,13 +88,7 @@ class Gemma3MLP(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
if hidden_activation != "gelu_pytorch_tanh":
raise ValueError(
"Gemma3 uses `gelu_pytorch_tanh` as the hidden activation "
"function. Please set `hidden_act` and `hidden_activation` to "
"`gelu_pytorch_tanh`."
)
self.act_fn = GeluAndMul(approximate="tanh")
self.act_fn = get_act_and_mul_fn(hidden_activation)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
+2 -8
View File
@@ -35,7 +35,7 @@ from vllm.distributed import (
)
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import GeluAndMul
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
@@ -238,13 +238,7 @@ class Gemma4MLP(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
if hidden_activation != "gelu_pytorch_tanh":
raise ValueError(
"Gemma4 uses `gelu_pytorch_tanh` as the hidden activation "
"function. Please set `hidden_act` and `hidden_activation` to "
"`gelu_pytorch_tanh`."
)
self.act_fn = GeluAndMul(approximate="tanh")
self.act_fn = get_act_and_mul_fn(hidden_activation)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)