[Models][Gemma3/Gemma4] Support hidden_act variants in gated MLP (#40588)

Signed-off-by: Hiroaki Mikami <hiroaki8270+github@gmail.com>
This commit is contained in:
Hiroaki Mikami
2026-05-08 11:29:11 -07:00
committed by GitHub
parent 4140faa4a5
commit 90f145aaf7
4 changed files with 67 additions and 16 deletions
+2 -8
View File
@@ -26,7 +26,7 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import GeluAndMul
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import (
Attention,
EncoderOnlyAttention,
@@ -88,13 +88,7 @@ class Gemma3MLP(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
if hidden_activation != "gelu_pytorch_tanh":
raise ValueError(
"Gemma3 uses `gelu_pytorch_tanh` as the hidden activation "
"function. Please set `hidden_act` and `hidden_activation` to "
"`gelu_pytorch_tanh`."
)
self.act_fn = GeluAndMul(approximate="tanh")
self.act_fn = get_act_and_mul_fn(hidden_activation)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)
+2 -8
View File
@@ -35,7 +35,7 @@ from vllm.distributed import (
)
from vllm.forward_context import get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import GeluAndMul
from vllm.model_executor.layers.activation import get_act_and_mul_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
@@ -238,13 +238,7 @@ class Gemma4MLP(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
if hidden_activation != "gelu_pytorch_tanh":
raise ValueError(
"Gemma4 uses `gelu_pytorch_tanh` as the hidden activation "
"function. Please set `hidden_act` and `hidden_activation` to "
"`gelu_pytorch_tanh`."
)
self.act_fn = GeluAndMul(approximate="tanh")
self.act_fn = get_act_and_mul_fn(hidden_activation)
def forward(self, x: torch.Tensor) -> torch.Tensor:
gate_up, _ = self.gate_up_proj(x)