[Models][Gemma3/Gemma4] Support hidden_act variants in gated MLP (#40588)

Signed-off-by: Hiroaki Mikami <hiroaki8270+github@gmail.com>
This commit is contained in:
Hiroaki Mikami
2026-05-08 11:29:11 -07:00
committed by GitHub
parent 4140faa4a5
commit 90f145aaf7
4 changed files with 67 additions and 16 deletions
+3
View File
@@ -712,6 +712,7 @@ _ACTIVATION_REGISTRY = LazyDict(
"relu": lambda: nn.ReLU(),
"relu2": lambda: ReLUSquaredActivation(),
"silu": lambda: nn.SiLU(),
"swish": lambda: nn.SiLU(),
"quick_gelu": lambda: QuickGELU(),
"tanh": lambda: nn.Tanh(),
"sigmoid": lambda: nn.Sigmoid(),
@@ -751,7 +752,9 @@ def get_act_fn(act_fn_name: str) -> nn.Module:
_ACTIVATION_AND_MUL_REGISTRY: LazyDict[nn.Module] = LazyDict(
{
"gelu": lambda: GeluAndMul(),
"gelu_pytorch_tanh": lambda: GeluAndMul(approximate="tanh"),
"silu": lambda: SiluAndMul(),
"swish": lambda: SiluAndMul(),
"geglu": lambda: GeluAndMul(),
"swigluoai": lambda: SwigluOAIAndMul(),
}