[Model] Fix Gemma4 MoE activation mismatch (#41574)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com>
This commit is contained in:
co-authored by
Luciano Martins
parent
eaec7be446
commit
6bb924bbf3
@@ -360,7 +360,7 @@ class Gemma4MoE(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
custom_routing_function=routing_function,
|
||||
activation="gelu",
|
||||
activation="gelu_tanh",
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor, router_logits: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
Reference in New Issue
Block a user