From 5794c65f8c36d9f7f486277cc5ed40e65b6e6100 Mon Sep 17 00:00:00 2001 From: Noelia Bentancor <71080743+NoeliaBentancor@users.noreply.github.com> Date: Wed, 13 May 2026 14:43:12 -0300 Subject: [PATCH] [Bugfix][Model] Gemma4 MoE routing closure captures per_expert_scale, breaking functional_call substitution (#42250) Signed-off-by: Noelia Signed-off-by: Noelia Bentancor <71080743+NoeliaBentancor@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/models/gemma4.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 1b50db61c2..75f6945ccc 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -326,8 +326,9 @@ class Gemma4MoE(nn.Module): # Gemma4 routing: softmax over ALL experts → top-k → renormalize. # FusedMoE's built-in fused_topk scopes softmax differently, so # a custom routing function is needed for numerical correctness. - per_expert_scale = self.per_expert_scale - + # NOTE: self.per_expert_scale is read at call time (not captured into + # a local) so that torch.func.functional_call parameter substitution + # reaches the routing function correctly. def routing_function( hidden_states: torch.Tensor, gating_output: torch.Tensor, @@ -336,10 +337,12 @@ class Gemma4MoE(nn.Module): ) -> tuple[torch.Tensor, torch.Tensor]: if current_platform.is_cuda_alike() or current_platform.is_xpu(): return gemma4_fused_routing_kernel_triton( - gating_output, topk, per_expert_scale + gating_output, topk, self.per_expert_scale ) - return gemma4_routing_function_torch(gating_output, topk, per_expert_scale) + return gemma4_routing_function_torch( + gating_output, topk, self.per_expert_scale + ) # FusedMoE experts with custom Gemma4 routing self.experts = FusedMoE(