From 8cc9aa5ad350f6f80cf3bebb298d410803ff75a9 Mon Sep 17 00:00:00 2001 From: Yannick Schnider Date: Mon, 28 Sep 2026 12:56:31 +0200 Subject: [PATCH] [Bugfix][Model] Gemma4: register aliased embedding scalars as buffers (#54213) Signed-off-by: Yannick Schnider --- vllm/model_executor/models/gemma4.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index e3dfe65c2f..17b8251e9a 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -846,13 +846,22 @@ class Gemma4SelfDecoderLayers(nn.Module): # Shared references to modules owned by Gemma4Model — must be # inside this nn.Module so torch.compile captures them. self.embed_tokens = embed_tokens - self.normalizer = normalizer self.embed_tokens_per_layer = embed_tokens_per_layer - self.embed_scale_per_layer = embed_scale_per_layer self.per_layer_model_projection = per_layer_model_projection self.per_layer_projection_norm = per_layer_projection_norm - self.per_layer_input_scale = per_layer_input_scale - self.per_layer_projection_scale = per_layer_projection_scale + + # Register the shared scalars as buffers so they follow the model on + # .to(device); plain attributes stay on CPU. + self.register_buffer("normalizer", normalizer, persistent=False) + self.register_buffer( + "embed_scale_per_layer", embed_scale_per_layer, persistent=False + ) + self.register_buffer( + "per_layer_input_scale", per_layer_input_scale, persistent=False + ) + self.register_buffer( + "per_layer_projection_scale", per_layer_projection_scale, persistent=False + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) * self.normalizer