From d29f7f5c9294be8e489dac34d45a939b95a06336 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 20 Aug 2026 17:47:08 -0700 Subject: [PATCH] [Bugfix] Load untied Gemma LM head weights (#53170) Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> --- .../models/language/generation/test_gemma.py | 45 +++++++++++++++++++ vllm/model_executor/models/gemma.py | 15 ++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/tests/models/language/generation/test_gemma.py b/tests/models/language/generation/test_gemma.py index 246b893be3..2375fc2ab7 100644 --- a/tests/models/language/generation/test_gemma.py +++ b/tests/models/language/generation/test_gemma.py @@ -1,11 +1,56 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from types import SimpleNamespace +from typing import cast + import numpy as np import pytest +import torch + +from vllm.config import VllmConfig +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.model_executor.models import gemma MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"] +@pytest.mark.cpu_test +@pytest.mark.usefixtures("dist_init") +def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None: + """A physical LM head must load after checkpoint-driven untying.""" + + class StubGemmaModel(torch.nn.Module): + def __init__(self, *, vllm_config, prefix): + super().__init__() + self.embed_tokens = VocabParallelEmbedding(4, 2) + self.make_empty_intermediate_tensors = None + + monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel) + config = SimpleNamespace( + vocab_size=4, + hidden_size=2, + tie_word_embeddings=False, + ) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace(hf_config=config), + quant_config=None, + ) + model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config)) + embedding_weight = torch.full((4, 2), 1.0) + lm_head_weight = torch.full((4, 2), 2.0) + + loaded = model.load_weights( + [ + ("model.embed_tokens.weight", embedding_weight), + ("lm_head.weight", lm_head_weight), + ] + ) + + assert loaded == {"model.embed_tokens.weight", "lm_head.weight"} + assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight) + assert torch.equal(model.lm_head.weight[:4], lm_head_weight) + + @pytest.mark.parametrize("model", MODELS) def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None: with monkeypatch.context() as m: diff --git a/vllm/model_executor/models/gemma.py b/vllm/model_executor/models/gemma.py index 0f1282fd96..7058290865 100644 --- a/vllm/model_executor/models/gemma.py +++ b/vllm/model_executor/models/gemma.py @@ -41,7 +41,10 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant @@ -351,6 +354,14 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): self.model = GemmaModel( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) self.logits_processor = LogitsProcessor(config.vocab_size) self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors @@ -375,7 +386,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant): self, hidden_states: torch.Tensor, ) -> torch.Tensor | None: - logits = self.logits_processor(self.model.embed_tokens, hidden_states) + logits = self.logits_processor(self.lm_head, hidden_states) return logits def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: