[Bugfix] Load untied Gemma LM head weights (#53170)

Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
Kevin H. Luu
2026-08-21 00:47:08 +00:00
committed by GitHub
parent f32b17b6d6
commit d29f7f5c92
2 changed files with 58 additions and 2 deletions
@@ -1,11 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import cast
import numpy as np
import pytest
import torch
from vllm.config import VllmConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models import gemma
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
"""A physical LM head must load after checkpoint-driven untying."""
class StubGemmaModel(torch.nn.Module):
def __init__(self, *, vllm_config, prefix):
super().__init__()
self.embed_tokens = VocabParallelEmbedding(4, 2)
self.make_empty_intermediate_tensors = None
monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel)
config = SimpleNamespace(
vocab_size=4,
hidden_size=2,
tie_word_embeddings=False,
)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(hf_config=config),
quant_config=None,
)
model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
embedding_weight = torch.full((4, 2), 1.0)
lm_head_weight = torch.full((4, 2), 2.0)
loaded = model.load_weights(
[
("model.embed_tokens.weight", embedding_weight),
("lm_head.weight", lm_head_weight),
]
)
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight)
assert torch.equal(model.lm_head.weight[:4], lm_head_weight)
@pytest.mark.parametrize("model", MODELS)
def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
with monkeypatch.context() as m:
+13 -2
View File
@@ -41,7 +41,10 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
@@ -351,6 +354,14 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
self.model = GemmaModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
@@ -375,7 +386,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: