[Bugfix] Load untied Gemma LM head weights (#53170)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
@@ -1,11 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models import gemma
|
||||
|
||||
MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_checkpoint_lm_head_can_override_tied_config(monkeypatch) -> None:
|
||||
"""A physical LM head must load after checkpoint-driven untying."""
|
||||
|
||||
class StubGemmaModel(torch.nn.Module):
|
||||
def __init__(self, *, vllm_config, prefix):
|
||||
super().__init__()
|
||||
self.embed_tokens = VocabParallelEmbedding(4, 2)
|
||||
self.make_empty_intermediate_tensors = None
|
||||
|
||||
monkeypatch.setattr(gemma, "GemmaModel", StubGemmaModel)
|
||||
config = SimpleNamespace(
|
||||
vocab_size=4,
|
||||
hidden_size=2,
|
||||
tie_word_embeddings=False,
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(hf_config=config),
|
||||
quant_config=None,
|
||||
)
|
||||
model = gemma.GemmaForCausalLM(vllm_config=cast(VllmConfig, vllm_config))
|
||||
embedding_weight = torch.full((4, 2), 1.0)
|
||||
lm_head_weight = torch.full((4, 2), 2.0)
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
("model.embed_tokens.weight", embedding_weight),
|
||||
("lm_head.weight", lm_head_weight),
|
||||
]
|
||||
)
|
||||
|
||||
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
|
||||
assert torch.equal(model.model.embed_tokens.weight[:4], embedding_weight)
|
||||
assert torch.equal(model.lm_head.weight[:4], lm_head_weight)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
|
||||
with monkeypatch.context() as m:
|
||||
|
||||
@@ -41,7 +41,10 @@ from vllm.model_executor.layers.linear import (
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP, SupportsQuant
|
||||
@@ -351,6 +354,14 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
self.model = GemmaModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
@@ -375,7 +386,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
|
||||
Reference in New Issue
Block a user