Fix weight tying (#51665)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-20 10:31:55 +00:00
committed by GitHub
parent 6e85feb1b7
commit f0c14b4f77
70 changed files with 526 additions and 238 deletions
@@ -0,0 +1,86 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from torch import nn
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
VOCAB_SIZE = 16
HIDDEN_SIZE = 4
class UntiedModel(nn.Module):
"""Nests the head like a multimodal model, which is the harder case."""
def __init__(self):
super().__init__()
self.language_model = nn.Module()
self.language_model.model = nn.Module()
self.language_model.model.embed_tokens = VocabParallelEmbedding(
VOCAB_SIZE, HIDDEN_SIZE
)
self.language_model.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
self.embed_tokens.weight.data.fill_(1.0)
self.lm_head.weight.data.fill_(1.0)
@property
def embed_tokens(self) -> VocabParallelEmbedding:
return self.language_model.model.embed_tokens
@property
def lm_head(self) -> ParallelLMHead:
return self.language_model.lm_head
def make_model_config(untied_by_checkpoint=False):
return SimpleNamespace(
model="dummy-model",
word_embeddings_untied_by_checkpoint=untied_by_checkpoint,
)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("identical", [True, False])
def test_retie_only_when_identical(identical: bool):
"""A redundant copy of a tied lm_head is shared again to reclaim memory."""
model = UntiedModel()
if not identical:
model.lm_head.weight.data.fill_(2.0)
maybe_retie_word_embeddings(model, make_model_config(untied_by_checkpoint=True))
assert (model.lm_head.weight is model.embed_tokens.weight) is identical
if not identical:
assert torch.all(model.lm_head.weight == 2.0)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_quantized_lm_head_is_left_alone():
"""A quantized head may store its weights packed under another name."""
model = UntiedModel()
model.lm_head.quant_method = SimpleNamespace()
maybe_retie_word_embeddings(model, make_model_config(untied_by_checkpoint=True))
assert model.lm_head.weight is not model.embed_tokens.weight
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_no_retie_without_checkpoint_override():
"""Word embeddings the config genuinely unties are left alone."""
model = UntiedModel()
maybe_retie_word_embeddings(model, make_model_config())
assert model.lm_head.weight is not model.embed_tokens.weight
+78
View File
@@ -4,6 +4,10 @@
import pytest
import torch
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
_merge_multimodal_embeddings,
@@ -165,6 +169,80 @@ def test_module_skip_substr():
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 1
VOCAB_SIZE = 16
HIDDEN_SIZE = 2
class ModuleWithTiedWeights(torch.nn.Module):
"""Mimics how models tie `lm_head` to the input embeddings."""
def __init__(self, tie: bool):
super().__init__()
self.model = torch.nn.Module()
self.model.embed_tokens = VocabParallelEmbedding(VOCAB_SIZE, HIDDEN_SIZE)
self.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
if tie:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
def make_embedding_weights(value: float) -> torch.Tensor:
return torch.full((VOCAB_SIZE, HIDDEN_SIZE), value)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("tie", [True, False])
def test_module_skip_tied_weights(tie: bool):
"""Tied weights must be loaded once, under the first of their names."""
mod = ModuleWithTiedWeights(tie)
weights = [
("model.embed_tokens.weight", make_embedding_weights(1.0)),
("lm_head.weight", make_embedding_weights(2.0)),
]
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
if tie:
assert loaded == {"model.embed_tokens.weight"}
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 1.0)
else:
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 2.0)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_module_skip_tied_weights_without_canonical():
"""Skipping a tied weight must not leave the shared weight uninitialized."""
mod = ModuleWithTiedWeights(tie=True)
weights = [("lm_head.weight", make_embedding_weights(2.0))]
with pytest.raises(ValueError, match="model.embed_tokens.weight"):
AutoWeightsLoader(mod).load_weights(iter(weights))
class ModuleWithSharedParam(torch.nn.Module):
"""Mimics an MoE router shared between the MLP and its fused experts."""
def __init__(self):
super().__init__()
self.experts = torch.nn.Module()
self.experts.gate = torch.nn.Linear(2, 2, bias=False)
self.gate = self.experts.gate
@pytest.mark.cpu_test
def test_module_load_shared_params_that_are_not_tied_embeddings():
"""Only tied embeddings are skipped; other shared params must still load."""
mod = ModuleWithSharedParam()
weights = [("gate.weight", torch.Tensor([[1, 2], [3, 4]]))]
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
assert loaded == {"gate.weight"}
assert torch.all(mod.gate.weight == torch.Tensor([[1, 2], [3, 4]]))
class raise_if_cuda_sync:
def __enter__(self):
self.previous_debug_mode = torch.cuda.get_sync_debug_mode()
+39
View File
@@ -604,6 +604,45 @@ def test_with_hf_config_leaves_unknown_model_type_without_architectures(
assert updated.model_config.hf_config.architectures is None
@pytest.mark.parametrize(
"checkpoint_tensors,tied",
[
# The checkpoint has an lm_head of its own, so it must win over the config
(["model.embed_tokens.weight", "lm_head.weight"], False),
(["model.embed_tokens.weight"], True),
# Contents unknown (not safetensors), so the config must be left alone
([], True),
],
)
def test_maybe_untie_word_embeddings(tmp_path, checkpoint_tensors, tied):
import torch
from safetensors.torch import save_file
if checkpoint_tensors:
save_file(
{name: torch.zeros(2, 2) for name in checkpoint_tensors},
tmp_path / "model.safetensors",
)
text_config = SimpleNamespace(tie_word_embeddings=True)
model_config = SimpleNamespace(
model=str(tmp_path),
revision=None,
hf_config=SimpleNamespace(
tie_word_embeddings=True,
get_text_config=lambda: text_config,
),
word_embeddings_untied_by_checkpoint=False,
)
ModelConfig.maybe_untie_word_embeddings(model_config)
# Both levels must agree, since different callers read different ones
assert model_config.hf_config.tie_word_embeddings is tied
assert text_config.tie_word_embeddings is tied
assert model_config.word_embeddings_untied_by_checkpoint is not tied
def test_async_scheduling_with_pipeline_parallelism_is_allowed():
cfg = VllmConfig(
scheduler_config=SchedulerConfig(
+24 -2
View File
@@ -8,14 +8,17 @@ only get the `eos_token_id` from the tokenizer as defined by
from types import SimpleNamespace
from typing import cast
from unittest.mock import patch
from unittest.mock import MagicMock, patch
from transformers import PretrainedConfig
from vllm.config.model import ModelConfig
from vllm.tokenizers import get_tokenizer
from vllm.transformers_utils import config as config_module
from vllm.transformers_utils.config import try_get_generation_config
from vllm.transformers_utils.config import (
get_safetensors_params_metadata,
try_get_generation_config,
)
def test_get_llama3_eos_token():
@@ -77,3 +80,22 @@ def test_model_config_generation_fallback_forwards_code_revision():
config_format="auto",
token=None,
)
def test_safetensors_metadata_of_repo_without_safetensors():
"""A repo storing its weights in another format is an answer, not a failure,
so it must not be retried."""
from huggingface_hub.errors import LocalEntryNotFoundError, NotASafetensorsRepoError
get_safetensors_metadata = MagicMock(
side_effect=NotASafetensorsRepoError("not a safetensors repo")
)
api = SimpleNamespace(
get_safetensors_metadata=get_safetensors_metadata,
snapshot_download=MagicMock(side_effect=LocalEntryNotFoundError("no cache")),
)
with patch.object(config_module, "hf_api", lambda: api):
assert get_safetensors_params_metadata("some/pytorch-only-model") == {}
get_safetensors_metadata.assert_called_once()
@@ -14,6 +14,7 @@ from vllm.transformers_utils.repo_utils import (
get_hf_file_to_dict,
is_mistral_model_repo,
list_filtered_repo_files,
with_retry,
)
@@ -185,3 +186,13 @@ def test_is_mistral_model_repo(files: list[str], expected_bool: bool):
repo_type="model",
token="token",
)
def test_with_retry_does_not_retry_fatal_errors():
"""A definitive answer must not be retried, which costs a delay per attempt."""
func = MagicMock(side_effect=FileNotFoundError("no such file"))
with pytest.raises(FileNotFoundError):
with_retry(func, "Error", fatal_errors=(FileNotFoundError,))
func.assert_called_once()