Fix weight tying (#51665)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
|
||||
|
||||
VOCAB_SIZE = 16
|
||||
HIDDEN_SIZE = 4
|
||||
|
||||
|
||||
class UntiedModel(nn.Module):
|
||||
"""Nests the head like a multimodal model, which is the harder case."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.language_model = nn.Module()
|
||||
self.language_model.model = nn.Module()
|
||||
self.language_model.model.embed_tokens = VocabParallelEmbedding(
|
||||
VOCAB_SIZE, HIDDEN_SIZE
|
||||
)
|
||||
self.language_model.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
self.embed_tokens.weight.data.fill_(1.0)
|
||||
self.lm_head.weight.data.fill_(1.0)
|
||||
|
||||
@property
|
||||
def embed_tokens(self) -> VocabParallelEmbedding:
|
||||
return self.language_model.model.embed_tokens
|
||||
|
||||
@property
|
||||
def lm_head(self) -> ParallelLMHead:
|
||||
return self.language_model.lm_head
|
||||
|
||||
|
||||
def make_model_config(untied_by_checkpoint=False):
|
||||
return SimpleNamespace(
|
||||
model="dummy-model",
|
||||
word_embeddings_untied_by_checkpoint=untied_by_checkpoint,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("identical", [True, False])
|
||||
def test_retie_only_when_identical(identical: bool):
|
||||
"""A redundant copy of a tied lm_head is shared again to reclaim memory."""
|
||||
model = UntiedModel()
|
||||
if not identical:
|
||||
model.lm_head.weight.data.fill_(2.0)
|
||||
|
||||
maybe_retie_word_embeddings(model, make_model_config(untied_by_checkpoint=True))
|
||||
|
||||
assert (model.lm_head.weight is model.embed_tokens.weight) is identical
|
||||
if not identical:
|
||||
assert torch.all(model.lm_head.weight == 2.0)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_quantized_lm_head_is_left_alone():
|
||||
"""A quantized head may store its weights packed under another name."""
|
||||
model = UntiedModel()
|
||||
model.lm_head.quant_method = SimpleNamespace()
|
||||
|
||||
maybe_retie_word_embeddings(model, make_model_config(untied_by_checkpoint=True))
|
||||
|
||||
assert model.lm_head.weight is not model.embed_tokens.weight
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_no_retie_without_checkpoint_override():
|
||||
"""Word embeddings the config genuinely unties are left alone."""
|
||||
model = UntiedModel()
|
||||
|
||||
maybe_retie_word_embeddings(model, make_model_config())
|
||||
|
||||
assert model.lm_head.weight is not model.embed_tokens.weight
|
||||
@@ -4,6 +4,10 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
_merge_multimodal_embeddings,
|
||||
@@ -165,6 +169,80 @@ def test_module_skip_substr():
|
||||
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 1
|
||||
|
||||
|
||||
VOCAB_SIZE = 16
|
||||
HIDDEN_SIZE = 2
|
||||
|
||||
|
||||
class ModuleWithTiedWeights(torch.nn.Module):
|
||||
"""Mimics how models tie `lm_head` to the input embeddings."""
|
||||
|
||||
def __init__(self, tie: bool):
|
||||
super().__init__()
|
||||
self.model = torch.nn.Module()
|
||||
self.model.embed_tokens = VocabParallelEmbedding(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
self.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
if tie:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
|
||||
|
||||
def make_embedding_weights(value: float) -> torch.Tensor:
|
||||
return torch.full((VOCAB_SIZE, HIDDEN_SIZE), value)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("tie", [True, False])
|
||||
def test_module_skip_tied_weights(tie: bool):
|
||||
"""Tied weights must be loaded once, under the first of their names."""
|
||||
mod = ModuleWithTiedWeights(tie)
|
||||
|
||||
weights = [
|
||||
("model.embed_tokens.weight", make_embedding_weights(1.0)),
|
||||
("lm_head.weight", make_embedding_weights(2.0)),
|
||||
]
|
||||
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
if tie:
|
||||
assert loaded == {"model.embed_tokens.weight"}
|
||||
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 1.0)
|
||||
else:
|
||||
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
|
||||
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 2.0)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_module_skip_tied_weights_without_canonical():
|
||||
"""Skipping a tied weight must not leave the shared weight uninitialized."""
|
||||
mod = ModuleWithTiedWeights(tie=True)
|
||||
|
||||
weights = [("lm_head.weight", make_embedding_weights(2.0))]
|
||||
with pytest.raises(ValueError, match="model.embed_tokens.weight"):
|
||||
AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
|
||||
class ModuleWithSharedParam(torch.nn.Module):
|
||||
"""Mimics an MoE router shared between the MLP and its fused experts."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = torch.nn.Module()
|
||||
self.experts.gate = torch.nn.Linear(2, 2, bias=False)
|
||||
self.gate = self.experts.gate
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_module_load_shared_params_that_are_not_tied_embeddings():
|
||||
"""Only tied embeddings are skipped; other shared params must still load."""
|
||||
mod = ModuleWithSharedParam()
|
||||
|
||||
weights = [("gate.weight", torch.Tensor([[1, 2], [3, 4]]))]
|
||||
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
assert loaded == {"gate.weight"}
|
||||
assert torch.all(mod.gate.weight == torch.Tensor([[1, 2], [3, 4]]))
|
||||
|
||||
|
||||
class raise_if_cuda_sync:
|
||||
def __enter__(self):
|
||||
self.previous_debug_mode = torch.cuda.get_sync_debug_mode()
|
||||
|
||||
@@ -604,6 +604,45 @@ def test_with_hf_config_leaves_unknown_model_type_without_architectures(
|
||||
assert updated.model_config.hf_config.architectures is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_tensors,tied",
|
||||
[
|
||||
# The checkpoint has an lm_head of its own, so it must win over the config
|
||||
(["model.embed_tokens.weight", "lm_head.weight"], False),
|
||||
(["model.embed_tokens.weight"], True),
|
||||
# Contents unknown (not safetensors), so the config must be left alone
|
||||
([], True),
|
||||
],
|
||||
)
|
||||
def test_maybe_untie_word_embeddings(tmp_path, checkpoint_tensors, tied):
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
if checkpoint_tensors:
|
||||
save_file(
|
||||
{name: torch.zeros(2, 2) for name in checkpoint_tensors},
|
||||
tmp_path / "model.safetensors",
|
||||
)
|
||||
|
||||
text_config = SimpleNamespace(tie_word_embeddings=True)
|
||||
model_config = SimpleNamespace(
|
||||
model=str(tmp_path),
|
||||
revision=None,
|
||||
hf_config=SimpleNamespace(
|
||||
tie_word_embeddings=True,
|
||||
get_text_config=lambda: text_config,
|
||||
),
|
||||
word_embeddings_untied_by_checkpoint=False,
|
||||
)
|
||||
|
||||
ModelConfig.maybe_untie_word_embeddings(model_config)
|
||||
|
||||
# Both levels must agree, since different callers read different ones
|
||||
assert model_config.hf_config.tie_word_embeddings is tied
|
||||
assert text_config.tie_word_embeddings is tied
|
||||
assert model_config.word_embeddings_untied_by_checkpoint is not tied
|
||||
|
||||
|
||||
def test_async_scheduling_with_pipeline_parallelism_is_allowed():
|
||||
cfg = VllmConfig(
|
||||
scheduler_config=SchedulerConfig(
|
||||
|
||||
@@ -8,14 +8,17 @@ only get the `eos_token_id` from the tokenizer as defined by
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.transformers_utils import config as config_module
|
||||
from vllm.transformers_utils.config import try_get_generation_config
|
||||
from vllm.transformers_utils.config import (
|
||||
get_safetensors_params_metadata,
|
||||
try_get_generation_config,
|
||||
)
|
||||
|
||||
|
||||
def test_get_llama3_eos_token():
|
||||
@@ -77,3 +80,22 @@ def test_model_config_generation_fallback_forwards_code_revision():
|
||||
config_format="auto",
|
||||
token=None,
|
||||
)
|
||||
|
||||
|
||||
def test_safetensors_metadata_of_repo_without_safetensors():
|
||||
"""A repo storing its weights in another format is an answer, not a failure,
|
||||
so it must not be retried."""
|
||||
from huggingface_hub.errors import LocalEntryNotFoundError, NotASafetensorsRepoError
|
||||
|
||||
get_safetensors_metadata = MagicMock(
|
||||
side_effect=NotASafetensorsRepoError("not a safetensors repo")
|
||||
)
|
||||
api = SimpleNamespace(
|
||||
get_safetensors_metadata=get_safetensors_metadata,
|
||||
snapshot_download=MagicMock(side_effect=LocalEntryNotFoundError("no cache")),
|
||||
)
|
||||
|
||||
with patch.object(config_module, "hf_api", lambda: api):
|
||||
assert get_safetensors_params_metadata("some/pytorch-only-model") == {}
|
||||
|
||||
get_safetensors_metadata.assert_called_once()
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.transformers_utils.repo_utils import (
|
||||
get_hf_file_to_dict,
|
||||
is_mistral_model_repo,
|
||||
list_filtered_repo_files,
|
||||
with_retry,
|
||||
)
|
||||
|
||||
|
||||
@@ -185,3 +186,13 @@ def test_is_mistral_model_repo(files: list[str], expected_bool: bool):
|
||||
repo_type="model",
|
||||
token="token",
|
||||
)
|
||||
|
||||
|
||||
def test_with_retry_does_not_retry_fatal_errors():
|
||||
"""A definitive answer must not be retried, which costs a delay per attempt."""
|
||||
func = MagicMock(side_effect=FileNotFoundError("no such file"))
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
with_retry(func, "Error", fatal_errors=(FileNotFoundError,))
|
||||
|
||||
func.assert_called_once()
|
||||
|
||||
Reference in New Issue
Block a user