Fix weight tying (#51665)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-20 10:31:55 +00:00
committed by GitHub
parent 6e85feb1b7
commit f0c14b4f77
70 changed files with 526 additions and 238 deletions
+78
View File
@@ -4,6 +4,10 @@
import pytest
import torch
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
_merge_multimodal_embeddings,
@@ -165,6 +169,80 @@ def test_module_skip_substr():
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 1
VOCAB_SIZE = 16
HIDDEN_SIZE = 2
class ModuleWithTiedWeights(torch.nn.Module):
"""Mimics how models tie `lm_head` to the input embeddings."""
def __init__(self, tie: bool):
super().__init__()
self.model = torch.nn.Module()
self.model.embed_tokens = VocabParallelEmbedding(VOCAB_SIZE, HIDDEN_SIZE)
self.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
if tie:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
def make_embedding_weights(value: float) -> torch.Tensor:
return torch.full((VOCAB_SIZE, HIDDEN_SIZE), value)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("tie", [True, False])
def test_module_skip_tied_weights(tie: bool):
"""Tied weights must be loaded once, under the first of their names."""
mod = ModuleWithTiedWeights(tie)
weights = [
("model.embed_tokens.weight", make_embedding_weights(1.0)),
("lm_head.weight", make_embedding_weights(2.0)),
]
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
if tie:
assert loaded == {"model.embed_tokens.weight"}
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 1.0)
else:
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 2.0)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_module_skip_tied_weights_without_canonical():
"""Skipping a tied weight must not leave the shared weight uninitialized."""
mod = ModuleWithTiedWeights(tie=True)
weights = [("lm_head.weight", make_embedding_weights(2.0))]
with pytest.raises(ValueError, match="model.embed_tokens.weight"):
AutoWeightsLoader(mod).load_weights(iter(weights))
class ModuleWithSharedParam(torch.nn.Module):
"""Mimics an MoE router shared between the MLP and its fused experts."""
def __init__(self):
super().__init__()
self.experts = torch.nn.Module()
self.experts.gate = torch.nn.Linear(2, 2, bias=False)
self.gate = self.experts.gate
@pytest.mark.cpu_test
def test_module_load_shared_params_that_are_not_tied_embeddings():
"""Only tied embeddings are skipped; other shared params must still load."""
mod = ModuleWithSharedParam()
weights = [("gate.weight", torch.Tensor([[1, 2], [3, 4]]))]
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
assert loaded == {"gate.weight"}
assert torch.all(mod.gate.weight == torch.Tensor([[1, 2], [3, 4]]))
class raise_if_cuda_sync:
def __enter__(self):
self.previous_debug_mode = torch.cuda.get_sync_debug_mode()