Fix weight tying (#51665)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,6 +4,10 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
_merge_multimodal_embeddings,
|
||||
@@ -165,6 +169,80 @@ def test_module_skip_substr():
|
||||
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 1
|
||||
|
||||
|
||||
VOCAB_SIZE = 16
|
||||
HIDDEN_SIZE = 2
|
||||
|
||||
|
||||
class ModuleWithTiedWeights(torch.nn.Module):
|
||||
"""Mimics how models tie `lm_head` to the input embeddings."""
|
||||
|
||||
def __init__(self, tie: bool):
|
||||
super().__init__()
|
||||
self.model = torch.nn.Module()
|
||||
self.model.embed_tokens = VocabParallelEmbedding(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
self.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
if tie:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
|
||||
|
||||
def make_embedding_weights(value: float) -> torch.Tensor:
|
||||
return torch.full((VOCAB_SIZE, HIDDEN_SIZE), value)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("tie", [True, False])
|
||||
def test_module_skip_tied_weights(tie: bool):
|
||||
"""Tied weights must be loaded once, under the first of their names."""
|
||||
mod = ModuleWithTiedWeights(tie)
|
||||
|
||||
weights = [
|
||||
("model.embed_tokens.weight", make_embedding_weights(1.0)),
|
||||
("lm_head.weight", make_embedding_weights(2.0)),
|
||||
]
|
||||
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
if tie:
|
||||
assert loaded == {"model.embed_tokens.weight"}
|
||||
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 1.0)
|
||||
else:
|
||||
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
|
||||
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 2.0)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_module_skip_tied_weights_without_canonical():
|
||||
"""Skipping a tied weight must not leave the shared weight uninitialized."""
|
||||
mod = ModuleWithTiedWeights(tie=True)
|
||||
|
||||
weights = [("lm_head.weight", make_embedding_weights(2.0))]
|
||||
with pytest.raises(ValueError, match="model.embed_tokens.weight"):
|
||||
AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
|
||||
class ModuleWithSharedParam(torch.nn.Module):
|
||||
"""Mimics an MoE router shared between the MLP and its fused experts."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = torch.nn.Module()
|
||||
self.experts.gate = torch.nn.Linear(2, 2, bias=False)
|
||||
self.gate = self.experts.gate
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_module_load_shared_params_that_are_not_tied_embeddings():
|
||||
"""Only tied embeddings are skipped; other shared params must still load."""
|
||||
mod = ModuleWithSharedParam()
|
||||
|
||||
weights = [("gate.weight", torch.Tensor([[1, 2], [3, 4]]))]
|
||||
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
assert loaded == {"gate.weight"}
|
||||
assert torch.all(mod.gate.weight == torch.Tensor([[1, 2], [3, 4]]))
|
||||
|
||||
|
||||
class raise_if_cuda_sync:
|
||||
def __enter__(self):
|
||||
self.previous_debug_mode = torch.cuda.get_sync_debug_mode()
|
||||
|
||||
Reference in New Issue
Block a user