Fully generalise input embedding handling in Transformers modelling backend (#51247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -2,9 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test the functionality of the Transformers modeling backend."""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from ...conftest import HfRunner, VllmRunner
|
||||
from ...utils import multi_gpu_test, prep_prompts
|
||||
@@ -299,3 +304,174 @@ def test_pooling(hf_runner, vllm_runner, example_prompts, arch):
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
VOCAB_SIZE = 64
|
||||
HIDDEN_SIZE = 8
|
||||
EMBED_SCALE = 3.0
|
||||
|
||||
|
||||
class ScaledWordEmbedding(nn.Embedding):
|
||||
"""Mirrors Transformers' `*ScaledWordEmbedding` classes."""
|
||||
|
||||
def __init__(
|
||||
self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0
|
||||
):
|
||||
super().__init__(num_embeddings, embedding_dim, padding_idx)
|
||||
self.scalar_embed_scale = embed_scale
|
||||
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
|
||||
|
||||
def forward(self, input_ids):
|
||||
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
|
||||
|
||||
|
||||
class ComposedWordEmbedding(nn.Module):
|
||||
"""Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it."""
|
||||
|
||||
def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0):
|
||||
super().__init__()
|
||||
self.embed = nn.Embedding(num_embeddings, embedding_dim)
|
||||
self.embed_scale = embed_scale
|
||||
|
||||
def forward(self, input_ids):
|
||||
return self.embed(input_ids) * self.embed_scale
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tp_init():
|
||||
"""Single rank tensor parallel state, so vLLM layers can be constructed."""
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import ensure_current_vllm_config
|
||||
|
||||
fd, temp_file = tempfile.mkstemp()
|
||||
os.close(fd)
|
||||
try:
|
||||
with ensure_current_vllm_config():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=0,
|
||||
backend=current_platform.dist_backend,
|
||||
)
|
||||
initialize_model_parallel(1, 1)
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temp_file)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vpe(tp_init):
|
||||
"""`VocabParallelEmbedding`, imported late so collection does not import vLLM."""
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
return VocabParallelEmbedding
|
||||
|
||||
|
||||
def replace(embedding):
|
||||
"""Replace `embedding` and fill the new weights with recognisable values."""
|
||||
from vllm.model_executor.models.transformers.utils import replace_embedding_class
|
||||
|
||||
new_embedding = replace_embedding_class(embedding)
|
||||
for _, param in new_embedding.named_parameters():
|
||||
param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape)
|
||||
return new_embedding
|
||||
|
||||
|
||||
def assert_scaled(vpe, module, embedding=None):
|
||||
"""`module`'s output is `embedding`'s unscaled output times `EMBED_SCALE`."""
|
||||
input_ids = torch.arange(VOCAB_SIZE)
|
||||
unscaled = vpe.forward(embedding if embedding is not None else module, input_ids)
|
||||
torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE)
|
||||
|
||||
|
||||
def test_replace_plain_embedding(vpe):
|
||||
"""A plain `nn.Embedding` is replaced outright, leaving no subclass behind."""
|
||||
assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe
|
||||
|
||||
|
||||
def test_replace_infers_shape_and_dtype(tp_init):
|
||||
"""Shape and dtype come from the replaced module, not from the config."""
|
||||
embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16)
|
||||
new_embedding = replace(embedding)
|
||||
|
||||
assert new_embedding.num_embeddings == VOCAB_SIZE * 2
|
||||
assert new_embedding.org_vocab_size == VOCAB_SIZE * 2
|
||||
assert new_embedding.embedding_dim == HIDDEN_SIZE + 1
|
||||
assert new_embedding.weight.dtype == torch.float16
|
||||
|
||||
|
||||
def test_replace_inherited_embedding(vpe):
|
||||
"""Subclasses keep their extra state and their scaled `forward`."""
|
||||
new_embedding = replace(
|
||||
ScaledWordEmbedding(
|
||||
VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(new_embedding, vpe)
|
||||
assert new_embedding.scalar_embed_scale == EMBED_SCALE
|
||||
assert "embed_scale" in new_embedding._non_persistent_buffers_set
|
||||
assert_scaled(vpe, new_embedding)
|
||||
|
||||
|
||||
def test_replace_composed_embedding(vpe):
|
||||
"""Wrappers are left alone; only the `nn.Embedding` they hold is replaced."""
|
||||
embedding = ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
|
||||
new_embedding = replace(embedding)
|
||||
|
||||
assert new_embedding is embedding
|
||||
assert type(embedding.embed) is vpe
|
||||
assert_scaled(vpe, new_embedding, embedding.embed)
|
||||
|
||||
|
||||
def test_replace_nested_embedding(vpe):
|
||||
"""The composed `nn.Embedding` is found and set however deeply it is nested."""
|
||||
wrapper = nn.Module()
|
||||
wrapper.add_module("inner", nn.Module())
|
||||
wrapper.inner.add_module(
|
||||
"embed", ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
|
||||
)
|
||||
replace(wrapper)
|
||||
|
||||
assert isinstance(wrapper.inner.embed, vpe)
|
||||
assert_scaled(vpe, wrapper.inner.embed)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_embeddings", [0, 2])
|
||||
def test_replace_ambiguous_embedding(tp_init, num_embeddings):
|
||||
"""Composing anything but one `nn.Embedding` is an error, not a silent guess."""
|
||||
wrapper = nn.Module()
|
||||
for i in range(num_embeddings):
|
||||
wrapper.add_module(f"embed_{i}", nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))
|
||||
|
||||
with pytest.raises(ValueError, match=f"found {num_embeddings}"):
|
||||
replace(wrapper)
|
||||
|
||||
|
||||
def test_replaced_embedding_exposes_one_vpe(vpe):
|
||||
"""`CausalMixin` ties `lm_head` to the one `VocabParallelEmbedding` it can find.
|
||||
|
||||
`tie_weights` reads `.weight`, which a composing module does not have, so it must
|
||||
be handed the composed embedding instead.
|
||||
"""
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
inherited = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE))
|
||||
composed = replace(ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, EMBED_SCALE))
|
||||
|
||||
assert [m for m in inherited.modules() if isinstance(m, vpe)] == [inherited]
|
||||
assert [m for m in composed.modules() if isinstance(m, vpe)] == [composed.embed]
|
||||
|
||||
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
assert lm_head.tie_weights(composed.embed).weight is composed.embed.weight
|
||||
|
||||
Reference in New Issue
Block a user