Fully generalise input embedding handling in Transformers modelling backend (#51247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-06 16:21:24 +00:00
committed by GitHub
parent 566c80edf9
commit e7b8d59460
6 changed files with 300 additions and 47 deletions
+176
View File
@@ -2,9 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
import contextlib
import os
import tempfile
from typing import Any
import pytest
import torch
import torch.nn as nn
from ...conftest import HfRunner, VllmRunner
from ...utils import multi_gpu_test, prep_prompts
@@ -299,3 +304,174 @@ def test_pooling(hf_runner, vllm_runner, example_prompts, arch):
name_0="hf",
name_1="vllm",
)
VOCAB_SIZE = 64
HIDDEN_SIZE = 8
EMBED_SCALE = 3.0
class ScaledWordEmbedding(nn.Embedding):
"""Mirrors Transformers' `*ScaledWordEmbedding` classes."""
def __init__(
self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0
):
super().__init__(num_embeddings, embedding_dim, padding_idx)
self.scalar_embed_scale = embed_scale
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
def forward(self, input_ids):
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
class ComposedWordEmbedding(nn.Module):
"""Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it."""
def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0):
super().__init__()
self.embed = nn.Embedding(num_embeddings, embedding_dim)
self.embed_scale = embed_scale
def forward(self, input_ids):
return self.embed(input_ids) * self.embed_scale
@pytest.fixture
def tp_init():
"""Single rank tensor parallel state, so vLLM layers can be constructed."""
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from ...utils import ensure_current_vllm_config
fd, temp_file = tempfile.mkstemp()
os.close(fd)
try:
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend=current_platform.dist_backend,
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
finally:
with contextlib.suppress(OSError):
os.unlink(temp_file)
@pytest.fixture
def vpe(tp_init):
"""`VocabParallelEmbedding`, imported late so collection does not import vLLM."""
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
return VocabParallelEmbedding
def replace(embedding):
"""Replace `embedding` and fill the new weights with recognisable values."""
from vllm.model_executor.models.transformers.utils import replace_embedding_class
new_embedding = replace_embedding_class(embedding)
for _, param in new_embedding.named_parameters():
param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape)
return new_embedding
def assert_scaled(vpe, module, embedding=None):
"""`module`'s output is `embedding`'s unscaled output times `EMBED_SCALE`."""
input_ids = torch.arange(VOCAB_SIZE)
unscaled = vpe.forward(embedding if embedding is not None else module, input_ids)
torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE)
def test_replace_plain_embedding(vpe):
"""A plain `nn.Embedding` is replaced outright, leaving no subclass behind."""
assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe
def test_replace_infers_shape_and_dtype(tp_init):
"""Shape and dtype come from the replaced module, not from the config."""
embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16)
new_embedding = replace(embedding)
assert new_embedding.num_embeddings == VOCAB_SIZE * 2
assert new_embedding.org_vocab_size == VOCAB_SIZE * 2
assert new_embedding.embedding_dim == HIDDEN_SIZE + 1
assert new_embedding.weight.dtype == torch.float16
def test_replace_inherited_embedding(vpe):
"""Subclasses keep their extra state and their scaled `forward`."""
new_embedding = replace(
ScaledWordEmbedding(
VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE
)
)
assert isinstance(new_embedding, vpe)
assert new_embedding.scalar_embed_scale == EMBED_SCALE
assert "embed_scale" in new_embedding._non_persistent_buffers_set
assert_scaled(vpe, new_embedding)
def test_replace_composed_embedding(vpe):
"""Wrappers are left alone; only the `nn.Embedding` they hold is replaced."""
embedding = ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
new_embedding = replace(embedding)
assert new_embedding is embedding
assert type(embedding.embed) is vpe
assert_scaled(vpe, new_embedding, embedding.embed)
def test_replace_nested_embedding(vpe):
"""The composed `nn.Embedding` is found and set however deeply it is nested."""
wrapper = nn.Module()
wrapper.add_module("inner", nn.Module())
wrapper.inner.add_module(
"embed", ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
)
replace(wrapper)
assert isinstance(wrapper.inner.embed, vpe)
assert_scaled(vpe, wrapper.inner.embed)
@pytest.mark.parametrize("num_embeddings", [0, 2])
def test_replace_ambiguous_embedding(tp_init, num_embeddings):
"""Composing anything but one `nn.Embedding` is an error, not a silent guess."""
wrapper = nn.Module()
for i in range(num_embeddings):
wrapper.add_module(f"embed_{i}", nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))
with pytest.raises(ValueError, match=f"found {num_embeddings}"):
replace(wrapper)
def test_replaced_embedding_exposes_one_vpe(vpe):
"""`CausalMixin` ties `lm_head` to the one `VocabParallelEmbedding` it can find.
`tie_weights` reads `.weight`, which a composing module does not have, so it must
be handed the composed embedding instead.
"""
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
inherited = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE))
composed = replace(ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, EMBED_SCALE))
assert [m for m in inherited.modules() if isinstance(m, vpe)] == [inherited]
assert [m for m in composed.modules() if isinstance(m, vpe)] == [composed.embed]
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
assert lm_head.tie_weights(composed.embed).weight is composed.embed.weight