Fully generalise input embedding handling in Transformers modelling backend (#51247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-06 16:21:24 +00:00
committed by GitHub
parent 566c80edf9
commit e7b8d59460
6 changed files with 300 additions and 47 deletions
+176
View File
@@ -2,9 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
import contextlib
import os
import tempfile
from typing import Any
import pytest
import torch
import torch.nn as nn
from ...conftest import HfRunner, VllmRunner
from ...utils import multi_gpu_test, prep_prompts
@@ -299,3 +304,174 @@ def test_pooling(hf_runner, vllm_runner, example_prompts, arch):
name_0="hf",
name_1="vllm",
)
VOCAB_SIZE = 64
HIDDEN_SIZE = 8
EMBED_SCALE = 3.0
class ScaledWordEmbedding(nn.Embedding):
"""Mirrors Transformers' `*ScaledWordEmbedding` classes."""
def __init__(
self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0
):
super().__init__(num_embeddings, embedding_dim, padding_idx)
self.scalar_embed_scale = embed_scale
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
def forward(self, input_ids):
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
class ComposedWordEmbedding(nn.Module):
"""Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it."""
def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0):
super().__init__()
self.embed = nn.Embedding(num_embeddings, embedding_dim)
self.embed_scale = embed_scale
def forward(self, input_ids):
return self.embed(input_ids) * self.embed_scale
@pytest.fixture
def tp_init():
"""Single rank tensor parallel state, so vLLM layers can be constructed."""
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from ...utils import ensure_current_vllm_config
fd, temp_file = tempfile.mkstemp()
os.close(fd)
try:
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend=current_platform.dist_backend,
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
finally:
with contextlib.suppress(OSError):
os.unlink(temp_file)
@pytest.fixture
def vpe(tp_init):
"""`VocabParallelEmbedding`, imported late so collection does not import vLLM."""
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
return VocabParallelEmbedding
def replace(embedding):
"""Replace `embedding` and fill the new weights with recognisable values."""
from vllm.model_executor.models.transformers.utils import replace_embedding_class
new_embedding = replace_embedding_class(embedding)
for _, param in new_embedding.named_parameters():
param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape)
return new_embedding
def assert_scaled(vpe, module, embedding=None):
"""`module`'s output is `embedding`'s unscaled output times `EMBED_SCALE`."""
input_ids = torch.arange(VOCAB_SIZE)
unscaled = vpe.forward(embedding if embedding is not None else module, input_ids)
torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE)
def test_replace_plain_embedding(vpe):
"""A plain `nn.Embedding` is replaced outright, leaving no subclass behind."""
assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe
def test_replace_infers_shape_and_dtype(tp_init):
"""Shape and dtype come from the replaced module, not from the config."""
embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16)
new_embedding = replace(embedding)
assert new_embedding.num_embeddings == VOCAB_SIZE * 2
assert new_embedding.org_vocab_size == VOCAB_SIZE * 2
assert new_embedding.embedding_dim == HIDDEN_SIZE + 1
assert new_embedding.weight.dtype == torch.float16
def test_replace_inherited_embedding(vpe):
"""Subclasses keep their extra state and their scaled `forward`."""
new_embedding = replace(
ScaledWordEmbedding(
VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE
)
)
assert isinstance(new_embedding, vpe)
assert new_embedding.scalar_embed_scale == EMBED_SCALE
assert "embed_scale" in new_embedding._non_persistent_buffers_set
assert_scaled(vpe, new_embedding)
def test_replace_composed_embedding(vpe):
"""Wrappers are left alone; only the `nn.Embedding` they hold is replaced."""
embedding = ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
new_embedding = replace(embedding)
assert new_embedding is embedding
assert type(embedding.embed) is vpe
assert_scaled(vpe, new_embedding, embedding.embed)
def test_replace_nested_embedding(vpe):
"""The composed `nn.Embedding` is found and set however deeply it is nested."""
wrapper = nn.Module()
wrapper.add_module("inner", nn.Module())
wrapper.inner.add_module(
"embed", ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
)
replace(wrapper)
assert isinstance(wrapper.inner.embed, vpe)
assert_scaled(vpe, wrapper.inner.embed)
@pytest.mark.parametrize("num_embeddings", [0, 2])
def test_replace_ambiguous_embedding(tp_init, num_embeddings):
"""Composing anything but one `nn.Embedding` is an error, not a silent guess."""
wrapper = nn.Module()
for i in range(num_embeddings):
wrapper.add_module(f"embed_{i}", nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))
with pytest.raises(ValueError, match=f"found {num_embeddings}"):
replace(wrapper)
def test_replaced_embedding_exposes_one_vpe(vpe):
"""`CausalMixin` ties `lm_head` to the one `VocabParallelEmbedding` it can find.
`tie_weights` reads `.weight`, which a composing module does not have, so it must
be handed the composed embedding instead.
"""
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
inherited = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE))
composed = replace(ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, EMBED_SCALE))
assert [m for m in inherited.modules() if isinstance(m, vpe)] == [inherited]
assert [m for m in composed.modules() if isinstance(m, vpe)] == [composed.embed]
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
assert lm_head.tie_weights(composed.embed).weight is composed.embed.weight
+15 -2
View File
@@ -8,7 +8,10 @@ from transformers import PretrainedConfig
from vllm.config.lora import LoRAConfig
from vllm.model_executor.custom_op import maybe_get_oot_by_class
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.platforms import current_platform
from .base import BaseLayerWithLoRA
@@ -133,7 +136,17 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
packed_modules_list: list,
model_config: PretrainedConfig | None = None,
) -> bool:
return type(source_layer) is maybe_get_oot_by_class(VocabParallelEmbedding)
# Accept both the in-tree and any OOT class, because layers built on top of
# `VocabParallelEmbedding` (e.g. by the Transformers modelling backend) use the
# former while `PluggableLayer.__new__` instantiates the latter
embedding_classes = (
VocabParallelEmbedding,
maybe_get_oot_by_class(VocabParallelEmbedding),
)
lm_head_classes = (ParallelLMHead, maybe_get_oot_by_class(ParallelLMHead))
return isinstance(source_layer, embedding_classes) and not isinstance(
source_layer, lm_head_classes
)
@property
def weight(self):
@@ -290,7 +290,7 @@ class VocabParallelEmbedding(PluggableLayer):
# If we are making an embedding layer, then our quantization linear
# method must implement the embedding operation. If we are another
# layer type like ParallelLMHead, this is not important.
is_embedding_layer = type(self) is VocabParallelEmbedding
is_embedding_layer = not isinstance(self, ParallelLMHead)
quant_method_implements_embedding = method_has_implemented_embedding(
type(quant_method)
)
@@ -46,7 +46,6 @@ from vllm.model_executor.layers.attention import (
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import (
SupportsEagle,
SupportsEagle3,
@@ -58,12 +57,14 @@ from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
named_state,
replace_conv_class,
replace_embedding_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
@@ -84,17 +85,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class ScaledVocabParallelEmbedding(VocabParallelEmbedding):
"""`VocabParallelEmbedding` that scales its output."""
def __init__(self, *args, embed_scale: float, **kwargs):
super().__init__(*args, **kwargs)
self.embed_scale = embed_scale
def forward(self, input_: torch.Tensor) -> torch.Tensor:
return super().forward(input_) * self.embed_scale
class Base(
nn.Module,
VllmModel,
@@ -179,24 +169,9 @@ class Base(
# Input embeddings
input_embeddings = self.model.get_input_embeddings()
if not isinstance(input_embeddings, PPMissingLayer):
names = ("embedding_size", "hidden_size")
embedding_dim = getattr_iter(self.text_config, names, None)
assert embedding_dim is not None
embedding_kwargs = dict(
num_embeddings=self.text_config.vocab_size,
embedding_dim=embedding_dim,
org_num_embeddings=self.text_config.vocab_size,
quant_config=self.quant_config,
self.model.set_input_embeddings(
replace_embedding_class(input_embeddings, self.quant_config)
)
embed_scale = getattr(input_embeddings, "embed_scale", None)
if embed_scale is not None:
# Some models scale embeddings inside the input embedding layer
new_input_embeddings = ScaledVocabParallelEmbedding(
**embedding_kwargs, embed_scale=float(embed_scale)
)
else:
new_input_embeddings = VocabParallelEmbedding(**embedding_kwargs)
self.model.set_input_embeddings(new_input_embeddings)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
@@ -380,16 +355,6 @@ class Base(
tip,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
def setter(obj: object, value: object):
attr_parent = attrgetter(parent)(obj) if parent else obj
setattr(attr_parent, name, value)
return setter
module_lists = []
module_list_idx = None
for i, name in enumerate(names):
@@ -20,7 +20,10 @@ from collections.abc import Iterable
from typing import TYPE_CHECKING
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
@@ -51,9 +54,10 @@ class CausalMixin(VllmModelForTextGeneration):
prefix=maybe_prefix(prefix, "lm_head"),
)
if tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(
self.model.get_input_embeddings()
)
for module in self.model.get_input_embeddings().modules():
if isinstance(module, VocabParallelEmbedding):
self.lm_head = self.lm_head.tie_weights(module)
break
logit_scale = getattr(self.text_config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
@@ -16,9 +16,11 @@
# limitations under the License.
"""Transformers modeling backend utilities."""
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from functools import lru_cache
from itertools import chain
from operator import attrgetter
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -32,6 +34,7 @@ from vllm.model_executor.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.utils import maybe_prefix
from vllm.transformers_utils.config import is_rope_parameters_nested
@@ -183,6 +186,98 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
def setter(obj: object, value: object):
attr_parent = attrgetter(parent)(obj) if parent else obj
setattr(attr_parent, name, value)
return setter
class _UninitializedEmbedding(nn.Embedding):
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
def __init__(self):
pass
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
@lru_cache
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
Args:
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
maps to the same class.
Returns:
The new class, to assign to `__class__` of an instance of `cls`.
"""
return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
def replace_embedding_class(
embedding: nn.Module,
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> nn.Module:
"""Replace the `nn.Embedding` in `embedding` with `VocabParallelEmbedding`.
Args:
embedding: The module returned by `model.get_input_embeddings()`.
quant_config: Quantization config for the new embedding.
prefix: Qualname of `embedding`, used to look up its quantization method.
Returns:
The module to install with `model.set_input_embeddings()`. Composing and
inheriting modules are mutated in place and returned as-is.
Raises:
ValueError: If `embedding` composes anything other than one `nn.Embedding`,
which would leave the input embedding weights ambiguous.
"""
# If `embedding` composes its `nn.Embedding`, recurse into it
if not isinstance(embedding, nn.Embedding):
composed = [
(name, module)
for name, module in embedding.named_modules()
if isinstance(module, nn.Embedding)
]
if len(composed) != 1:
raise ValueError(
f"Expected {type(embedding).__name__} to be an `nn.Embedding` or to "
f"compose exactly one, but found {len(composed)}."
)
name, module = composed[0]
new_embedding = replace_embedding_class(
module, quant_config, prefix=maybe_prefix(prefix, name)
)
attrsetter(name)(embedding, new_embedding)
return embedding
kwargs = dict(
num_embeddings=embedding.num_embeddings,
embedding_dim=embedding.embedding_dim,
params_dtype=embedding.weight.dtype,
quant_config=quant_config,
prefix=prefix,
)
# If `embedding` is a bare `nn.Embedding`, simple replace
if type(embedding) is nn.Embedding:
return VocabParallelEmbedding(**kwargs)
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
VocabParallelEmbedding.__init__(embedding, **kwargs)
return embedding
def recursive_replace_linear(
model: nn.Module,
quant_config: "QuantizationConfig | None",