From e7b8d5946095a594af2cf7ca3c314b9806cb7c32 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:21:24 +0100 Subject: [PATCH] Fully generalise input embedding handling in Transformers modelling backend (#51247) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- tests/models/transformers/test_backend.py | 176 ++++++++++++++++++ vllm/lora/layers/vocal_parallel_embedding.py | 17 +- .../layers/vocab_parallel_embedding.py | 2 +- .../models/transformers/base.py | 43 +---- .../models/transformers/causal.py | 12 +- .../models/transformers/utils.py | 97 +++++++++- 6 files changed, 300 insertions(+), 47 deletions(-) diff --git a/tests/models/transformers/test_backend.py b/tests/models/transformers/test_backend.py index 1945fd706b..323ec760bf 100644 --- a/tests/models/transformers/test_backend.py +++ b/tests/models/transformers/test_backend.py @@ -2,9 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Test the functionality of the Transformers modeling backend.""" +import contextlib +import os +import tempfile from typing import Any import pytest +import torch +import torch.nn as nn from ...conftest import HfRunner, VllmRunner from ...utils import multi_gpu_test, prep_prompts @@ -299,3 +304,174 @@ def test_pooling(hf_runner, vllm_runner, example_prompts, arch): name_0="hf", name_1="vllm", ) + + +VOCAB_SIZE = 64 +HIDDEN_SIZE = 8 +EMBED_SCALE = 3.0 + + +class ScaledWordEmbedding(nn.Embedding): + """Mirrors Transformers' `*ScaledWordEmbedding` classes.""" + + def __init__( + self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0 + ): + super().__init__(num_embeddings, embedding_dim, padding_idx) + self.scalar_embed_scale = embed_scale + self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False) + + def forward(self, input_ids): + return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype) + + +class ComposedWordEmbedding(nn.Module): + """Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it.""" + + def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0): + super().__init__() + self.embed = nn.Embedding(num_embeddings, embedding_dim) + self.embed_scale = embed_scale + + def forward(self, input_ids): + return self.embed(input_ids) * self.embed_scale + + +@pytest.fixture +def tp_init(): + """Single rank tensor parallel state, so vLLM layers can be constructed.""" + from vllm.distributed import ( + cleanup_dist_env_and_memory, + init_distributed_environment, + initialize_model_parallel, + ) + from vllm.platforms import current_platform + + from ...utils import ensure_current_vllm_config + + fd, temp_file = tempfile.mkstemp() + os.close(fd) + try: + with ensure_current_vllm_config(): + init_distributed_environment( + world_size=1, + rank=0, + distributed_init_method=f"file://{temp_file}", + local_rank=0, + backend=current_platform.dist_backend, + ) + initialize_model_parallel(1, 1) + yield + cleanup_dist_env_and_memory() + finally: + with contextlib.suppress(OSError): + os.unlink(temp_file) + + +@pytest.fixture +def vpe(tp_init): + """`VocabParallelEmbedding`, imported late so collection does not import vLLM.""" + from vllm.model_executor.layers.vocab_parallel_embedding import ( + VocabParallelEmbedding, + ) + + return VocabParallelEmbedding + + +def replace(embedding): + """Replace `embedding` and fill the new weights with recognisable values.""" + from vllm.model_executor.models.transformers.utils import replace_embedding_class + + new_embedding = replace_embedding_class(embedding) + for _, param in new_embedding.named_parameters(): + param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape) + return new_embedding + + +def assert_scaled(vpe, module, embedding=None): + """`module`'s output is `embedding`'s unscaled output times `EMBED_SCALE`.""" + input_ids = torch.arange(VOCAB_SIZE) + unscaled = vpe.forward(embedding if embedding is not None else module, input_ids) + torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE) + + +def test_replace_plain_embedding(vpe): + """A plain `nn.Embedding` is replaced outright, leaving no subclass behind.""" + assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe + + +def test_replace_infers_shape_and_dtype(tp_init): + """Shape and dtype come from the replaced module, not from the config.""" + embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16) + new_embedding = replace(embedding) + + assert new_embedding.num_embeddings == VOCAB_SIZE * 2 + assert new_embedding.org_vocab_size == VOCAB_SIZE * 2 + assert new_embedding.embedding_dim == HIDDEN_SIZE + 1 + assert new_embedding.weight.dtype == torch.float16 + + +def test_replace_inherited_embedding(vpe): + """Subclasses keep their extra state and their scaled `forward`.""" + new_embedding = replace( + ScaledWordEmbedding( + VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE + ) + ) + + assert isinstance(new_embedding, vpe) + assert new_embedding.scalar_embed_scale == EMBED_SCALE + assert "embed_scale" in new_embedding._non_persistent_buffers_set + assert_scaled(vpe, new_embedding) + + +def test_replace_composed_embedding(vpe): + """Wrappers are left alone; only the `nn.Embedding` they hold is replaced.""" + embedding = ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE) + new_embedding = replace(embedding) + + assert new_embedding is embedding + assert type(embedding.embed) is vpe + assert_scaled(vpe, new_embedding, embedding.embed) + + +def test_replace_nested_embedding(vpe): + """The composed `nn.Embedding` is found and set however deeply it is nested.""" + wrapper = nn.Module() + wrapper.add_module("inner", nn.Module()) + wrapper.inner.add_module( + "embed", ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE) + ) + replace(wrapper) + + assert isinstance(wrapper.inner.embed, vpe) + assert_scaled(vpe, wrapper.inner.embed) + + +@pytest.mark.parametrize("num_embeddings", [0, 2]) +def test_replace_ambiguous_embedding(tp_init, num_embeddings): + """Composing anything but one `nn.Embedding` is an error, not a silent guess.""" + wrapper = nn.Module() + for i in range(num_embeddings): + wrapper.add_module(f"embed_{i}", nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE)) + + with pytest.raises(ValueError, match=f"found {num_embeddings}"): + replace(wrapper) + + +def test_replaced_embedding_exposes_one_vpe(vpe): + """`CausalMixin` ties `lm_head` to the one `VocabParallelEmbedding` it can find. + + `tie_weights` reads `.weight`, which a composing module does not have, so it must + be handed the composed embedding instead. + """ + from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead + + inherited = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE)) + composed = replace(ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, EMBED_SCALE)) + + assert [m for m in inherited.modules() if isinstance(m, vpe)] == [inherited] + assert [m for m in composed.modules() if isinstance(m, vpe)] == [composed.embed] + + lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE) + assert lm_head.tie_weights(composed.embed).weight is composed.embed.weight diff --git a/vllm/lora/layers/vocal_parallel_embedding.py b/vllm/lora/layers/vocal_parallel_embedding.py index 05e7cfa06c..6aade587cd 100644 --- a/vllm/lora/layers/vocal_parallel_embedding.py +++ b/vllm/lora/layers/vocal_parallel_embedding.py @@ -8,7 +8,10 @@ from transformers import PretrainedConfig from vllm.config.lora import LoRAConfig from vllm.model_executor.custom_op import maybe_get_oot_by_class -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) from vllm.platforms import current_platform from .base import BaseLayerWithLoRA @@ -133,7 +136,17 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA): packed_modules_list: list, model_config: PretrainedConfig | None = None, ) -> bool: - return type(source_layer) is maybe_get_oot_by_class(VocabParallelEmbedding) + # Accept both the in-tree and any OOT class, because layers built on top of + # `VocabParallelEmbedding` (e.g. by the Transformers modelling backend) use the + # former while `PluggableLayer.__new__` instantiates the latter + embedding_classes = ( + VocabParallelEmbedding, + maybe_get_oot_by_class(VocabParallelEmbedding), + ) + lm_head_classes = (ParallelLMHead, maybe_get_oot_by_class(ParallelLMHead)) + return isinstance(source_layer, embedding_classes) and not isinstance( + source_layer, lm_head_classes + ) @property def weight(self): diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index 1096f5aac0..79695084e4 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -290,7 +290,7 @@ class VocabParallelEmbedding(PluggableLayer): # If we are making an embedding layer, then our quantization linear # method must implement the embedding operation. If we are another # layer type like ParallelLMHead, this is not important. - is_embedding_layer = type(self) is VocabParallelEmbedding + is_embedding_layer = not isinstance(self, ParallelLMHead) quant_method_implements_embedding = method_has_implemented_embedding( type(quant_method) ) diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index b54cb5aa25..6803eb33ee 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -46,7 +46,6 @@ from vllm.model_executor.layers.attention import ( from vllm.model_executor.layers.attention.mla_attention import get_mla_dims from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.fused_moe import MoERunner -from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.models.interfaces import ( SupportsEagle, SupportsEagle3, @@ -58,12 +57,14 @@ from vllm.model_executor.models.interfaces_base import VllmModel from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers from vllm.model_executor.models.transformers.fusers import MLAFuser from vllm.model_executor.models.transformers.utils import ( + attrsetter, can_enable_torch_compile, get_feature_request_tip, init_on_device_without_buffers, log_replacement, named_state, replace_conv_class, + replace_embedding_class, replace_linear_class, ) from vllm.model_executor.models.utils import ( @@ -84,17 +85,6 @@ if TYPE_CHECKING: logger = init_logger(__name__) -class ScaledVocabParallelEmbedding(VocabParallelEmbedding): - """`VocabParallelEmbedding` that scales its output.""" - - def __init__(self, *args, embed_scale: float, **kwargs): - super().__init__(*args, **kwargs) - self.embed_scale = embed_scale - - def forward(self, input_: torch.Tensor) -> torch.Tensor: - return super().forward(input_) * self.embed_scale - - class Base( nn.Module, VllmModel, @@ -179,24 +169,9 @@ class Base( # Input embeddings input_embeddings = self.model.get_input_embeddings() if not isinstance(input_embeddings, PPMissingLayer): - names = ("embedding_size", "hidden_size") - embedding_dim = getattr_iter(self.text_config, names, None) - assert embedding_dim is not None - embedding_kwargs = dict( - num_embeddings=self.text_config.vocab_size, - embedding_dim=embedding_dim, - org_num_embeddings=self.text_config.vocab_size, - quant_config=self.quant_config, + self.model.set_input_embeddings( + replace_embedding_class(input_embeddings, self.quant_config) ) - embed_scale = getattr(input_embeddings, "embed_scale", None) - if embed_scale is not None: - # Some models scale embeddings inside the input embedding layer - new_input_embeddings = ScaledVocabParallelEmbedding( - **embedding_kwargs, embed_scale=float(embed_scale) - ) - else: - new_input_embeddings = VocabParallelEmbedding(**embedding_kwargs) - self.model.set_input_embeddings(new_input_embeddings) # Initialize any parameters that have not had their modules replaced self.init_parameters(self.model) @@ -380,16 +355,6 @@ class Base( tip, ) - def attrsetter(attr: str) -> Callable[[object, object], None]: - """Set a possibly nested attribute, like the inverse of attrgetter.""" - parent, _, name = attr.rpartition(".") - - def setter(obj: object, value: object): - attr_parent = attrgetter(parent)(obj) if parent else obj - setattr(attr_parent, name, value) - - return setter - module_lists = [] module_list_idx = None for i, name in enumerate(names): diff --git a/vllm/model_executor/models/transformers/causal.py b/vllm/model_executor/models/transformers/causal.py index 01a7e41983..b637df1164 100644 --- a/vllm/model_executor/models/transformers/causal.py +++ b/vllm/model_executor/models/transformers/causal.py @@ -20,7 +20,10 @@ from collections.abc import Iterable from typing import TYPE_CHECKING from vllm.model_executor.layers.logits_processor import LogitsProcessor -from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix @@ -51,9 +54,10 @@ class CausalMixin(VllmModelForTextGeneration): prefix=maybe_prefix(prefix, "lm_head"), ) if tie_word_embeddings: - self.lm_head = self.lm_head.tie_weights( - self.model.get_input_embeddings() - ) + for module in self.model.get_input_embeddings().modules(): + if isinstance(module, VocabParallelEmbedding): + self.lm_head = self.lm_head.tie_weights(module) + break logit_scale = getattr(self.text_config, "logit_scale", 1.0) self.logits_processor = LogitsProcessor( diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py index 9000ace905..0d5d5f9a06 100644 --- a/vllm/model_executor/models/transformers/utils.py +++ b/vllm/model_executor/models/transformers/utils.py @@ -16,9 +16,11 @@ # limitations under the License. """Transformers modeling backend utilities.""" -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager +from functools import lru_cache from itertools import chain +from operator import attrgetter from pathlib import Path from typing import TYPE_CHECKING, Literal @@ -32,6 +34,7 @@ from vllm.model_executor.layers.linear import ( ReplicatedLinear, RowParallelLinear, ) +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.models.utils import maybe_prefix from vllm.transformers_utils.config import is_rope_parameters_nested @@ -183,6 +186,98 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv: ) +def attrsetter(attr: str) -> Callable[[object, object], None]: + """Set a possibly nested attribute, like the inverse of attrgetter.""" + parent, _, name = attr.rpartition(".") + + def setter(obj: object, value: object): + attr_parent = attrgetter(parent)(obj) if parent else obj + setattr(attr_parent, name, value) + + return setter + + +class _UninitializedEmbedding(nn.Embedding): + """Make `__init__` inert, so that `VocabParallelEmbedding.__init__`' + call to `super().__init__` does not invoke `nn.Embedding.__init__`.""" + + def __init__(self): + pass + + +class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding): + """Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that + `super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding.""" + + +@lru_cache +def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]: + """Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`. + + Args: + cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always + maps to the same class. + Returns: + The new class, to assign to `__class__` of an instance of `cls`. + """ + return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {}) + + +def replace_embedding_class( + embedding: nn.Module, + quant_config: "QuantizationConfig | None" = None, + *, + prefix: str = "", +) -> nn.Module: + """Replace the `nn.Embedding` in `embedding` with `VocabParallelEmbedding`. + + Args: + embedding: The module returned by `model.get_input_embeddings()`. + quant_config: Quantization config for the new embedding. + prefix: Qualname of `embedding`, used to look up its quantization method. + Returns: + The module to install with `model.set_input_embeddings()`. Composing and + inheriting modules are mutated in place and returned as-is. + Raises: + ValueError: If `embedding` composes anything other than one `nn.Embedding`, + which would leave the input embedding weights ambiguous. + """ + # If `embedding` composes its `nn.Embedding`, recurse into it + if not isinstance(embedding, nn.Embedding): + composed = [ + (name, module) + for name, module in embedding.named_modules() + if isinstance(module, nn.Embedding) + ] + if len(composed) != 1: + raise ValueError( + f"Expected {type(embedding).__name__} to be an `nn.Embedding` or to " + f"compose exactly one, but found {len(composed)}." + ) + name, module = composed[0] + new_embedding = replace_embedding_class( + module, quant_config, prefix=maybe_prefix(prefix, name) + ) + attrsetter(name)(embedding, new_embedding) + return embedding + + kwargs = dict( + num_embeddings=embedding.num_embeddings, + embedding_dim=embedding.embedding_dim, + params_dtype=embedding.weight.dtype, + quant_config=quant_config, + prefix=prefix, + ) + # If `embedding` is a bare `nn.Embedding`, simple replace + if type(embedding) is nn.Embedding: + return VocabParallelEmbedding(**kwargs) + + # Otherwise `embedding` inherits `nn.Embedding`, rebase it in place + embedding.__class__ = _rebase_on_vocab_parallel(type(embedding)) + VocabParallelEmbedding.__init__(embedding, **kwargs) + return embedding + + def recursive_replace_linear( model: nn.Module, quant_config: "QuantizationConfig | None",