Fully generalise input embedding handling in Transformers modelling backend (#51247)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-06 16:21:24 +00:00
committed by GitHub
parent 566c80edf9
commit e7b8d59460
6 changed files with 300 additions and 47 deletions
@@ -46,7 +46,6 @@ from vllm.model_executor.layers.attention import (
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import (
SupportsEagle,
SupportsEagle3,
@@ -58,12 +57,14 @@ from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import MLAFuser
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
named_state,
replace_conv_class,
replace_embedding_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
@@ -84,17 +85,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class ScaledVocabParallelEmbedding(VocabParallelEmbedding):
"""`VocabParallelEmbedding` that scales its output."""
def __init__(self, *args, embed_scale: float, **kwargs):
super().__init__(*args, **kwargs)
self.embed_scale = embed_scale
def forward(self, input_: torch.Tensor) -> torch.Tensor:
return super().forward(input_) * self.embed_scale
class Base(
nn.Module,
VllmModel,
@@ -179,24 +169,9 @@ class Base(
# Input embeddings
input_embeddings = self.model.get_input_embeddings()
if not isinstance(input_embeddings, PPMissingLayer):
names = ("embedding_size", "hidden_size")
embedding_dim = getattr_iter(self.text_config, names, None)
assert embedding_dim is not None
embedding_kwargs = dict(
num_embeddings=self.text_config.vocab_size,
embedding_dim=embedding_dim,
org_num_embeddings=self.text_config.vocab_size,
quant_config=self.quant_config,
self.model.set_input_embeddings(
replace_embedding_class(input_embeddings, self.quant_config)
)
embed_scale = getattr(input_embeddings, "embed_scale", None)
if embed_scale is not None:
# Some models scale embeddings inside the input embedding layer
new_input_embeddings = ScaledVocabParallelEmbedding(
**embedding_kwargs, embed_scale=float(embed_scale)
)
else:
new_input_embeddings = VocabParallelEmbedding(**embedding_kwargs)
self.model.set_input_embeddings(new_input_embeddings)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
@@ -380,16 +355,6 @@ class Base(
tip,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
def setter(obj: object, value: object):
attr_parent = attrgetter(parent)(obj) if parent else obj
setattr(attr_parent, name, value)
return setter
module_lists = []
module_list_idx = None
for i, name in enumerate(names):
@@ -20,7 +20,10 @@ from collections.abc import Iterable
from typing import TYPE_CHECKING
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
@@ -51,9 +54,10 @@ class CausalMixin(VllmModelForTextGeneration):
prefix=maybe_prefix(prefix, "lm_head"),
)
if tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(
self.model.get_input_embeddings()
)
for module in self.model.get_input_embeddings().modules():
if isinstance(module, VocabParallelEmbedding):
self.lm_head = self.lm_head.tie_weights(module)
break
logit_scale = getattr(self.text_config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
@@ -16,9 +16,11 @@
# limitations under the License.
"""Transformers modeling backend utilities."""
from collections.abc import Iterator
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from functools import lru_cache
from itertools import chain
from operator import attrgetter
from pathlib import Path
from typing import TYPE_CHECKING, Literal
@@ -32,6 +34,7 @@ from vllm.model_executor.layers.linear import (
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.utils import maybe_prefix
from vllm.transformers_utils.config import is_rope_parameters_nested
@@ -183,6 +186,98 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
def setter(obj: object, value: object):
attr_parent = attrgetter(parent)(obj) if parent else obj
setattr(attr_parent, name, value)
return setter
class _UninitializedEmbedding(nn.Embedding):
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
def __init__(self):
pass
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
@lru_cache
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
Args:
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
maps to the same class.
Returns:
The new class, to assign to `__class__` of an instance of `cls`.
"""
return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
def replace_embedding_class(
embedding: nn.Module,
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> nn.Module:
"""Replace the `nn.Embedding` in `embedding` with `VocabParallelEmbedding`.
Args:
embedding: The module returned by `model.get_input_embeddings()`.
quant_config: Quantization config for the new embedding.
prefix: Qualname of `embedding`, used to look up its quantization method.
Returns:
The module to install with `model.set_input_embeddings()`. Composing and
inheriting modules are mutated in place and returned as-is.
Raises:
ValueError: If `embedding` composes anything other than one `nn.Embedding`,
which would leave the input embedding weights ambiguous.
"""
# If `embedding` composes its `nn.Embedding`, recurse into it
if not isinstance(embedding, nn.Embedding):
composed = [
(name, module)
for name, module in embedding.named_modules()
if isinstance(module, nn.Embedding)
]
if len(composed) != 1:
raise ValueError(
f"Expected {type(embedding).__name__} to be an `nn.Embedding` or to "
f"compose exactly one, but found {len(composed)}."
)
name, module = composed[0]
new_embedding = replace_embedding_class(
module, quant_config, prefix=maybe_prefix(prefix, name)
)
attrsetter(name)(embedding, new_embedding)
return embedding
kwargs = dict(
num_embeddings=embedding.num_embeddings,
embedding_dim=embedding.embedding_dim,
params_dtype=embedding.weight.dtype,
quant_config=quant_config,
prefix=prefix,
)
# If `embedding` is a bare `nn.Embedding`, simple replace
if type(embedding) is nn.Embedding:
return VocabParallelEmbedding(**kwargs)
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
VocabParallelEmbedding.__init__(embedding, **kwargs)
return embedding
def recursive_replace_linear(
model: nn.Module,
quant_config: "QuantizationConfig | None",