Fully generalise input embedding handling in Transformers modelling backend (#51247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -2,9 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test the functionality of the Transformers modeling backend."""
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from ...conftest import HfRunner, VllmRunner
|
||||
from ...utils import multi_gpu_test, prep_prompts
|
||||
@@ -299,3 +304,174 @@ def test_pooling(hf_runner, vllm_runner, example_prompts, arch):
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
VOCAB_SIZE = 64
|
||||
HIDDEN_SIZE = 8
|
||||
EMBED_SCALE = 3.0
|
||||
|
||||
|
||||
class ScaledWordEmbedding(nn.Embedding):
|
||||
"""Mirrors Transformers' `*ScaledWordEmbedding` classes."""
|
||||
|
||||
def __init__(
|
||||
self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0
|
||||
):
|
||||
super().__init__(num_embeddings, embedding_dim, padding_idx)
|
||||
self.scalar_embed_scale = embed_scale
|
||||
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
|
||||
|
||||
def forward(self, input_ids):
|
||||
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
|
||||
|
||||
|
||||
class ComposedWordEmbedding(nn.Module):
|
||||
"""Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it."""
|
||||
|
||||
def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0):
|
||||
super().__init__()
|
||||
self.embed = nn.Embedding(num_embeddings, embedding_dim)
|
||||
self.embed_scale = embed_scale
|
||||
|
||||
def forward(self, input_ids):
|
||||
return self.embed(input_ids) * self.embed_scale
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tp_init():
|
||||
"""Single rank tensor parallel state, so vLLM layers can be constructed."""
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import ensure_current_vllm_config
|
||||
|
||||
fd, temp_file = tempfile.mkstemp()
|
||||
os.close(fd)
|
||||
try:
|
||||
with ensure_current_vllm_config():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=0,
|
||||
backend=current_platform.dist_backend,
|
||||
)
|
||||
initialize_model_parallel(1, 1)
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temp_file)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vpe(tp_init):
|
||||
"""`VocabParallelEmbedding`, imported late so collection does not import vLLM."""
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
return VocabParallelEmbedding
|
||||
|
||||
|
||||
def replace(embedding):
|
||||
"""Replace `embedding` and fill the new weights with recognisable values."""
|
||||
from vllm.model_executor.models.transformers.utils import replace_embedding_class
|
||||
|
||||
new_embedding = replace_embedding_class(embedding)
|
||||
for _, param in new_embedding.named_parameters():
|
||||
param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape)
|
||||
return new_embedding
|
||||
|
||||
|
||||
def assert_scaled(vpe, module, embedding=None):
|
||||
"""`module`'s output is `embedding`'s unscaled output times `EMBED_SCALE`."""
|
||||
input_ids = torch.arange(VOCAB_SIZE)
|
||||
unscaled = vpe.forward(embedding if embedding is not None else module, input_ids)
|
||||
torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE)
|
||||
|
||||
|
||||
def test_replace_plain_embedding(vpe):
|
||||
"""A plain `nn.Embedding` is replaced outright, leaving no subclass behind."""
|
||||
assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe
|
||||
|
||||
|
||||
def test_replace_infers_shape_and_dtype(tp_init):
|
||||
"""Shape and dtype come from the replaced module, not from the config."""
|
||||
embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16)
|
||||
new_embedding = replace(embedding)
|
||||
|
||||
assert new_embedding.num_embeddings == VOCAB_SIZE * 2
|
||||
assert new_embedding.org_vocab_size == VOCAB_SIZE * 2
|
||||
assert new_embedding.embedding_dim == HIDDEN_SIZE + 1
|
||||
assert new_embedding.weight.dtype == torch.float16
|
||||
|
||||
|
||||
def test_replace_inherited_embedding(vpe):
|
||||
"""Subclasses keep their extra state and their scaled `forward`."""
|
||||
new_embedding = replace(
|
||||
ScaledWordEmbedding(
|
||||
VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(new_embedding, vpe)
|
||||
assert new_embedding.scalar_embed_scale == EMBED_SCALE
|
||||
assert "embed_scale" in new_embedding._non_persistent_buffers_set
|
||||
assert_scaled(vpe, new_embedding)
|
||||
|
||||
|
||||
def test_replace_composed_embedding(vpe):
|
||||
"""Wrappers are left alone; only the `nn.Embedding` they hold is replaced."""
|
||||
embedding = ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
|
||||
new_embedding = replace(embedding)
|
||||
|
||||
assert new_embedding is embedding
|
||||
assert type(embedding.embed) is vpe
|
||||
assert_scaled(vpe, new_embedding, embedding.embed)
|
||||
|
||||
|
||||
def test_replace_nested_embedding(vpe):
|
||||
"""The composed `nn.Embedding` is found and set however deeply it is nested."""
|
||||
wrapper = nn.Module()
|
||||
wrapper.add_module("inner", nn.Module())
|
||||
wrapper.inner.add_module(
|
||||
"embed", ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
|
||||
)
|
||||
replace(wrapper)
|
||||
|
||||
assert isinstance(wrapper.inner.embed, vpe)
|
||||
assert_scaled(vpe, wrapper.inner.embed)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_embeddings", [0, 2])
|
||||
def test_replace_ambiguous_embedding(tp_init, num_embeddings):
|
||||
"""Composing anything but one `nn.Embedding` is an error, not a silent guess."""
|
||||
wrapper = nn.Module()
|
||||
for i in range(num_embeddings):
|
||||
wrapper.add_module(f"embed_{i}", nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))
|
||||
|
||||
with pytest.raises(ValueError, match=f"found {num_embeddings}"):
|
||||
replace(wrapper)
|
||||
|
||||
|
||||
def test_replaced_embedding_exposes_one_vpe(vpe):
|
||||
"""`CausalMixin` ties `lm_head` to the one `VocabParallelEmbedding` it can find.
|
||||
|
||||
`tie_weights` reads `.weight`, which a composing module does not have, so it must
|
||||
be handed the composed embedding instead.
|
||||
"""
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
inherited = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE))
|
||||
composed = replace(ComposedWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, EMBED_SCALE))
|
||||
|
||||
assert [m for m in inherited.modules() if isinstance(m, vpe)] == [inherited]
|
||||
assert [m for m in composed.modules() if isinstance(m, vpe)] == [composed.embed]
|
||||
|
||||
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
assert lm_head.tie_weights(composed.embed).weight is composed.embed.weight
|
||||
|
||||
@@ -8,7 +8,10 @@ from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.model_executor.custom_op import maybe_get_oot_by_class
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import BaseLayerWithLoRA
|
||||
@@ -133,7 +136,17 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
packed_modules_list: list,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> bool:
|
||||
return type(source_layer) is maybe_get_oot_by_class(VocabParallelEmbedding)
|
||||
# Accept both the in-tree and any OOT class, because layers built on top of
|
||||
# `VocabParallelEmbedding` (e.g. by the Transformers modelling backend) use the
|
||||
# former while `PluggableLayer.__new__` instantiates the latter
|
||||
embedding_classes = (
|
||||
VocabParallelEmbedding,
|
||||
maybe_get_oot_by_class(VocabParallelEmbedding),
|
||||
)
|
||||
lm_head_classes = (ParallelLMHead, maybe_get_oot_by_class(ParallelLMHead))
|
||||
return isinstance(source_layer, embedding_classes) and not isinstance(
|
||||
source_layer, lm_head_classes
|
||||
)
|
||||
|
||||
@property
|
||||
def weight(self):
|
||||
|
||||
@@ -290,7 +290,7 @@ class VocabParallelEmbedding(PluggableLayer):
|
||||
# If we are making an embedding layer, then our quantization linear
|
||||
# method must implement the embedding operation. If we are another
|
||||
# layer type like ParallelLMHead, this is not important.
|
||||
is_embedding_layer = type(self) is VocabParallelEmbedding
|
||||
is_embedding_layer = not isinstance(self, ParallelLMHead)
|
||||
quant_method_implements_embedding = method_has_implemented_embedding(
|
||||
type(quant_method)
|
||||
)
|
||||
|
||||
@@ -46,7 +46,6 @@ from vllm.model_executor.layers.attention import (
|
||||
from vllm.model_executor.layers.attention.mla_attention import get_mla_dims
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
@@ -58,12 +57,14 @@ from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
|
||||
from vllm.model_executor.models.transformers.fusers import MLAFuser
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
named_state,
|
||||
replace_conv_class,
|
||||
replace_embedding_class,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
@@ -84,17 +85,6 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ScaledVocabParallelEmbedding(VocabParallelEmbedding):
|
||||
"""`VocabParallelEmbedding` that scales its output."""
|
||||
|
||||
def __init__(self, *args, embed_scale: float, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.embed_scale = embed_scale
|
||||
|
||||
def forward(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
return super().forward(input_) * self.embed_scale
|
||||
|
||||
|
||||
class Base(
|
||||
nn.Module,
|
||||
VllmModel,
|
||||
@@ -179,24 +169,9 @@ class Base(
|
||||
# Input embeddings
|
||||
input_embeddings = self.model.get_input_embeddings()
|
||||
if not isinstance(input_embeddings, PPMissingLayer):
|
||||
names = ("embedding_size", "hidden_size")
|
||||
embedding_dim = getattr_iter(self.text_config, names, None)
|
||||
assert embedding_dim is not None
|
||||
embedding_kwargs = dict(
|
||||
num_embeddings=self.text_config.vocab_size,
|
||||
embedding_dim=embedding_dim,
|
||||
org_num_embeddings=self.text_config.vocab_size,
|
||||
quant_config=self.quant_config,
|
||||
self.model.set_input_embeddings(
|
||||
replace_embedding_class(input_embeddings, self.quant_config)
|
||||
)
|
||||
embed_scale = getattr(input_embeddings, "embed_scale", None)
|
||||
if embed_scale is not None:
|
||||
# Some models scale embeddings inside the input embedding layer
|
||||
new_input_embeddings = ScaledVocabParallelEmbedding(
|
||||
**embedding_kwargs, embed_scale=float(embed_scale)
|
||||
)
|
||||
else:
|
||||
new_input_embeddings = VocabParallelEmbedding(**embedding_kwargs)
|
||||
self.model.set_input_embeddings(new_input_embeddings)
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
@@ -380,16 +355,6 @@ class Base(
|
||||
tip,
|
||||
)
|
||||
|
||||
def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
"""Set a possibly nested attribute, like the inverse of attrgetter."""
|
||||
parent, _, name = attr.rpartition(".")
|
||||
|
||||
def setter(obj: object, value: object):
|
||||
attr_parent = attrgetter(parent)(obj) if parent else obj
|
||||
setattr(attr_parent, name, value)
|
||||
|
||||
return setter
|
||||
|
||||
module_lists = []
|
||||
module_list_idx = None
|
||||
for i, name in enumerate(names):
|
||||
|
||||
@@ -20,7 +20,10 @@ from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
|
||||
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
|
||||
|
||||
@@ -51,9 +54,10 @@ class CausalMixin(VllmModelForTextGeneration):
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(
|
||||
self.model.get_input_embeddings()
|
||||
)
|
||||
for module in self.model.get_input_embeddings().modules():
|
||||
if isinstance(module, VocabParallelEmbedding):
|
||||
self.lm_head = self.lm_head.tie_weights(module)
|
||||
break
|
||||
|
||||
logit_scale = getattr(self.text_config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
|
||||
@@ -16,9 +16,11 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend utilities."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from functools import lru_cache
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal
|
||||
|
||||
@@ -32,6 +34,7 @@ from vllm.model_executor.layers.linear import (
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.transformers_utils.config import is_rope_parameters_nested
|
||||
|
||||
@@ -183,6 +186,98 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
|
||||
)
|
||||
|
||||
|
||||
def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
"""Set a possibly nested attribute, like the inverse of attrgetter."""
|
||||
parent, _, name = attr.rpartition(".")
|
||||
|
||||
def setter(obj: object, value: object):
|
||||
attr_parent = attrgetter(parent)(obj) if parent else obj
|
||||
setattr(attr_parent, name, value)
|
||||
|
||||
return setter
|
||||
|
||||
|
||||
class _UninitializedEmbedding(nn.Embedding):
|
||||
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
|
||||
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
|
||||
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
|
||||
|
||||
Args:
|
||||
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
|
||||
maps to the same class.
|
||||
Returns:
|
||||
The new class, to assign to `__class__` of an instance of `cls`.
|
||||
"""
|
||||
return type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
|
||||
|
||||
|
||||
def replace_embedding_class(
|
||||
embedding: nn.Module,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> nn.Module:
|
||||
"""Replace the `nn.Embedding` in `embedding` with `VocabParallelEmbedding`.
|
||||
|
||||
Args:
|
||||
embedding: The module returned by `model.get_input_embeddings()`.
|
||||
quant_config: Quantization config for the new embedding.
|
||||
prefix: Qualname of `embedding`, used to look up its quantization method.
|
||||
Returns:
|
||||
The module to install with `model.set_input_embeddings()`. Composing and
|
||||
inheriting modules are mutated in place and returned as-is.
|
||||
Raises:
|
||||
ValueError: If `embedding` composes anything other than one `nn.Embedding`,
|
||||
which would leave the input embedding weights ambiguous.
|
||||
"""
|
||||
# If `embedding` composes its `nn.Embedding`, recurse into it
|
||||
if not isinstance(embedding, nn.Embedding):
|
||||
composed = [
|
||||
(name, module)
|
||||
for name, module in embedding.named_modules()
|
||||
if isinstance(module, nn.Embedding)
|
||||
]
|
||||
if len(composed) != 1:
|
||||
raise ValueError(
|
||||
f"Expected {type(embedding).__name__} to be an `nn.Embedding` or to "
|
||||
f"compose exactly one, but found {len(composed)}."
|
||||
)
|
||||
name, module = composed[0]
|
||||
new_embedding = replace_embedding_class(
|
||||
module, quant_config, prefix=maybe_prefix(prefix, name)
|
||||
)
|
||||
attrsetter(name)(embedding, new_embedding)
|
||||
return embedding
|
||||
|
||||
kwargs = dict(
|
||||
num_embeddings=embedding.num_embeddings,
|
||||
embedding_dim=embedding.embedding_dim,
|
||||
params_dtype=embedding.weight.dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
# If `embedding` is a bare `nn.Embedding`, simple replace
|
||||
if type(embedding) is nn.Embedding:
|
||||
return VocabParallelEmbedding(**kwargs)
|
||||
|
||||
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
|
||||
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
|
||||
VocabParallelEmbedding.__init__(embedding, **kwargs)
|
||||
return embedding
|
||||
|
||||
|
||||
def recursive_replace_linear(
|
||||
model: nn.Module,
|
||||
quant_config: "QuantizationConfig | None",
|
||||
|
||||
Reference in New Issue
Block a user