[Transformers backend] Replace vocab embeddings in recursive_replace (#54760)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-01 15:35:24 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 82b7d49a6e
commit f1e5fdd7f2
3 changed files with 128 additions and 70 deletions
@@ -29,7 +29,7 @@ import torch
import transformers
from packaging.version import Version
from torch import nn
from transformers import AutoModel
from transformers import AutoModel, PretrainedConfig
from transformers.conversion_mapping import (
WeightRenaming,
get_model_conversion_mapping,
@@ -169,13 +169,6 @@ class Base(
# Create attention instances for KV cache allocation
self.attention_instances = self.create_attention_instances()
# Input embeddings
input_embeddings = self.model.get_input_embeddings()
if not isinstance(input_embeddings, PPMissingLayer):
self.model.set_input_embeddings(
replace_embedding_class(input_embeddings, self.quant_config)
)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
@@ -423,6 +416,31 @@ class Base(
# attrsetter in case the module is nested (e.g. "text_model.norm")
attrsetter(name)(module, PPMissingLayer())
def _vocab_embedding_ids(self) -> set[int]:
"""ids of the `nn.Embedding`s in `self.model` which hold a vocab table."""
def vocab_sizes(config: PretrainedConfig):
for key, value in vars(config).items():
if isinstance(value, PretrainedConfig):
yield from vocab_sizes(value)
elif "vocab_size" in key and isinstance(value, int):
yield value
sizes = set(vocab_sizes(self.config))
# `get_input_embeddings` may return a module which composes the `nn.Embedding`,
# or a `PPMissingLayer` if the embeddings are not on this pipeline stage
ids = {
id(module)
for module in self.model.get_input_embeddings().modules()
if isinstance(module, nn.Embedding)
}
ids.update(
id(module)
for module in self.model.modules()
if isinstance(module, nn.Embedding) and module.num_embeddings in sizes
)
return ids
def recursive_replace(self):
"""Recursively replace modules in the model as needed.
@@ -432,6 +450,7 @@ class Base(
- Attention QKV projections with a fused `QKVParallelLinear` + split
- `nn.Linear` with vLLM's tensor parallel linear classes
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
"""
tp_plan = self.model.tp_plan or {}
@@ -454,6 +473,8 @@ class Base(
# Detect fusable patterns once per module class (cached, so this is cheap)
fusers = Fusers(self.model, self.vllm_config)
vocab_embedding_ids = self._vocab_embedding_ids()
def register_fusion(fuser: BaseFuser, prefix: str):
"""Register a fused layer's mappings just before it is built."""
self.fusers[prefix] = fuser
@@ -503,6 +524,10 @@ class Base(
)
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
new_module = replace_conv_class(child_module)
elif id(child_module) in vocab_embedding_ids:
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif (fuser := fusers[child_module]) is not None:
register_fusion(fuser, qual_name)
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
@@ -224,41 +224,22 @@ def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbe
def replace_embedding_class(
embedding: nn.Module,
embedding: nn.Embedding,
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> nn.Module:
"""Replace the `nn.Embedding` in `embedding` with `VocabParallelEmbedding`.
"""Replace `nn.Embedding` with `VocabParallelEmbedding`.
Args:
embedding: The module returned by `model.get_input_embeddings()`.
embedding: The `nn.Embedding` holding a vocab table.
quant_config: Quantization config for the new embedding.
prefix: Qualname of `embedding`, used to look up its quantization method.
Returns:
The module to install with `model.set_input_embeddings()`. Composing and
inheriting modules are mutated in place and returned as-is.
Raises:
ValueError: If `embedding` composes anything other than one `nn.Embedding`,
which would leave the input embedding weights ambiguous.
The module to install in `embedding`'s place.
"""
# If `embedding` composes its `nn.Embedding`, recurse into it
if not isinstance(embedding, nn.Embedding):
composed = [
(name, module)
for name, module in embedding.named_modules()
if isinstance(module, nn.Embedding)
]
if len(composed) != 1:
raise ValueError(
f"Expected {type(embedding).__name__} to be an `nn.Embedding` or to "
f"compose exactly one, but found {len(composed)}."
)
name, module = composed[0]
new_embedding = replace_embedding_class(
module, quant_config, prefix=maybe_prefix(prefix, name)
)
attrsetter(name)(embedding, new_embedding)
# Tied tables are reached more than once, so don't rebase an already new embedding
if isinstance(embedding, VocabParallelEmbedding):
return embedding
kwargs = dict(