[Transformers backend] Replace vocab embeddings in recursive_replace (#54760)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
82b7d49a6e
commit
f1e5fdd7f2
@@ -29,7 +29,7 @@ import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from torch import nn
|
||||
from transformers import AutoModel
|
||||
from transformers import AutoModel, PretrainedConfig
|
||||
from transformers.conversion_mapping import (
|
||||
WeightRenaming,
|
||||
get_model_conversion_mapping,
|
||||
@@ -169,13 +169,6 @@ class Base(
|
||||
# Create attention instances for KV cache allocation
|
||||
self.attention_instances = self.create_attention_instances()
|
||||
|
||||
# Input embeddings
|
||||
input_embeddings = self.model.get_input_embeddings()
|
||||
if not isinstance(input_embeddings, PPMissingLayer):
|
||||
self.model.set_input_embeddings(
|
||||
replace_embedding_class(input_embeddings, self.quant_config)
|
||||
)
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
|
||||
@@ -423,6 +416,31 @@ class Base(
|
||||
# attrsetter in case the module is nested (e.g. "text_model.norm")
|
||||
attrsetter(name)(module, PPMissingLayer())
|
||||
|
||||
def _vocab_embedding_ids(self) -> set[int]:
|
||||
"""ids of the `nn.Embedding`s in `self.model` which hold a vocab table."""
|
||||
|
||||
def vocab_sizes(config: PretrainedConfig):
|
||||
for key, value in vars(config).items():
|
||||
if isinstance(value, PretrainedConfig):
|
||||
yield from vocab_sizes(value)
|
||||
elif "vocab_size" in key and isinstance(value, int):
|
||||
yield value
|
||||
|
||||
sizes = set(vocab_sizes(self.config))
|
||||
# `get_input_embeddings` may return a module which composes the `nn.Embedding`,
|
||||
# or a `PPMissingLayer` if the embeddings are not on this pipeline stage
|
||||
ids = {
|
||||
id(module)
|
||||
for module in self.model.get_input_embeddings().modules()
|
||||
if isinstance(module, nn.Embedding)
|
||||
}
|
||||
ids.update(
|
||||
id(module)
|
||||
for module in self.model.modules()
|
||||
if isinstance(module, nn.Embedding) and module.num_embeddings in sizes
|
||||
)
|
||||
return ids
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Recursively replace modules in the model as needed.
|
||||
|
||||
@@ -432,6 +450,7 @@ class Base(
|
||||
- Attention QKV projections with a fused `QKVParallelLinear` + split
|
||||
- `nn.Linear` with vLLM's tensor parallel linear classes
|
||||
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
|
||||
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
|
||||
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
|
||||
"""
|
||||
tp_plan = self.model.tp_plan or {}
|
||||
@@ -454,6 +473,8 @@ class Base(
|
||||
# Detect fusable patterns once per module class (cached, so this is cheap)
|
||||
fusers = Fusers(self.model, self.vllm_config)
|
||||
|
||||
vocab_embedding_ids = self._vocab_embedding_ids()
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
self.fusers[prefix] = fuser
|
||||
@@ -503,6 +524,10 @@ class Base(
|
||||
)
|
||||
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
|
||||
new_module = replace_conv_class(child_module)
|
||||
elif id(child_module) in vocab_embedding_ids:
|
||||
new_module = replace_embedding_class(
|
||||
child_module, self.quant_config, prefix=qual_name
|
||||
)
|
||||
elif (fuser := fusers[child_module]) is not None:
|
||||
register_fusion(fuser, qual_name)
|
||||
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
|
||||
|
||||
@@ -224,41 +224,22 @@ def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbe
|
||||
|
||||
|
||||
def replace_embedding_class(
|
||||
embedding: nn.Module,
|
||||
embedding: nn.Embedding,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> nn.Module:
|
||||
"""Replace the `nn.Embedding` in `embedding` with `VocabParallelEmbedding`.
|
||||
"""Replace `nn.Embedding` with `VocabParallelEmbedding`.
|
||||
|
||||
Args:
|
||||
embedding: The module returned by `model.get_input_embeddings()`.
|
||||
embedding: The `nn.Embedding` holding a vocab table.
|
||||
quant_config: Quantization config for the new embedding.
|
||||
prefix: Qualname of `embedding`, used to look up its quantization method.
|
||||
Returns:
|
||||
The module to install with `model.set_input_embeddings()`. Composing and
|
||||
inheriting modules are mutated in place and returned as-is.
|
||||
Raises:
|
||||
ValueError: If `embedding` composes anything other than one `nn.Embedding`,
|
||||
which would leave the input embedding weights ambiguous.
|
||||
The module to install in `embedding`'s place.
|
||||
"""
|
||||
# If `embedding` composes its `nn.Embedding`, recurse into it
|
||||
if not isinstance(embedding, nn.Embedding):
|
||||
composed = [
|
||||
(name, module)
|
||||
for name, module in embedding.named_modules()
|
||||
if isinstance(module, nn.Embedding)
|
||||
]
|
||||
if len(composed) != 1:
|
||||
raise ValueError(
|
||||
f"Expected {type(embedding).__name__} to be an `nn.Embedding` or to "
|
||||
f"compose exactly one, but found {len(composed)}."
|
||||
)
|
||||
name, module = composed[0]
|
||||
new_embedding = replace_embedding_class(
|
||||
module, quant_config, prefix=maybe_prefix(prefix, name)
|
||||
)
|
||||
attrsetter(name)(embedding, new_embedding)
|
||||
# Tied tables are reached more than once, so don't rebase an already new embedding
|
||||
if isinstance(embedding, VocabParallelEmbedding):
|
||||
return embedding
|
||||
|
||||
kwargs = dict(
|
||||
|
||||
Reference in New Issue
Block a user