Fix weight tying (#51665)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-08-20 10:31:55 +00:00
committed by GitHub
parent 6e85feb1b7
commit f0c14b4f77
70 changed files with 526 additions and 238 deletions
@@ -24,6 +24,7 @@ from vllm.model_executor.model_loader.reload import (
record_metadata_for_reloading,
set_torchao_reload_attrs,
)
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.tracing import instrument
from vllm.utils.mem_utils import release_device_memory_under_pressure
@@ -96,6 +97,10 @@ def initialize_model(
def process_weights_after_loading(
model: nn.Module, model_config: ModelConfig, target_device: torch.device
) -> None:
# Reclaim memory when an explicit lm_head has been
# loaded, but it is identical to the input embeddings.
maybe_retie_word_embeddings(model, model_config)
for _, module in model.named_modules():
quant_method = getattr(module, "quant_method", None)
if isinstance(quant_method, QuantizeMethodBase):
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reconcile word embedding tying with what the checkpoint actually contains."""
from dataclasses import dataclass
import torch
from torch import nn
from vllm.config import ModelConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
logger = init_logger(__name__)
@dataclass
class _UntiedLMHead:
name: str
lm_head: ParallelLMHead
embed_tokens: VocabParallelEmbedding
@property
def weight_name(self) -> str:
return f"{self.name}.weight"
def tie(self, model: nn.Module) -> None:
parent_name, _, attr = self.name.rpartition(".")
parent = model.get_submodule(parent_name) if parent_name else model
setattr(parent, attr, self.lm_head.tie_weights(self.embed_tokens))
def _get_untied_lm_head(model: nn.Module) -> _UntiedLMHead | None:
"""Locate an `lm_head` that could be tied to the input embeddings.
Returns `None` unless the model has exactly one of each, so that models with
several heads (such as MTP) or with already tied weights are left alone.
Both are found by type rather than by name, because `get_input_embeddings`
takes different arguments on multimodal models and the `lm_head` of a
multimodal model is nested inside its language model.
A quantized `lm_head` is also left alone. Its weights may be packed under
another name, and online quantization creates them after loading, so
neither their contents nor whether they were loaded can be established here.
Note that this is a property of the layer, not of the model: most quantized
models leave the `lm_head` and the input embeddings unquantized.
"""
heads = list[tuple[str, ParallelLMHead]]()
embeddings = list[VocabParallelEmbedding]()
for name, module in model.named_modules():
if not isinstance(module, VocabParallelEmbedding):
continue
if not isinstance(module.quant_method, UnquantizedEmbeddingMethod):
continue
if isinstance(module, ParallelLMHead):
heads.append((name, module))
else:
embeddings.append(module)
if len(heads) != 1 or len(embeddings) != 1:
return None
(name, lm_head), embed_tokens = heads[0], embeddings[0]
if lm_head.weight.shape != embed_tokens.weight.shape:
return None
return _UntiedLMHead(name, lm_head, embed_tokens)
def maybe_retie_word_embeddings(model: nn.Module, model_config: ModelConfig) -> None:
"""Re-tie word embeddings that
[ModelConfig.maybe_untie_word_embeddings][vllm.config.ModelConfig.maybe_untie_word_embeddings]
untied, if the loaded `lm_head` turned out to be identical to the input embeddings
after all.
Checkpoints produced by quantization or fine-tuning tooling often keep a redundant
copy of the tied `lm_head`. Sharing the storage again reclaims the memory it would
otherwise cost."""
if not model_config.word_embeddings_untied_by_checkpoint:
return
if (untied := _get_untied_lm_head(model)) is None:
return
# On device, torch.equal segfaults on ROCm when sleep mode is enabled
if not torch.equal(untied.lm_head.weight.cpu(), untied.embed_tokens.weight.cpu()):
logger.warning(
"The config for %s says the word embeddings are tied, but the checkpoint "
"contains a different %s, which has been used instead of tying. "
"Set `tie_word_embeddings=False` in the config to silence this warning.",
model_config.model,
untied.weight_name,
)
return
logger.debug("Re-tying %s, which is identical to the input embeddings", untied.name)
untied.tie(model)
+1 -4
View File
@@ -491,8 +491,5 @@ class ApertusForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -5
View File
@@ -356,11 +356,7 @@ class ArceeForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights into the model (delegates to inner model and handles
tied embeddings)."""
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
skip_substrs=["gate_proj"],
)
loader = AutoWeightsLoader(self, skip_substrs=["gate_proj"])
# AutoWeightLoader handles weight name remapping, including fusing
# separate q_proj, k_proj, v_proj into qkv_proj
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -589,8 +589,5 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -557,10 +557,7 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(self._normalize_lm_head(weights))
+1 -1
View File
@@ -360,7 +360,7 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self, skip_prefixes=["lm_head.weight"])
loader = AutoWeightsLoader(self)
weights = _add_transformer_prefix(weights)
return loader.load_weights(weights)
+1 -2
View File
@@ -1059,6 +1059,5 @@ class ChameleonForConditionalGeneration(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@@ -493,7 +493,6 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant):
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
assert getattr(config, "tie_word_embeddings", True)
self.unpadded_vocab_size = config.vocab_size
self.quant_config = quant_config
self.logits_scale = config.logit_scale
-3
View File
@@ -429,9 +429,6 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
quant_config = vllm_config.quant_config
self.config = config
# currently all existing command R models have `tie_word_embeddings`
# enabled
assert config.tie_word_embeddings
self.quant_config = quant_config
self.logits_processor = LogitsProcessor(
+1 -4
View File
@@ -629,8 +629,5 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -2
View File
@@ -214,6 +214,5 @@ class ErnieMTP(nn.Module):
if any(k in name for k in ("mtp", "embed_tokens", "lm_head")):
yield name, weight
skip_prefixes = ["lm_head"] if self.config.tie_word_embeddings else []
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
+1 -7
View File
@@ -446,11 +446,5 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
# With tie_word_embeddings, we can skip lm_head.weight
# The weight might appear unnecessarily in the files if the model is
# processed with quantization, LoRA, fine-tuning, etc.
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -7
View File
@@ -446,11 +446,5 @@ class Exaone4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
# With tie_word_embeddings, we can skip lm_head.weight
# The weight might appear unnecessarily in the files if the model is
# processed with quantization, LoRA, fine-tuning, etc.
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -6
View File
@@ -560,12 +560,7 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
# With tie_word_embeddings, we can skip lm_head.weight
# The weight might appear unnecessarily in the files if the model is
# processed with quantization, LoRA, fine-tuning, etc.
skip_prefixes=(
["lm_head.", "mtp."] if self.config.tie_word_embeddings else ["mtp."]
),
skip_prefixes=["mtp."],
# Skip loading extra parameters for GPTQ/modelopt models.
ignore_unexpected_suffixes=[
".bias",
+1 -4
View File
@@ -74,10 +74,7 @@ class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
params = dict(self.named_parameters())
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(
self.reshape_fairseq2_weights(name, loaded_weight, params)
for name, loaded_weight in weights
+1 -4
View File
@@ -535,8 +535,5 @@ class FalconForCausalLM(nn.Module, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -643,8 +643,5 @@ class FalconH1ForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -6
View File
@@ -346,8 +346,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
quant_config = vllm_config.quant_config
self.config = config
# currently all existing Gemma models have `tie_word_embeddings` enabled
assert config.tie_word_embeddings
self.quant_config = quant_config
self.model = GemmaModel(
@@ -381,8 +379,5 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -6
View File
@@ -335,8 +335,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
super().__init__()
self.config = config
# currently all existing Gemma models have `tie_word_embeddings` enabled
assert config.tie_word_embeddings
self.quant_config = quant_config
self.model = Gemma2Model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
@@ -371,8 +369,5 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -442,8 +442,5 @@ class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-3
View File
@@ -1694,15 +1694,12 @@ class Gemma4ForCausalLM(
yield name, weight
# Skip multimodal weights — handled by the multimodal wrapper.
# Also skip lm_head when weights are tied.
skip = [
"audio_tower.",
"vision_tower.",
"embed_audio.",
"embed_vision.",
]
if self.config.tie_word_embeddings:
skip.append("lm_head.")
loader = AutoWeightsLoader(self, skip_substrs=skip)
return loader.load_weights(_weight_iterator())
+1 -2
View File
@@ -292,11 +292,10 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else []
# Skip the speculative (MTP) layers, which are loaded by the
# draft model instead.
num_nextn_layers = getattr(self.config, "num_nextn_predict_layers", 0)
skip_prefixes += [
skip_prefixes = [
f"model.layers.{self.config.num_hidden_layers + i}."
for i in range(num_nextn_layers)
]
+1 -4
View File
@@ -1242,8 +1242,5 @@ class GptOssForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -6
View File
@@ -467,10 +467,5 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# With tie_word_embeddings, we can skip lm_head.weight
# The weight might appear unnecessarily in the files if the model is
# processed with quantization, LoRA, fine-tuning, etc.
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -2
View File
@@ -452,6 +452,5 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -289,6 +289,5 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -2
View File
@@ -521,6 +521,5 @@ class HrmTextForCausalLM(nn.Module):
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -970,10 +970,7 @@ class HunyuanV1ModelBase(
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+1 -4
View File
@@ -1034,10 +1034,7 @@ class HunYuanVLForConditionalGeneration(
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
+1 -4
View File
@@ -710,10 +710,7 @@ class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
continue
yield name, weight
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(_filter_weights(weights))
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
+1 -4
View File
@@ -478,8 +478,5 @@ class HyperCLOVAXForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=["lm_head."] if self.config.tie_word_embeddings else None,
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -381,10 +381,7 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsQuant):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["output."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
@@ -575,8 +575,5 @@ class IQuestLoopCoderForCausalLM(nn.Module):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -437,8 +437,5 @@ class Jais2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -763,8 +763,5 @@ class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -508,8 +508,5 @@ class Lfm2ForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -671,8 +671,5 @@ class Lfm2MoeForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -534,10 +534,7 @@ class LlamaForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -792,10 +792,7 @@ class Llama4ForCausalLM(LlamaForCausalLM, MixtureOfExperts):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
# Use a generator (not a list comprehension) so the weights iterator is
# consumed lazily by AutoWeightsLoader. Materializing it here would hold
# the entire language-model checkpoint in host memory at once, which can
+1 -3
View File
@@ -119,10 +119,8 @@ class MiMoForCausalLM(Qwen2ForCausalLM, nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else []
# MTP layers are loaded by the draft model, not the main model.
skip_prefixes.append("model.mtp_layers.")
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
loader = AutoWeightsLoader(self, skip_prefixes=["model.mtp_layers."])
return loader.load_weights(weights)
def compute_logits(
+1 -4
View File
@@ -648,8 +648,5 @@ class MiniCPMForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -387,8 +387,5 @@ class EagleMiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle
process_eagle_weight(self, name)
return name, loaded_weight
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(map(transform, weights))
+1 -4
View File
@@ -277,10 +277,7 @@ class MistralForCausalLM(LlamaForCausalLM):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(
self.maybe_remap_mistral(name, loaded_weight)
for name, loaded_weight in weights
-1
View File
@@ -279,7 +279,6 @@ class MPTForCausalLM(nn.Module, SupportsPP):
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
assert config.tie_word_embeddings
self.quant_config = quant_config
self.transformer = MPTModel(
+1 -4
View File
@@ -412,8 +412,5 @@ class DeciLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, HasNoOps):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -6
View File
@@ -408,10 +408,5 @@ class Olmo3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(
self,
skip_prefixes=(
["lm_head.weight"] if self.config.tie_word_embeddings else None
),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -6
View File
@@ -474,10 +474,5 @@ class OlmoHybridForCausalLM(
return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(
self,
skip_prefixes=(
["lm_head.weight"] if self.config.tie_word_embeddings else None
),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -1153,10 +1153,7 @@ class OpenPanguModelBase(nn.Module, SupportsPP, SupportsLoRA):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -6
View File
@@ -384,10 +384,5 @@ class OPTForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(
["lm_head.weight"] if self.config.tie_word_embeddings else None
),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -430,8 +430,5 @@ class Plamo3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -500,8 +500,5 @@ class Qwen2ForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -335,8 +335,5 @@ class Qwen3ForCausalLM(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -395,8 +395,5 @@ class Rnj1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -679,10 +679,7 @@ class SarvamMLAForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SarvamMixtureOfE
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+1 -4
View File
@@ -428,8 +428,5 @@ class SeedOssForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+1 -4
View File
@@ -122,8 +122,5 @@ class TeleChat2ForCausalLM(LlamaForCausalLM):
return TeleChat2Model(vllm_config=vllm_config, prefix=prefix)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@@ -40,11 +40,7 @@ class CausalMixin(VllmModelForTextGeneration):
vllm_config=vllm_config, prefix=prefix
)
# Tell `Base.load_weights` to skip
# `lm_head` if the model has tied word embeddings
tie_word_embeddings = self._get_tie_word_embeddings()
if tie_word_embeddings:
self.skip_prefixes.append("lm_head.")
if self.pp_group.is_last_rank:
self.lm_head = ParallelLMHead(
+54 -6
View File
@@ -18,6 +18,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.reload import (
support_quantized_model_reload_from_hp_weights,
)
@@ -169,6 +170,20 @@ class WeightsMapper:
return replace(self, orig_to_new_stacked={})
def _get_tied_embedding_params(module: nn.Module) -> dict[str, str]:
"""Map each tied word embedding qualname to the first name it aliases."""
canonical = dict[int, str]()
aliased = dict[str, str]()
for prefix, submodule in module.named_modules(remove_duplicate=False):
if not isinstance(submodule, VocabParallelEmbedding):
continue
for name, param in submodule.named_parameters(remove_duplicate=False):
qualname = f"{prefix}.{name}" if prefix else name
if (first_name := canonical.setdefault(id(param), qualname)) != qualname:
aliased[qualname] = first_name
return aliased
class AutoWeightsLoader:
"""
Helper class to load weights into a [`torch.nn.Module`][]. It is able
@@ -213,6 +228,14 @@ class AutoWeightsLoader:
# update default skip_substrs
self.skip_substrs += self.ROTARY_EMBEDS_UNUSED_WEIGHTS
# Weight tying makes two qualnames point at the same `nn.Parameter`
# (e.g. `lm_head.weight` and `model.embed_tokens.weight`). Loading both
# would write the same storage twice, so only the first name reached by
# module traversal is loaded and the rest are skipped.
self.aliased_params = _get_tied_embedding_params(module)
self._skipped_aliases = dict[str, str]()
self._loaded_params_are_complete = True
def _groupby_prefix(
self,
weights: Iterable[tuple[str, torch.Tensor]],
@@ -242,8 +265,10 @@ class AutoWeightsLoader:
return ".".join((prefix, rest))
def _can_skip(self, qualname: str) -> bool:
return any(qualname.startswith(p) for p in self.skip_prefixes) or any(
substr in qualname for substr in self.skip_substrs
return (
qualname in self.aliased_params
or any(qualname.startswith(p) for p in self.skip_prefixes)
or any(substr in qualname for substr in self.skip_substrs)
)
def _can_ignore_unexpected(self, qualname: str) -> bool:
@@ -332,6 +357,7 @@ class AutoWeightsLoader:
logger.warning(
"Unable to collect loaded parameters for module %s", module
)
self._loaded_params_are_complete = False
else:
yield from map(
lambda x: self._get_qualname(base_prefix, x),
@@ -415,14 +441,36 @@ class AutoWeightsLoader:
self.ignore_unexpected_suffixes.extend(ignore_unexpected_suffixes)
if mapper is not None:
weights = mapper.apply(weights)
# filter out weights with first-prefix/substr to skip in name
weights = (
(name, weight) for name, weight in weights if not self._can_skip(name)
)
weights = self._filter_skipped(weights)
autoloaded_weights = set(self._load_module("", self.module, weights))
self._check_skipped_aliases(autoloaded_weights)
return autoloaded_weights
def _filter_skipped(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
for name, weight in weights:
if (canonical := self.aliased_params.get(name)) is not None:
self._skipped_aliases[name] = canonical
if self._can_skip(name):
continue
yield name, weight
def _check_skipped_aliases(self, autoloaded_weights: set[str]) -> None:
"""Guard against skipping an alias whose canonical name never loads."""
if not self._loaded_params_are_complete:
return
for alias, canonical in self._skipped_aliases.items():
if canonical not in autoloaded_weights:
raise ValueError(
f"{alias!r} was skipped because it is tied to {canonical!r} "
f"in {self.module._get_name()}, but {canonical!r} was not "
"found in the checkpoint, so the tied weight is "
"uninitialized."
)
def maybe_fuse_shared_experts(
weights: Iterable[tuple[str, torch.Tensor]],