Fix weight tying (#51665)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from vllm.platforms import current_platform
|
||||
from vllm.tasks import PoolingTask, ScoreType, SupportedTask
|
||||
from vllm.transformers_utils.config import (
|
||||
ConfigFormat,
|
||||
checkpoint_has_lm_head,
|
||||
get_config,
|
||||
get_hf_image_processor_config,
|
||||
get_hf_text_config,
|
||||
@@ -185,6 +186,10 @@ class ModelConfig:
|
||||
"""The Hugging Face config of the model."""
|
||||
hf_text_config: PretrainedConfig = field(init=False)
|
||||
"""The Hugging Face config of the text model (same as hf_config for text models)."""
|
||||
word_embeddings_untied_by_checkpoint: bool = field(default=False, init=False)
|
||||
"""Whether `tie_word_embeddings` was overridden to `False` because the checkpoint
|
||||
contains an `lm_head` of its own. The two may still turn out to be identical, in
|
||||
which case they are re-tied once the weights have been loaded."""
|
||||
hf_config_path: str | None = None
|
||||
"""Name or path of the Hugging Face config to use. If unspecified, model
|
||||
name or path will be used."""
|
||||
@@ -1109,6 +1114,34 @@ class ModelConfig:
|
||||
def _get_encoder_config(self) -> dict[str, Any] | None:
|
||||
return get_sentence_transformer_tokenizer_config(self.model, self.revision)
|
||||
|
||||
def maybe_untie_word_embeddings(self) -> None:
|
||||
"""Stop trusting `tie_word_embeddings` when the checkpoint disagrees.
|
||||
|
||||
A config may claim the word embeddings are tied while the checkpoint
|
||||
ships an `lm_head` of its own. Tying regardless would silently discard
|
||||
that tensor, so build the `lm_head` as if untied and let it load. The
|
||||
two are compared once loaded, and re-tied if they turn out to match, by
|
||||
[maybe_retie_word_embeddings][vllm.model_executor.model_loader.weight_tying.maybe_retie_word_embeddings].
|
||||
|
||||
Transformers makes the same decision in `PreTrainedModel.tie_weights`,
|
||||
where it can compare the two tensors directly.
|
||||
"""
|
||||
if not getattr(self.hf_config, "tie_word_embeddings", False):
|
||||
return
|
||||
if not checkpoint_has_lm_head(self.model, revision=self.revision):
|
||||
return
|
||||
|
||||
logger.debug(
|
||||
"The config for %s says the word embeddings are tied, but the checkpoint "
|
||||
"contains an lm_head. Loading it to find out whether they really are tied.",
|
||||
self.model,
|
||||
)
|
||||
# Both levels must be updated: `VllmConfig.with_hf_config` reads the top
|
||||
# level, and the text config is what the language model itself sees.
|
||||
self.hf_config.tie_word_embeddings = False
|
||||
self.hf_config.get_text_config().tie_word_embeddings = False
|
||||
self.word_embeddings_untied_by_checkpoint = True
|
||||
|
||||
def _get_default_runner_type(
|
||||
self,
|
||||
architectures: list[str],
|
||||
|
||||
@@ -1170,6 +1170,16 @@ class VllmConfig:
|
||||
self.model_config, self.load_config
|
||||
)
|
||||
|
||||
# "dummy" reads no weights at all, and the sharded formats read a vLLM
|
||||
# state dict, which stores tied word embeddings under the lm_head only.
|
||||
# Neither can tell us what the original checkpoint contained.
|
||||
if self.model_config is not None and self.load_config.load_format not in (
|
||||
"dummy",
|
||||
"sharded_state",
|
||||
"runai_streamer_sharded",
|
||||
):
|
||||
self.model_config.maybe_untie_word_embeddings()
|
||||
|
||||
if (
|
||||
self.quant_config is not None
|
||||
and self.model_config is not None
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.model_loader.reload import (
|
||||
record_metadata_for_reloading,
|
||||
set_torchao_reload_attrs,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_tying import maybe_retie_word_embeddings
|
||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_utils import release_device_memory_under_pressure
|
||||
@@ -96,6 +97,10 @@ def initialize_model(
|
||||
def process_weights_after_loading(
|
||||
model: nn.Module, model_config: ModelConfig, target_device: torch.device
|
||||
) -> None:
|
||||
# Reclaim memory when an explicit lm_head has been
|
||||
# loaded, but it is identical to the input embeddings.
|
||||
maybe_retie_word_embeddings(model, model_config)
|
||||
|
||||
for _, module in model.named_modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reconcile word embedding tying with what the checkpoint actually contains."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _UntiedLMHead:
|
||||
name: str
|
||||
lm_head: ParallelLMHead
|
||||
embed_tokens: VocabParallelEmbedding
|
||||
|
||||
@property
|
||||
def weight_name(self) -> str:
|
||||
return f"{self.name}.weight"
|
||||
|
||||
def tie(self, model: nn.Module) -> None:
|
||||
parent_name, _, attr = self.name.rpartition(".")
|
||||
parent = model.get_submodule(parent_name) if parent_name else model
|
||||
setattr(parent, attr, self.lm_head.tie_weights(self.embed_tokens))
|
||||
|
||||
|
||||
def _get_untied_lm_head(model: nn.Module) -> _UntiedLMHead | None:
|
||||
"""Locate an `lm_head` that could be tied to the input embeddings.
|
||||
|
||||
Returns `None` unless the model has exactly one of each, so that models with
|
||||
several heads (such as MTP) or with already tied weights are left alone.
|
||||
Both are found by type rather than by name, because `get_input_embeddings`
|
||||
takes different arguments on multimodal models and the `lm_head` of a
|
||||
multimodal model is nested inside its language model.
|
||||
|
||||
A quantized `lm_head` is also left alone. Its weights may be packed under
|
||||
another name, and online quantization creates them after loading, so
|
||||
neither their contents nor whether they were loaded can be established here.
|
||||
Note that this is a property of the layer, not of the model: most quantized
|
||||
models leave the `lm_head` and the input embeddings unquantized.
|
||||
"""
|
||||
heads = list[tuple[str, ParallelLMHead]]()
|
||||
embeddings = list[VocabParallelEmbedding]()
|
||||
for name, module in model.named_modules():
|
||||
if not isinstance(module, VocabParallelEmbedding):
|
||||
continue
|
||||
if not isinstance(module.quant_method, UnquantizedEmbeddingMethod):
|
||||
continue
|
||||
if isinstance(module, ParallelLMHead):
|
||||
heads.append((name, module))
|
||||
else:
|
||||
embeddings.append(module)
|
||||
|
||||
if len(heads) != 1 or len(embeddings) != 1:
|
||||
return None
|
||||
|
||||
(name, lm_head), embed_tokens = heads[0], embeddings[0]
|
||||
if lm_head.weight.shape != embed_tokens.weight.shape:
|
||||
return None
|
||||
return _UntiedLMHead(name, lm_head, embed_tokens)
|
||||
|
||||
|
||||
def maybe_retie_word_embeddings(model: nn.Module, model_config: ModelConfig) -> None:
|
||||
"""Re-tie word embeddings that
|
||||
[ModelConfig.maybe_untie_word_embeddings][vllm.config.ModelConfig.maybe_untie_word_embeddings]
|
||||
untied, if the loaded `lm_head` turned out to be identical to the input embeddings
|
||||
after all.
|
||||
|
||||
Checkpoints produced by quantization or fine-tuning tooling often keep a redundant
|
||||
copy of the tied `lm_head`. Sharing the storage again reclaims the memory it would
|
||||
otherwise cost."""
|
||||
if not model_config.word_embeddings_untied_by_checkpoint:
|
||||
return
|
||||
if (untied := _get_untied_lm_head(model)) is None:
|
||||
return
|
||||
|
||||
# On device, torch.equal segfaults on ROCm when sleep mode is enabled
|
||||
if not torch.equal(untied.lm_head.weight.cpu(), untied.embed_tokens.weight.cpu()):
|
||||
logger.warning(
|
||||
"The config for %s says the word embeddings are tied, but the checkpoint "
|
||||
"contains a different %s, which has been used instead of tying. "
|
||||
"Set `tie_word_embeddings=False` in the config to silence this warning.",
|
||||
model_config.model,
|
||||
untied.weight_name,
|
||||
)
|
||||
return
|
||||
|
||||
logger.debug("Re-tying %s, which is identical to the input embeddings", untied.name)
|
||||
untied.tie(model)
|
||||
@@ -491,8 +491,5 @@ class ApertusForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -356,11 +356,7 @@ class ArceeForCausalLM(
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights into the model (delegates to inner model and handles
|
||||
tied embeddings)."""
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
skip_substrs=["gate_proj"],
|
||||
)
|
||||
loader = AutoWeightsLoader(self, skip_substrs=["gate_proj"])
|
||||
# AutoWeightLoader handles weight name remapping, including fusing
|
||||
# separate q_proj, k_proj, v_proj into qkv_proj
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -589,8 +589,5 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -557,10 +557,7 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(self._normalize_lm_head(weights))
|
||||
|
||||
|
||||
|
||||
@@ -360,7 +360,7 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["lm_head.weight"])
|
||||
loader = AutoWeightsLoader(self)
|
||||
weights = _add_transformer_prefix(weights)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
@@ -1059,6 +1059,5 @@ class ChameleonForConditionalGeneration(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -493,7 +493,6 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant):
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
assert getattr(config, "tie_word_embeddings", True)
|
||||
self.unpadded_vocab_size = config.vocab_size
|
||||
self.quant_config = quant_config
|
||||
self.logits_scale = config.logit_scale
|
||||
|
||||
@@ -429,9 +429,6 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
# currently all existing command R models have `tie_word_embeddings`
|
||||
# enabled
|
||||
assert config.tie_word_embeddings
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.logits_processor = LogitsProcessor(
|
||||
|
||||
@@ -629,8 +629,5 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -214,6 +214,5 @@ class ErnieMTP(nn.Module):
|
||||
if any(k in name for k in ("mtp", "embed_tokens", "lm_head")):
|
||||
yield name, weight
|
||||
|
||||
skip_prefixes = ["lm_head"] if self.config.tie_word_embeddings else []
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -446,11 +446,5 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
# With tie_word_embeddings, we can skip lm_head.weight
|
||||
# The weight might appear unnecessarily in the files if the model is
|
||||
# processed with quantization, LoRA, fine-tuning, etc.
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -446,11 +446,5 @@ class Exaone4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
# With tie_word_embeddings, we can skip lm_head.weight
|
||||
# The weight might appear unnecessarily in the files if the model is
|
||||
# processed with quantization, LoRA, fine-tuning, etc.
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -560,12 +560,7 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
# With tie_word_embeddings, we can skip lm_head.weight
|
||||
# The weight might appear unnecessarily in the files if the model is
|
||||
# processed with quantization, LoRA, fine-tuning, etc.
|
||||
skip_prefixes=(
|
||||
["lm_head.", "mtp."] if self.config.tie_word_embeddings else ["mtp."]
|
||||
),
|
||||
skip_prefixes=["mtp."],
|
||||
# Skip loading extra parameters for GPTQ/modelopt models.
|
||||
ignore_unexpected_suffixes=[
|
||||
".bias",
|
||||
|
||||
@@ -74,10 +74,7 @@ class Fairseq2LlamaForCausalLM(LlamaForCausalLM):
|
||||
|
||||
params = dict(self.named_parameters())
|
||||
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self.reshape_fairseq2_weights(name, loaded_weight, params)
|
||||
for name, loaded_weight in weights
|
||||
|
||||
@@ -535,8 +535,5 @@ class FalconForCausalLM(nn.Module, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -643,8 +643,5 @@ class FalconH1ForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -346,8 +346,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
# currently all existing Gemma models have `tie_word_embeddings` enabled
|
||||
assert config.tie_word_embeddings
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.model = GemmaModel(
|
||||
@@ -381,8 +379,5 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -335,8 +335,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
|
||||
super().__init__()
|
||||
self.config = config
|
||||
# currently all existing Gemma models have `tie_word_embeddings` enabled
|
||||
assert config.tie_word_embeddings
|
||||
self.quant_config = quant_config
|
||||
self.model = Gemma2Model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
@@ -371,8 +369,5 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -442,8 +442,5 @@ class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -1694,15 +1694,12 @@ class Gemma4ForCausalLM(
|
||||
yield name, weight
|
||||
|
||||
# Skip multimodal weights — handled by the multimodal wrapper.
|
||||
# Also skip lm_head when weights are tied.
|
||||
skip = [
|
||||
"audio_tower.",
|
||||
"vision_tower.",
|
||||
"embed_audio.",
|
||||
"embed_vision.",
|
||||
]
|
||||
if self.config.tie_word_embeddings:
|
||||
skip.append("lm_head.")
|
||||
|
||||
loader = AutoWeightsLoader(self, skip_substrs=skip)
|
||||
return loader.load_weights(_weight_iterator())
|
||||
|
||||
@@ -292,11 +292,10 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else []
|
||||
# Skip the speculative (MTP) layers, which are loaded by the
|
||||
# draft model instead.
|
||||
num_nextn_layers = getattr(self.config, "num_nextn_predict_layers", 0)
|
||||
skip_prefixes += [
|
||||
skip_prefixes = [
|
||||
f"model.layers.{self.config.num_hidden_layers + i}."
|
||||
for i in range(num_nextn_layers)
|
||||
]
|
||||
|
||||
@@ -1242,8 +1242,5 @@ class GptOssForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -467,10 +467,5 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
# With tie_word_embeddings, we can skip lm_head.weight
|
||||
# The weight might appear unnecessarily in the files if the model is
|
||||
# processed with quantization, LoRA, fine-tuning, etc.
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -452,6 +452,5 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -289,6 +289,5 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -521,6 +521,5 @@ class HrmTextForCausalLM(nn.Module):
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -970,10 +970,7 @@ class HunyuanV1ModelBase(
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -1034,10 +1034,7 @@ class HunYuanVLForConditionalGeneration(
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
|
||||
@@ -710,10 +710,7 @@ class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
continue
|
||||
yield name, weight
|
||||
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(_filter_weights(weights))
|
||||
|
||||
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
|
||||
|
||||
@@ -478,8 +478,5 @@ class HyperCLOVAXForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=["lm_head."] if self.config.tie_word_embeddings else None,
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -381,10 +381,7 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsQuant):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["output."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
|
||||
@@ -575,8 +575,5 @@ class IQuestLoopCoderForCausalLM(nn.Module):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -437,8 +437,5 @@ class Jais2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -763,8 +763,5 @@ class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -508,8 +508,5 @@ class Lfm2ForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -671,8 +671,5 @@ class Lfm2MoeForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -534,10 +534,7 @@ class LlamaForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
|
||||
@@ -792,10 +792,7 @@ class Llama4ForCausalLM(LlamaForCausalLM, MixtureOfExperts):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
# Use a generator (not a list comprehension) so the weights iterator is
|
||||
# consumed lazily by AutoWeightsLoader. Materializing it here would hold
|
||||
# the entire language-model checkpoint in host memory at once, which can
|
||||
|
||||
@@ -119,10 +119,8 @@ class MiMoForCausalLM(Qwen2ForCausalLM, nn.Module):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else []
|
||||
# MTP layers are loaded by the draft model, not the main model.
|
||||
skip_prefixes.append("model.mtp_layers.")
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
|
||||
loader = AutoWeightsLoader(self, skip_prefixes=["model.mtp_layers."])
|
||||
return loader.load_weights(weights)
|
||||
|
||||
def compute_logits(
|
||||
|
||||
@@ -648,8 +648,5 @@ class MiniCPMForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -387,8 +387,5 @@ class EagleMiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle
|
||||
process_eagle_weight(self, name)
|
||||
return name, loaded_weight
|
||||
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(map(transform, weights))
|
||||
|
||||
@@ -277,10 +277,7 @@ class MistralForCausalLM(LlamaForCausalLM):
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self.maybe_remap_mistral(name, loaded_weight)
|
||||
for name, loaded_weight in weights
|
||||
|
||||
@@ -279,7 +279,6 @@ class MPTForCausalLM(nn.Module, SupportsPP):
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
assert config.tie_word_embeddings
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.transformer = MPTModel(
|
||||
|
||||
@@ -412,8 +412,5 @@ class DeciLMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, HasNoOps):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -408,10 +408,5 @@ class Olmo3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(
|
||||
["lm_head.weight"] if self.config.tie_word_embeddings else None
|
||||
),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -474,10 +474,5 @@ class OlmoHybridForCausalLM(
|
||||
return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(
|
||||
["lm_head.weight"] if self.config.tie_word_embeddings else None
|
||||
),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -1153,10 +1153,7 @@ class OpenPanguModelBase(nn.Module, SupportsPP, SupportsLoRA):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
|
||||
@@ -384,10 +384,5 @@ class OPTForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(
|
||||
["lm_head.weight"] if self.config.tie_word_embeddings else None
|
||||
),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -430,8 +430,5 @@ class Plamo3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -500,8 +500,5 @@ class Qwen2ForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -335,8 +335,5 @@ class Qwen3ForCausalLM(
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -395,8 +395,5 @@ class Rnj1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -679,10 +679,7 @@ class SarvamMLAForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SarvamMixtureOfE
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
|
||||
|
||||
@@ -428,8 +428,5 @@ class SeedOssForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -122,8 +122,5 @@ class TeleChat2ForCausalLM(LlamaForCausalLM):
|
||||
return TeleChat2Model(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@@ -40,11 +40,7 @@ class CausalMixin(VllmModelForTextGeneration):
|
||||
vllm_config=vllm_config, prefix=prefix
|
||||
)
|
||||
|
||||
# Tell `Base.load_weights` to skip
|
||||
# `lm_head` if the model has tied word embeddings
|
||||
tie_word_embeddings = self._get_tie_word_embeddings()
|
||||
if tie_word_embeddings:
|
||||
self.skip_prefixes.append("lm_head.")
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
support_quantized_model_reload_from_hp_weights,
|
||||
)
|
||||
@@ -169,6 +170,20 @@ class WeightsMapper:
|
||||
return replace(self, orig_to_new_stacked={})
|
||||
|
||||
|
||||
def _get_tied_embedding_params(module: nn.Module) -> dict[str, str]:
|
||||
"""Map each tied word embedding qualname to the first name it aliases."""
|
||||
canonical = dict[int, str]()
|
||||
aliased = dict[str, str]()
|
||||
for prefix, submodule in module.named_modules(remove_duplicate=False):
|
||||
if not isinstance(submodule, VocabParallelEmbedding):
|
||||
continue
|
||||
for name, param in submodule.named_parameters(remove_duplicate=False):
|
||||
qualname = f"{prefix}.{name}" if prefix else name
|
||||
if (first_name := canonical.setdefault(id(param), qualname)) != qualname:
|
||||
aliased[qualname] = first_name
|
||||
return aliased
|
||||
|
||||
|
||||
class AutoWeightsLoader:
|
||||
"""
|
||||
Helper class to load weights into a [`torch.nn.Module`][]. It is able
|
||||
@@ -213,6 +228,14 @@ class AutoWeightsLoader:
|
||||
# update default skip_substrs
|
||||
self.skip_substrs += self.ROTARY_EMBEDS_UNUSED_WEIGHTS
|
||||
|
||||
# Weight tying makes two qualnames point at the same `nn.Parameter`
|
||||
# (e.g. `lm_head.weight` and `model.embed_tokens.weight`). Loading both
|
||||
# would write the same storage twice, so only the first name reached by
|
||||
# module traversal is loaded and the rest are skipped.
|
||||
self.aliased_params = _get_tied_embedding_params(module)
|
||||
self._skipped_aliases = dict[str, str]()
|
||||
self._loaded_params_are_complete = True
|
||||
|
||||
def _groupby_prefix(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
@@ -242,8 +265,10 @@ class AutoWeightsLoader:
|
||||
return ".".join((prefix, rest))
|
||||
|
||||
def _can_skip(self, qualname: str) -> bool:
|
||||
return any(qualname.startswith(p) for p in self.skip_prefixes) or any(
|
||||
substr in qualname for substr in self.skip_substrs
|
||||
return (
|
||||
qualname in self.aliased_params
|
||||
or any(qualname.startswith(p) for p in self.skip_prefixes)
|
||||
or any(substr in qualname for substr in self.skip_substrs)
|
||||
)
|
||||
|
||||
def _can_ignore_unexpected(self, qualname: str) -> bool:
|
||||
@@ -332,6 +357,7 @@ class AutoWeightsLoader:
|
||||
logger.warning(
|
||||
"Unable to collect loaded parameters for module %s", module
|
||||
)
|
||||
self._loaded_params_are_complete = False
|
||||
else:
|
||||
yield from map(
|
||||
lambda x: self._get_qualname(base_prefix, x),
|
||||
@@ -415,14 +441,36 @@ class AutoWeightsLoader:
|
||||
self.ignore_unexpected_suffixes.extend(ignore_unexpected_suffixes)
|
||||
if mapper is not None:
|
||||
weights = mapper.apply(weights)
|
||||
# filter out weights with first-prefix/substr to skip in name
|
||||
weights = (
|
||||
(name, weight) for name, weight in weights if not self._can_skip(name)
|
||||
)
|
||||
weights = self._filter_skipped(weights)
|
||||
|
||||
autoloaded_weights = set(self._load_module("", self.module, weights))
|
||||
self._check_skipped_aliases(autoloaded_weights)
|
||||
return autoloaded_weights
|
||||
|
||||
def _filter_skipped(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
for name, weight in weights:
|
||||
if (canonical := self.aliased_params.get(name)) is not None:
|
||||
self._skipped_aliases[name] = canonical
|
||||
if self._can_skip(name):
|
||||
continue
|
||||
|
||||
yield name, weight
|
||||
|
||||
def _check_skipped_aliases(self, autoloaded_weights: set[str]) -> None:
|
||||
"""Guard against skipping an alias whose canonical name never loads."""
|
||||
if not self._loaded_params_are_complete:
|
||||
return
|
||||
for alias, canonical in self._skipped_aliases.items():
|
||||
if canonical not in autoloaded_weights:
|
||||
raise ValueError(
|
||||
f"{alias!r} was skipped because it is tied to {canonical!r} "
|
||||
f"in {self.module._get_name()}, but {canonical!r} was not "
|
||||
"found in the checkpoint, so the tied weight is "
|
||||
"uninitialized."
|
||||
)
|
||||
|
||||
|
||||
def maybe_fuse_shared_experts(
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
|
||||
@@ -1082,8 +1082,5 @@ class KimiLinearForCausalLM(
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -1676,10 +1676,7 @@ class KimiLinearForCausalLM(
|
||||
return self.logits_processor(self.lm_head, hidden_states)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
|
||||
)
|
||||
loader = AutoWeightsLoader(self)
|
||||
loaded = loader.load_weights(weights)
|
||||
self.model.finalize_mega_moe_weights()
|
||||
# The fused MultiHeadLatentAttention's process_weights_after_loading
|
||||
|
||||
@@ -1112,7 +1112,9 @@ def try_get_safetensors_metadata(
|
||||
|
||||
try:
|
||||
return with_retry(
|
||||
get_safetensors_metadata_partial, "Error retrieving safetensors"
|
||||
get_safetensors_metadata_partial,
|
||||
"Error retrieving safetensors",
|
||||
fatal_errors=(huggingface_hub.errors.NotASafetensorsRepoError,),
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -1218,6 +1220,24 @@ def get_safetensors_params_metadata(
|
||||
return _read_safetensors_metadata_in_dir(Path(local_dir))
|
||||
|
||||
|
||||
@cache
|
||||
def checkpoint_has_lm_head(model: str, *, revision: str | None = None) -> bool | None:
|
||||
"""Whether the checkpoint contains an `lm_head` tensor of its own.
|
||||
|
||||
Args:
|
||||
model: Name or path of the model repository.
|
||||
revision: The specific model version to use.
|
||||
|
||||
Returns:
|
||||
`None` if the checkpoint contents could not be determined, for example
|
||||
because it is not stored as safetensors.
|
||||
"""
|
||||
metadata = get_safetensors_params_metadata(model, revision=revision)
|
||||
if not metadata:
|
||||
return None
|
||||
return any(name.endswith("lm_head.weight") for name in metadata)
|
||||
|
||||
|
||||
def _download_mistral_config_file(model, revision) -> dict:
|
||||
config_file_name = "params.json"
|
||||
config_dict = get_hf_file_to_dict(config_file_name, model, revision)
|
||||
|
||||
@@ -95,10 +95,23 @@ def with_retry(
|
||||
log_msg: str,
|
||||
max_retries: int = 2,
|
||||
retry_delay: int = 2,
|
||||
fatal_errors: tuple[type[Exception], ...] = (),
|
||||
) -> _R:
|
||||
"""Call `func`, retrying transient failures.
|
||||
|
||||
Args:
|
||||
func: The call to make.
|
||||
log_msg: Prefix for the message logged when a call fails.
|
||||
max_retries: How many times to call `func` before giving up.
|
||||
retry_delay: Seconds to wait after the first failure, doubled each time.
|
||||
fatal_errors: Exceptions that are a definitive answer rather than a
|
||||
transient failure. These are raised immediately, without logging.
|
||||
"""
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
return func()
|
||||
except fatal_errors:
|
||||
raise
|
||||
except Exception as e:
|
||||
if attempt == max_retries - 1:
|
||||
logger.error("%s: %s", log_msg, e)
|
||||
|
||||
Reference in New Issue
Block a user