diff --git a/vllm/model_executor/layers/vocab_parallel_embedding.py b/vllm/model_executor/layers/vocab_parallel_embedding.py index 61f33591b8..8d9a7ccbac 100644 --- a/vllm/model_executor/layers/vocab_parallel_embedding.py +++ b/vllm/model_executor/layers/vocab_parallel_embedding.py @@ -542,19 +542,16 @@ class ParallelLMHead(VocabParallelEmbedding): ) self.quant_config = quant_config if bias: - self.bias = Parameter( - torch.empty(self.num_embeddings_per_partition, dtype=params_dtype) - ) - set_weight_attrs( - self.bias, - { - "output_dim": 0, - "weight_loader": self.weight_loader, - }, - ) + self._register_bias() else: self.register_parameter("bias", None) + def _register_bias(self): + data = torch.empty(self.num_embeddings_per_partition, dtype=self.params_dtype) + self.bias = Parameter(data, requires_grad=False) + weight_attrs = dict(output_dim=0, weight_loader=self.weight_loader) + set_weight_attrs(weight=self.bias, weight_attrs=weight_attrs) + def tie_weights(self, embed_tokens: VocabParallelEmbedding): """Tie the weights with word embeddings.""" return self.quant_method.tie_weights(self, embed_tokens) diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 4402d180ca..bcda62918f 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -640,10 +640,7 @@ class Base( return hidden_states, aux_hidden_states return hidden_states - def load_weights( - self, - weights: Iterable[tuple[str, torch.Tensor]], - ) -> set[str]: + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader( self, skip_prefixes=self.skip_prefixes, diff --git a/vllm/model_executor/models/transformers/causal.py b/vllm/model_executor/models/transformers/causal.py index b6ceb2d677..01a7e41983 100644 --- a/vllm/model_executor/models/transformers/causal.py +++ b/vllm/model_executor/models/transformers/causal.py @@ -16,6 +16,7 @@ # limitations under the License. """Transformers modeling backend mixin for causal language models.""" +from collections.abc import Iterable from typing import TYPE_CHECKING from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -61,6 +62,22 @@ class CausalMixin(VllmModelForTextGeneration): else: self.lm_head = PPMissingLayer() + def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]: + """A thin wrapper around `Base.load_weights` to handle the lm_head bias.""" + + lm_head_bias = set() + + def auto_load_lm_head_bias(weights): + for name, weight in weights: + if name.endswith("lm_head.bias") and self.pp_group.is_last_rank: + self.lm_head._register_bias() + self.lm_head.bias.weight_loader(self.lm_head.bias, weight) + lm_head_bias.add(name) + else: + yield name, weight + + return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias + def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None": - logits = self.logits_processor(self.lm_head, hidden_states) + logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias) return logits