[Bugfix] Transformers backend: apply learned lm_head.bias for tied-embedding models (#46835)
Signed-off-by: John Langford <jl@hunch.net> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -640,10 +640,7 @@ class Base(
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=self.skip_prefixes,
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for causal language models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -61,6 +62,22 @@ class CausalMixin(VllmModelForTextGeneration):
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
|
||||
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
|
||||
|
||||
lm_head_bias = set()
|
||||
|
||||
def auto_load_lm_head_bias(weights):
|
||||
for name, weight in weights:
|
||||
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
|
||||
self.lm_head._register_bias()
|
||||
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
|
||||
lm_head_bias.add(name)
|
||||
else:
|
||||
yield name, weight
|
||||
|
||||
return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias
|
||||
|
||||
def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
|
||||
return logits
|
||||
|
||||
Reference in New Issue
Block a user