[Core][LoRA] Support fp32 lm_head (head_dtype) on the LoRA path (#48525)
Signed-off-by: Karthik Kothuri <karthikkothuri2009@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
6e073440b1
commit
4f7fffb92f
@@ -45,15 +45,6 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
self.hidden_size = hidden_size
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
# The fp32 lm_head path lives in the base LogitsProcessor._get_logits,
|
||||
# which this wrapper bypasses. Rather than silently emit model-dtype
|
||||
# logits, reject the combination until the LoRA path supports it.
|
||||
head_dtype = getattr(base_layer, "head_dtype", None)
|
||||
if head_dtype is not None and head_dtype != dtype:
|
||||
raise ValueError(
|
||||
"A head_dtype different from the model dtype (e.g. an fp32 "
|
||||
"lm_head) is not yet supported with LoRA."
|
||||
)
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.sharded_to_full_mapping = sharded_to_full_mapping
|
||||
@@ -158,9 +149,13 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
actual_lm_head = lm_head.base_layer
|
||||
else:
|
||||
actual_lm_head = lm_head
|
||||
logits = actual_lm_head.quant_method.apply(actual_lm_head, hidden_states)
|
||||
if embedding_bias is not None:
|
||||
logits += embedding_bias
|
||||
# Run the base projection through the LogitsProcessor so head_dtype
|
||||
# (e.g. an fp32 lm_head) is honored on the LoRA path too. The LoRA
|
||||
# delta is accumulated into these logits by add_lora_logits below,
|
||||
# whose internal buffer is already fp32.
|
||||
logits = self.base_layer._apply_head(
|
||||
actual_lm_head, hidden_states, embedding_bias
|
||||
)
|
||||
|
||||
# Gather logits for TP
|
||||
logits = self.base_layer._gather_logits(logits)
|
||||
|
||||
Reference in New Issue
Block a user