[Core] Support fp32 lm_head for generation models via head_dtype (RFC #48305 §3.6) (#48390)

Signed-off-by: Karthik Kothuri <karthikkothuri2009@gmail.com>
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
Karthik Kothuri
2026-07-13 16:43:34 +08:00
committed by GitHub
co-authored by Claude wang.yuqi
parent 56a357ed33
commit 107a03ba63
6 changed files with 249 additions and 15 deletions
+51 -3
View File
@@ -3,14 +3,19 @@
"""A layer that compute logits from hidden_stats."""
import torch
import torch.nn.functional as F
from vllm.config import get_current_vllm_config
from vllm.distributed import (
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
tensor_model_parallel_gather,
)
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.layers.vocab_parallel_embedding import (
UnquantizedEmbeddingMethod,
VocabParallelEmbedding,
)
from vllm.platforms import current_platform
@@ -50,6 +55,11 @@ class LogitsProcessor(PluggableLayer):
self.soft_cap = soft_cap
# Whether to use gather or all-gather to gather the logits.
self.use_all_gather = current_platform.use_all_gather()
# Dtype of the lm_head projection. Defaults to the model dtype; an
# fp32 head (via `--hf-overrides '{"head_dtype": "float32"}'`) is
# required for RL training-inference consistency.
model_config = get_current_vllm_config().model_config
self.head_dtype = model_config.head_dtype if model_config is not None else None
def forward(
self,
@@ -86,6 +96,44 @@ class LogitsProcessor(PluggableLayer):
logits = tensor_model_parallel_gather(logits)
return logits
def _apply_head(
self,
lm_head: VocabParallelEmbedding,
hidden_states: torch.Tensor,
embedding_bias: torch.Tensor | None,
) -> torch.Tensor:
"""Project hidden states through the lm_head, honoring head_dtype."""
if self.head_dtype is None or self.head_dtype == hidden_states.dtype:
return lm_head.quant_method.apply(
lm_head, hidden_states, bias=embedding_bias
)
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
raise ValueError(
"A head_dtype different from the model dtype is only "
"supported for an unquantized lm_head."
)
if (
self.head_dtype == torch.float32
and current_platform.is_cuda()
and hidden_states.is_cuda
):
# Accumulate the projection directly into fp32. This avoids
# materializing an fp32 copy of the lm_head weight on every step,
# unlike casting both operands. `torch.mm(out_dtype=...)` is
# CUDA-only and only supports fp32 output for fp16/bf16 inputs, so
# other cases fall back to the cast path below.
flat = hidden_states.reshape(-1, hidden_states.shape[-1])
logits = torch.mm(flat, lm_head.weight.t(), out_dtype=self.head_dtype)
if embedding_bias is not None:
logits = logits + embedding_bias.to(self.head_dtype)
return logits.reshape(*hidden_states.shape[:-1], -1)
return F.linear(
hidden_states.to(self.head_dtype),
lm_head.weight.to(self.head_dtype),
embedding_bias.to(self.head_dtype) if embedding_bias is not None else None,
)
def _get_logits(
self,
hidden_states: torch.Tensor,
@@ -93,7 +141,7 @@ class LogitsProcessor(PluggableLayer):
embedding_bias: torch.Tensor | None,
) -> torch.Tensor | None:
# Get the logits for the next tokens.
logits = lm_head.quant_method.apply(lm_head, hidden_states, bias=embedding_bias)
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
# Gather logits for TP
logits = self._gather_logits(logits)
@@ -122,7 +170,7 @@ class LogitsProcessor(PluggableLayer):
)
tp_size = get_tensor_model_parallel_world_size()
logits = lm_head.quant_method.apply(lm_head, hidden_states, bias=embedding_bias)
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
if self.soft_cap is not None:
logits = torch.tanh(logits / self.soft_cap) * self.soft_cap
if self.scale != 1.0: