Signed-off-by: Karthik Kothuri <karthikkothuri2009@gmail.com> Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
co-authored by
Claude
wang.yuqi
parent
56a357ed33
commit
107a03ba63
@@ -3,14 +3,19 @@
|
||||
"""A layer that compute logits from hidden_stats."""
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
tensor_model_parallel_gather,
|
||||
)
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
UnquantizedEmbeddingMethod,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -50,6 +55,11 @@ class LogitsProcessor(PluggableLayer):
|
||||
self.soft_cap = soft_cap
|
||||
# Whether to use gather or all-gather to gather the logits.
|
||||
self.use_all_gather = current_platform.use_all_gather()
|
||||
# Dtype of the lm_head projection. Defaults to the model dtype; an
|
||||
# fp32 head (via `--hf-overrides '{"head_dtype": "float32"}'`) is
|
||||
# required for RL training-inference consistency.
|
||||
model_config = get_current_vllm_config().model_config
|
||||
self.head_dtype = model_config.head_dtype if model_config is not None else None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -86,6 +96,44 @@ class LogitsProcessor(PluggableLayer):
|
||||
logits = tensor_model_parallel_gather(logits)
|
||||
return logits
|
||||
|
||||
def _apply_head(
|
||||
self,
|
||||
lm_head: VocabParallelEmbedding,
|
||||
hidden_states: torch.Tensor,
|
||||
embedding_bias: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Project hidden states through the lm_head, honoring head_dtype."""
|
||||
if self.head_dtype is None or self.head_dtype == hidden_states.dtype:
|
||||
return lm_head.quant_method.apply(
|
||||
lm_head, hidden_states, bias=embedding_bias
|
||||
)
|
||||
|
||||
if not isinstance(lm_head.quant_method, UnquantizedEmbeddingMethod):
|
||||
raise ValueError(
|
||||
"A head_dtype different from the model dtype is only "
|
||||
"supported for an unquantized lm_head."
|
||||
)
|
||||
if (
|
||||
self.head_dtype == torch.float32
|
||||
and current_platform.is_cuda()
|
||||
and hidden_states.is_cuda
|
||||
):
|
||||
# Accumulate the projection directly into fp32. This avoids
|
||||
# materializing an fp32 copy of the lm_head weight on every step,
|
||||
# unlike casting both operands. `torch.mm(out_dtype=...)` is
|
||||
# CUDA-only and only supports fp32 output for fp16/bf16 inputs, so
|
||||
# other cases fall back to the cast path below.
|
||||
flat = hidden_states.reshape(-1, hidden_states.shape[-1])
|
||||
logits = torch.mm(flat, lm_head.weight.t(), out_dtype=self.head_dtype)
|
||||
if embedding_bias is not None:
|
||||
logits = logits + embedding_bias.to(self.head_dtype)
|
||||
return logits.reshape(*hidden_states.shape[:-1], -1)
|
||||
return F.linear(
|
||||
hidden_states.to(self.head_dtype),
|
||||
lm_head.weight.to(self.head_dtype),
|
||||
embedding_bias.to(self.head_dtype) if embedding_bias is not None else None,
|
||||
)
|
||||
|
||||
def _get_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -93,7 +141,7 @@ class LogitsProcessor(PluggableLayer):
|
||||
embedding_bias: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
# Get the logits for the next tokens.
|
||||
logits = lm_head.quant_method.apply(lm_head, hidden_states, bias=embedding_bias)
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
|
||||
# Gather logits for TP
|
||||
logits = self._gather_logits(logits)
|
||||
@@ -122,7 +170,7 @@ class LogitsProcessor(PluggableLayer):
|
||||
)
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
|
||||
logits = lm_head.quant_method.apply(lm_head, hidden_states, bias=embedding_bias)
|
||||
logits = self._apply_head(lm_head, hidden_states, embedding_bias)
|
||||
if self.soft_cap is not None:
|
||||
logits = torch.tanh(logits / self.soft_cap) * self.soft_cap
|
||||
if self.scale != 1.0:
|
||||
|
||||
Reference in New Issue
Block a user