[Bugfix][V1/V2] Fix prompt_logprobs to respect logprobs_mode (#47680)
Signed-off-by: Wojciech Wais <wojciech.wais@gmail.com> Signed-off-by: Federico Kamelhar <209537060+fede-kamel@users.noreply.github.com> Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: Wojciech Wais <wojciech.wais@gmail.com> Co-authored-by: Federico Kamelhar <209537060+fede-kamel@users.noreply.github.com>
This commit is contained in:
co-authored by
Wojciech Wais
Federico Kamelhar
parent
088c0be268
commit
41ea2dd44a
@@ -106,7 +106,7 @@ def compute_token_logprobs(
|
||||
return logprobs
|
||||
|
||||
|
||||
def compute_topk_logprobs(
|
||||
def compute_topk_scores(
|
||||
logits: torch.Tensor,
|
||||
num_logprobs: int,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
@@ -114,6 +114,7 @@ def compute_topk_logprobs(
|
||||
logprob_token_ids_state: "LogprobTokenIdsState | None" = None,
|
||||
expanded_idx_mapping: torch.Tensor | None = None,
|
||||
max_per_req_token_ids: int = 0,
|
||||
logits_mode: bool = False,
|
||||
) -> LogprobsTensors:
|
||||
assert num_logprobs >= 0
|
||||
batch_size, vocab_size = logits.shape
|
||||
@@ -124,7 +125,10 @@ def compute_topk_logprobs(
|
||||
if num_logprobs > 0:
|
||||
topk_indices = torch.topk(logits, num_logprobs, dim=-1).indices
|
||||
logprob_token_ids = torch.cat((logprob_token_ids, topk_indices), dim=1)
|
||||
logprobs = compute_token_logprobs(logits, logprob_token_ids)
|
||||
if logits_mode:
|
||||
scores = logits.gather(-1, logprob_token_ids).to(torch.float32)
|
||||
else:
|
||||
scores = compute_token_logprobs(logits, logprob_token_ids)
|
||||
else:
|
||||
# Some requests specified logprob_token_ids. Build the [batch_size,
|
||||
# 1 + max_cols] token_ids matrix and validity mask on the GPU via a
|
||||
@@ -158,8 +162,11 @@ def compute_topk_logprobs(
|
||||
NUM_TOPK=num_logprobs,
|
||||
PADDED_COLS=triton.next_power_of_2(num_cols),
|
||||
)
|
||||
logprobs = compute_token_logprobs(logits, logprob_token_ids)
|
||||
logprobs = logprobs.masked_fill(~valid_mask, float("-inf"))
|
||||
if logits_mode:
|
||||
scores = logits.gather(-1, logprob_token_ids).to(torch.float32)
|
||||
else:
|
||||
scores = compute_token_logprobs(logits, logprob_token_ids)
|
||||
scores = scores.masked_fill(~valid_mask, float("-inf"))
|
||||
|
||||
token_ranks = torch.empty(batch_size, dtype=torch.int64, device=logits.device)
|
||||
_ranks_kernel[(batch_size,)](
|
||||
@@ -172,7 +179,7 @@ def compute_topk_logprobs(
|
||||
)
|
||||
return LogprobsTensors(
|
||||
logprob_token_ids=logprob_token_ids,
|
||||
logprobs=logprobs,
|
||||
logprobs=scores,
|
||||
selected_token_ranks=token_ranks,
|
||||
cu_num_generated_tokens=cu_num_logits,
|
||||
)
|
||||
|
||||
@@ -5,16 +5,18 @@ from collections.abc import Callable
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
|
||||
|
||||
|
||||
class PromptLogprobsWorker:
|
||||
def __init__(self, max_num_reqs: int):
|
||||
def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"):
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.logprobs_mode = logprobs_mode
|
||||
|
||||
self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool)
|
||||
self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32)
|
||||
@@ -82,6 +84,7 @@ class PromptLogprobsWorker:
|
||||
hidden_states[: input_batch.num_tokens],
|
||||
logits_fn,
|
||||
max_num_prompt_logprobs,
|
||||
self.logprobs_mode,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -206,33 +209,36 @@ def compute_prompt_logprobs_with_chunking(
|
||||
prompt_hidden_states: torch.Tensor,
|
||||
logits_fn: Callable[[torch.Tensor], torch.Tensor],
|
||||
num_prompt_logprobs: int,
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Since materializing the full prompt logits can take too much memory,
|
||||
# we compute it in chunks.
|
||||
CHUNK_SIZE = 1024
|
||||
token_ids = []
|
||||
logprobs = []
|
||||
scores = []
|
||||
ranks = []
|
||||
logits_mode = logprobs_mode in ("raw_logits", "processed_logits")
|
||||
prompt_token_ids = prompt_token_ids.to(torch.int64)
|
||||
for start_idx in range(0, prompt_token_ids.shape[0], CHUNK_SIZE):
|
||||
end_idx = start_idx + CHUNK_SIZE
|
||||
# NOTE(woosuk): logits_fn can be slow because it involves all-gather.
|
||||
prompt_logits = logits_fn(prompt_hidden_states[start_idx:end_idx])
|
||||
requested_num_prompt_logprobs = (
|
||||
requested_num = (
|
||||
prompt_logits.shape[-1]
|
||||
if num_prompt_logprobs == -1
|
||||
else num_prompt_logprobs
|
||||
)
|
||||
prompt_logprobs = compute_topk_logprobs(
|
||||
result = compute_topk_scores(
|
||||
prompt_logits,
|
||||
requested_num_prompt_logprobs,
|
||||
requested_num,
|
||||
prompt_token_ids[start_idx:end_idx],
|
||||
logits_mode=logits_mode,
|
||||
)
|
||||
token_ids.append(prompt_logprobs.logprob_token_ids)
|
||||
logprobs.append(prompt_logprobs.logprobs)
|
||||
ranks.append(prompt_logprobs.selected_token_ranks)
|
||||
token_ids.append(result.logprob_token_ids)
|
||||
scores.append(result.logprobs)
|
||||
ranks.append(result.selected_token_ranks)
|
||||
|
||||
token_ids = torch.cat(token_ids, dim=0) if len(token_ids) > 1 else token_ids[0]
|
||||
logprobs = torch.cat(logprobs, dim=0) if len(logprobs) > 1 else logprobs[0]
|
||||
scores = torch.cat(scores, dim=0) if len(scores) > 1 else scores[0]
|
||||
ranks = torch.cat(ranks, dim=0) if len(ranks) > 1 else ranks[0]
|
||||
return token_ids, logprobs, ranks
|
||||
return token_ids, scores, ranks
|
||||
|
||||
@@ -19,7 +19,7 @@ from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
|
||||
from vllm.v1.worker.gpu.sample.logprob import (
|
||||
LogprobTokenIdsState,
|
||||
compute_topk_logprobs,
|
||||
compute_topk_scores,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
|
||||
@@ -38,8 +38,6 @@ class Sampler:
|
||||
num_speculative_tokens: int = 1,
|
||||
use_fp64_gumbel: bool = False,
|
||||
):
|
||||
if logprobs_mode not in ("processed_logprobs", "raw_logprobs"):
|
||||
raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}")
|
||||
self.logprobs_mode = logprobs_mode
|
||||
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
|
||||
self.use_fp64_gumbel = use_fp64_gumbel
|
||||
@@ -102,12 +100,12 @@ class Sampler:
|
||||
)
|
||||
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode == "processed_logprobs":
|
||||
if self.logprobs_mode in ("processed_logprobs", "processed_logits"):
|
||||
logits = processed_logits
|
||||
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
|
||||
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
|
||||
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
|
||||
logprobs_tensors = compute_topk_logprobs(
|
||||
logprobs_tensors = compute_topk_scores(
|
||||
logits,
|
||||
num_logprobs,
|
||||
sampled,
|
||||
@@ -115,6 +113,7 @@ class Sampler:
|
||||
logprob_token_ids_state=self.logprob_token_ids_state,
|
||||
expanded_idx_mapping=input_batch.expanded_idx_mapping,
|
||||
max_per_req_token_ids=max_per_req_token_ids,
|
||||
logits_mode=self.logprobs_mode in ("raw_logits", "processed_logits"),
|
||||
)
|
||||
else:
|
||||
logprobs_tensors = None
|
||||
@@ -222,7 +221,10 @@ class Sampler:
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
# logprobs need to be returned for any requests.
|
||||
(top_k is None and top_p is None)
|
||||
or (return_logprobs and self.logprobs_mode == "processed_logprobs")
|
||||
or (
|
||||
return_logprobs
|
||||
and self.logprobs_mode in ("processed_logprobs", "processed_logits")
|
||||
)
|
||||
or self.sampling_states.any_greedy(idx_mapping_np)
|
||||
or self.sampling_states.any_explicit_seed(idx_mapping_np)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user