[Bugfix][V1/V2] Fix prompt_logprobs to respect logprobs_mode (#47680)
Signed-off-by: Wojciech Wais <wojciech.wais@gmail.com> Signed-off-by: Federico Kamelhar <209537060+fede-kamel@users.noreply.github.com> Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: Wojciech Wais <wojciech.wais@gmail.com> Co-authored-by: Federico Kamelhar <209537060+fede-kamel@users.noreply.github.com>
This commit is contained in:
co-authored by
Wojciech Wais
Federico Kamelhar
parent
088c0be268
commit
41ea2dd44a
@@ -19,7 +19,7 @@ from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample
|
||||
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
|
||||
from vllm.v1.worker.gpu.sample.logprob import (
|
||||
LogprobTokenIdsState,
|
||||
compute_topk_logprobs,
|
||||
compute_topk_scores,
|
||||
)
|
||||
from vllm.v1.worker.gpu.sample.output import SamplerOutput
|
||||
from vllm.v1.worker.gpu.sample.penalties import PenaltiesState
|
||||
@@ -38,8 +38,6 @@ class Sampler:
|
||||
num_speculative_tokens: int = 1,
|
||||
use_fp64_gumbel: bool = False,
|
||||
):
|
||||
if logprobs_mode not in ("processed_logprobs", "raw_logprobs"):
|
||||
raise NotImplementedError(f"Unsupported logprobs_mode: {logprobs_mode}")
|
||||
self.logprobs_mode = logprobs_mode
|
||||
self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default.
|
||||
self.use_fp64_gumbel = use_fp64_gumbel
|
||||
@@ -102,12 +100,12 @@ class Sampler:
|
||||
)
|
||||
|
||||
if return_logprobs:
|
||||
if self.logprobs_mode == "processed_logprobs":
|
||||
if self.logprobs_mode in ("processed_logprobs", "processed_logits"):
|
||||
logits = processed_logits
|
||||
expanded_logits = logits.shape[0] != idx_mapping_np.shape[0]
|
||||
cu_num_logits = cu_num_logits_np.tolist() if expanded_logits else None
|
||||
num_logprobs = max_num_logprobs if max_num_logprobs != NO_LOGPROBS else 0
|
||||
logprobs_tensors = compute_topk_logprobs(
|
||||
logprobs_tensors = compute_topk_scores(
|
||||
logits,
|
||||
num_logprobs,
|
||||
sampled,
|
||||
@@ -115,6 +113,7 @@ class Sampler:
|
||||
logprob_token_ids_state=self.logprob_token_ids_state,
|
||||
expanded_idx_mapping=input_batch.expanded_idx_mapping,
|
||||
max_per_req_token_ids=max_per_req_token_ids,
|
||||
logits_mode=self.logprobs_mode in ("raw_logits", "processed_logits"),
|
||||
)
|
||||
else:
|
||||
logprobs_tensors = None
|
||||
@@ -222,7 +221,10 @@ class Sampler:
|
||||
# any greedy requests or per-request seeds, or if post-processed
|
||||
# logprobs need to be returned for any requests.
|
||||
(top_k is None and top_p is None)
|
||||
or (return_logprobs and self.logprobs_mode == "processed_logprobs")
|
||||
or (
|
||||
return_logprobs
|
||||
and self.logprobs_mode in ("processed_logprobs", "processed_logits")
|
||||
)
|
||||
or self.sampling_states.any_greedy(idx_mapping_np)
|
||||
or self.sampling_states.any_explicit_seed(idx_mapping_np)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user