[Bugfix][V1/V2] Fix prompt_logprobs to respect logprobs_mode (#47680)
Signed-off-by: Wojciech Wais <wojciech.wais@gmail.com> Signed-off-by: Federico Kamelhar <209537060+fede-kamel@users.noreply.github.com> Signed-off-by: Allen Shen <aoshen@inferact.ai> Co-authored-by: Wojciech Wais <wojciech.wais@gmail.com> Co-authored-by: Federico Kamelhar <209537060+fede-kamel@users.noreply.github.com>
This commit is contained in:
co-authored by
Wojciech Wais
Federico Kamelhar
parent
088c0be268
commit
41ea2dd44a
@@ -233,6 +233,8 @@ class ModelConfig:
|
||||
Raw means the values before applying any logit processors, like bad words.
|
||||
Processed means the values after applying all processors, including
|
||||
temperature and top_k/top_p.
|
||||
Note: for prompt_logprobs, processed_* and raw_* yield identical results
|
||||
because prompt tokens do not go through sampling processors.
|
||||
"""
|
||||
use_fp64_gumbel: bool = False
|
||||
"""Whether to use FP64 (instead of FP32) random noise for Gumbel-max and
|
||||
|
||||
@@ -2202,13 +2202,6 @@ class VllmConfig:
|
||||
if model_config is not None and model_config.enable_prompt_embeds:
|
||||
unsupported.append("prompt embeds")
|
||||
|
||||
if (
|
||||
model_config is not None
|
||||
and model_config.runner_type == "generate"
|
||||
and model_config.logprobs_mode in ("raw_logits", "processed_logits")
|
||||
):
|
||||
unsupported.append(f"logprobs mode '{model_config.logprobs_mode}'")
|
||||
|
||||
if self.cache_config.kv_sharing_fast_prefill:
|
||||
# Will be added by https://github.com/vllm-project/vllm/pull/35045
|
||||
unsupported.append("KV sharing fast prefill")
|
||||
|
||||
Reference in New Issue
Block a user