[Core][Bugfix] Support prompt_logprobs returned with speculative decoding (#8047)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
This commit is contained in:
+1
-3
@@ -675,8 +675,6 @@ class VllmRunner:
|
||||
videos: Optional[PromptVideoInput] = None,
|
||||
) -> Union[List[TokensTextLogprobs],
|
||||
List[TokensTextLogprobsPromptLogprobs]]:
|
||||
assert sampling_params.logprobs is not None
|
||||
|
||||
if images is not None:
|
||||
assert len(prompts) == len(images)
|
||||
|
||||
@@ -754,7 +752,7 @@ class VllmRunner:
|
||||
temperature=0.0,
|
||||
max_tokens=max_tokens,
|
||||
logprobs=num_logprobs,
|
||||
prompt_logprobs=(num_prompt_logprobs),
|
||||
prompt_logprobs=num_prompt_logprobs,
|
||||
stop_token_ids=stop_token_ids)
|
||||
|
||||
return self.generate_w_logprobs(prompts,
|
||||
|
||||
Reference in New Issue
Block a user