[Bugfix] Support prompt_logprobs==0 (#5217)

This commit is contained in:
Toshiki Kataoka
2024-06-03 17:59:30 -07:00
committed by GitHub
parent f775a07e30
commit 06b2550cbb
4 changed files with 11 additions and 7 deletions
@@ -312,7 +312,7 @@ class OpenAIServingCompletion(OpenAIServing):
elif request.echo and request.max_tokens > 0:
token_ids = prompt_token_ids + output.token_ids
top_logprobs = (prompt_logprobs + output.logprobs
if request.logprobs else None)
if request.logprobs is not None else None)
output_text = prompt_text + output.text
else:
token_ids = output.token_ids
+1 -1
View File
@@ -233,7 +233,7 @@ def _prepare_seq_groups(
logits = hidden_states[selected_token_indices]
"""
if sampling_params.prompt_logprobs:
if sampling_params.prompt_logprobs is not None:
selected_token_indices.extend(
range(model_output_idx, model_output_idx + prompt_logprob_len))
model_output_idx += prompt_logprob_len
+1 -1
View File
@@ -427,7 +427,7 @@ class ModelRunner:
[lora_id] *
(query_len if seq_group_metadata.sampling_params
and seq_group_metadata.sampling_params.prompt_logprobs
else 1))
is not None else 1))
mm_data = seq_group_metadata.multi_modal_data
if mm_data is not None: