[Bugfix][Frontend] Return streaming errors before the first token (#40986)
Signed-off-by: 1fanwang <1fannnw@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
89439db727
commit
b0eb87fe49
@@ -631,6 +631,8 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
if finish_reason_sent[i]:
|
||||
continue
|
||||
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
@@ -758,10 +760,6 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
|
||||
# if the model is finished generating
|
||||
else:
|
||||
# check for error finish reason and abort streaming
|
||||
# finish_reason='error' indicates a retryable error
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
# Send the finish response for each request.n only once
|
||||
# In OpenAI's API, when a tool is called, the
|
||||
# finish_reason is:
|
||||
|
||||
@@ -326,6 +326,8 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
for output in res.outputs:
|
||||
i = output.index + prompt_idx * num_choices
|
||||
finish_reason = output.finish_reason
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
# Useful when request.return_token_ids is True
|
||||
# Returning prompt token IDs shares the same logic
|
||||
@@ -392,11 +394,8 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
|
||||
previous_text_lens[i] += len(delta_text)
|
||||
previous_num_tokens[i] += len(output.token_ids)
|
||||
finish_reason = output.finish_reason
|
||||
stop_reason = output.stop_reason
|
||||
|
||||
self._raise_if_error(finish_reason, request_id)
|
||||
|
||||
chunk = CompletionStreamResponse(
|
||||
id=request_id,
|
||||
object="text_completion",
|
||||
|
||||
Reference in New Issue
Block a user