[Bugfix][Frontend] Return streaming errors before the first token (#40986)

Signed-off-by: 1fanwang <1fannnw@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Stefan Wang
2026-10-04 16:42:49 +00:00
committed by GitHub
co-authored by Robert Shaw
parent 89439db727
commit b0eb87fe49
4 changed files with 138 additions and 13 deletions
@@ -326,6 +326,8 @@ class OpenAIServingCompletion(GenerateBaseServing):
for output in res.outputs:
i = output.index + prompt_idx * num_choices
finish_reason = output.finish_reason
self._raise_if_error(finish_reason, request_id)
# Useful when request.return_token_ids is True
# Returning prompt token IDs shares the same logic
@@ -392,11 +394,8 @@ class OpenAIServingCompletion(GenerateBaseServing):
previous_text_lens[i] += len(delta_text)
previous_num_tokens[i] += len(output.token_ids)
finish_reason = output.finish_reason
stop_reason = output.stop_reason
self._raise_if_error(finish_reason, request_id)
chunk = CompletionStreamResponse(
id=request_id,
object="text_completion",