[Bugfix][Frontend] Return streaming errors before the first token (#40986)

Signed-off-by: 1fanwang <1fannnw@gmail.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Stefan Wang
2026-10-04 16:42:49 +00:00
committed by GitHub
co-authored by Robert Shaw
parent 89439db727
commit b0eb87fe49
4 changed files with 138 additions and 13 deletions
@@ -631,6 +631,8 @@ class OpenAIServingChat(GenerateBaseServing):
if finish_reason_sent[i]:
continue
self._raise_if_error(output.finish_reason, request_id)
if request.logprobs and (
request.top_logprobs is not None or request.logprob_token_ids
):
@@ -758,10 +760,6 @@ class OpenAIServingChat(GenerateBaseServing):
# if the model is finished generating
else:
# check for error finish reason and abort streaming
# finish_reason='error' indicates a retryable error
self._raise_if_error(output.finish_reason, request_id)
# Send the finish response for each request.n only once
# In OpenAI's API, when a tool is called, the
# finish_reason is: