[Bugfix][Frontend] Preserve Cohere and Anthropic streaming errors (#61947)

Signed-off-by: Kevin <jia__kuan@163.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Jovan
2026-10-17 06:10:16 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 90d6d871d6
commit 5be4266f5c
6 changed files with 531 additions and 375 deletions
@@ -478,6 +478,44 @@ class OpenAIServingChat(GenerateBaseServing):
chat_template_kwargs: dict[str, Any] | None = None,
mm_token_counts: dict[str, int] | None = None,
) -> AsyncGenerator[str, None]:
try:
async for chunk in self.chat_completion_stream_generator_raw(
request,
result_generator,
request_id,
model_name,
conversation,
tokenizer,
request_metadata,
chat_template_kwargs=chat_template_kwargs,
mm_token_counts=mm_token_counts,
):
if isinstance(chunk, ErrorResponse):
data = chunk.model_dump_json()
else:
data = chunk.model_dump_json(
exclude_unset=True, exclude_none=not chunk.choices
)
yield f"data: {data}\n\n"
except Exception as e:
logger.exception("Error serializing chat completion stream.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
yield "data: [DONE]\n\n"
async def chat_completion_stream_generator_raw(
self,
request: ChatCompletionRequest,
result_generator: AsyncIterator[RequestOutput],
request_id: str,
model_name: str,
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
chat_template_kwargs: dict[str, Any] | None = None,
mm_token_counts: dict[str, int] | None = None,
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
"""Yield chat chunks or errors without serializing them into SSE frames."""
created_time = int(time.time())
chunk_object_type: Final = "chat.completion.chunk"
first_iteration = True
@@ -510,9 +548,7 @@ class OpenAIServingChat(GenerateBaseServing):
parsers = [None] * num_choices
except Exception as e:
logger.exception("Error in parser creation.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
yield "data: [DONE]\n\n"
yield self.create_error_response(e)
return
stream_options = request.stream_options
@@ -583,8 +619,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
yield chunk
# Send response to echo the input portion of the
# last message
@@ -624,8 +659,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
yield chunk
first_iteration = False
for output in res.outputs:
@@ -821,8 +855,7 @@ class OpenAIServingChat(GenerateBaseServing):
),
)
data = chunk.model_dump_json(exclude_unset=True)
yield f"data: {data}\n\n"
yield chunk
if self._include_reasoning_tokens_details and not include_continuous_usage:
for i, parser in enumerate(parsers):
@@ -882,10 +915,7 @@ class OpenAIServingChat(GenerateBaseServing):
system_fingerprint=self.system_fingerprint,
metrics=stream_per_request_metrics,
)
final_usage_data = final_usage_chunk.model_dump_json(
exclude_unset=True, exclude_none=True
)
yield f"data: {final_usage_data}\n\n"
yield final_usage_chunk
# report to FastAPI middleware aggregate usage across all choices
num_completion_tokens = sum(previous_num_tokens)
@@ -919,13 +949,12 @@ class OpenAIServingChat(GenerateBaseServing):
)
except GenerationError as e:
yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
yield self.create_error_response(
str(e), err_type="InternalServerError", status_code=e.status_code
)
except Exception as e:
logger.exception("Error in chat completion stream generator.")
data = self.create_streaming_error_response(e)
yield f"data: {data}\n\n"
# Send the final done message after all response.n are finished
yield "data: [DONE]\n\n"
yield self.create_error_response(e)
async def chat_completion_full_generator(
self,