[Bugfix][Frontend] Preserve Cohere and Anthropic streaming errors (#61947)
Signed-off-by: Kevin <jia__kuan@163.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
@@ -478,6 +478,44 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_token_counts: dict[str, int] | None = None,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
try:
|
||||
async for chunk in self.chat_completion_stream_generator_raw(
|
||||
request,
|
||||
result_generator,
|
||||
request_id,
|
||||
model_name,
|
||||
conversation,
|
||||
tokenizer,
|
||||
request_metadata,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
mm_token_counts=mm_token_counts,
|
||||
):
|
||||
if isinstance(chunk, ErrorResponse):
|
||||
data = chunk.model_dump_json()
|
||||
else:
|
||||
data = chunk.model_dump_json(
|
||||
exclude_unset=True, exclude_none=not chunk.choices
|
||||
)
|
||||
yield f"data: {data}\n\n"
|
||||
except Exception as e:
|
||||
logger.exception("Error serializing chat completion stream.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
async def chat_completion_stream_generator_raw(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
result_generator: AsyncIterator[RequestOutput],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
conversation: list[ConversationMessage],
|
||||
tokenizer: TokenizerLike,
|
||||
request_metadata: RequestResponseMetadata,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
mm_token_counts: dict[str, int] | None = None,
|
||||
) -> AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None]:
|
||||
"""Yield chat chunks or errors without serializing them into SSE frames."""
|
||||
created_time = int(time.time())
|
||||
chunk_object_type: Final = "chat.completion.chunk"
|
||||
first_iteration = True
|
||||
@@ -510,9 +548,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
parsers = [None] * num_choices
|
||||
except Exception as e:
|
||||
logger.exception("Error in parser creation.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
yield "data: [DONE]\n\n"
|
||||
yield self.create_error_response(e)
|
||||
return
|
||||
|
||||
stream_options = request.stream_options
|
||||
@@ -583,8 +619,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield chunk
|
||||
|
||||
# Send response to echo the input portion of the
|
||||
# last message
|
||||
@@ -624,8 +659,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield chunk
|
||||
first_iteration = False
|
||||
|
||||
for output in res.outputs:
|
||||
@@ -821,8 +855,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
),
|
||||
)
|
||||
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
yield f"data: {data}\n\n"
|
||||
yield chunk
|
||||
|
||||
if self._include_reasoning_tokens_details and not include_continuous_usage:
|
||||
for i, parser in enumerate(parsers):
|
||||
@@ -882,10 +915,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
system_fingerprint=self.system_fingerprint,
|
||||
metrics=stream_per_request_metrics,
|
||||
)
|
||||
final_usage_data = final_usage_chunk.model_dump_json(
|
||||
exclude_unset=True, exclude_none=True
|
||||
)
|
||||
yield f"data: {final_usage_data}\n\n"
|
||||
yield final_usage_chunk
|
||||
|
||||
# report to FastAPI middleware aggregate usage across all choices
|
||||
num_completion_tokens = sum(previous_num_tokens)
|
||||
@@ -919,13 +949,12 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
except GenerationError as e:
|
||||
yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
|
||||
yield self.create_error_response(
|
||||
str(e), err_type="InternalServerError", status_code=e.status_code
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("Error in chat completion stream generator.")
|
||||
data = self.create_streaming_error_response(e)
|
||||
yield f"data: {data}\n\n"
|
||||
# Send the final done message after all response.n are finished
|
||||
yield "data: [DONE]\n\n"
|
||||
yield self.create_error_response(e)
|
||||
|
||||
async def chat_completion_full_generator(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user