[Frontend] Cohere chat v2 api support (#47189)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6c91de3689
commit
9c110fa522
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
return self.response_role
|
||||
return request.messages[-1]["role"]
|
||||
|
||||
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
|
||||
"""Construct the response :class:`ChatMessage` for the non-streaming path.
|
||||
|
||||
The full-generator calls this at every construction site so
|
||||
subclasses can swap in a specialized :class:`ChatMessage`
|
||||
subclass (e.g. :class:`CohereServingChatV2` returning
|
||||
:class:`CohereChatMessage`) without duplicating the branchy
|
||||
tool-choice / auto-tools logic that decides which fields are
|
||||
populated. The default returns a plain :class:`ChatMessage`.
|
||||
"""
|
||||
return ChatMessage(*args, **kwargs)
|
||||
|
||||
def _finalize_response_message(
|
||||
self,
|
||||
message: ChatMessage,
|
||||
*,
|
||||
parser: Parser | None,
|
||||
) -> ChatMessage:
|
||||
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
|
||||
|
||||
Default is a no-op. Subclasses that need to surface parser-side
|
||||
extras (e.g. :class:`CohereServingChatV2` reading grounding
|
||||
citations off the reasoning parser and populating
|
||||
:class:`CohereChatMessage.citations`) override this to inspect
|
||||
``parser`` and mutate/replace ``message``.
|
||||
"""
|
||||
return message
|
||||
|
||||
async def chat_completion_stream_generator(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
is_required_tool_choice = request.tool_choice == "required"
|
||||
|
||||
# All six construction sites route through ``self._create_chat_message``
|
||||
# so subclasses can swap in a specialized :class:`ChatMessage`
|
||||
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
|
||||
# having to duplicate this branch logic.
|
||||
if (not self.enable_auto_tools or not tool_parser_cls) and (
|
||||
not is_named_tool_choice and not is_required_tool_choice
|
||||
):
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
elif is_named_tool_choice or is_required_tool_choice:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content or "",
|
||||
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# if the request doesn't use tool choice
|
||||
# OR specifies to not use a tool
|
||||
elif not request.tool_choice or request.tool_choice == "none":
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# handle when there are tools and tool choice is auto
|
||||
elif (
|
||||
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
):
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
else:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
" if tools should be extracted. Returning a standard chat "
|
||||
"completion."
|
||||
)
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# Subclass hook: enrich the constructed message with any
|
||||
# parser-side extras that don't fit through the plain
|
||||
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
|
||||
# citation-aware handlers use this to surface grounding
|
||||
# metadata cached on the reasoning parser.
|
||||
message = self._finalize_response_message(message, parser=parser)
|
||||
|
||||
# In OpenAI's API, when a tool is called, the finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
|
||||
Reference in New Issue
Block a user