[Frontend] Cohere chat v2 api support (#47189)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
andrewbcohere
2026-07-31 22:53:54 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 6c91de3689
commit 9c110fa522
31 changed files with 8770 additions and 20 deletions
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
return self.response_role
return request.messages[-1]["role"]
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
"""Construct the response :class:`ChatMessage` for the non-streaming path.
The full-generator calls this at every construction site so
subclasses can swap in a specialized :class:`ChatMessage`
subclass (e.g. :class:`CohereServingChatV2` returning
:class:`CohereChatMessage`) without duplicating the branchy
tool-choice / auto-tools logic that decides which fields are
populated. The default returns a plain :class:`ChatMessage`.
"""
return ChatMessage(*args, **kwargs)
def _finalize_response_message(
self,
message: ChatMessage,
*,
parser: Parser | None,
) -> ChatMessage:
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
Default is a no-op. Subclasses that need to surface parser-side
extras (e.g. :class:`CohereServingChatV2` reading grounding
citations off the reasoning parser and populating
:class:`CohereChatMessage.citations`) override this to inspect
``parser`` and mutate/replace ``message``.
"""
return message
async def chat_completion_stream_generator(
self,
request: ChatCompletionRequest,
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
)
is_required_tool_choice = request.tool_choice == "required"
# All six construction sites route through ``self._create_chat_message``
# so subclasses can swap in a specialized :class:`ChatMessage`
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
# having to duplicate this branch logic.
if (not self.enable_auto_tools or not tool_parser_cls) and (
not is_named_tool_choice and not is_required_tool_choice
):
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
elif is_named_tool_choice or is_required_tool_choice:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content or "",
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
# if the request doesn't use tool choice
# OR specifies to not use a tool
elif not request.tool_choice or request.tool_choice == "none":
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
# handle when there are tools and tool choice is auto
elif (
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
):
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content,
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
)
else:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content,
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
" if tools should be extracted. Returning a standard chat "
"completion."
)
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
# Subclass hook: enrich the constructed message with any
# parser-side extras that don't fit through the plain
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
# citation-aware handlers use this to surface grounding
# metadata cached on the reasoning parser.
message = self._finalize_response_message(message, parser=parser)
# In OpenAI's API, when a tool is called, the finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.