[Frontend] Cohere chat v2 api support (#47189)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6c91de3689
commit
9c110fa522
@@ -10,7 +10,13 @@ from openai.types.chat.chat_completion_audio import (
|
||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
|
||||
from pydantic import Field, PrivateAttr, model_serializer, model_validator
|
||||
from pydantic import (
|
||||
Field,
|
||||
PrivateAttr,
|
||||
SerializeAsAny,
|
||||
model_serializer,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -91,7 +97,12 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||
|
||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
message: ChatMessage
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
|
||||
# so that added fields like ``citations`` survive JSON serialization
|
||||
# instead of being stripped down to the base schema. Plain
|
||||
# ``ChatMessage`` instances serialize identically to before.
|
||||
message: SerializeAsAny[ChatMessage]
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
@@ -139,7 +150,11 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
delta: DeltaMessage
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
|
||||
# so streaming ``citations`` survive JSON serialization. Plain
|
||||
# ``DeltaMessage`` instances serialize identically to before.
|
||||
delta: SerializeAsAny[DeltaMessage]
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
finish_reason: str | None = None
|
||||
stop_reason: int | str | None = None
|
||||
|
||||
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
return self.response_role
|
||||
return request.messages[-1]["role"]
|
||||
|
||||
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
|
||||
"""Construct the response :class:`ChatMessage` for the non-streaming path.
|
||||
|
||||
The full-generator calls this at every construction site so
|
||||
subclasses can swap in a specialized :class:`ChatMessage`
|
||||
subclass (e.g. :class:`CohereServingChatV2` returning
|
||||
:class:`CohereChatMessage`) without duplicating the branchy
|
||||
tool-choice / auto-tools logic that decides which fields are
|
||||
populated. The default returns a plain :class:`ChatMessage`.
|
||||
"""
|
||||
return ChatMessage(*args, **kwargs)
|
||||
|
||||
def _finalize_response_message(
|
||||
self,
|
||||
message: ChatMessage,
|
||||
*,
|
||||
parser: Parser | None,
|
||||
) -> ChatMessage:
|
||||
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
|
||||
|
||||
Default is a no-op. Subclasses that need to surface parser-side
|
||||
extras (e.g. :class:`CohereServingChatV2` reading grounding
|
||||
citations off the reasoning parser and populating
|
||||
:class:`CohereChatMessage.citations`) override this to inspect
|
||||
``parser`` and mutate/replace ``message``.
|
||||
"""
|
||||
return message
|
||||
|
||||
async def chat_completion_stream_generator(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
is_required_tool_choice = request.tool_choice == "required"
|
||||
|
||||
# All six construction sites route through ``self._create_chat_message``
|
||||
# so subclasses can swap in a specialized :class:`ChatMessage`
|
||||
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
|
||||
# having to duplicate this branch logic.
|
||||
if (not self.enable_auto_tools or not tool_parser_cls) and (
|
||||
not is_named_tool_choice and not is_required_tool_choice
|
||||
):
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
elif is_named_tool_choice or is_required_tool_choice:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content or "",
|
||||
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# if the request doesn't use tool choice
|
||||
# OR specifies to not use a tool
|
||||
elif not request.tool_choice or request.tool_choice == "none":
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# handle when there are tools and tool choice is auto
|
||||
elif (
|
||||
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
):
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
else:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
" if tools should be extracted. Returning a standard chat "
|
||||
"completion."
|
||||
)
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# Subclass hook: enrich the constructed message with any
|
||||
# parser-side extras that don't fit through the plain
|
||||
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
|
||||
# citation-aware handlers use this to surface grounding
|
||||
# metadata cached on the reasoning parser.
|
||||
message = self._finalize_response_message(message, parser=parser)
|
||||
|
||||
# In OpenAI's API, when a tool is called, the finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
|
||||
Reference in New Issue
Block a user