[Frontend] Cohere chat v2 api support (#47189)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6c91de3689
commit
9c110fa522
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handler
|
||||
itself - are gated on a one-shot probe at module load. If the SDK isn't
|
||||
installed, :func:`attach_router` becomes a no-op (with an info log) and
|
||||
vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the route. This keeps non-Cohere deployments that pull in the SDK for
|
||||
unrelated reasons (e.g. test dependencies) from accidentally exposing
|
||||
the api.
|
||||
|
||||
Note: the handler must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining it locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
"""
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
sanitize_message,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
_COHERE_PATH_PREFIX = "/cohere/"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
import cohere # noqa: F401 -- dependency probe
|
||||
except ImportError:
|
||||
_SDK_AVAILABLE = False
|
||||
else:
|
||||
_SDK_AVAILABLE = True
|
||||
|
||||
|
||||
if _SDK_AVAILABLE:
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
Prefers the id the underlying chat handler stamped onto
|
||||
``raw_request.state.request_metadata`` (if it got that far before
|
||||
failing), falling back to the ``X-Request-Id`` HTTP header. May
|
||||
return ``None`` if neither is available, in which case the field
|
||||
is omitted from the response.
|
||||
"""
|
||||
if raw_request is None:
|
||||
return None
|
||||
meta = getattr(raw_request.state, "request_metadata", None)
|
||||
if meta is not None and getattr(meta, "request_id", None):
|
||||
return meta.request_id
|
||||
return raw_request.headers.get("X-Request-Id")
|
||||
|
||||
def _error_response(
|
||||
error: ErrorResponse,
|
||||
raw_request: Request | None,
|
||||
*,
|
||||
fallback_status: int = HTTPStatus.BAD_REQUEST,
|
||||
) -> JSONResponse:
|
||||
"""Translate vLLM's internal error envelope into Cohere's shape."""
|
||||
info = error.error
|
||||
status = info.code or fallback_status
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content=CohereError(
|
||||
message=sanitize_message(info.message),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
|
||||
handler = _serving(raw_request)
|
||||
if handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message="The model does not support the Cohere v2 chat API.",
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
result = await handler.create_chat_v2(request, raw_request)
|
||||
except Exception as e: # noqa: BLE001 - report as 500 for parity
|
||||
logger.exception("Error in /cohere/v2/chat: %s", e)
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
content=CohereError(
|
||||
message=sanitize_message(str(e)),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
match result:
|
||||
case ErrorResponse():
|
||||
return _error_response(result, raw_request)
|
||||
case CohereChatV2Response():
|
||||
return JSONResponse(content=result.model_dump(exclude_none=True))
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
The endpoint handler above already returns :class:`CohereError` for
|
||||
errors it owns, but globally-registered exception handlers (e.g.
|
||||
:func:`validation_exception_handler` for pydantic body errors,
|
||||
:func:`http_exception_handler`, engine error handlers) fire *before*
|
||||
the handler runs and produce vLLM's internal
|
||||
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
|
||||
shape doesn't match the ``CohereError`` schema advertised on the
|
||||
route's OpenAPI ``responses``, so clients (and schema-conformance
|
||||
tests like ``test_openai_schema.py``) would see a mismatch on
|
||||
those paths. This middleware normalises any error body on
|
||||
``/cohere/*`` responses to :class:`CohereError`.
|
||||
"""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
|
||||
return response
|
||||
if response.status_code < 400:
|
||||
return response
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if not content_type.startswith("application/json"):
|
||||
return response
|
||||
|
||||
body = b"".join([chunk async for chunk in response.body_iterator])
|
||||
translated = _translate_vllm_error_body(body, request)
|
||||
if translated is not None:
|
||||
return translated
|
||||
passthrough_headers = {
|
||||
k: v
|
||||
for k, v in response.headers.items()
|
||||
if k.lower() != "content-length"
|
||||
}
|
||||
return Response(
|
||||
content=body,
|
||||
status_code=response.status_code,
|
||||
headers=passthrough_headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
|
||||
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
|
||||
|
||||
Returns ``None`` if ``raw`` does not match the vLLM error envelope
|
||||
(which signals the middleware to pass the body through unchanged).
|
||||
"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
if not (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("error"), dict)
|
||||
and "message" in data["error"]
|
||||
):
|
||||
return None
|
||||
try:
|
||||
err = ErrorResponse.model_validate(data)
|
||||
except Exception: # noqa: BLE001 - malformed envelope; pass through
|
||||
return None
|
||||
return _error_response(err, request)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register ``POST /cohere/v2/chat`` on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoint is opt-in because it carries Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
* the optional ``cohere`` SDK isn't installed (the v2 protocol
|
||||
models live there)
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoint and it's silently
|
||||
absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoint."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
|
||||
|
||||
This module keeps the OpenAI chat completion protocol classes
|
||||
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
|
||||
It provides two things:
|
||||
|
||||
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
|
||||
representation produced by the Cohere reasoning parser
|
||||
(:mod:`vllm.reasoning.cohere_command_reasoning_parser`) and consumed by
|
||||
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
|
||||
SDK shape -- that conversion happens in the serving layer.
|
||||
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
|
||||
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
|
||||
are only instantiated by the citation-aware serving handler
|
||||
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
|
||||
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
|
||||
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
|
||||
|
||||
The module intentionally does *not* import the ``cohere`` Python SDK so the
|
||||
reasoning parser (which is Cohere-model-specific but ships without the SDK
|
||||
dependency) can import from here freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_serializer
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
|
||||
|
||||
class CitationSource(OpenAIBaseModel):
|
||||
"""Source attribution for a :class:`Citation`.
|
||||
|
||||
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
|
||||
discriminator (``document`` or ``tool``); ``id`` is the citing
|
||||
document/tool-output identifier; ``document`` and ``tool_output`` carry
|
||||
the original payload that produced the citation.
|
||||
|
||||
Sources are fully resolved by the reasoning parser at emit time
|
||||
(see :func:`_melody_sources_to_vllm` in
|
||||
:mod:`vllm.reasoning.cohere_command_reasoning_parser`) using the
|
||||
``POSITION_TO_SOURCE_KEY`` map forwarded through
|
||||
``chat_template_kwargs`` by
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
|
||||
That means every instance that reaches the serving / wire layer
|
||||
already has ``type`` / ``id`` populated; the serving layer only has
|
||||
to coerce to :class:`cohere.types.Citation` and rewrite
|
||||
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
|
||||
"""
|
||||
|
||||
type: Literal["document", "tool"] | None = None
|
||||
id: str | None = None
|
||||
document: dict[str, Any] | None = None
|
||||
tool_output: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Citation(OpenAIBaseModel):
|
||||
"""A citation grounding a span of generated text in source material.
|
||||
|
||||
vLLM-internal representation used by the Cohere reasoning parser and
|
||||
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
|
||||
conversion to the SDK's ``cohere.types.Citation`` happens in
|
||||
:mod:`vllm.entrypoints.cohere.serving`.
|
||||
"""
|
||||
|
||||
start: int | None = None
|
||||
"""Start character offset in the surrounding text content."""
|
||||
end: int | None = None
|
||||
"""End character offset (exclusive) in the surrounding text content."""
|
||||
text: str | None = None
|
||||
"""The cited text snippet."""
|
||||
sources: list[CitationSource] = Field(default_factory=list)
|
||||
"""Source documents / tool outputs that ground this citation."""
|
||||
content_index: int | None = None
|
||||
"""Index of the content block this citation refers to (when the message
|
||||
has multiple content blocks)."""
|
||||
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
|
||||
"""Which kind of content block this citation grounds: the user-visible
|
||||
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
|
||||
tool-plan block (``PLAN``). ``None`` means unspecified."""
|
||||
|
||||
|
||||
class CohereChatMessage(ChatMessage):
|
||||
""":class:`ChatMessage` extension carrying grounding citations.
|
||||
|
||||
Only instantiated by :class:`CohereServingChatV2` (and any other
|
||||
:class:`OpenAIServingChat` subclass that opts in by overriding
|
||||
``_create_chat_message``). Regular OpenAI-compatible handlers keep
|
||||
emitting plain :class:`ChatMessage` so their response schema is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
|
||||
so pydantic serializes this subclass with its own schema (including
|
||||
``citations``) when it flows through
|
||||
:class:`ChatCompletionResponseChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# ``mode="wrap"`` fully overrides (rather than chains) the
|
||||
# parent's ``@model_serializer``, so we explicitly delegate via
|
||||
# ``super()._serialize(handler)`` to preserve the parent's
|
||||
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
|
||||
# unset ``citations`` field so the wire matches the OpenAI-style
|
||||
# contract that optional vLLM extensions are omitted rather than
|
||||
# serialized as ``null``.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
|
||||
|
||||
class CohereDeltaMessage(DeltaMessage):
|
||||
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
|
||||
|
||||
Emitted by
|
||||
:class:`vllm.reasoning.cohere_command_reasoning_parser.BaseCohereCommandReasoningParser`
|
||||
on delta events whose payload includes citations. Non-Cohere parsers
|
||||
return plain :class:`DeltaMessage`, so their streamed shape is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares
|
||||
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
|
||||
subclass with its own schema when it flows through
|
||||
:class:`ChatCompletionResponseStreamChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# See ``CohereChatMessage._serialize`` for why we delegate via
|
||||
# ``super()`` instead of calling ``handler(self)`` directly.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
@@ -0,0 +1,362 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere Chat v2 API protocol.
|
||||
|
||||
The bulk of the wire types come straight from the official ``cohere``
|
||||
Python SDK so we stay in sync with the upstream specification and
|
||||
avoid re-declaring the schema. We only own three things locally:
|
||||
|
||||
1. The top-level request body model (the SDK doesn't ship one — its
|
||||
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
|
||||
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
|
||||
2. The non-streaming response envelope (the SDK exposes the message
|
||||
shape via :class:`AssistantMessageResponse` but no full response
|
||||
wrapper).
|
||||
3. The streaming discriminated union (the SDK exports each event type
|
||||
individually but not as a combined ``Annotated[Union[...],
|
||||
discriminator]``).
|
||||
|
||||
Importing this module pulls in the ``cohere`` package. The router that
|
||||
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
|
||||
boots without the SDK installed.
|
||||
|
||||
See https://docs.cohere.com/reference/chat for the upstream spec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from cohere import types as _sdk
|
||||
from cohere.types import (
|
||||
AssistantChatMessageV2,
|
||||
AssistantMessageResponse,
|
||||
ChatMessageV2,
|
||||
ChatRequestSafetyMode,
|
||||
Citation,
|
||||
CitationOptions,
|
||||
Document,
|
||||
ResponseFormatV2,
|
||||
SystemChatMessageV2,
|
||||
Thinking,
|
||||
ToolCallV2,
|
||||
ToolChatMessageV2,
|
||||
ToolV2,
|
||||
UserChatMessageV2,
|
||||
)
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
# Re-export the SDK wire-format types alongside our local extensions so
|
||||
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
|
||||
# they need from this module.
|
||||
__all__ = [
|
||||
"AssistantChatMessageV2",
|
||||
"AssistantMessageResponse",
|
||||
"ChatMessageV2",
|
||||
"Citation",
|
||||
"CitationEndEvent",
|
||||
"CitationOptions",
|
||||
"CitationStartEvent",
|
||||
"CohereChatV2Request",
|
||||
"CohereChatV2Response",
|
||||
"CohereError",
|
||||
"CohereFinishReason",
|
||||
"CohereLogprobItem",
|
||||
"CohereUsage",
|
||||
"CohereUsageBilledUnits",
|
||||
"CohereUsageTokens",
|
||||
"ContentDeltaEvent",
|
||||
"ContentEndEvent",
|
||||
"ContentStartEvent",
|
||||
"Document",
|
||||
"MessageEndEvent",
|
||||
"MessageStartEvent",
|
||||
"ResponseFormatV2",
|
||||
"SystemChatMessageV2",
|
||||
"Thinking",
|
||||
"ToolCallDeltaEvent",
|
||||
"ToolCallEndEvent",
|
||||
"ToolCallStartEvent",
|
||||
"ToolCallV2",
|
||||
"ToolChatMessageV2",
|
||||
"ToolPlanDeltaEvent",
|
||||
"ToolV2",
|
||||
"UserChatMessageV2",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereError(BaseModel):
|
||||
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
|
||||
|
||||
Cohere's documented error schemas are uniform: ``{message, id}``.
|
||||
"""
|
||||
|
||||
message: str
|
||||
id: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool choice / finish reasons
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# These literals aren't first-class enums in the SDK but are documented at
|
||||
# https://docs.cohere.com/reference/chat. We declare them here so the
|
||||
# request/response models can validate them.
|
||||
|
||||
CohereToolChoice = Literal["REQUIRED", "NONE"]
|
||||
|
||||
CohereFinishReason = Literal[
|
||||
"COMPLETE",
|
||||
"STOP_SEQUENCE",
|
||||
"MAX_TOKENS",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
"TIMEOUT",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Request(BaseModel):
|
||||
"""Cohere Chat v2 request body.
|
||||
|
||||
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
|
||||
All structured fields delegate to the official SDK types so the body
|
||||
schema stays in sync with the upstream spec.
|
||||
"""
|
||||
|
||||
model: str
|
||||
messages: list[ChatMessageV2]
|
||||
stream: bool | None = False
|
||||
|
||||
# Tooling
|
||||
tools: list[ToolV2] | None = None
|
||||
strict_tools: bool | None = None
|
||||
tool_choice: CohereToolChoice | None = None
|
||||
|
||||
# Grounding
|
||||
documents: list[str | Document] | None = None
|
||||
citation_options: CitationOptions | None = None
|
||||
|
||||
# Output
|
||||
response_format: ResponseFormatV2 | None = None
|
||||
safety_mode: ChatRequestSafetyMode | None = None
|
||||
max_tokens: int | None = None
|
||||
stop_sequences: list[str] | None = None
|
||||
|
||||
# Sampling
|
||||
temperature: float | None = None
|
||||
seed: int | None = None
|
||||
frequency_penalty: float | None = None
|
||||
presence_penalty: float | None = None
|
||||
k: int | None = None
|
||||
p: float | None = None
|
||||
logprobs: bool | None = None
|
||||
|
||||
# Reasoning
|
||||
thinking: Thinking | None = None
|
||||
|
||||
# Scheduling
|
||||
priority: int | None = None
|
||||
|
||||
# vLLM-specific extensions (not in Cohere spec). These mirror what the
|
||||
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
|
||||
# the same engine knobs when needed.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def _validate_model(cls, v: str) -> str:
|
||||
if not v:
|
||||
raise ValueError("model is required")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def _validate_max_tokens(cls, v: int | None) -> int | None:
|
||||
if v is not None and v < 0:
|
||||
raise ValueError("max_tokens must be non-negative")
|
||||
return v
|
||||
|
||||
@field_validator("messages", mode="before")
|
||||
@classmethod
|
||||
def _normalize_message_roles(cls, v: Any) -> Any:
|
||||
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
|
||||
|
||||
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
|
||||
the four literal roles ``user`` / ``assistant`` / ``system`` /
|
||||
``tool``. OpenAI's ``developer`` role is documented as
|
||||
high-priority system instructions, so we alias it onto
|
||||
``system`` *before* the SDK discriminator runs (otherwise it
|
||||
rejects the message with a ``literal_error`` against each union
|
||||
member). Mirrors ``_role_to_melody`` in the renderer so the
|
||||
same alias is honoured no matter which surface the message
|
||||
arrives through.
|
||||
|
||||
``mode="before"`` is required so the rewrite happens before the
|
||||
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
|
||||
union; a default-mode validator would never see ``developer``
|
||||
because validation would have already failed. On any
|
||||
structural malformation (non-iterable input, items without a
|
||||
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
|
||||
and let Pydantic's normal coercion surface a precise error.
|
||||
"""
|
||||
try:
|
||||
return [
|
||||
{**msg, "role": "system"}
|
||||
if msg.get("role", "").lower() == "developer"
|
||||
else msg
|
||||
for msg in v
|
||||
]
|
||||
except (AttributeError, TypeError):
|
||||
return v
|
||||
|
||||
@field_validator("messages")
|
||||
@classmethod
|
||||
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
|
||||
if not v:
|
||||
raise ValueError("messages must contain at least one message")
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / Logprobs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
|
||||
# envelope is documented separately in the OpenAPI spec and we declare it
|
||||
# here.
|
||||
|
||||
|
||||
class CohereUsageBilledUnits(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
search_units: float | None = None
|
||||
classifications: float | None = None
|
||||
|
||||
|
||||
class CohereUsageTokens(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereUsage(BaseModel):
|
||||
billed_units: CohereUsageBilledUnits | None = None
|
||||
tokens: CohereUsageTokens | None = None
|
||||
cached_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereLogprobItem(BaseModel):
|
||||
text: str | None = None
|
||||
token_ids: list[int]
|
||||
logprobs: list[float] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Response(BaseModel):
|
||||
"""Cohere Chat v2 non-streaming response body.
|
||||
|
||||
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
|
||||
the documented v2 response envelope (``id``, ``finish_reason``,
|
||||
``usage``, ``logprobs``). The single constructor in
|
||||
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
|
||||
supplying a non-empty ``id`` (falling back to a synthesized one if
|
||||
the upstream response is missing it) to this model.
|
||||
"""
|
||||
|
||||
id: str
|
||||
finish_reason: CohereFinishReason
|
||||
message: AssistantMessageResponse
|
||||
usage: CohereUsage | None = None
|
||||
logprobs: list[CohereLogprobItem] | None = None
|
||||
|
||||
# vLLM-specific extension.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming events
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
|
||||
# Sent Events; each event's ``type`` field carries the discriminator. The
|
||||
# SDK exposes a Pydantic model per event but none of them declare ``type``
|
||||
# as a field (the SDK relies on its own deserializer for discrimination),
|
||||
# so a naive ``model_dump_json()`` would silently drop the discriminator
|
||||
# and break clients that demux on ``type``.
|
||||
#
|
||||
# We therefore subclass each SDK event and bake the wire-format ``type``
|
||||
# string in as a ``Literal`` field with a default. ``model_dump()`` now
|
||||
# emits ``type`` for free, and surfaces can simply construct the event
|
||||
# class and serialize it -- no manual ``type`` parameter needed.
|
||||
#
|
||||
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
|
||||
# ``StreamedChatResponseV2`` schema for the wire-format reference.
|
||||
|
||||
|
||||
class MessageStartEvent(_sdk.ChatMessageStartEvent):
|
||||
type: Literal["message-start"] = "message-start"
|
||||
|
||||
|
||||
class ContentStartEvent(_sdk.ChatContentStartEvent):
|
||||
type: Literal["content-start"] = "content-start"
|
||||
|
||||
|
||||
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
|
||||
type: Literal["content-delta"] = "content-delta"
|
||||
|
||||
|
||||
class ContentEndEvent(_sdk.ChatContentEndEvent):
|
||||
type: Literal["content-end"] = "content-end"
|
||||
|
||||
|
||||
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
|
||||
type: Literal["tool-plan-delta"] = "tool-plan-delta"
|
||||
|
||||
|
||||
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
|
||||
type: Literal["tool-call-start"] = "tool-call-start"
|
||||
|
||||
|
||||
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
|
||||
type: Literal["tool-call-delta"] = "tool-call-delta"
|
||||
|
||||
|
||||
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
|
||||
type: Literal["tool-call-end"] = "tool-call-end"
|
||||
|
||||
|
||||
class CitationStartEvent(_sdk.CitationStartEvent):
|
||||
type: Literal["citation-start"] = "citation-start"
|
||||
|
||||
|
||||
class CitationEndEvent(_sdk.CitationEndEvent):
|
||||
type: Literal["citation-end"] = "citation-end"
|
||||
|
||||
|
||||
class MessageEndEvent(_sdk.ChatMessageEndEvent):
|
||||
type: Literal["message-end"] = "message-end"
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user