[Frontend] Cohere chat v2 api support (#47189)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
andrewbcohere
2026-07-31 22:53:54 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 6c91de3689
commit 9c110fa522
31 changed files with 8770 additions and 20 deletions
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+244
View File
@@ -0,0 +1,244 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
The Cohere v2 protocol models are sourced from the official ``cohere``
Python SDK (``pip install cohere``). To keep that an *optional*
dependency for vLLM, the SDK-dependent imports - and the route handler
itself - are gated on a one-shot probe at module load. If the SDK isn't
installed, :func:`attach_router` becomes a no-op (with an info log) and
vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the route. This keeps non-Cohere deployments that pull in the SDK for
unrelated reasons (e.g. test dependencies) from accidentally exposing
the api.
Note: the handler must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
the ``CohereChatV2Request`` body annotation against the module's
globals. Defining it locally inside ``attach_router`` would hide the
type from ``get_type_hints``, causing FastAPI to silently degrade the
body parameter into a query parameter and reject every request with
422.
"""
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.middleware.base import BaseHTTPMiddleware
import vllm.envs as envs
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
sanitize_message,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
_COHERE_PATH_PREFIX = "/cohere/"
logger = init_logger(__name__)
try:
import cohere # noqa: F401 -- dependency probe
except ImportError:
_SDK_AVAILABLE = False
else:
_SDK_AVAILABLE = True
if _SDK_AVAILABLE:
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
CohereChatV2Response,
CohereError,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
router = APIRouter()
def _serving(request: Request) -> CohereServingChatV2 | None:
return getattr(request.app.state, "cohere_serving_chat_v2", None)
def _request_id(raw_request: Request | None) -> str | None:
"""Best-effort lookup of the active request id.
Prefers the id the underlying chat handler stamped onto
``raw_request.state.request_metadata`` (if it got that far before
failing), falling back to the ``X-Request-Id`` HTTP header. May
return ``None`` if neither is available, in which case the field
is omitted from the response.
"""
if raw_request is None:
return None
meta = getattr(raw_request.state, "request_metadata", None)
if meta is not None and getattr(meta, "request_id", None):
return meta.request_id
return raw_request.headers.get("X-Request-Id")
def _error_response(
error: ErrorResponse,
raw_request: Request | None,
*,
fallback_status: int = HTTPStatus.BAD_REQUEST,
) -> JSONResponse:
"""Translate vLLM's internal error envelope into Cohere's shape."""
info = error.error
status = info.code or fallback_status
return JSONResponse(
status_code=status,
content=CohereError(
message=sanitize_message(info.message),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
@router.post(
"/cohere/v2/chat",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
@load_aware_call
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
handler = _serving(raw_request)
if handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message="The model does not support the Cohere v2 chat API.",
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
result = await handler.create_chat_v2(request, raw_request)
except Exception as e: # noqa: BLE001 - report as 500 for parity
logger.exception("Error in /cohere/v2/chat: %s", e)
return JSONResponse(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
content=CohereError(
message=sanitize_message(str(e)),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
match result:
case ErrorResponse():
return _error_response(result, raw_request)
case CohereChatV2Response():
return JSONResponse(content=result.model_dump(exclude_none=True))
case _:
return StreamingResponse(content=result, media_type="text/event-stream")
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
The endpoint handler above already returns :class:`CohereError` for
errors it owns, but globally-registered exception handlers (e.g.
:func:`validation_exception_handler` for pydantic body errors,
:func:`http_exception_handler`, engine error handlers) fire *before*
the handler runs and produce vLLM's internal
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
shape doesn't match the ``CohereError`` schema advertised on the
route's OpenAPI ``responses``, so clients (and schema-conformance
tests like ``test_openai_schema.py``) would see a mismatch on
those paths. This middleware normalises any error body on
``/cohere/*`` responses to :class:`CohereError`.
"""
async def dispatch(self, request: Request, call_next):
response = await call_next(request)
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
return response
if response.status_code < 400:
return response
content_type = response.headers.get("content-type", "")
if not content_type.startswith("application/json"):
return response
body = b"".join([chunk async for chunk in response.body_iterator])
translated = _translate_vllm_error_body(body, request)
if translated is not None:
return translated
passthrough_headers = {
k: v
for k, v in response.headers.items()
if k.lower() != "content-length"
}
return Response(
content=body,
status_code=response.status_code,
headers=passthrough_headers,
media_type=content_type,
)
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
Returns ``None`` if ``raw`` does not match the vLLM error envelope
(which signals the middleware to pass the body through unchanged).
"""
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError, ValueError):
return None
if not (
isinstance(data, dict)
and isinstance(data.get("error"), dict)
and "message" in data["error"]
):
return None
try:
err = ErrorResponse.model_validate(data)
except Exception: # noqa: BLE001 - malformed envelope; pass through
return None
return _error_response(err, request)
def attach_router(app: FastAPI) -> None:
"""Register ``POST /cohere/v2/chat`` on ``app``.
No-op when either:
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
Cohere v2 endpoint is opt-in because it carries Cohere-specific
request/response semantics (grounding citations, tool_plan,
PLAN/THINKING_CONTENT blocks) that are only meaningful when
serving a Cohere Command-family model.
* the optional ``cohere`` SDK isn't installed (the v2 protocol
models live there)
The two skip paths log at different levels: an operator who set
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
a WARNING (they explicitly asked for the endpoint and it's silently
absent), whereas the default-off skip logs at debug.
"""
enabled = envs.VLLM_ENABLE_COHERE_API
if not enabled:
logger.debug(
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
)
return
if not _SDK_AVAILABLE:
logger.warning(
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
"installed; /cohere/v2/chat will not be exposed. Install "
"with `pip install cohere` to enable the endpoint."
)
return
app.include_router(router)
app.add_middleware(CohereErrorEnvelopeMiddleware)
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
This module keeps the OpenAI chat completion protocol classes
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
It provides two things:
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
representation produced by the Cohere reasoning parser
(:mod:`vllm.reasoning.cohere_command_reasoning_parser`) and consumed by
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
SDK shape -- that conversion happens in the serving layer.
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
are only instantiated by the citation-aware serving handler
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
The module intentionally does *not* import the ``cohere`` Python SDK so the
reasoning parser (which is Cohere-model-specific but ships without the SDK
dependency) can import from here freely.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import Field, model_serializer
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
OpenAIBaseModel,
)
class CitationSource(OpenAIBaseModel):
"""Source attribution for a :class:`Citation`.
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
discriminator (``document`` or ``tool``); ``id`` is the citing
document/tool-output identifier; ``document`` and ``tool_output`` carry
the original payload that produced the citation.
Sources are fully resolved by the reasoning parser at emit time
(see :func:`_melody_sources_to_vllm` in
:mod:`vllm.reasoning.cohere_command_reasoning_parser`) using the
``POSITION_TO_SOURCE_KEY`` map forwarded through
``chat_template_kwargs`` by
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
That means every instance that reaches the serving / wire layer
already has ``type`` / ``id`` populated; the serving layer only has
to coerce to :class:`cohere.types.Citation` and rewrite
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
"""
type: Literal["document", "tool"] | None = None
id: str | None = None
document: dict[str, Any] | None = None
tool_output: dict[str, Any] | None = None
class Citation(OpenAIBaseModel):
"""A citation grounding a span of generated text in source material.
vLLM-internal representation used by the Cohere reasoning parser and
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
conversion to the SDK's ``cohere.types.Citation`` happens in
:mod:`vllm.entrypoints.cohere.serving`.
"""
start: int | None = None
"""Start character offset in the surrounding text content."""
end: int | None = None
"""End character offset (exclusive) in the surrounding text content."""
text: str | None = None
"""The cited text snippet."""
sources: list[CitationSource] = Field(default_factory=list)
"""Source documents / tool outputs that ground this citation."""
content_index: int | None = None
"""Index of the content block this citation refers to (when the message
has multiple content blocks)."""
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
"""Which kind of content block this citation grounds: the user-visible
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
tool-plan block (``PLAN``). ``None`` means unspecified."""
class CohereChatMessage(ChatMessage):
""":class:`ChatMessage` extension carrying grounding citations.
Only instantiated by :class:`CohereServingChatV2` (and any other
:class:`OpenAIServingChat` subclass that opts in by overriding
``_create_chat_message``). Regular OpenAI-compatible handlers keep
emitting plain :class:`ChatMessage` so their response schema is
unchanged.
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
so pydantic serializes this subclass with its own schema (including
``citations``) when it flows through
:class:`ChatCompletionResponseChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# ``mode="wrap"`` fully overrides (rather than chains) the
# parent's ``@model_serializer``, so we explicitly delegate via
# ``super()._serialize(handler)`` to preserve the parent's
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
# unset ``citations`` field so the wire matches the OpenAI-style
# contract that optional vLLM extensions are omitted rather than
# serialized as ``null``.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
class CohereDeltaMessage(DeltaMessage):
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
Emitted by
:class:`vllm.reasoning.cohere_command_reasoning_parser.BaseCohereCommandReasoningParser`
on delta events whose payload includes citations. Non-Cohere parsers
return plain :class:`DeltaMessage`, so their streamed shape is
unchanged.
The response envelope declares
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
subclass with its own schema when it flows through
:class:`ChatCompletionResponseStreamChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# See ``CohereChatMessage._serialize`` for why we delegate via
# ``super()`` instead of calling ``handler(self)`` directly.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
+362
View File
@@ -0,0 +1,362 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cohere Chat v2 API protocol.
The bulk of the wire types come straight from the official ``cohere``
Python SDK so we stay in sync with the upstream specification and
avoid re-declaring the schema. We only own three things locally:
1. The top-level request body model (the SDK doesn't ship one — its
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
2. The non-streaming response envelope (the SDK exposes the message
shape via :class:`AssistantMessageResponse` but no full response
wrapper).
3. The streaming discriminated union (the SDK exports each event type
individually but not as a combined ``Annotated[Union[...],
discriminator]``).
Importing this module pulls in the ``cohere`` package. The router that
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
boots without the SDK installed.
See https://docs.cohere.com/reference/chat for the upstream spec.
"""
from __future__ import annotations
from typing import Any, Literal
from cohere import types as _sdk
from cohere.types import (
AssistantChatMessageV2,
AssistantMessageResponse,
ChatMessageV2,
ChatRequestSafetyMode,
Citation,
CitationOptions,
Document,
ResponseFormatV2,
SystemChatMessageV2,
Thinking,
ToolCallV2,
ToolChatMessageV2,
ToolV2,
UserChatMessageV2,
)
from pydantic import BaseModel, Field, field_validator
# Re-export the SDK wire-format types alongside our local extensions so
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
# they need from this module.
__all__ = [
"AssistantChatMessageV2",
"AssistantMessageResponse",
"ChatMessageV2",
"Citation",
"CitationEndEvent",
"CitationOptions",
"CitationStartEvent",
"CohereChatV2Request",
"CohereChatV2Response",
"CohereError",
"CohereFinishReason",
"CohereLogprobItem",
"CohereUsage",
"CohereUsageBilledUnits",
"CohereUsageTokens",
"ContentDeltaEvent",
"ContentEndEvent",
"ContentStartEvent",
"Document",
"MessageEndEvent",
"MessageStartEvent",
"ResponseFormatV2",
"SystemChatMessageV2",
"Thinking",
"ToolCallDeltaEvent",
"ToolCallEndEvent",
"ToolCallStartEvent",
"ToolCallV2",
"ToolChatMessageV2",
"ToolPlanDeltaEvent",
"ToolV2",
"UserChatMessageV2",
]
# ---------------------------------------------------------------------------
# Errors
# ---------------------------------------------------------------------------
class CohereError(BaseModel):
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
Cohere's documented error schemas are uniform: ``{message, id}``.
"""
message: str
id: str | None = None
# ---------------------------------------------------------------------------
# Tool choice / finish reasons
# ---------------------------------------------------------------------------
#
# These literals aren't first-class enums in the SDK but are documented at
# https://docs.cohere.com/reference/chat. We declare them here so the
# request/response models can validate them.
CohereToolChoice = Literal["REQUIRED", "NONE"]
CohereFinishReason = Literal[
"COMPLETE",
"STOP_SEQUENCE",
"MAX_TOKENS",
"TOOL_CALL",
"ERROR",
"TIMEOUT",
]
# ---------------------------------------------------------------------------
# Request
# ---------------------------------------------------------------------------
class CohereChatV2Request(BaseModel):
"""Cohere Chat v2 request body.
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
All structured fields delegate to the official SDK types so the body
schema stays in sync with the upstream spec.
"""
model: str
messages: list[ChatMessageV2]
stream: bool | None = False
# Tooling
tools: list[ToolV2] | None = None
strict_tools: bool | None = None
tool_choice: CohereToolChoice | None = None
# Grounding
documents: list[str | Document] | None = None
citation_options: CitationOptions | None = None
# Output
response_format: ResponseFormatV2 | None = None
safety_mode: ChatRequestSafetyMode | None = None
max_tokens: int | None = None
stop_sequences: list[str] | None = None
# Sampling
temperature: float | None = None
seed: int | None = None
frequency_penalty: float | None = None
presence_penalty: float | None = None
k: int | None = None
p: float | None = None
logprobs: bool | None = None
# Reasoning
thinking: Thinking | None = None
# Scheduling
priority: int | None = None
# vLLM-specific extensions (not in Cohere spec). These mirror what the
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
# the same engine knobs when needed.
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def _validate_model(cls, v: str) -> str:
if not v:
raise ValueError("model is required")
return v
@field_validator("max_tokens")
@classmethod
def _validate_max_tokens(cls, v: int | None) -> int | None:
if v is not None and v < 0:
raise ValueError("max_tokens must be non-negative")
return v
@field_validator("messages", mode="before")
@classmethod
def _normalize_message_roles(cls, v: Any) -> Any:
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
the four literal roles ``user`` / ``assistant`` / ``system`` /
``tool``. OpenAI's ``developer`` role is documented as
high-priority system instructions, so we alias it onto
``system`` *before* the SDK discriminator runs (otherwise it
rejects the message with a ``literal_error`` against each union
member). Mirrors ``_role_to_melody`` in the renderer so the
same alias is honoured no matter which surface the message
arrives through.
``mode="before"`` is required so the rewrite happens before the
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
union; a default-mode validator would never see ``developer``
because validation would have already failed. On any
structural malformation (non-iterable input, items without a
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
and let Pydantic's normal coercion surface a precise error.
"""
try:
return [
{**msg, "role": "system"}
if msg.get("role", "").lower() == "developer"
else msg
for msg in v
]
except (AttributeError, TypeError):
return v
@field_validator("messages")
@classmethod
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
if not v:
raise ValueError("messages must contain at least one message")
return v
# ---------------------------------------------------------------------------
# Usage / Logprobs
# ---------------------------------------------------------------------------
#
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
# envelope is documented separately in the OpenAPI spec and we declare it
# here.
class CohereUsageBilledUnits(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
search_units: float | None = None
classifications: float | None = None
class CohereUsageTokens(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
class CohereUsage(BaseModel):
billed_units: CohereUsageBilledUnits | None = None
tokens: CohereUsageTokens | None = None
cached_tokens: float | None = None
class CohereLogprobItem(BaseModel):
text: str | None = None
token_ids: list[int]
logprobs: list[float] | None = None
# ---------------------------------------------------------------------------
# Non-streaming response
# ---------------------------------------------------------------------------
class CohereChatV2Response(BaseModel):
"""Cohere Chat v2 non-streaming response body.
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
the documented v2 response envelope (``id``, ``finish_reason``,
``usage``, ``logprobs``). The single constructor in
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
supplying a non-empty ``id`` (falling back to a synthesized one if
the upstream response is missing it) to this model.
"""
id: str
finish_reason: CohereFinishReason
message: AssistantMessageResponse
usage: CohereUsage | None = None
logprobs: list[CohereLogprobItem] | None = None
# vLLM-specific extension.
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
# ---------------------------------------------------------------------------
# Streaming events
# ---------------------------------------------------------------------------
#
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
# Sent Events; each event's ``type`` field carries the discriminator. The
# SDK exposes a Pydantic model per event but none of them declare ``type``
# as a field (the SDK relies on its own deserializer for discrimination),
# so a naive ``model_dump_json()`` would silently drop the discriminator
# and break clients that demux on ``type``.
#
# We therefore subclass each SDK event and bake the wire-format ``type``
# string in as a ``Literal`` field with a default. ``model_dump()`` now
# emits ``type`` for free, and surfaces can simply construct the event
# class and serialize it -- no manual ``type`` parameter needed.
#
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
# ``StreamedChatResponseV2`` schema for the wire-format reference.
class MessageStartEvent(_sdk.ChatMessageStartEvent):
type: Literal["message-start"] = "message-start"
class ContentStartEvent(_sdk.ChatContentStartEvent):
type: Literal["content-start"] = "content-start"
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
type: Literal["content-delta"] = "content-delta"
class ContentEndEvent(_sdk.ChatContentEndEvent):
type: Literal["content-end"] = "content-end"
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
type: Literal["tool-plan-delta"] = "tool-plan-delta"
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
type: Literal["tool-call-start"] = "tool-call-start"
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
type: Literal["tool-call-delta"] = "tool-call-delta"
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
type: Literal["tool-call-end"] = "tool-call-end"
class CitationStartEvent(_sdk.CitationStartEvent):
type: Literal["citation-start"] = "citation-start"
class CitationEndEvent(_sdk.CitationEndEvent):
type: Literal["citation-end"] = "citation-end"
class MessageEndEvent(_sdk.ChatMessageEndEvent):
type: Literal["message-end"] = "message-end"
File diff suppressed because it is too large Load Diff
+62 -3
View File
@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING
from fastapi import FastAPI
import vllm.envs as envs
if TYPE_CHECKING:
from argparse import Namespace
@@ -41,6 +43,12 @@ def register_generate_api_routers(app: FastAPI):
register_anthropic_api_router(app)
from vllm.entrypoints.cohere.api_router import (
attach_router as register_cohere_api_router,
)
register_cohere_api_router(app)
from .generative_scoring.api_router import register_generative_scoring_api_router
register_generative_scoring_api_router(app)
@@ -55,6 +63,23 @@ async def init_generate_state(
):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
# The Cohere serving handler depends on the optional `cohere` SDK for
# its wire-format protocol models, and is additionally gated on the
# `VLLM_ENABLE_COHERE_API` env flag (see
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
# entirely when the endpoint isn't going to be exposed, both because
# the SDK may not be installed and because the serving object holds
# nontrivial state (chat handler, warmup) that would otherwise be
# unused.
if envs.VLLM_ENABLE_COHERE_API:
try:
from vllm.entrypoints.cohere.serving import CohereServingChatV2
except ImportError:
CohereServingChatV2 = None # type: ignore[assignment,misc]
else:
CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.mcp.tool_server import (
DemoToolServer,
MCPToolServer,
@@ -86,6 +111,19 @@ async def init_generate_state(
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template)
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
# default chat-template kwargs. The cohere renderer reads
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
# rendering; making this a first-class flag keeps the right format
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
# without forcing them to hand-construct a JSON dict for
# ``--default-chat-template-kwargs``. Per-request overrides still
# take precedence (see ``merge_kwargs`` in
# ``ChatCompletionRequest.build_chat_params``).
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
@@ -106,7 +144,7 @@ async def init_generate_state(
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
)
if "generate" in supported_tasks
else None
@@ -119,7 +157,7 @@ async def init_generate_state(
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
@@ -169,11 +207,32 @@ async def init_generate_state(
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
)
if "generate" in supported_tasks
else None
)
state.cohere_serving_chat_v2 = (
CohereServingChatV2(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
default_chat_template_kwargs=default_chat_template_kwargs,
is_reasoning_model=args.cohere_is_reasoning_model,
)
if CohereServingChatV2 is not None and "generate" in supported_tasks
else None
)
from .generative_scoring.serving import ServingGenerativeScoring
@@ -10,7 +10,13 @@ from openai.types.chat.chat_completion_audio import (
ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
from pydantic import Field, PrivateAttr, model_serializer, model_validator
from pydantic import (
Field,
PrivateAttr,
SerializeAsAny,
model_serializer,
model_validator,
)
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
@@ -91,7 +97,12 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
class ChatCompletionResponseChoice(OpenAIBaseModel):
index: int
message: ChatMessage
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
# so that added fields like ``citations`` survive JSON serialization
# instead of being stripped down to the base schema. Plain
# ``ChatMessage`` instances serialize identically to before.
message: SerializeAsAny[ChatMessage]
logprobs: ChatCompletionLogProbs | None = None
# per OpenAI spec this is the default
finish_reason: str | None = "stop"
@@ -139,7 +150,11 @@ class ChatCompletionResponse(OpenAIBaseModel):
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
index: int
delta: DeltaMessage
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
# so streaming ``citations`` survive JSON serialization. Plain
# ``DeltaMessage`` instances serialize identically to before.
delta: SerializeAsAny[DeltaMessage]
logprobs: ChatCompletionLogProbs | None = None
finish_reason: str | None = None
stop_reason: int | str | None = None
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
return self.response_role
return request.messages[-1]["role"]
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
"""Construct the response :class:`ChatMessage` for the non-streaming path.
The full-generator calls this at every construction site so
subclasses can swap in a specialized :class:`ChatMessage`
subclass (e.g. :class:`CohereServingChatV2` returning
:class:`CohereChatMessage`) without duplicating the branchy
tool-choice / auto-tools logic that decides which fields are
populated. The default returns a plain :class:`ChatMessage`.
"""
return ChatMessage(*args, **kwargs)
def _finalize_response_message(
self,
message: ChatMessage,
*,
parser: Parser | None,
) -> ChatMessage:
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
Default is a no-op. Subclasses that need to surface parser-side
extras (e.g. :class:`CohereServingChatV2` reading grounding
citations off the reasoning parser and populating
:class:`CohereChatMessage.citations`) override this to inspect
``parser`` and mutate/replace ``message``.
"""
return message
async def chat_completion_stream_generator(
self,
request: ChatCompletionRequest,
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
)
is_required_tool_choice = request.tool_choice == "required"
# All six construction sites route through ``self._create_chat_message``
# so subclasses can swap in a specialized :class:`ChatMessage`
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
# having to duplicate this branch logic.
if (not self.enable_auto_tools or not tool_parser_cls) and (
not is_named_tool_choice and not is_required_tool_choice
):
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
elif is_named_tool_choice or is_required_tool_choice:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content or "",
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
# if the request doesn't use tool choice
# OR specifies to not use a tool
elif not request.tool_choice or request.tool_choice == "none":
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
# handle when there are tools and tool choice is auto
elif (
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
):
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content,
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
)
else:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content,
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
" if tools should be extracted. Returning a standard chat "
"completion."
)
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
# Subclass hook: enrich the constructed message with any
# parser-side extras that don't fit through the plain
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
# citation-aware handlers use this to surface grounding
# metadata cached on the reasoning parser.
message = self._finalize_response_message(message, parser=parser)
# In OpenAI's API, when a tool is called, the finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.
+19
View File
@@ -148,6 +148,25 @@ class BaseFrontendArgs:
"""If set to False, output deltas will not be logged. Relevant only if
--enable-log-outputs is set.
"""
cohere_is_reasoning_model: bool = True
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
reasoning Command-family model. When True (default), the assistant's
chain-of-thought is surfaced as a ``thinking`` content block (or
``content-*`` events on the stream). When False, reasoning is
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
events) whenever the model emits tool calls, matching older non-
reasoning Command models. Has no effect on the non-Cohere
endpoints."""
cohere_format: str = "cmd4"
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
format to render: ``cmd4`` (current Command A+ models; default) or
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
wrong format silently produces a prompt the model wasn't trained
on, which most commonly manifests as the model emitting text but no
citations / tool calls / thinking blocks. Equivalent to passing
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
explicit request-level ``chat_template_kwargs.cohere_format`` takes
priority."""
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
+1 -1
View File
@@ -39,7 +39,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
logger = init_logger("vllm.entrypoints.openai.server_utils")
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
class AuthenticationMiddleware: