[Frontend] Cohere chat v2 api support (#47189)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6c91de3689
commit
9c110fa522
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handler
|
||||
itself - are gated on a one-shot probe at module load. If the SDK isn't
|
||||
installed, :func:`attach_router` becomes a no-op (with an info log) and
|
||||
vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the route. This keeps non-Cohere deployments that pull in the SDK for
|
||||
unrelated reasons (e.g. test dependencies) from accidentally exposing
|
||||
the api.
|
||||
|
||||
Note: the handler must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining it locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
"""
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
sanitize_message,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
_COHERE_PATH_PREFIX = "/cohere/"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
import cohere # noqa: F401 -- dependency probe
|
||||
except ImportError:
|
||||
_SDK_AVAILABLE = False
|
||||
else:
|
||||
_SDK_AVAILABLE = True
|
||||
|
||||
|
||||
if _SDK_AVAILABLE:
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
Prefers the id the underlying chat handler stamped onto
|
||||
``raw_request.state.request_metadata`` (if it got that far before
|
||||
failing), falling back to the ``X-Request-Id`` HTTP header. May
|
||||
return ``None`` if neither is available, in which case the field
|
||||
is omitted from the response.
|
||||
"""
|
||||
if raw_request is None:
|
||||
return None
|
||||
meta = getattr(raw_request.state, "request_metadata", None)
|
||||
if meta is not None and getattr(meta, "request_id", None):
|
||||
return meta.request_id
|
||||
return raw_request.headers.get("X-Request-Id")
|
||||
|
||||
def _error_response(
|
||||
error: ErrorResponse,
|
||||
raw_request: Request | None,
|
||||
*,
|
||||
fallback_status: int = HTTPStatus.BAD_REQUEST,
|
||||
) -> JSONResponse:
|
||||
"""Translate vLLM's internal error envelope into Cohere's shape."""
|
||||
info = error.error
|
||||
status = info.code or fallback_status
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content=CohereError(
|
||||
message=sanitize_message(info.message),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
|
||||
handler = _serving(raw_request)
|
||||
if handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message="The model does not support the Cohere v2 chat API.",
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
result = await handler.create_chat_v2(request, raw_request)
|
||||
except Exception as e: # noqa: BLE001 - report as 500 for parity
|
||||
logger.exception("Error in /cohere/v2/chat: %s", e)
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
content=CohereError(
|
||||
message=sanitize_message(str(e)),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
match result:
|
||||
case ErrorResponse():
|
||||
return _error_response(result, raw_request)
|
||||
case CohereChatV2Response():
|
||||
return JSONResponse(content=result.model_dump(exclude_none=True))
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
The endpoint handler above already returns :class:`CohereError` for
|
||||
errors it owns, but globally-registered exception handlers (e.g.
|
||||
:func:`validation_exception_handler` for pydantic body errors,
|
||||
:func:`http_exception_handler`, engine error handlers) fire *before*
|
||||
the handler runs and produce vLLM's internal
|
||||
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
|
||||
shape doesn't match the ``CohereError`` schema advertised on the
|
||||
route's OpenAPI ``responses``, so clients (and schema-conformance
|
||||
tests like ``test_openai_schema.py``) would see a mismatch on
|
||||
those paths. This middleware normalises any error body on
|
||||
``/cohere/*`` responses to :class:`CohereError`.
|
||||
"""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
|
||||
return response
|
||||
if response.status_code < 400:
|
||||
return response
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if not content_type.startswith("application/json"):
|
||||
return response
|
||||
|
||||
body = b"".join([chunk async for chunk in response.body_iterator])
|
||||
translated = _translate_vllm_error_body(body, request)
|
||||
if translated is not None:
|
||||
return translated
|
||||
passthrough_headers = {
|
||||
k: v
|
||||
for k, v in response.headers.items()
|
||||
if k.lower() != "content-length"
|
||||
}
|
||||
return Response(
|
||||
content=body,
|
||||
status_code=response.status_code,
|
||||
headers=passthrough_headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
|
||||
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
|
||||
|
||||
Returns ``None`` if ``raw`` does not match the vLLM error envelope
|
||||
(which signals the middleware to pass the body through unchanged).
|
||||
"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
if not (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("error"), dict)
|
||||
and "message" in data["error"]
|
||||
):
|
||||
return None
|
||||
try:
|
||||
err = ErrorResponse.model_validate(data)
|
||||
except Exception: # noqa: BLE001 - malformed envelope; pass through
|
||||
return None
|
||||
return _error_response(err, request)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register ``POST /cohere/v2/chat`` on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoint is opt-in because it carries Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
* the optional ``cohere`` SDK isn't installed (the v2 protocol
|
||||
models live there)
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoint and it's silently
|
||||
absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoint."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
|
||||
|
||||
This module keeps the OpenAI chat completion protocol classes
|
||||
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
|
||||
It provides two things:
|
||||
|
||||
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
|
||||
representation produced by the Cohere reasoning parser
|
||||
(:mod:`vllm.reasoning.cohere_command_reasoning_parser`) and consumed by
|
||||
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
|
||||
SDK shape -- that conversion happens in the serving layer.
|
||||
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
|
||||
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
|
||||
are only instantiated by the citation-aware serving handler
|
||||
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
|
||||
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
|
||||
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
|
||||
|
||||
The module intentionally does *not* import the ``cohere`` Python SDK so the
|
||||
reasoning parser (which is Cohere-model-specific but ships without the SDK
|
||||
dependency) can import from here freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_serializer
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
|
||||
|
||||
class CitationSource(OpenAIBaseModel):
|
||||
"""Source attribution for a :class:`Citation`.
|
||||
|
||||
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
|
||||
discriminator (``document`` or ``tool``); ``id`` is the citing
|
||||
document/tool-output identifier; ``document`` and ``tool_output`` carry
|
||||
the original payload that produced the citation.
|
||||
|
||||
Sources are fully resolved by the reasoning parser at emit time
|
||||
(see :func:`_melody_sources_to_vllm` in
|
||||
:mod:`vllm.reasoning.cohere_command_reasoning_parser`) using the
|
||||
``POSITION_TO_SOURCE_KEY`` map forwarded through
|
||||
``chat_template_kwargs`` by
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
|
||||
That means every instance that reaches the serving / wire layer
|
||||
already has ``type`` / ``id`` populated; the serving layer only has
|
||||
to coerce to :class:`cohere.types.Citation` and rewrite
|
||||
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
|
||||
"""
|
||||
|
||||
type: Literal["document", "tool"] | None = None
|
||||
id: str | None = None
|
||||
document: dict[str, Any] | None = None
|
||||
tool_output: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Citation(OpenAIBaseModel):
|
||||
"""A citation grounding a span of generated text in source material.
|
||||
|
||||
vLLM-internal representation used by the Cohere reasoning parser and
|
||||
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
|
||||
conversion to the SDK's ``cohere.types.Citation`` happens in
|
||||
:mod:`vllm.entrypoints.cohere.serving`.
|
||||
"""
|
||||
|
||||
start: int | None = None
|
||||
"""Start character offset in the surrounding text content."""
|
||||
end: int | None = None
|
||||
"""End character offset (exclusive) in the surrounding text content."""
|
||||
text: str | None = None
|
||||
"""The cited text snippet."""
|
||||
sources: list[CitationSource] = Field(default_factory=list)
|
||||
"""Source documents / tool outputs that ground this citation."""
|
||||
content_index: int | None = None
|
||||
"""Index of the content block this citation refers to (when the message
|
||||
has multiple content blocks)."""
|
||||
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
|
||||
"""Which kind of content block this citation grounds: the user-visible
|
||||
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
|
||||
tool-plan block (``PLAN``). ``None`` means unspecified."""
|
||||
|
||||
|
||||
class CohereChatMessage(ChatMessage):
|
||||
""":class:`ChatMessage` extension carrying grounding citations.
|
||||
|
||||
Only instantiated by :class:`CohereServingChatV2` (and any other
|
||||
:class:`OpenAIServingChat` subclass that opts in by overriding
|
||||
``_create_chat_message``). Regular OpenAI-compatible handlers keep
|
||||
emitting plain :class:`ChatMessage` so their response schema is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
|
||||
so pydantic serializes this subclass with its own schema (including
|
||||
``citations``) when it flows through
|
||||
:class:`ChatCompletionResponseChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# ``mode="wrap"`` fully overrides (rather than chains) the
|
||||
# parent's ``@model_serializer``, so we explicitly delegate via
|
||||
# ``super()._serialize(handler)`` to preserve the parent's
|
||||
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
|
||||
# unset ``citations`` field so the wire matches the OpenAI-style
|
||||
# contract that optional vLLM extensions are omitted rather than
|
||||
# serialized as ``null``.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
|
||||
|
||||
class CohereDeltaMessage(DeltaMessage):
|
||||
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
|
||||
|
||||
Emitted by
|
||||
:class:`vllm.reasoning.cohere_command_reasoning_parser.BaseCohereCommandReasoningParser`
|
||||
on delta events whose payload includes citations. Non-Cohere parsers
|
||||
return plain :class:`DeltaMessage`, so their streamed shape is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares
|
||||
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
|
||||
subclass with its own schema when it flows through
|
||||
:class:`ChatCompletionResponseStreamChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# See ``CohereChatMessage._serialize`` for why we delegate via
|
||||
# ``super()`` instead of calling ``handler(self)`` directly.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
@@ -0,0 +1,362 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere Chat v2 API protocol.
|
||||
|
||||
The bulk of the wire types come straight from the official ``cohere``
|
||||
Python SDK so we stay in sync with the upstream specification and
|
||||
avoid re-declaring the schema. We only own three things locally:
|
||||
|
||||
1. The top-level request body model (the SDK doesn't ship one — its
|
||||
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
|
||||
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
|
||||
2. The non-streaming response envelope (the SDK exposes the message
|
||||
shape via :class:`AssistantMessageResponse` but no full response
|
||||
wrapper).
|
||||
3. The streaming discriminated union (the SDK exports each event type
|
||||
individually but not as a combined ``Annotated[Union[...],
|
||||
discriminator]``).
|
||||
|
||||
Importing this module pulls in the ``cohere`` package. The router that
|
||||
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
|
||||
boots without the SDK installed.
|
||||
|
||||
See https://docs.cohere.com/reference/chat for the upstream spec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from cohere import types as _sdk
|
||||
from cohere.types import (
|
||||
AssistantChatMessageV2,
|
||||
AssistantMessageResponse,
|
||||
ChatMessageV2,
|
||||
ChatRequestSafetyMode,
|
||||
Citation,
|
||||
CitationOptions,
|
||||
Document,
|
||||
ResponseFormatV2,
|
||||
SystemChatMessageV2,
|
||||
Thinking,
|
||||
ToolCallV2,
|
||||
ToolChatMessageV2,
|
||||
ToolV2,
|
||||
UserChatMessageV2,
|
||||
)
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
# Re-export the SDK wire-format types alongside our local extensions so
|
||||
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
|
||||
# they need from this module.
|
||||
__all__ = [
|
||||
"AssistantChatMessageV2",
|
||||
"AssistantMessageResponse",
|
||||
"ChatMessageV2",
|
||||
"Citation",
|
||||
"CitationEndEvent",
|
||||
"CitationOptions",
|
||||
"CitationStartEvent",
|
||||
"CohereChatV2Request",
|
||||
"CohereChatV2Response",
|
||||
"CohereError",
|
||||
"CohereFinishReason",
|
||||
"CohereLogprobItem",
|
||||
"CohereUsage",
|
||||
"CohereUsageBilledUnits",
|
||||
"CohereUsageTokens",
|
||||
"ContentDeltaEvent",
|
||||
"ContentEndEvent",
|
||||
"ContentStartEvent",
|
||||
"Document",
|
||||
"MessageEndEvent",
|
||||
"MessageStartEvent",
|
||||
"ResponseFormatV2",
|
||||
"SystemChatMessageV2",
|
||||
"Thinking",
|
||||
"ToolCallDeltaEvent",
|
||||
"ToolCallEndEvent",
|
||||
"ToolCallStartEvent",
|
||||
"ToolCallV2",
|
||||
"ToolChatMessageV2",
|
||||
"ToolPlanDeltaEvent",
|
||||
"ToolV2",
|
||||
"UserChatMessageV2",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereError(BaseModel):
|
||||
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
|
||||
|
||||
Cohere's documented error schemas are uniform: ``{message, id}``.
|
||||
"""
|
||||
|
||||
message: str
|
||||
id: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool choice / finish reasons
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# These literals aren't first-class enums in the SDK but are documented at
|
||||
# https://docs.cohere.com/reference/chat. We declare them here so the
|
||||
# request/response models can validate them.
|
||||
|
||||
CohereToolChoice = Literal["REQUIRED", "NONE"]
|
||||
|
||||
CohereFinishReason = Literal[
|
||||
"COMPLETE",
|
||||
"STOP_SEQUENCE",
|
||||
"MAX_TOKENS",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
"TIMEOUT",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Request(BaseModel):
|
||||
"""Cohere Chat v2 request body.
|
||||
|
||||
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
|
||||
All structured fields delegate to the official SDK types so the body
|
||||
schema stays in sync with the upstream spec.
|
||||
"""
|
||||
|
||||
model: str
|
||||
messages: list[ChatMessageV2]
|
||||
stream: bool | None = False
|
||||
|
||||
# Tooling
|
||||
tools: list[ToolV2] | None = None
|
||||
strict_tools: bool | None = None
|
||||
tool_choice: CohereToolChoice | None = None
|
||||
|
||||
# Grounding
|
||||
documents: list[str | Document] | None = None
|
||||
citation_options: CitationOptions | None = None
|
||||
|
||||
# Output
|
||||
response_format: ResponseFormatV2 | None = None
|
||||
safety_mode: ChatRequestSafetyMode | None = None
|
||||
max_tokens: int | None = None
|
||||
stop_sequences: list[str] | None = None
|
||||
|
||||
# Sampling
|
||||
temperature: float | None = None
|
||||
seed: int | None = None
|
||||
frequency_penalty: float | None = None
|
||||
presence_penalty: float | None = None
|
||||
k: int | None = None
|
||||
p: float | None = None
|
||||
logprobs: bool | None = None
|
||||
|
||||
# Reasoning
|
||||
thinking: Thinking | None = None
|
||||
|
||||
# Scheduling
|
||||
priority: int | None = None
|
||||
|
||||
# vLLM-specific extensions (not in Cohere spec). These mirror what the
|
||||
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
|
||||
# the same engine knobs when needed.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def _validate_model(cls, v: str) -> str:
|
||||
if not v:
|
||||
raise ValueError("model is required")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def _validate_max_tokens(cls, v: int | None) -> int | None:
|
||||
if v is not None and v < 0:
|
||||
raise ValueError("max_tokens must be non-negative")
|
||||
return v
|
||||
|
||||
@field_validator("messages", mode="before")
|
||||
@classmethod
|
||||
def _normalize_message_roles(cls, v: Any) -> Any:
|
||||
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
|
||||
|
||||
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
|
||||
the four literal roles ``user`` / ``assistant`` / ``system`` /
|
||||
``tool``. OpenAI's ``developer`` role is documented as
|
||||
high-priority system instructions, so we alias it onto
|
||||
``system`` *before* the SDK discriminator runs (otherwise it
|
||||
rejects the message with a ``literal_error`` against each union
|
||||
member). Mirrors ``_role_to_melody`` in the renderer so the
|
||||
same alias is honoured no matter which surface the message
|
||||
arrives through.
|
||||
|
||||
``mode="before"`` is required so the rewrite happens before the
|
||||
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
|
||||
union; a default-mode validator would never see ``developer``
|
||||
because validation would have already failed. On any
|
||||
structural malformation (non-iterable input, items without a
|
||||
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
|
||||
and let Pydantic's normal coercion surface a precise error.
|
||||
"""
|
||||
try:
|
||||
return [
|
||||
{**msg, "role": "system"}
|
||||
if msg.get("role", "").lower() == "developer"
|
||||
else msg
|
||||
for msg in v
|
||||
]
|
||||
except (AttributeError, TypeError):
|
||||
return v
|
||||
|
||||
@field_validator("messages")
|
||||
@classmethod
|
||||
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
|
||||
if not v:
|
||||
raise ValueError("messages must contain at least one message")
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / Logprobs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
|
||||
# envelope is documented separately in the OpenAPI spec and we declare it
|
||||
# here.
|
||||
|
||||
|
||||
class CohereUsageBilledUnits(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
search_units: float | None = None
|
||||
classifications: float | None = None
|
||||
|
||||
|
||||
class CohereUsageTokens(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereUsage(BaseModel):
|
||||
billed_units: CohereUsageBilledUnits | None = None
|
||||
tokens: CohereUsageTokens | None = None
|
||||
cached_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereLogprobItem(BaseModel):
|
||||
text: str | None = None
|
||||
token_ids: list[int]
|
||||
logprobs: list[float] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Response(BaseModel):
|
||||
"""Cohere Chat v2 non-streaming response body.
|
||||
|
||||
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
|
||||
the documented v2 response envelope (``id``, ``finish_reason``,
|
||||
``usage``, ``logprobs``). The single constructor in
|
||||
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
|
||||
supplying a non-empty ``id`` (falling back to a synthesized one if
|
||||
the upstream response is missing it) to this model.
|
||||
"""
|
||||
|
||||
id: str
|
||||
finish_reason: CohereFinishReason
|
||||
message: AssistantMessageResponse
|
||||
usage: CohereUsage | None = None
|
||||
logprobs: list[CohereLogprobItem] | None = None
|
||||
|
||||
# vLLM-specific extension.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming events
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
|
||||
# Sent Events; each event's ``type`` field carries the discriminator. The
|
||||
# SDK exposes a Pydantic model per event but none of them declare ``type``
|
||||
# as a field (the SDK relies on its own deserializer for discrimination),
|
||||
# so a naive ``model_dump_json()`` would silently drop the discriminator
|
||||
# and break clients that demux on ``type``.
|
||||
#
|
||||
# We therefore subclass each SDK event and bake the wire-format ``type``
|
||||
# string in as a ``Literal`` field with a default. ``model_dump()`` now
|
||||
# emits ``type`` for free, and surfaces can simply construct the event
|
||||
# class and serialize it -- no manual ``type`` parameter needed.
|
||||
#
|
||||
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
|
||||
# ``StreamedChatResponseV2`` schema for the wire-format reference.
|
||||
|
||||
|
||||
class MessageStartEvent(_sdk.ChatMessageStartEvent):
|
||||
type: Literal["message-start"] = "message-start"
|
||||
|
||||
|
||||
class ContentStartEvent(_sdk.ChatContentStartEvent):
|
||||
type: Literal["content-start"] = "content-start"
|
||||
|
||||
|
||||
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
|
||||
type: Literal["content-delta"] = "content-delta"
|
||||
|
||||
|
||||
class ContentEndEvent(_sdk.ChatContentEndEvent):
|
||||
type: Literal["content-end"] = "content-end"
|
||||
|
||||
|
||||
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
|
||||
type: Literal["tool-plan-delta"] = "tool-plan-delta"
|
||||
|
||||
|
||||
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
|
||||
type: Literal["tool-call-start"] = "tool-call-start"
|
||||
|
||||
|
||||
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
|
||||
type: Literal["tool-call-delta"] = "tool-call-delta"
|
||||
|
||||
|
||||
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
|
||||
type: Literal["tool-call-end"] = "tool-call-end"
|
||||
|
||||
|
||||
class CitationStartEvent(_sdk.CitationStartEvent):
|
||||
type: Literal["citation-start"] = "citation-start"
|
||||
|
||||
|
||||
class CitationEndEvent(_sdk.CitationEndEvent):
|
||||
type: Literal["citation-end"] = "citation-end"
|
||||
|
||||
|
||||
class MessageEndEvent(_sdk.ChatMessageEndEvent):
|
||||
type: Literal["message-end"] = "message-end"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
@@ -41,6 +43,12 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_anthropic_api_router(app)
|
||||
|
||||
from vllm.entrypoints.cohere.api_router import (
|
||||
attach_router as register_cohere_api_router,
|
||||
)
|
||||
|
||||
register_cohere_api_router(app)
|
||||
|
||||
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||
|
||||
register_generative_scoring_api_router(app)
|
||||
@@ -55,6 +63,23 @@ async def init_generate_state(
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
|
||||
# The Cohere serving handler depends on the optional `cohere` SDK for
|
||||
# its wire-format protocol models, and is additionally gated on the
|
||||
# `VLLM_ENABLE_COHERE_API` env flag (see
|
||||
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
|
||||
# entirely when the endpoint isn't going to be exposed, both because
|
||||
# the SDK may not be installed and because the serving object holds
|
||||
# nontrivial state (chat handler, warmup) that would otherwise be
|
||||
# unused.
|
||||
if envs.VLLM_ENABLE_COHERE_API:
|
||||
try:
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
except ImportError:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import (
|
||||
DemoToolServer,
|
||||
MCPToolServer,
|
||||
@@ -86,6 +111,19 @@ async def init_generate_state(
|
||||
tool_server = None
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
|
||||
# default chat-template kwargs. The cohere renderer reads
|
||||
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
|
||||
# rendering; making this a first-class flag keeps the right format
|
||||
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
|
||||
# without forcing them to hand-construct a JSON dict for
|
||||
# ``--default-chat-template-kwargs``. Per-request overrides still
|
||||
# take precedence (see ``merge_kwargs`` in
|
||||
# ``ChatCompletionRequest.build_chat_params``).
|
||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
|
||||
# Render endpoints are always backed by OnlineRenderer so that
|
||||
# /v1/chat/completions/render and /v1/completions/render work on both
|
||||
# generate-mode and render-only servers. Created in init_app_state.
|
||||
@@ -106,7 +144,7 @@ async def init_generate_state(
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
@@ -119,7 +157,7 @@ async def init_generate_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
@@ -169,11 +207,32 @@ async def init_generate_state(
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.cohere_serving_chat_v2 = (
|
||||
CohereServingChatV2(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
is_reasoning_model=args.cohere_is_reasoning_model,
|
||||
)
|
||||
if CohereServingChatV2 is not None and "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
|
||||
from .generative_scoring.serving import ServingGenerativeScoring
|
||||
|
||||
|
||||
@@ -10,7 +10,13 @@ from openai.types.chat.chat_completion_audio import (
|
||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
|
||||
from pydantic import Field, PrivateAttr, model_serializer, model_validator
|
||||
from pydantic import (
|
||||
Field,
|
||||
PrivateAttr,
|
||||
SerializeAsAny,
|
||||
model_serializer,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -91,7 +97,12 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||
|
||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
message: ChatMessage
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
|
||||
# so that added fields like ``citations`` survive JSON serialization
|
||||
# instead of being stripped down to the base schema. Plain
|
||||
# ``ChatMessage`` instances serialize identically to before.
|
||||
message: SerializeAsAny[ChatMessage]
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
@@ -139,7 +150,11 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
delta: DeltaMessage
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
|
||||
# so streaming ``citations`` survive JSON serialization. Plain
|
||||
# ``DeltaMessage`` instances serialize identically to before.
|
||||
delta: SerializeAsAny[DeltaMessage]
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
finish_reason: str | None = None
|
||||
stop_reason: int | str | None = None
|
||||
|
||||
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
return self.response_role
|
||||
return request.messages[-1]["role"]
|
||||
|
||||
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
|
||||
"""Construct the response :class:`ChatMessage` for the non-streaming path.
|
||||
|
||||
The full-generator calls this at every construction site so
|
||||
subclasses can swap in a specialized :class:`ChatMessage`
|
||||
subclass (e.g. :class:`CohereServingChatV2` returning
|
||||
:class:`CohereChatMessage`) without duplicating the branchy
|
||||
tool-choice / auto-tools logic that decides which fields are
|
||||
populated. The default returns a plain :class:`ChatMessage`.
|
||||
"""
|
||||
return ChatMessage(*args, **kwargs)
|
||||
|
||||
def _finalize_response_message(
|
||||
self,
|
||||
message: ChatMessage,
|
||||
*,
|
||||
parser: Parser | None,
|
||||
) -> ChatMessage:
|
||||
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
|
||||
|
||||
Default is a no-op. Subclasses that need to surface parser-side
|
||||
extras (e.g. :class:`CohereServingChatV2` reading grounding
|
||||
citations off the reasoning parser and populating
|
||||
:class:`CohereChatMessage.citations`) override this to inspect
|
||||
``parser`` and mutate/replace ``message``.
|
||||
"""
|
||||
return message
|
||||
|
||||
async def chat_completion_stream_generator(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
is_required_tool_choice = request.tool_choice == "required"
|
||||
|
||||
# All six construction sites route through ``self._create_chat_message``
|
||||
# so subclasses can swap in a specialized :class:`ChatMessage`
|
||||
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
|
||||
# having to duplicate this branch logic.
|
||||
if (not self.enable_auto_tools or not tool_parser_cls) and (
|
||||
not is_named_tool_choice and not is_required_tool_choice
|
||||
):
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
elif is_named_tool_choice or is_required_tool_choice:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content or "",
|
||||
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# if the request doesn't use tool choice
|
||||
# OR specifies to not use a tool
|
||||
elif not request.tool_choice or request.tool_choice == "none":
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# handle when there are tools and tool choice is auto
|
||||
elif (
|
||||
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
):
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
else:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
" if tools should be extracted. Returning a standard chat "
|
||||
"completion."
|
||||
)
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# Subclass hook: enrich the constructed message with any
|
||||
# parser-side extras that don't fit through the plain
|
||||
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
|
||||
# citation-aware handlers use this to surface grounding
|
||||
# metadata cached on the reasoning parser.
|
||||
message = self._finalize_response_message(message, parser=parser)
|
||||
|
||||
# In OpenAI's API, when a tool is called, the finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
|
||||
@@ -148,6 +148,25 @@ class BaseFrontendArgs:
|
||||
"""If set to False, output deltas will not be logged. Relevant only if
|
||||
--enable-log-outputs is set.
|
||||
"""
|
||||
cohere_is_reasoning_model: bool = True
|
||||
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
|
||||
reasoning Command-family model. When True (default), the assistant's
|
||||
chain-of-thought is surfaced as a ``thinking`` content block (or
|
||||
``content-*`` events on the stream). When False, reasoning is
|
||||
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
|
||||
events) whenever the model emits tool calls, matching older non-
|
||||
reasoning Command models. Has no effect on the non-Cohere
|
||||
endpoints."""
|
||||
cohere_format: str = "cmd4"
|
||||
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
|
||||
format to render: ``cmd4`` (current Command A+ models; default) or
|
||||
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
|
||||
wrong format silently produces a prompt the model wasn't trained
|
||||
on, which most commonly manifests as the model emitting text but no
|
||||
citations / tool calls / thinking blocks. Equivalent to passing
|
||||
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
|
||||
explicit request-level ``chat_template_kwargs.cohere_format`` takes
|
||||
priority."""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
|
||||
@@ -39,7 +39,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
|
||||
Reference in New Issue
Block a user