Add Cohere ChatV2 render endpoint (#53219)

This commit is contained in:
andrewbcohere
2026-08-24 12:27:57 +00:00
committed by GitHub
parent e239947777
commit 41ae917c12
5 changed files with 380 additions and 24 deletions
+79 -15
View File
@@ -1,24 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
"""FastAPI router for the Cohere Chat v2 API.
Exposes two routes:
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
generation, mirroring ``POST /v1/chat/completions/render``.
The Cohere v2 protocol models are sourced from the official ``cohere``
Python SDK (``pip install cohere``). To keep that an *optional*
dependency for vLLM, the SDK-dependent imports - and the route handler
itself - are gated on a one-shot probe at module load. If the SDK isn't
installed, :func:`attach_router` becomes a no-op (with an info log) and
vLLM continues to boot normally.
dependency for vLLM, the SDK-dependent imports - and the route handlers
themselves - are gated on a one-shot probe at module load. If the SDK
isn't installed, :func:`attach_router` becomes a no-op (with an info
log) and vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the route. This keeps non-Cohere deployments that pull in the SDK for
the routes. This keeps non-Cohere deployments that pull in the SDK for
unrelated reasons (e.g. test dependencies) from accidentally exposing
the api.
Note: the handler must live at module scope (not inside
Note: the handlers must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
the ``CohereChatV2Request`` body annotation against the module's
globals. Defining it locally inside ``attach_router`` would hide the
globals. Defining them locally inside ``attach_router`` would hide the
type from ``get_type_hints``, causing FastAPI to silently degrade the
body parameter into a query parameter and reject every request with
422.
@@ -61,12 +67,17 @@ if _SDK_AVAILABLE:
CohereError,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
router = APIRouter()
def _serving(request: Request) -> CohereServingChatV2 | None:
return getattr(request.app.state, "cohere_serving_chat_v2", None)
def _serving_render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
def _request_id(raw_request: Request | None) -> str | None:
"""Best-effort lookup of the active request id.
@@ -143,6 +154,59 @@ if _SDK_AVAILABLE:
case _:
return StreamingResponse(content=result, media_type="text/event-stream")
@router.post(
"/cohere/v2/chat/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
"""Tokenize a Cohere v2 chat request without running generation.
The Cohere counterpart to ``POST /v1/chat/completions/render``.
The v2 body goes through the same conversion
``/cohere/v2/chat`` uses and is then handed to the shared
:class:`ServingRender`, so the returned ``GenerateRequest``
carries the prompt tokens and sampling params the chat endpoint
would have sent to the engine.
"""
handler = _serving(raw_request)
render_handler = _serving_render(raw_request)
if handler is None or render_handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message=(
"The model does not support the Cohere v2 chat render API."
),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
chat_request = handler.to_chat_completion_request(request)
result = await render_handler.render_chat_request(chat_request)
except Exception as e: # noqa: BLE001 - report as 500 for parity
logger.exception("Error in /cohere/v2/chat/render: %s", e)
return JSONResponse(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
content=CohereError(
message=sanitize_message(str(e)),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
if isinstance(result, ErrorResponse):
return _error_response(result, raw_request)
return JSONResponse(content=result.model_dump())
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
@@ -209,12 +273,12 @@ if _SDK_AVAILABLE:
def attach_router(app: FastAPI) -> None:
"""Register ``POST /cohere/v2/chat`` on ``app``.
"""Register the ``/cohere/v2/chat`` routes on ``app``.
No-op when either:
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
Cohere v2 endpoint is opt-in because it carries Cohere-specific
Cohere v2 endpoints are opt-in because they carry Cohere-specific
request/response semantics (grounding citations, tool_plan,
PLAN/THINKING_CONTENT blocks) that are only meaningful when
serving a Cohere Command-family model.
@@ -223,21 +287,21 @@ def attach_router(app: FastAPI) -> None:
The two skip paths log at different levels: an operator who set
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
a WARNING (they explicitly asked for the endpoint and it's silently
absent), whereas the default-off skip logs at debug.
a WARNING (they explicitly asked for the endpoints and they're
silently absent), whereas the default-off skip logs at debug.
"""
enabled = envs.VLLM_ENABLE_COHERE_API
if not enabled:
logger.debug(
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
)
return
if not _SDK_AVAILABLE:
logger.warning(
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
"installed; /cohere/v2/chat will not be exposed. Install "
"with `pip install cohere` to enable the endpoint."
"with `pip install cohere` to enable the endpoints."
)
return
app.include_router(router)