Add Cohere ChatV2 render endpoint (#53219)

This commit is contained in:
andrewbcohere
2026-08-24 12:27:57 +00:00
committed by GitHub
parent e239947777
commit 41ae917c12
5 changed files with 380 additions and 24 deletions
+195
View File
@@ -11,6 +11,9 @@ Covers:
* The router wiring: response shapes (JSON + SSE), error translation,
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
Implemented).
* The render route (``POST /cohere/v2/chat/render``): that the v2 body
reaches :class:`ServingRender` as a converted ``ChatCompletionRequest``
and that the resulting ``GenerateRequest`` is returned verbatim.
"""
import json
@@ -29,7 +32,9 @@ from vllm.entrypoints.cohere.protocol import (
AssistantMessageResponse,
CohereChatV2Response,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
from vllm.entrypoints.serve.exception_handling.handlers.http import (
http_exception_handler,
)
@@ -40,6 +45,7 @@ from vllm.entrypoints.serve.exception_handling.handlers.vllm_error import (
vllm_error_handler,
)
from vllm.exceptions import VLLMError
from vllm.sampling_params import SamplingParams
@pytest.fixture(autouse=True)
@@ -78,6 +84,49 @@ class _Handler:
return self.result
class _RenderChatHandler:
"""Stand-in for :class:`CohereServingChatV2` on the render route.
Only ``to_chat_completion_request`` is reachable from
``/cohere/v2/chat/render``, and it delegates to the real conversion
classmethod (which needs no engine or tokenizer) so the tests
exercise the actual v2 -> ``ChatCompletionRequest`` translation
rather than a stub of it.
"""
def to_chat_completion_request(self, request):
return CohereServingChatV2._convert_v2_to_chat_completion(request)
class _RenderHandler:
"""Minimal stand-in for :class:`ServingRender`.
Records the ``ChatCompletionRequest`` it was handed so tests can
assert on the conversion, and returns ``self.result`` - either a
:class:`GenerateRequest`, an :class:`ErrorResponse`, or an exception
to raise.
"""
def __init__(self, result):
self.result = result
self.seen_request = None
async def render_chat_request(self, request):
self.seen_request = request
if isinstance(self.result, Exception):
raise self.result
return self.result
def _generate_request() -> GenerateRequest:
return GenerateRequest(
request_id="chatcmpl-abc",
token_ids=[5, 6, 7],
sampling_params=SamplingParams(temperature=0.25),
model="m",
)
def _build_app(handler: _Handler | None) -> FastAPI:
app = FastAPI()
attach_router(app)
@@ -85,6 +134,14 @@ def _build_app(handler: _Handler | None) -> FastAPI:
return app
def _build_render_app(chat_handler, render_handler) -> FastAPI:
app = FastAPI()
attach_router(app)
app.state.cohere_serving_chat_v2 = chat_handler
app.state.serving_render = render_handler
return app
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
"""Build a FastAPI app that mirrors the real vLLM setup by installing
validation, HTTP, and vLLM exception handlers. The
@@ -321,6 +378,144 @@ class TestEndpoint:
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
# ----------------------------------------------------------------------
# Render endpoint
# ----------------------------------------------------------------------
class TestRenderEndpoint:
"""``POST /cohere/v2/chat/render`` is the Cohere counterpart to
``POST /v1/chat/completions/render``: it converts the v2 body to a
``ChatCompletionRequest`` and hands it to the shared
``ServingRender``, returning the resulting ``GenerateRequest``
instead of running generation.
The route therefore spans *two* app-state handlers, and these tests
pin both halves: that the conversion preserves the Cohere-specific
request surface, and that the render handler's output (or error)
reaches the client unchanged.
"""
def test_route_registered(self):
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" in paths
def test_route_not_registered_when_flag_unset(self, monkeypatch):
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat/render" not in paths
def test_returns_generate_request_json(self):
render_handler = _RenderHandler(_generate_request())
app = _build_render_app(_RenderChatHandler(), render_handler)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("application/json")
body = r.json()
assert body["request_id"] == "chatcmpl-abc"
assert body["token_ids"] == [5, 6, 7]
assert body["sampling_params"]["temperature"] == 0.25
# The body must be a wire-valid GenerateRequest so callers can
# feed it straight back to /v1/generate.
assert GenerateRequest.model_validate(body).token_ids == [5, 6, 7]
def test_cohere_request_surface_survives_conversion(self):
"""The point of routing through ``CohereServingChatV2`` rather
than asking callers to pre-convert: Cohere-only fields
(``documents``, ``safety_mode``, ``citation_options``) must land
in ``chat_template_kwargs`` so the rendered prompt matches what
``/cohere/v2/chat`` would have produced.
"""
render_handler = _RenderHandler(_generate_request())
app = _build_render_app(_RenderChatHandler(), render_handler)
body = {
**_minimal_request_body(),
"documents": [{"id": "doc-1", "data": {"text": "Paris is in France."}}],
"safety_mode": "STRICT",
"max_tokens": 32,
}
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=body)
assert r.status_code == HTTPStatus.OK
seen = render_handler.seen_request
assert seen is not None
assert seen.model == "m"
assert seen.messages[0]["content"] == "hi"
# ``max_tokens`` matters because ServingRender feeds it to
# ``get_max_tokens`` when building sampling params.
assert seen.max_completion_tokens == 32
kwargs = seen.chat_template_kwargs or {}
assert kwargs["documents"][0]["id"] == "doc-1"
# The conversion lower-cases safety_mode for the renderer.
assert kwargs["safety_mode"] == "strict"
def test_501_when_chat_handler_missing(self):
app = _build_render_app(None, _RenderHandler(_generate_request()))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
assert "does not support" in r.json()["message"]
def test_501_when_render_handler_missing(self):
# A server without ``serving_render`` in app state (e.g. a
# non-generate task set) can't render, even though the Cohere
# chat handler exists.
app = _build_render_app(_RenderChatHandler(), None)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
assert "does not support" in r.json()["message"]
def test_error_response_translated_to_cohere_envelope(self):
err = ErrorResponse(
error=ErrorInfo(message="unknown model", type="NotFound", code=404)
)
app = _build_render_app(_RenderChatHandler(), _RenderHandler(err))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_FOUND
assert r.json() == {"message": "unknown model"}
def test_render_exception_returns_500_envelope(self):
app = _build_render_app(
_RenderChatHandler(), _RenderHandler(RuntimeError("tokenizer exploded"))
)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert r.json() == {"message": "tokenizer exploded"}
def test_conversion_exception_returns_500_envelope(self):
"""Conversion runs inside the same ``try`` as the render call, so
a bad v2 body that slips past Pydantic still yields the Cohere
error envelope rather than an unhandled 500 with vLLM's shape.
"""
class _Boom:
def to_chat_completion_request(self, request):
raise ValueError("unconvertible message")
app = _build_render_app(_Boom(), _RenderHandler(_generate_request()))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert r.json() == {"message": "unconvertible message"}
def test_invalid_body_returns_422(self):
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat/render",
json={"messages": [{"role": "user", "content": "hi"}]},
)
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
# ----------------------------------------------------------------------
# CohereErrorEnvelopeMiddleware
# ----------------------------------------------------------------------
+83 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
"""End-to-end integration tests for the Cohere Chat v2 endpoints
(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
These tests spin up a real ``vllm serve`` process via
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
@@ -173,6 +174,87 @@ async def test_cohere_v2_chat_validation_error_returns_400(
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
"""``/cohere/v2/chat/render`` tokenizes without running the model.
Beyond the ``GenerateRequest`` shape, this pins the cross-layer
contract that makes the endpoint useful: v2 sampling fields
(``temperature``, ``stop_sequences``) survive the v2 ->
``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
render once and generate later without re-deriving them.
"""
resp = await httpx_client.post(
"/cohere/v2/chat/render",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Say hi."}],
"max_tokens": 16,
"temperature": 0.125,
"stop_sequences": ["END"],
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
token_ids = data["token_ids"]
assert isinstance(token_ids, list) and len(token_ids) > 0
assert all(isinstance(t, int) and t >= 0 for t in token_ids)
assert data["request_id"]
sampling_params = data["sampling_params"]
assert sampling_params["temperature"] == 0.125
assert sampling_params["stop"] == ["END"]
# No generation happened, so the response carries none of the chat
# envelope the /cohere/v2/chat endpoint would return.
assert "message" not in data
assert "finish_reason" not in data
@pytest.mark.asyncio
async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
httpx_client: httpx.AsyncClient,
):
"""The rendered ids are the real templated prompt, not a debug dump.
Round-tripping them through ``/detokenize`` and finding the user
message back proves the endpoint applied the chat template and
tokenized it, which is what a caller splitting prompt construction
from generation depends on.
"""
render_resp = await httpx_client.post(
"/cohere/v2/chat/render",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"max_tokens": 8,
},
)
assert render_resp.status_code == 200, render_resp.text
token_ids = render_resp.json()["token_ids"]
detok_resp = await httpx_client.post(
"/detokenize",
json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
)
assert detok_resp.status_code == 200, detok_resp.text
prompt = detok_resp.json()["prompt"]
assert "What is the capital of France?" in prompt
@pytest.mark.asyncio
async def test_cohere_v2_chat_render_validation_error_returns_400(
httpx_client: httpx.AsyncClient,
):
"""Bad bodies get the Cohere error envelope, not vLLM's."""
resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
assert resp.status_code == 400
body = resp.json()
assert "error" not in body
assert body.get("message")
@pytest.mark.asyncio
async def test_cohere_v2_chat_documents_field_accepted(
httpx_client: httpx.AsyncClient,
+79 -15
View File
@@ -1,24 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
"""FastAPI router for the Cohere Chat v2 API.
Exposes two routes:
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
generation, mirroring ``POST /v1/chat/completions/render``.
The Cohere v2 protocol models are sourced from the official ``cohere``
Python SDK (``pip install cohere``). To keep that an *optional*
dependency for vLLM, the SDK-dependent imports - and the route handler
itself - are gated on a one-shot probe at module load. If the SDK isn't
installed, :func:`attach_router` becomes a no-op (with an info log) and
vLLM continues to boot normally.
dependency for vLLM, the SDK-dependent imports - and the route handlers
themselves - are gated on a one-shot probe at module load. If the SDK
isn't installed, :func:`attach_router` becomes a no-op (with an info
log) and vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the route. This keeps non-Cohere deployments that pull in the SDK for
the routes. This keeps non-Cohere deployments that pull in the SDK for
unrelated reasons (e.g. test dependencies) from accidentally exposing
the api.
Note: the handler must live at module scope (not inside
Note: the handlers must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
the ``CohereChatV2Request`` body annotation against the module's
globals. Defining it locally inside ``attach_router`` would hide the
globals. Defining them locally inside ``attach_router`` would hide the
type from ``get_type_hints``, causing FastAPI to silently degrade the
body parameter into a query parameter and reject every request with
422.
@@ -61,12 +67,17 @@ if _SDK_AVAILABLE:
CohereError,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
from vllm.entrypoints.scale_out.render.serving import ServingRender
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
router = APIRouter()
def _serving(request: Request) -> CohereServingChatV2 | None:
return getattr(request.app.state, "cohere_serving_chat_v2", None)
def _serving_render(request: Request) -> ServingRender | None:
return getattr(request.app.state, "serving_render", None)
def _request_id(raw_request: Request | None) -> str | None:
"""Best-effort lookup of the active request id.
@@ -143,6 +154,59 @@ if _SDK_AVAILABLE:
case _:
return StreamingResponse(content=result, media_type="text/event-stream")
@router.post(
"/cohere/v2/chat/render",
dependencies=[Depends(validate_json_request)],
response_model=GenerateRequest,
responses={
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
"""Tokenize a Cohere v2 chat request without running generation.
The Cohere counterpart to ``POST /v1/chat/completions/render``.
The v2 body goes through the same conversion
``/cohere/v2/chat`` uses and is then handed to the shared
:class:`ServingRender`, so the returned ``GenerateRequest``
carries the prompt tokens and sampling params the chat endpoint
would have sent to the engine.
"""
handler = _serving(raw_request)
render_handler = _serving_render(raw_request)
if handler is None or render_handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message=(
"The model does not support the Cohere v2 chat render API."
),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
chat_request = handler.to_chat_completion_request(request)
result = await render_handler.render_chat_request(chat_request)
except Exception as e: # noqa: BLE001 - report as 500 for parity
logger.exception("Error in /cohere/v2/chat/render: %s", e)
return JSONResponse(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
content=CohereError(
message=sanitize_message(str(e)),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
if isinstance(result, ErrorResponse):
return _error_response(result, raw_request)
return JSONResponse(content=result.model_dump())
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
@@ -209,12 +273,12 @@ if _SDK_AVAILABLE:
def attach_router(app: FastAPI) -> None:
"""Register ``POST /cohere/v2/chat`` on ``app``.
"""Register the ``/cohere/v2/chat`` routes on ``app``.
No-op when either:
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
Cohere v2 endpoint is opt-in because it carries Cohere-specific
Cohere v2 endpoints are opt-in because they carry Cohere-specific
request/response semantics (grounding citations, tool_plan,
PLAN/THINKING_CONTENT blocks) that are only meaningful when
serving a Cohere Command-family model.
@@ -223,21 +287,21 @@ def attach_router(app: FastAPI) -> None:
The two skip paths log at different levels: an operator who set
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
a WARNING (they explicitly asked for the endpoint and it's silently
absent), whereas the default-off skip logs at debug.
a WARNING (they explicitly asked for the endpoints and they're
silently absent), whereas the default-off skip logs at debug.
"""
enabled = envs.VLLM_ENABLE_COHERE_API
if not enabled:
logger.debug(
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
)
return
if not _SDK_AVAILABLE:
logger.warning(
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
"installed; /cohere/v2/chat will not be exposed. Install "
"with `pip install cohere` to enable the endpoint."
"with `pip install cohere` to enable the endpoints."
)
return
app.include_router(router)
+21 -7
View File
@@ -304,13 +304,7 @@ class CohereServingChatV2(OpenAIServingChat):
"Received Cohere v2 chat request %s", request.model_dump_json()
)
chat_req = self._convert_v2_to_chat_completion(request)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"Converted Cohere v2 -> ChatCompletion: %s",
chat_req.model_dump_json(),
)
chat_req = self.to_chat_completion_request(request)
generator = await self.create_chat_completion(chat_req, raw_request)
match generator:
@@ -321,6 +315,26 @@ class CohereServingChatV2(OpenAIServingChat):
case _:
return self._chat_completion_stream_to_v2(generator, request)
def to_chat_completion_request(
self, request: CohereChatV2Request
) -> ChatCompletionRequest:
"""Convert a Cohere v2 request into its ``ChatCompletionRequest``.
Exposed for callers that need the converted request without
running generation - notably the render endpoint
(``POST /cohere/v2/chat/render``), which hands the result to
:meth:`vllm.entrypoints.scale_out.render.serving.ServingRender.render_chat_request`
so that Cohere requests tokenize through exactly the same path
as ``/v1/chat/completions/render``.
"""
chat_req = self._convert_v2_to_chat_completion(request)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"Converted Cohere v2 -> ChatCompletion: %s",
chat_req.model_dump_json(),
)
return chat_req
def _engine_chat_template_kwargs(
self, chat_template_kwargs: dict[str, Any]
) -> dict[str, Any]:
+2 -1
View File
@@ -1868,7 +1868,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
),
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``.
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
# and its render endpoint at ``POST /cohere/v2/chat/render``.
# Default off
"VLLM_ENABLE_COHERE_API": lambda: bool(
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))