Add Cohere ChatV2 render endpoint (#53219)

This commit is contained in:
andrewbcohere
2026-08-24 12:27:57 +00:00
committed by GitHub
parent e239947777
commit 41ae917c12
5 changed files with 380 additions and 24 deletions
+83 -1
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
"""End-to-end integration tests for the Cohere Chat v2 endpoints
(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
These tests spin up a real ``vllm serve`` process via
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
@@ -173,6 +174,87 @@ async def test_cohere_v2_chat_validation_error_returns_400(
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
"""``/cohere/v2/chat/render`` tokenizes without running the model.
Beyond the ``GenerateRequest`` shape, this pins the cross-layer
contract that makes the endpoint useful: v2 sampling fields
(``temperature``, ``stop_sequences``) survive the v2 ->
``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
render once and generate later without re-deriving them.
"""
resp = await httpx_client.post(
"/cohere/v2/chat/render",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Say hi."}],
"max_tokens": 16,
"temperature": 0.125,
"stop_sequences": ["END"],
},
)
assert resp.status_code == 200, resp.text
data = resp.json()
token_ids = data["token_ids"]
assert isinstance(token_ids, list) and len(token_ids) > 0
assert all(isinstance(t, int) and t >= 0 for t in token_ids)
assert data["request_id"]
sampling_params = data["sampling_params"]
assert sampling_params["temperature"] == 0.125
assert sampling_params["stop"] == ["END"]
# No generation happened, so the response carries none of the chat
# envelope the /cohere/v2/chat endpoint would return.
assert "message" not in data
assert "finish_reason" not in data
@pytest.mark.asyncio
async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
httpx_client: httpx.AsyncClient,
):
"""The rendered ids are the real templated prompt, not a debug dump.
Round-tripping them through ``/detokenize`` and finding the user
message back proves the endpoint applied the chat template and
tokenized it, which is what a caller splitting prompt construction
from generation depends on.
"""
render_resp = await httpx_client.post(
"/cohere/v2/chat/render",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"max_tokens": 8,
},
)
assert render_resp.status_code == 200, render_resp.text
token_ids = render_resp.json()["token_ids"]
detok_resp = await httpx_client.post(
"/detokenize",
json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
)
assert detok_resp.status_code == 200, detok_resp.text
prompt = detok_resp.json()["prompt"]
assert "What is the capital of France?" in prompt
@pytest.mark.asyncio
async def test_cohere_v2_chat_render_validation_error_returns_400(
httpx_client: httpx.AsyncClient,
):
"""Bad bodies get the Cohere error envelope, not vLLM's."""
resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
assert resp.status_code == 400
body = resp.json()
assert "error" not in body
assert body.get("message")
@pytest.mark.asyncio
async def test_cohere_v2_chat_documents_field_accepted(
httpx_client: httpx.AsyncClient,