Add Cohere ChatV2 render endpoint (#53219)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
|
||||
"""End-to-end integration tests for the Cohere Chat v2 endpoints
|
||||
(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
|
||||
|
||||
These tests spin up a real ``vllm serve`` process via
|
||||
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
|
||||
@@ -173,6 +174,87 @@ async def test_cohere_v2_chat_validation_error_returns_400(
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
|
||||
"""``/cohere/v2/chat/render`` tokenizes without running the model.
|
||||
|
||||
Beyond the ``GenerateRequest`` shape, this pins the cross-layer
|
||||
contract that makes the endpoint useful: v2 sampling fields
|
||||
(``temperature``, ``stop_sequences``) survive the v2 ->
|
||||
``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
|
||||
render once and generate later without re-deriving them.
|
||||
"""
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Say hi."}],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0.125,
|
||||
"stop_sequences": ["END"],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
|
||||
token_ids = data["token_ids"]
|
||||
assert isinstance(token_ids, list) and len(token_ids) > 0
|
||||
assert all(isinstance(t, int) and t >= 0 for t in token_ids)
|
||||
assert data["request_id"]
|
||||
|
||||
sampling_params = data["sampling_params"]
|
||||
assert sampling_params["temperature"] == 0.125
|
||||
assert sampling_params["stop"] == ["END"]
|
||||
|
||||
# No generation happened, so the response carries none of the chat
|
||||
# envelope the /cohere/v2/chat endpoint would return.
|
||||
assert "message" not in data
|
||||
assert "finish_reason" not in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""The rendered ids are the real templated prompt, not a debug dump.
|
||||
|
||||
Round-tripping them through ``/detokenize`` and finding the user
|
||||
message back proves the endpoint applied the chat template and
|
||||
tokenized it, which is what a caller splitting prompt construction
|
||||
from generation depends on.
|
||||
"""
|
||||
render_resp = await httpx_client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"max_tokens": 8,
|
||||
},
|
||||
)
|
||||
assert render_resp.status_code == 200, render_resp.text
|
||||
token_ids = render_resp.json()["token_ids"]
|
||||
|
||||
detok_resp = await httpx_client.post(
|
||||
"/detokenize",
|
||||
json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
|
||||
)
|
||||
assert detok_resp.status_code == 200, detok_resp.text
|
||||
prompt = detok_resp.json()["prompt"]
|
||||
assert "What is the capital of France?" in prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render_validation_error_returns_400(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""Bad bodies get the Cohere error envelope, not vLLM's."""
|
||||
resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
|
||||
assert resp.status_code == 400
|
||||
body = resp.json()
|
||||
assert "error" not in body
|
||||
assert body.get("message")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_documents_field_accepted(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
|
||||
Reference in New Issue
Block a user