[Frontend] Cohere chat v2 api support (#47189)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
andrewbcohere
2026-07-31 22:53:54 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 6c91de3689
commit 9c110fa522
31 changed files with 8770 additions and 20 deletions
+425
View File
@@ -0,0 +1,425 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ``vllm/entrypoints/cohere/api_router.py``.
Covers:
* The optional-import guard: ``attach_router`` is a no-op when the
``cohere`` SDK isn't installed.
* The env-var opt-in gate: ``attach_router`` is a no-op unless
``VLLM_ENABLE_COHERE_API=1`` is set.
* The router wiring: response shapes (JSON + SSE), error translation,
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
Implemented).
"""
import json
from argparse import Namespace
from collections.abc import AsyncGenerator
from http import HTTPStatus
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.exceptions import RequestValidationError
from fastapi.testclient import TestClient
from vllm.entrypoints.cohere import api_router as api_router_mod
from vllm.entrypoints.cohere.api_router import attach_router
from vllm.entrypoints.cohere.protocol import (
AssistantMessageResponse,
CohereChatV2Response,
)
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
from vllm.entrypoints.serve.utils.server_utils import (
http_exception_handler,
validation_exception_handler,
)
@pytest.fixture(autouse=True)
def _enable_cohere_api(monkeypatch):
"""Auto-enable the Cohere API gate for every test in this module.
The endpoint is opt-in in production (``VLLM_ENABLE_COHERE_API=1``);
every test in this file exercises the enabled path *except* the
dedicated gate test in :class:`TestEnvVarGate`, which unsets the
flag inside the test body.
"""
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
# ----------------------------------------------------------------------
# Fakes
# ----------------------------------------------------------------------
class _Handler:
"""Minimal stand-in for :class:`CohereServingChatV2` used by the
router. Each test sets ``self.result`` to either:
* a :class:`CohereChatV2Response` (non-streaming JSON path);
* an async generator yielding SSE frames (streaming path);
* an :class:`ErrorResponse` (error envelope path); or
* an exception (router-level 500 path).
"""
def __init__(self, result):
self.result = result
async def create_chat_v2(self, request, raw_request):
if isinstance(self.result, Exception):
raise self.result
return self.result
def _build_app(handler: _Handler | None) -> FastAPI:
app = FastAPI()
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
return app
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
"""Build a FastAPI app that mirrors the real vLLM setup by installing
``validation_exception_handler`` and ``http_exception_handler``. The
:class:`CohereErrorEnvelopeMiddleware` registered by ``attach_router``
is expected to translate any resulting vLLM ``ErrorResponse`` body
into the ``CohereError`` wire shape.
"""
app = FastAPI()
attach_router(app)
app.state.cohere_serving_chat_v2 = handler
# ``validation_exception_handler`` reads ``req.app.state.args``; the
# real cli builds this via argparse.
app.state.args = Namespace(log_error_stack=False)
app.exception_handler(RequestValidationError)(validation_exception_handler)
app.exception_handler(HTTPException)(http_exception_handler)
return app
def _minimal_request_body() -> dict:
return {
"model": "m",
"messages": [{"role": "user", "content": "hi"}],
}
# ----------------------------------------------------------------------
# Optional-import guard
# ----------------------------------------------------------------------
class TestOptionalCohereImport:
"""``attach_router`` probes for the SDK once at module load (because
the route handler uses types imported from ``cohere``) and stashes
the result in ``_SDK_AVAILABLE``. Tests simulate the "SDK missing"
state by flipping that flag for the duration of the test.
``attach_router`` checks the env-var gate before the SDK probe, so
the SDK-missing branch is only reachable when the operator opts in
via ``VLLM_ENABLE_COHERE_API=1``. The flag-off-and-SDK-missing case
below exists to pin down that ordering — the flag-off short-circuits
"""
def test_flag_off_and_sdk_missing_stays_silent_about_sdk(self, monkeypatch, caplog):
"""Flag off doesn't do SDK-missing check.
When the operator hasn't opted in, ``attach_router`` must not
warn about the ``cohere`` SDK being missing: they never asked
for the endpoint, so surfacing the SDK gap is misleading noise.
Only the flag-off DEBUG message should fire.
"""
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
# The flag-off short-circuit ran; the SDK check never did.
assert not any(
"SDK is not installed" in rec.message for rec in caplog.records
), "SDK-missing log leaked despite the flag being off"
def test_flag_on_but_sdk_missing_logs_warning(self, monkeypatch, caplog):
"""Misconfiguration path: the operator explicitly opted into the
endpoint via ``VLLM_ENABLE_COHERE_API=1`` (already set by the
autouse fixture) but forgot to install ``cohere``.
"""
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
warn_sdk_records = [
rec
for rec in caplog.records
if "VLLM_ENABLE_COHERE_API=1" in rec.message
and "SDK is not installed" in rec.message
]
assert warn_sdk_records, (
"expected a WARNING that pairs the opt-in flag with the "
"missing SDK so operators notice the misconfiguration"
)
assert all(rec.levelname == "WARNING" for rec in warn_sdk_records)
def test_attach_router_registers_route_when_cohere_present(self):
app = _build_app(handler=None)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" in paths
# ----------------------------------------------------------------------
# VLLM_ENABLE_COHERE_API gate
# ----------------------------------------------------------------------
class TestEnvVarGate:
"""The Cohere v2 endpoint is opt-in via ``VLLM_ENABLE_COHERE_API``.
Even with the SDK installed, :func:`attach_router` must skip route
registration and middleware installation unless the env flag is
set. The autouse fixture on this module enables the flag by
default, so each test here explicitly disables it.
"""
def test_attach_router_noop_when_flag_unset(self, monkeypatch, caplog):
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
# The flag-off skip logs at DEBUG on purpose: this is the default
# state for every non-Cohere vLLM deployment, so an INFO log on
# every server startup would be pointless noise. The test raises
# caplog's level accordingly.
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
debug_flag_records = [
rec
for rec in caplog.records
if "VLLM_ENABLE_COHERE_API is not set" in rec.message
]
assert debug_flag_records, (
"expected a DEBUG message that the cohere flag is off"
)
assert all(rec.levelname == "DEBUG" for rec in debug_flag_records)
def test_attach_router_noop_when_flag_zero(self, monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "0")
app = FastAPI()
attach_router(app)
paths = [getattr(r, "path", None) for r in app.routes]
assert "/cohere/v2/chat" not in paths
# ----------------------------------------------------------------------
# Endpoint behavior
# ----------------------------------------------------------------------
class TestEndpoint:
def test_501_when_handler_missing(self):
app = _build_app(handler=None)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
body = r.json()
assert "does not support" in body["message"]
assert "id" not in body # excluded by ``exclude_none=True``
def test_non_streaming_response_is_json(self):
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hello"}])
result = CohereChatV2Response(id="r1", finish_reason="COMPLETE", message=msg)
app = _build_app(handler=_Handler(result))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("application/json")
body = r.json()
assert body["id"] == "r1"
assert body["finish_reason"] == "COMPLETE"
assert body["message"]["content"][0]["text"] == "hello"
def test_streaming_response_is_sse(self):
async def _gen() -> AsyncGenerator[str, None]:
yield 'data: {"type":"message-start"}\n\n'
yield "data: [DONE]\n\n"
app = _build_app(handler=_Handler(_gen()))
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={**_minimal_request_body(), "stream": True},
)
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("text/event-stream")
body = r.text
assert "message-start" in body
assert body.rstrip().endswith("[DONE]")
def test_error_response_translated_to_cohere_envelope(self):
err = ErrorResponse(
error=ErrorInfo(
message="bad request",
type="bad_request",
code=400,
)
)
app = _build_app(handler=_Handler(err))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
assert body == {"message": "bad request"}
def test_handler_exception_returns_500_envelope(self):
app = _build_app(handler=_Handler(RuntimeError("kaboom")))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
body = r.json()
assert body == {"message": "kaboom"}
def test_non_json_content_type_rejected(self):
"""The ``validate_json_request`` dependency raises
``RequestValidationError`` (HTTP 422) for non-JSON content
types, matching the behavior of the other vLLM API routers.
"""
app = _build_app(handler=None)
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
content=json.dumps(_minimal_request_body()),
headers={"content-type": "text/plain"},
)
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
def test_invalid_body_returns_422(self):
# ``model`` is required; omit it to trip Pydantic validation.
app = _build_app(handler=None)
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={"messages": [{"role": "user", "content": "hi"}]},
)
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
# ----------------------------------------------------------------------
# CohereErrorEnvelopeMiddleware
# ----------------------------------------------------------------------
class TestCohereErrorEnvelope:
"""When the app installs vLLM's global exception handlers, validation
and HTTP errors escape as ``ErrorResponse`` bodies before the route
handler runs. The middleware installed by ``attach_router`` must
normalise those bodies to the ``CohereError`` shape declared on the
endpoint's OpenAPI ``responses`` map so schema-conformance tests
(``test_openai_schema.py``) don't see a mismatch on ``/cohere/*``
responses.
"""
def test_validation_error_body_is_cohere_shaped(self):
# ``model=""`` and ``messages=[]`` trip our custom field
# validators, which raise pydantic ValueErrors and are routed
# through ``validation_exception_handler`` in the real vLLM
# server (producing the ``{"error": {...}}`` shape).
app = _build_app_with_vllm_handlers(handler=None)
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json={"messages": [], "model": ""})
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
# ``CohereError`` has ``message`` at the top level, not nested
# under an ``error`` envelope.
assert "error" not in body
assert "message" in body
assert isinstance(body["message"], str) and body["message"]
def test_http_error_body_is_cohere_shaped(self):
# A raised ``HTTPException`` from anywhere in the request cycle
# is routed through ``http_exception_handler`` (producing the
# ``ErrorResponse`` shape) and must be translated.
app = _build_app_with_vllm_handlers(handler=None)
@app.get("/cohere/v2/boom")
async def _boom():
raise HTTPException(status_code=418, detail="teapot")
with TestClient(app) as client:
r = client.get("/cohere/v2/boom")
assert r.status_code == 418
body = r.json()
assert body == {"message": "teapot"}
def test_non_cohere_path_is_not_translated(self):
app = _build_app_with_vllm_handlers(handler=None)
@app.get("/v1/other")
async def _other():
raise HTTPException(status_code=400, detail="nope")
with TestClient(app) as client:
r = client.get("/v1/other")
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
# Non-cohere paths keep the vLLM ``ErrorResponse`` shape.
assert "error" in body
assert body["error"]["message"] == "nope"
def test_streaming_response_passes_through(self):
# SSE responses have content-type text/event-stream; the
# middleware must never buffer these (which would break
# streaming) even though they're on ``/cohere/*``.
async def _gen() -> AsyncGenerator[str, None]:
yield 'data: {"type":"message-start"}\n\n'
yield "data: [DONE]\n\n"
app = _build_app_with_vllm_handlers(handler=_Handler(_gen()))
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={**_minimal_request_body(), "stream": True},
)
assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"].startswith("text/event-stream")
assert "message-start" in r.text
assert r.text.rstrip().endswith("[DONE]")
def test_already_cohere_shaped_body_passes_through(self):
# When the handler returns an ``ErrorResponse`` the route
# itself translates it to ``CohereError``; the middleware sees
# the ``CohereError`` shape and must leave it alone.
err = ErrorResponse(
error=ErrorInfo(message="already cohere", type="Bad Request", code=400)
)
app = _build_app_with_vllm_handlers(handler=_Handler(err))
with TestClient(app) as client:
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
# No ``error`` wrapper: the route already emitted the wire shape.
assert body == {"message": "already cohere"}
def test_request_id_preserved_in_translated_body(self):
# Client-provided ``X-Request-Id`` should be echoed as
# ``CohereError.id`` so callers can correlate failures.
app = _build_app_with_vllm_handlers(handler=None)
with TestClient(app) as client:
r = client.post(
"/cohere/v2/chat",
json={"messages": [], "model": ""},
headers={"X-Request-Id": "req-abc"},
)
assert r.status_code == HTTPStatus.BAD_REQUEST
body = r.json()
assert body.get("id") == "req-abc"
+259
View File
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
These tests spin up a real ``vllm serve`` process via
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
contract over HTTP. They mirror the pattern used by
:mod:`tests.entrypoints.anthropic.test_messages`.
Two layers of integration are covered:
1. Raw HTTP via :mod:`httpx` — always runs, verifies the wire contract.
2. Cohere SDK (``pip install cohere``) — auto-skipped when the optional
dependency isn't installed, verifies SDK-level interop.
We use a tiny generic chat model (``HuggingFaceTB/SmolLM2-135M-Instruct``,
≈135M params) so the fixture stays runnable on CPU-only laptops. The
Cohere v2 endpoint is model-agnostic and just performs the v2 ↔ OpenAI
chat-completion translation, so the choice of model only matters for
response *shape* — the translation logic itself is unit-tested
elsewhere.
"""
import json
import httpx
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
# Tiny chat-tuned model so the fixture is cheap to boot on CPU-only
# hosts (Mac, CI runners without GPUs). The Cohere v2 translation is
# completely independent of which underlying chat model is loaded.
MODEL_NAME = "HuggingFaceTB/SmolLM2-135M-Instruct"
SERVED_MODEL_NAME = "command-r-plus-08-2024"
@pytest.fixture(scope="module")
def server():
args = [
# 1k tokens is plenty for the prompts these tests send; trimming
# it from the default keeps the KV-cache footprint tiny.
"--max-model-len",
"1024",
"--max-num-seqs",
"4",
"--dtype",
"bfloat16",
"--enforce-eager",
# Advertise a Cohere model name so Cohere SDK ``model=`` calls
# round-trip without the ``model not found`` check.
"--served-model-name",
SERVED_MODEL_NAME,
# Disable the reasoning-model path so the test doesn't require
# the conversation to surface a thinking block (SmolLM2 doesn't
# emit Cohere-style reasoning tokens out of the box).
"--no-cohere-is-reasoning-model",
]
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
# startup. The default (4 GB) trips Mac's RAM check on smaller
# machines; 1 GB is more than enough for max_model_len=1024 and
# max_num_seqs=4.
# ``VLLM_ENABLE_COHERE_API=1`` flips the opt-in gate that
# ``vllm.entrypoints.cohere.api_router.attach_router`` reads to
# decide whether to register ``POST /cohere/v2/chat`` at server
# startup. Without it the fixture would boot a server that returns
# 404 for every request the tests below make.
env_dict = {
"VLLM_CPU_KVCACHE_SPACE": "1",
"VLLM_ENABLE_COHERE_API": "1",
}
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
yield remote_server
# ----------------------------------------------------------------------
# Layer 1: raw HTTP contract (no optional deps)
# ----------------------------------------------------------------------
@pytest_asyncio.fixture
async def httpx_client(server):
async with httpx.AsyncClient(
base_url=server.url_root, timeout=httpx.Timeout(120.0)
) as client:
yield client
@pytest.mark.asyncio
async def test_cohere_v2_chat_non_streaming(httpx_client: httpx.AsyncClient):
resp = await httpx_client.post(
"/cohere/v2/chat",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Say hi."}],
"max_tokens": 16,
"stream": False,
},
)
assert resp.status_code == 200, resp.text
payload = resp.json()
# The response envelope follows Cohere v2's schema.
assert "message" in payload
assert payload["message"]["role"] == "assistant"
# ``content`` is a list of content blocks; the synthesized
# ``CohereServingChatV2`` should always emit at least one ``text``
# block (it falls back to an empty block when the model returned
# nothing).
content = payload["message"]["content"]
assert isinstance(content, list) and len(content) >= 1
assert content[0]["type"] == "text"
# ``usage`` is always populated by the translator.
assert "usage" in payload
assert "billed_units" in payload["usage"]
assert "tokens" in payload["usage"]
# ``finish_reason`` is one of Cohere's enum values.
assert payload["finish_reason"] in {
"COMPLETE",
"MAX_TOKENS",
"STOP_SEQUENCE",
"TOOL_CALL",
"ERROR",
}
@pytest.mark.asyncio
async def test_cohere_v2_chat_streaming(httpx_client: httpx.AsyncClient):
"""Streaming returns SSE frames in the v2 message-lifecycle shape."""
events: list[dict] = []
async with httpx_client.stream(
"POST",
"/cohere/v2/chat",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Hi"}],
"max_tokens": 8,
"stream": True,
},
) as resp:
assert resp.status_code == 200, await resp.aread()
assert resp.headers["content-type"].startswith("text/event-stream")
async for line in resp.aiter_lines():
if not line.startswith("data: "):
continue
data = line[len("data: ") :]
if data == "[DONE]":
events.append({"type": "_DONE_"})
continue
events.append(json.loads(data))
types = [ev["type"] for ev in events]
# The lifecycle always starts with message-start and ends with [DONE]
# preceded by message-end.
assert types[0] == "message-start"
assert types[-1] == "_DONE_"
assert types[-2] == "message-end"
# message-start must carry the chunk/message id.
assert events[0].get("id")
@pytest.mark.asyncio
async def test_cohere_v2_chat_validation_error_returns_400(
httpx_client: httpx.AsyncClient,
):
# Missing required ``model`` field → FastAPI/Pydantic returns 400.
resp = await httpx_client.post(
"/cohere/v2/chat",
json={"messages": []},
)
assert resp.status_code == 400
@pytest.mark.asyncio
async def test_cohere_v2_chat_documents_field_accepted(
httpx_client: httpx.AsyncClient,
):
"""The v2 endpoint forwards ``documents`` into chat_template_kwargs.
We only assert the request is accepted and produces a 200 response —
the renderer-level effect is covered by ``tests/renderers/test_cohere.py``.
"""
resp = await httpx_client.post(
"/cohere/v2/chat",
json={
"model": SERVED_MODEL_NAME,
"messages": [{"role": "user", "content": "Summarize."}],
"documents": [{"id": "d1", "data": {"title": "T", "snippet": "S"}}],
"max_tokens": 16,
"stream": False,
},
)
assert resp.status_code == 200, resp.text
# ----------------------------------------------------------------------
# Layer 2: Cohere SDK round-trip (auto-skipped if SDK absent)
# ----------------------------------------------------------------------
@pytest_asyncio.fixture
async def cohere_async_client(server):
cohere = pytest.importorskip("cohere")
# The vLLM endpoint is mounted at ``/cohere/v2/chat`` while the
# cohere SDK targets ``${base_url}/v2/chat``; point base_url at the
# ``/cohere`` prefix so paths line up.
client = cohere.AsyncClientV2(
api_key="dummy",
base_url=server.url_for("cohere"),
)
try:
yield client
finally:
# ``AsyncClientV2`` exposes a sync close; if a future version
# adds aclose we still close cleanly.
close = getattr(client, "aclose", None) or getattr(client, "close", None)
if close is not None:
result = close()
if hasattr(result, "__await__"):
await result
@pytest.mark.asyncio
async def test_cohere_sdk_non_streaming(cohere_async_client):
resp = await cohere_async_client.chat(
model=SERVED_MODEL_NAME,
messages=[{"role": "user", "content": "Say hi."}],
max_tokens=16,
)
# SDK parses our JSON into typed objects.
assert resp.message.role == "assistant"
assert resp.message.content is not None
assert len(resp.message.content) >= 1
assert resp.message.content[0].type == "text"
assert resp.finish_reason in {
"COMPLETE",
"MAX_TOKENS",
"STOP_SEQUENCE",
"TOOL_CALL",
"ERROR",
}
@pytest.mark.asyncio
async def test_cohere_sdk_streaming(cohere_async_client):
events: list[str] = []
stream = cohere_async_client.chat_stream(
model=SERVED_MODEL_NAME,
messages=[{"role": "user", "content": "Hi"}],
max_tokens=8,
)
async for ev in stream:
events.append(ev.type)
assert events, "SDK stream yielded no events"
assert events[0] == "message-start"
assert events[-1] == "message-end"
@@ -0,0 +1,135 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for :mod:`vllm.entrypoints.cohere.cohere_chat_message`.
The module hosts the citation-carrying subclasses of the OpenAI chat
completion protocol (``CohereChatMessage`` / ``CohereDeltaMessage``) plus
the shared ``Citation`` / ``CitationSource`` shapes. These tests pin:
1. Serialization behavior we own (empty-``citations`` cleanup on the
subclass ``_serialize``, ``exclude_none`` shape, round-trip).
2. The constraint that ``type`` discriminators reject unknown values.
"""
import pytest
from pydantic import ValidationError
from vllm.entrypoints.cohere.cohere_chat_message import (
Citation,
CitationSource,
CohereChatMessage,
CohereDeltaMessage,
)
# ======================================================================
# CitationSource
# ======================================================================
class TestCitationSource:
def test_invalid_type_rejected(self):
with pytest.raises(ValidationError):
CitationSource(type="other") # type: ignore[arg-type]
def test_none_fields_excluded_from_dump(self):
s = CitationSource(type="document", id="d1")
assert s.model_dump(exclude_none=True) == {
"type": "document",
"id": "d1",
}
def test_resolved_source_survives_json_round_trip(self):
# The parser produces fully-resolved sources (see
# ``_melody_sources_to_vllm`` in
# ``vllm/reasoning/cohere_command_reasoning_parser.py``) which
# then flow through the internal streaming pipeline (parser ->
# OpenAI stream chunk -> ``_chat_completion_stream_to_v2``).
# Every field must round-trip through ``model_dump_json`` at
# that layer or the wire event ends up missing pieces.
src = CitationSource(
type="tool",
id="res_a0",
tool_output={"id": "res_a0", "text": "r"},
)
src2 = CitationSource.model_validate_json(src.model_dump_json())
assert src2.type == "tool"
assert src2.id == "res_a0"
assert src2.tool_output == {"id": "res_a0", "text": "r"}
# ======================================================================
# Citation
# ======================================================================
class TestCitation:
def test_invalid_type_rejected(self):
with pytest.raises(ValidationError):
Citation(type="OTHER") # type: ignore[arg-type]
def test_dump_excludes_none_fields(self):
c = Citation(start=0, end=5, text="hello")
dumped = c.model_dump(exclude_none=True)
assert dumped == {
"start": 0,
"end": 5,
"text": "hello",
"sources": [],
}
# ======================================================================
# CohereDeltaMessage
# ======================================================================
class TestCohereDeltaMessage:
def test_citations_omitted_from_dump_when_none(self):
# Non-grounded deltas should never carry ``citations: null`` on
# the wire even when routed through the Cohere subclass.
d = CohereDeltaMessage(content="hello")
dumped = d.model_dump(exclude_none=True)
assert "citations" not in dumped
def test_citations_dump_round_trip(self):
d = CohereDeltaMessage(citations=[Citation(start=0, end=5, text="hi")])
dumped = d.model_dump(exclude_none=True)
assert dumped["citations"][0]["text"] == "hi"
# Round-trip through the subclass to confirm the field validates.
d2 = CohereDeltaMessage.model_validate(dumped)
assert d2.citations[0].text == "hi"
# ======================================================================
# CohereChatMessage
# ======================================================================
class TestCohereChatMessage:
def test_citations_omitted_from_dump_when_none(self):
m = CohereChatMessage(role="assistant", content="hi")
dumped = m.model_dump(exclude_none=True)
assert "citations" not in dumped
def test_citations_dump_round_trip(self):
m = CohereChatMessage(
role="assistant",
content="hello",
citations=[
Citation(
start=0,
end=5,
text="hello",
sources=[CitationSource(type="document", id="d1")],
)
],
)
dumped = m.model_dump(exclude_none=True)
assert dumped["role"] == "assistant"
assert dumped["content"] == "hello"
assert dumped["citations"][0]["text"] == "hello"
assert dumped["citations"][0]["sources"][0]["id"] == "d1"
# Round-trip through the subclass to confirm the field validates.
m2 = CohereChatMessage.model_validate(dumped)
assert m2.citations[0].text == "hello"
assert m2.citations[0].sources[0].id == "d1"
+217
View File
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ``vllm/entrypoints/cohere/protocol.py``.
The module is mostly a thin wrapper around the official ``cohere`` SDK
types plus a few local additions:
* :class:`CohereError` envelope.
* :class:`CohereChatV2Request` (model required, ``max_tokens`` non-negative).
* :class:`CohereChatV2Response` plus the usage / logprob helpers.
* The streaming event subclasses that bake a wire-format ``type``
discriminator into ``model_dump()`` so SSE consumers can demux on it.
"""
import pytest
from pydantic import ValidationError
from vllm.entrypoints.cohere.protocol import (
AssistantMessageResponse,
CitationEndEvent,
CitationStartEvent,
CohereChatV2Request,
CohereChatV2Response,
CohereError,
CohereUsageTokens,
ContentDeltaEvent,
ContentEndEvent,
ContentStartEvent,
MessageEndEvent,
MessageStartEvent,
ToolCallDeltaEvent,
ToolCallEndEvent,
ToolCallStartEvent,
ToolPlanDeltaEvent,
)
# ======================================================================
# CohereError
# ======================================================================
class TestCohereError:
def test_model_dump_excludes_none(self):
err = CohereError(message="boom")
assert err.model_dump(exclude_none=True) == {"message": "boom"}
# ======================================================================
# CohereChatV2Request
# ======================================================================
class TestCohereChatV2Request:
def test_empty_model_rejected(self):
with pytest.raises(ValidationError, match="model is required"):
CohereChatV2Request(model="", messages=[{"role": "user", "content": "hi"}])
def test_negative_max_tokens_rejected(self):
with pytest.raises(ValidationError, match="non-negative"):
CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
max_tokens=-1,
)
def test_zero_max_tokens_allowed(self):
# Zero is allowed (the docs allow 0 -> return prompt only).
req = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
max_tokens=0,
)
assert req.max_tokens == 0
def test_invalid_tool_choice_rejected(self):
with pytest.raises(ValidationError):
CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
tool_choice="ANY", # not REQUIRED/NONE
)
# ======================================================================
# Usage / Logprobs
# ======================================================================
class TestUsage:
def test_tokens_serialization(self):
u = CohereUsageTokens(input_tokens=10, output_tokens=5)
assert u.model_dump() == {"input_tokens": 10.0, "output_tokens": 5.0}
# ======================================================================
# CohereChatV2Response
# ======================================================================
class TestCohereChatV2Response:
def test_invalid_finish_reason_rejected(self):
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hi"}])
with pytest.raises(ValidationError):
CohereChatV2Response(
id="r1",
finish_reason="NOT_A_REASON", # type: ignore[arg-type]
message=msg,
)
# ======================================================================
# Streaming event ``type`` discriminator baked into model_dump()
# ======================================================================
class TestStreamingEventTypeField:
"""Each event subclass adds a ``type: Literal[...]`` field with a
default so ``model_dump()`` always emits the wire-format discriminator
(the parent SDK classes don't declare ``type`` as a Pydantic field).
"""
@pytest.mark.parametrize(
"cls, expected_type, kwargs",
[
(
MessageStartEvent,
"message-start",
{"id": "a", "delta": {"message": {"role": "assistant"}}},
),
(
ContentStartEvent,
"content-start",
{
"index": 0,
"delta": {"message": {"content": {"type": "text", "text": ""}}},
},
),
(
ContentDeltaEvent,
"content-delta",
{
"index": 0,
"delta": {"message": {"content": {"text": "hi"}}},
},
),
(ContentEndEvent, "content-end", {"index": 0}),
(
ToolPlanDeltaEvent,
"tool-plan-delta",
{"delta": {"message": {"tool_plan": "thinking"}}},
),
(
ToolCallStartEvent,
"tool-call-start",
{
"index": 0,
"delta": {
"message": {
"tool_calls": {
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": ""},
}
}
},
},
),
(
ToolCallDeltaEvent,
"tool-call-delta",
{
"index": 0,
"delta": {
"message": {"tool_calls": {"function": {"arguments": "{}"}}}
},
},
),
(ToolCallEndEvent, "tool-call-end", {"index": 0}),
(
CitationStartEvent,
"citation-start",
{
"index": 0,
"delta": {
"message": {
"citations": {"start": 0, "end": 5, "text": "hello"}
}
},
},
),
(CitationEndEvent, "citation-end", {"index": 0}),
(
MessageEndEvent,
"message-end",
{"id": "a", "delta": {"finish_reason": "COMPLETE"}},
),
],
)
def test_type_field_default(self, cls, expected_type, kwargs):
ev = cls(**kwargs)
# type field is auto-populated from the Literal default.
assert ev.type == expected_type
# The discriminator must be present in the serialized payload so
# clients reading the stream can demux on it.
dumped = ev.model_dump(exclude_none=True)
assert dumped["type"] == expected_type
# Same in JSON form (what ``_emit`` serializes).
assert f'"type":"{expected_type}"' in ev.model_dump_json(exclude_none=True)
def test_type_field_cannot_be_overridden_to_wrong_value(self):
# Literal types reject any value other than the bake-in default.
with pytest.raises(ValidationError):
MessageStartEvent(
id="a",
delta={"message": {"role": "assistant"}},
type="other", # type: ignore[arg-type]
)
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sanity tests for the small registry / CLI-arg / config additions made
by the Cohere v2 chat API:
* ``vllm/renderers/registry.py``: new ``"cohere"`` renderer entry.
* ``vllm/tokenizers/registry.py``: new ``"cohere"`` tokenizer entry
(aliased to the cached HF tokenizer).
* ``vllm/entrypoints/openai/cli_args.py``: new
``cohere_is_reasoning_model`` field.
* ``vllm/config/model.py``: ``"cohere"`` added to ``TokenizerMode``
Literal.
"""
import dataclasses
import typing
import pytest
from vllm.config.model import TokenizerMode
from vllm.entrypoints.openai.cli_args import BaseFrontendArgs, make_arg_parser
from vllm.renderers.registry import RENDERER_REGISTRY
from vllm.tokenizers.registry import TokenizerRegistry
from vllm.utils.argparse_utils import FlexibleArgumentParser
class TestRendererRegistry:
def test_cohere_renderer_registered(self):
# The registry resolves to the importable ``CohereRenderer`` class.
cls = RENDERER_REGISTRY.load_renderer_cls("cohere")
assert cls.__name__ == "CohereRenderer"
# Sanity: class lives in the cohere renderer module.
assert cls.__module__ == "vllm.renderers.cohere"
class TestTokenizerRegistry:
def test_cohere_aliased_to_cached_hf_tokenizer(self):
# ``cohere`` mode uses the standard HF tokenizer; only the
# renderer stage is replaced. This test guards against accidental
# divergence.
cls = TokenizerRegistry.load_tokenizer_cls("cohere")
assert cls.__name__ == "CachedHfTokenizer"
class TestTokenizerModeLiteral:
def test_cohere_is_a_valid_tokenizer_mode(self):
# The Literal must enumerate ``"cohere"`` so engine arg parsing
# accepts ``--tokenizer-mode cohere``.
modes = typing.get_args(TokenizerMode)
assert "cohere" in modes
# ----------------------------------------------------------------------
# ``--cohere-is-reasoning-model`` CLI flag
# ----------------------------------------------------------------------
class TestCohereCliArg:
"""Verifies the new ``--cohere-is-reasoning-model`` flag end-to-end
through ``make_arg_parser`` — mirrors the pattern used in
:mod:`tests.entrypoints.openai.test_cli_args`.
"""
def test_default_value_on_dataclass_is_true(self):
fields = {f.name: f for f in dataclasses.fields(BaseFrontendArgs)}
assert "cohere_is_reasoning_model" in fields
field = fields["cohere_is_reasoning_model"]
assert field.default is True
assert field.type is bool
@pytest.fixture
def serve_parser(self) -> FlexibleArgumentParser:
parser = FlexibleArgumentParser()
return make_arg_parser(parser)
def test_default_via_argparse_is_true(self, serve_parser: FlexibleArgumentParser):
# No flag supplied → dataclass default (True) wins.
args = serve_parser.parse_args(["--model", "m"])
assert args.cohere_is_reasoning_model is True
def test_explicit_false_via_argparse(self, serve_parser: FlexibleArgumentParser):
# Boolean dataclass fields are wired up as ``--flag value`` /
# ``--no-flag`` pairs by FlexibleArgumentParser.
args = serve_parser.parse_args(
["--model", "m", "--no-cohere-is-reasoning-model"]
)
assert args.cohere_is_reasoning_model is False
def test_explicit_true_via_argparse(self, serve_parser: FlexibleArgumentParser):
args = serve_parser.parse_args(["--model", "m", "--cohere-is-reasoning-model"])
assert args.cohere_is_reasoning_model is True
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,944 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Cohere v2 SSE stream conversion in
``vllm/entrypoints/cohere/serving.py``.
The stream-translation entry point is
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
async iterable of OpenAI SSE chunks into Cohere's
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
event stream.
We test the helpers (``_StreamState``, ``_handle_*_delta``, etc.) in
isolation, plus a handful of end-to-end scenarios that exercise the
state machine.
"""
import json
from collections.abc import AsyncGenerator
from typing import Any
import pytest
from vllm.entrypoints.cohere.cohere_chat_message import (
Citation as VLLMCitation,
)
from vllm.entrypoints.cohere.cohere_chat_message import (
CitationSource,
)
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
MessageStartEvent,
)
from vllm.entrypoints.cohere.serving import (
_DONE_FRAME,
CohereServingChatV2,
ContentBlockType,
_emit,
_sse,
_StreamState,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionStreamResponse,
)
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
class _FakeServing(CohereServingChatV2):
"""Lightweight stand-in for :class:`CohereServingChatV2` that skips
the heavy ``OpenAIServingChat.__init__`` chain (which would need a
real engine client, model registry, render service, etc.).
Only ``_is_reasoning_model`` is read by the methods under test
(``_chat_completion_stream_to_v2`` and the per-delta handlers); the
rest is dead weight for unit testing.
"""
def __init__(self, is_reasoning_model: bool = True) -> None:
# Intentionally skipping super().__init__ — see class docstring.
self._is_reasoning_model = is_reasoning_model
def _serving(is_reasoning_model: bool = True) -> CohereServingChatV2:
return _FakeServing(is_reasoning_model=is_reasoning_model)
def _parse_event(frame: str) -> dict[str, Any]:
"""Strip the ``data: ... \\n\\n`` wrapper and parse the JSON payload."""
assert frame.startswith("data: ")
assert frame.endswith("\n\n")
return json.loads(frame[len("data: ") : -2])
def _make_chunk(
*,
chunk_id: str = "chunk_0",
role: str | None = None,
content: str | None = None,
reasoning: str | None = None,
tool_calls: list[dict[str, Any]] | None = None,
finish_reason: str | None = None,
usage: dict[str, Any] | None = None,
omit_choices: bool = False,
citations: list[Any] | None = None,
) -> str:
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
consumes."""
delta: dict[str, Any] = {}
if role is not None:
delta["role"] = role
if content is not None:
delta["content"] = content
if reasoning is not None:
delta["reasoning"] = reasoning
if tool_calls is not None:
delta["tool_calls"] = tool_calls
if citations is not None:
delta["citations"] = citations
payload: dict[str, Any] = {
"id": chunk_id,
"object": "chat.completion.chunk",
"created": 0,
"model": "m",
}
if not omit_choices:
payload["choices"] = [
{"index": 0, "delta": delta, "finish_reason": finish_reason}
]
else:
payload["choices"] = []
if usage is not None:
payload["usage"] = usage
# Use ChatCompletionStreamResponse to normalize the payload shape.
chunk = ChatCompletionStreamResponse.model_validate(payload)
return f"data: {chunk.model_dump_json(exclude_none=False)}\n\n"
def _make_done() -> str:
return "data: [DONE]\n\n"
async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
for item in items:
yield item
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
the emitted SSE frames."""
request = CohereChatV2Request(
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
)
gen = serving._chat_completion_stream_to_v2(_stream_from(items), request)
return [frame async for frame in gen]
# ======================================================================
# Low-level helpers: _sse, _emit, _DONE_FRAME
# ======================================================================
class TestSSEHelpers:
def test_sse_wraps_payload(self):
assert _sse("hello") == "data: hello\n\n"
def test_done_frame_constant(self):
# cohere-python and Fern-generated clients key off this exact
# sentinel; keep it byte-for-byte stable.
assert _DONE_FRAME == "data: [DONE]\n\n"
def test_emit_serializes_event_with_type_discriminator(self):
frame = _emit(
MessageStartEvent(id="abc", delta={"message": {"role": "assistant"}})
)
payload = _parse_event(frame)
assert payload["type"] == "message-start"
assert payload["id"] == "abc"
assert payload["delta"] == {"message": {"role": "assistant"}}
# ======================================================================
# _StreamState
# ======================================================================
class TestStreamState:
def test_content_index_monotonic(self):
st = _StreamState()
assert st.next_content_index() == 0
assert st.next_content_index() == 1
assert st.next_content_index() == 2
def test_citation_index_separate_from_content_index(self):
st = _StreamState()
st.next_content_index() # 0
st.next_content_index() # 1
# Citation indexing is independent of content indexing.
assert st.next_citation_index() == 0
assert st.next_citation_index() == 1
# ======================================================================
# _close_open_blocks
# ======================================================================
class TestCloseOpenBlocks:
def test_no_block_open_emits_nothing(self):
serving = _serving()
state = _StreamState()
assert serving._close_open_blocks(state) == []
def test_text_block_emits_content_end(self):
serving = _serving()
state = _StreamState()
state.active_block = ContentBlockType.TEXT
state.active_block_index = 1
out = serving._close_open_blocks(state)
assert len(out) == 1
payload = _parse_event(out[0])
assert payload == {"type": "content-end", "index": 1}
assert state.active_block is None
assert state.active_block_index is None
def test_thinking_block_emits_content_end(self):
serving = _serving()
state = _StreamState()
state.active_block = ContentBlockType.THINKING
state.active_block_index = 3
out = serving._close_open_blocks(state)
payload = _parse_event(out[0])
assert payload == {"type": "content-end", "index": 3}
def test_tool_call_block_emits_tool_call_end(self):
serving = _serving()
state = _StreamState()
state.active_block = ContentBlockType.TOOL_CALL
state.active_tool_index = 7
out = serving._close_open_blocks(state)
payload = _parse_event(out[0])
assert payload == {"type": "tool-call-end", "index": 7}
assert state.active_tool_index is None
# ======================================================================
# _handle_text_delta
# ======================================================================
class TestHandleTextDelta:
def test_opens_block_first_time(self):
serving = _serving()
state = _StreamState()
events = serving._handle_text_delta(state, "Hi")
assert len(events) == 2
start = _parse_event(events[0])
delta = _parse_event(events[1])
assert start["type"] == "content-start"
assert start["index"] == 0
assert start["delta"]["message"]["content"]["type"] == "text"
assert delta["type"] == "content-delta"
assert delta["index"] == 0
assert delta["delta"]["message"]["content"]["text"] == "Hi"
assert state.active_block == ContentBlockType.TEXT
assert state.active_block_index == 0
def test_continues_block_with_just_delta(self):
serving = _serving()
state = _StreamState()
serving._handle_text_delta(state, "Hi")
events = serving._handle_text_delta(state, " there")
# Only a delta event, no new content-start.
assert len(events) == 1
delta = _parse_event(events[0])
assert delta["type"] == "content-delta"
assert delta["delta"]["message"]["content"]["text"] == " there"
def test_switches_from_thinking_block(self):
serving = _serving(is_reasoning_model=True)
state = _StreamState()
# Open a thinking block first, then switch to text.
serving._handle_thinking_delta(state, "ponder")
events = serving._handle_text_delta(state, "answer")
types = [_parse_event(ev)["type"] for ev in events]
assert types == ["content-end", "content-start", "content-delta"]
# The text block gets a new index (1), distinct from thinking's 0.
assert _parse_event(events[1])["index"] == 1
assert state.active_block == ContentBlockType.TEXT
# ======================================================================
# _handle_thinking_delta
# ======================================================================
class TestHandleThinkingDelta:
def test_reasoning_model_opens_thinking_block(self):
serving = _serving(is_reasoning_model=True)
state = _StreamState()
events = serving._handle_thinking_delta(state, "thought")
assert len(events) == 2
start = _parse_event(events[0])
delta = _parse_event(events[1])
assert start["type"] == "content-start"
assert start["delta"]["message"]["content"]["type"] == "thinking"
assert delta["type"] == "content-delta"
assert delta["delta"]["message"]["content"]["thinking"] == "thought"
assert state.active_block == ContentBlockType.THINKING
def test_reasoning_model_continues_thinking_block(self):
serving = _serving(is_reasoning_model=True)
state = _StreamState()
serving._handle_thinking_delta(state, "first")
events = serving._handle_thinking_delta(state, " more")
assert len(events) == 1
assert _parse_event(events[0])["type"] == "content-delta"
def test_non_reasoning_model_emits_tool_plan_delta(self):
# Older Command models stream reasoning as ``tool_plan`` deltas;
# no content-start/end pair is emitted.
serving = _serving(is_reasoning_model=False)
state = _StreamState()
events = serving._handle_thinking_delta(state, "planning")
assert len(events) == 1
payload = _parse_event(events[0])
assert payload["type"] == "tool-plan-delta"
assert payload["delta"]["message"]["tool_plan"] == "planning"
# ``tool_plan`` deltas don't claim an active content block.
assert state.active_block is None
def test_non_reasoning_model_closes_open_text_block(self):
serving = _serving(is_reasoning_model=False)
state = _StreamState()
# Open a text block first.
serving._handle_text_delta(state, "answer")
events = serving._handle_thinking_delta(state, "rethink")
types = [_parse_event(ev)["type"] for ev in events]
assert types == ["content-end", "tool-plan-delta"]
# ======================================================================
# _handle_tool_call_deltas
# ======================================================================
class TestHandleToolCallDeltas:
def test_new_tool_call_opens_tool_call_start(self):
serving = _serving()
state = _StreamState()
deltas = [
type(
"Delta",
(),
{
"index": 0,
"id": "c1",
"function": type(
"Fn", (), {"name": "calc", "arguments": '{"x":'}
)(),
},
)()
]
events = serving._handle_tool_call_deltas(state, deltas)
assert len(events) == 1
payload = _parse_event(events[0])
assert payload["type"] == "tool-call-start"
assert payload["index"] == 0
tc = payload["delta"]["message"]["tool_calls"]
assert tc["id"] == "c1"
assert tc["function"]["name"] == "calc"
assert tc["function"]["arguments"] == '{"x":'
assert state.active_block == ContentBlockType.TOOL_CALL
assert state.active_tool_index == 0
assert 0 in state.tool_calls_seen
def test_subsequent_arguments_emit_delta(self):
serving = _serving()
state = _StreamState()
# First call: start.
first = [
type(
"Delta",
(),
{
"index": 0,
"id": "c1",
"function": type("Fn", (), {"name": "calc", "arguments": ""})(),
},
)()
]
serving._handle_tool_call_deltas(state, first)
# Second call: same index, additional arguments fragment.
more = [
type(
"Delta",
(),
{
"index": 0,
"id": None,
"function": type("Fn", (), {"name": None, "arguments": "1}"})(),
},
)()
]
events = serving._handle_tool_call_deltas(state, more)
assert len(events) == 1
payload = _parse_event(events[0])
assert payload["type"] == "tool-call-delta"
assert payload["index"] == 0
assert (
payload["delta"]["message"]["tool_calls"]["function"]["arguments"] == "1}"
)
def test_new_tool_call_closes_existing_content_block(self):
serving = _serving()
state = _StreamState()
# Open a text block, then start a tool call.
serving._handle_text_delta(state, "I'll call:")
deltas = [
type(
"Delta",
(),
{
"index": 0,
"id": "c1",
"function": type("Fn", (), {"name": "calc", "arguments": "{}"})(),
},
)()
]
events = serving._handle_tool_call_deltas(state, deltas)
types = [_parse_event(ev)["type"] for ev in events]
assert types == ["content-end", "tool-call-start"]
# ======================================================================
# _handle_citation_deltas
# ======================================================================
class TestHandleCitationDeltas:
"""Test the per-delta emitter. Sources are pre-resolved by the
reasoning parser via the position map forwarded through
``chat_template_kwargs`` (see :func:`_melody_sources_to_vllm` in
``vllm/reasoning/cohere_command_reasoning_parser.py``), so the
handler only has to coerce to the SDK wire shape, apply the
``THINKING_CONTENT`` -> ``PLAN`` rewrite, and drop citations with
no attributable sources.
"""
@staticmethod
def _resolved_source(**overrides: Any) -> CitationSource:
base = dict(
type="document",
id="doc_hamlet",
document={"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
)
base.update(overrides)
return CitationSource(**base)
def test_resolved_citation_emits_start_and_end_with_real_id(self):
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=11,
text="Shakespeare",
sources=[self._resolved_source()],
type="TEXT_CONTENT",
)
]
events = serving._handle_citation_deltas(state, citations)
assert len(events) == 2
start = _parse_event(events[0])
end = _parse_event(events[1])
assert start["type"] == "citation-start"
assert start["index"] == 0
cit_payload = start["delta"]["message"]["citations"]
assert cit_payload["start"] == 0
assert cit_payload["end"] == 11
assert cit_payload["text"] == "Shakespeare"
assert cit_payload["sources"] == [
{
"type": "document",
"id": "doc_hamlet",
"document": {"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
}
]
assert end == {"type": "citation-end", "index": 0}
def test_multiple_sources_survive_to_wire(self):
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="works",
sources=[
self._resolved_source(id="d0", document={"id": "d0"}),
self._resolved_source(id="d1", document={"id": "d1"}),
],
type="TEXT_CONTENT",
)
]
events = serving._handle_citation_deltas(state, citations)
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
ids = [s["id"] for s in cit_payload["sources"]]
assert ids == ["d0", "d1"]
def test_source_without_id_streamed_with_payload(self):
# Match the cohere api: id-less tool sources (from text-only
# tool results) still ride through with their ``tool_output``
# payload; only the ``id`` field is omitted.
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="hello",
sources=[CitationSource(type="tool", tool_output={"content": "hi"})],
)
]
events = serving._handle_citation_deltas(state, citations)
assert len(events) >= 1
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
assert cit_payload["sources"] == [
{"type": "tool", "tool_output": {"content": "hi"}}
]
assert "id" not in cit_payload["sources"][0]
def test_citation_with_no_sources_streamed_dropped(self):
# Citations whose source list is entirely empty are still
# dropped so we don't emit an unattributed anchor.
serving = _serving()
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="hello",
sources=[],
)
]
events = serving._handle_citation_deltas(state, citations)
assert events == []
def test_plan_rewrite_on_non_reasoning_model(self):
serving = _serving(is_reasoning_model=False)
state = _StreamState()
citations = [
VLLMCitation(
start=0,
end=5,
text="hello",
sources=[self._resolved_source()],
type="THINKING_CONTENT",
)
]
events = serving._handle_citation_deltas(state, citations)
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
assert cit_payload["type"] == "PLAN"
# ======================================================================
# _build_message_end_event
# ======================================================================
class TestBuildMessageEndEvent:
def test_without_usage(self):
serving = _serving()
frame = serving._build_message_end_event(chunk_id="abc", finish_reason="stop")
payload = _parse_event(frame)
assert payload["type"] == "message-end"
assert payload["id"] == "abc"
assert payload["delta"]["finish_reason"] == "COMPLETE"
assert "usage" not in payload["delta"]
def test_with_usage(self):
serving = _serving()
chunk = ChatCompletionStreamResponse.model_validate(
{
"id": "abc",
"object": "chat.completion.chunk",
"created": 0,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
},
}
)
frame = serving._build_message_end_event(
chunk_id="abc", finish_reason="length", usage_chunk=chunk
)
payload = _parse_event(frame)
assert payload["delta"]["finish_reason"] == "MAX_TOKENS"
usage = payload["delta"]["usage"]
assert usage["billed_units"] == {"input_tokens": 10, "output_tokens": 5}
assert usage["tokens"] == {"input_tokens": 10, "output_tokens": 5}
assert "cached_tokens" not in usage
def test_with_cached_tokens(self):
serving = _serving()
chunk = ChatCompletionStreamResponse.model_validate(
{
"id": "abc",
"object": "chat.completion.chunk",
"created": 0,
"model": "m",
"choices": [],
"usage": {
"prompt_tokens": 10,
"completion_tokens": 5,
"total_tokens": 15,
"prompt_tokens_details": {"cached_tokens": 3},
},
}
)
frame = serving._build_message_end_event(
chunk_id="abc", finish_reason="stop", usage_chunk=chunk
)
payload = _parse_event(frame)
assert payload["delta"]["usage"]["cached_tokens"] == 3
# ======================================================================
# End-to-end: _chat_completion_stream_to_v2
# ======================================================================
class TestChatCompletionStreamToV2:
"""End-to-end stream lifecycle tests."""
@pytest.mark.asyncio
async def test_text_only_happy_path_emits_full_lifecycle(self):
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="Hi"),
_make_chunk(content=" there"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 4,
"completion_tokens": 2,
"total_tokens": 6,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
# message-start, content-start, content-delta, content-delta,
# content-end, message-end, then [DONE] as the last frame.
assert types == [
"message-start",
"content-start",
"content-delta",
"content-delta",
"content-end",
"message-end",
]
assert frames[-1] == _DONE_FRAME
# message-end should carry usage stats from the trailing chunk.
end_payload = _parse_event(frames[-2])
assert end_payload["delta"]["finish_reason"] == "COMPLETE"
assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4
@pytest.mark.asyncio
async def test_text_then_tool_call_closes_text_first(self):
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="planning..."),
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "c1",
"type": "function",
"function": {"name": "calc", "arguments": "{}"},
}
]
),
_make_chunk(finish_reason="tool_calls"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
# Text block is opened+delta, then closed before the tool-call
# opens, and the final close happens on the usage chunk path.
assert types == [
"message-start",
"content-start",
"content-delta",
"content-end",
"tool-call-start",
"tool-call-end",
"message-end",
]
# finish_reason captured from the prior chunk.
end = _parse_event(frames[-2])
assert end["delta"]["finish_reason"] == "TOOL_CALL"
@pytest.mark.asyncio
async def test_thinking_then_text_reasoning_model(self):
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(reasoning="thinking..."),
_make_chunk(content="answer"),
_make_chunk(finish_reason="stop"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
# Thinking block opens with index 0; text block reopens with index 1.
assert types == [
"message-start",
"content-start",
"content-delta",
"content-end",
"content-start",
"content-delta",
"content-end",
"message-end",
]
thinking_start = _parse_event(frames[1])
assert thinking_start["delta"]["message"]["content"]["type"] == "thinking"
text_start = _parse_event(frames[4])
assert text_start["delta"]["message"]["content"]["type"] == "text"
@pytest.mark.asyncio
async def test_reasoning_on_non_reasoning_model_emits_tool_plan_delta(self):
serving = _serving(is_reasoning_model=False)
items = [
_make_chunk(role="assistant"),
_make_chunk(reasoning="planning"),
_make_chunk(
tool_calls=[
{
"index": 0,
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
]
),
_make_chunk(finish_reason="tool_calls"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types == [
"message-start",
"tool-plan-delta",
"tool-call-start",
"tool-call-end",
"message-end",
]
# No thinking content blocks should be present.
assert "content-start" not in types
assert "content-end" not in types
@pytest.mark.asyncio
async def test_done_marker_in_middle_closes_open_block(self):
# Some upstreams send [DONE] without a trailing usage-only chunk.
# The translator must still emit message-end before [DONE] so
# Cohere clients don't hang.
serving = _serving(is_reasoning_model=True)
items = [
_make_chunk(role="assistant"),
_make_chunk(content="Hi"),
_make_done(),
]
frames = await _drain(serving, items)
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types == [
"message-start",
"content-start",
"content-delta",
"content-end",
"message-end",
]
assert frames[-1] == _DONE_FRAME
@pytest.mark.asyncio
async def test_skips_empty_and_non_data_lines(self):
serving = _serving()
items = [
"\n",
"event: ping\n\n",
_make_chunk(role="assistant"),
"data: \n\n", # empty data
_make_chunk(content="Hi"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
# Should still produce a complete lifecycle.
types = [_parse_event(f)["type"] for f in frames[:-1]]
assert types[0] == "message-start"
assert types[-1] == "message-end"
@pytest.mark.asyncio
async def test_exception_in_chunk_parsing_emits_error_message_end(self):
# An invalid JSON payload after the first chunk triggers the
# error path: a synthetic message-end with finish_reason=ERROR
# followed by [DONE].
serving = _serving()
items = [
_make_chunk(role="assistant"),
"data: {not valid json}\n\n",
]
frames = await _drain(serving, items)
assert frames[-1] == _DONE_FRAME
# Find the error-shaped message-end.
error_end = _parse_event(frames[-2])
assert error_end["type"] == "message-end"
assert error_end["delta"]["finish_reason"] == "ERROR"
assert "error" in error_end["delta"]
@pytest.mark.asyncio
async def test_error_message_end_is_sanitized(self):
# The client-visible ``error`` field on the terminal message-end
# must be routed through ``sanitize_message`` so tracebacks,
# host filesystem paths, and Python object memory addresses
# can't leak from a mid-stream exception into the SSE payload.
serving = _serving()
async def _raises_mid_stream() -> AsyncGenerator[str, None]:
yield _make_chunk(role="assistant")
raise RuntimeError(
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
)
request = CohereChatV2Request(
model="m",
messages=[{"role": "user", "content": "hi"}],
stream=True,
)
frames = [
f
async for f in serving._chat_completion_stream_to_v2(
_raises_mid_stream(), request
)
]
assert frames[-1] == _DONE_FRAME
error_end = _parse_event(frames[-2])
assert error_end["delta"]["finish_reason"] == "ERROR"
err_msg = error_end["delta"]["error"]
# Memory address stripped and path replaced with ``<path>``.
assert "0x7fabc0deface" not in err_msg
assert "/Users/dev/vllm/x.py" not in err_msg
assert "<path>" in err_msg
# The human-facing prefix should still survive sanitization.
assert "boom" in err_msg
@pytest.mark.asyncio
async def test_citations_in_delta_emit_citation_events(self):
# End-to-end streaming citation flow. The parser hands us
# already-resolved sources (see ``_melody_sources_to_vllm``
# in ``vllm/reasoning/cohere_command_reasoning_parser.py``);
# the streaming loop just has to coerce and emit.
serving = _serving()
items = [
_make_chunk(role="assistant"),
_make_chunk(content="hello"),
_make_chunk(
citations=[
{
"start": 0,
"end": 5,
"text": "hello",
"sources": [
{
"type": "document",
"id": "d1",
"document": {"id": "d1", "text": "cited"},
}
],
"type": "TEXT_CONTENT",
}
]
),
_make_chunk(finish_reason="stop"),
_make_chunk(
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
parsed = [_parse_event(f) for f in frames[:-1]]
types = [p["type"] for p in parsed]
assert "citation-start" in types
assert "citation-end" in types
cit_start = next(p for p in parsed if p["type"] == "citation-start")
assert cit_start["delta"]["message"]["citations"]["sources"] == [
{
"type": "document",
"id": "d1",
"document": {"id": "d1", "text": "cited"},
}
]
@pytest.mark.asyncio
async def test_first_chunk_emits_message_start_with_chunk_id(self):
serving = _serving()
items = [
_make_chunk(chunk_id="my-id", role="assistant"),
_make_chunk(chunk_id="my-id", content="hi"),
_make_chunk(
chunk_id="my-id",
omit_choices=True,
usage={
"prompt_tokens": 1,
"completion_tokens": 1,
"total_tokens": 2,
},
),
]
frames = await _drain(serving, items)
ms = _parse_event(frames[0])
assert ms["type"] == "message-start"
assert ms["id"] == "my-id"
assert ms["delta"]["message"]["role"] == "assistant"