[Frontend] Cohere chat v2 api support (#47189)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6c91de3689
commit
9c110fa522
@@ -0,0 +1,425 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``vllm/entrypoints/cohere/api_router.py``.
|
||||
|
||||
Covers:
|
||||
|
||||
* The optional-import guard: ``attach_router`` is a no-op when the
|
||||
``cohere`` SDK isn't installed.
|
||||
* The env-var opt-in gate: ``attach_router`` is a no-op unless
|
||||
``VLLM_ENABLE_COHERE_API=1`` is set.
|
||||
* The router wiring: response shapes (JSON + SSE), error translation,
|
||||
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
|
||||
Implemented).
|
||||
"""
|
||||
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from collections.abc import AsyncGenerator
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.cohere import api_router as api_router_mod
|
||||
from vllm.entrypoints.cohere.api_router import attach_router
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
AssistantMessageResponse,
|
||||
CohereChatV2Response,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.server_utils import (
|
||||
http_exception_handler,
|
||||
validation_exception_handler,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _enable_cohere_api(monkeypatch):
|
||||
"""Auto-enable the Cohere API gate for every test in this module.
|
||||
|
||||
The endpoint is opt-in in production (``VLLM_ENABLE_COHERE_API=1``);
|
||||
every test in this file exercises the enabled path *except* the
|
||||
dedicated gate test in :class:`TestEnvVarGate`, which unsets the
|
||||
flag inside the test body.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "1")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Fakes
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class _Handler:
|
||||
"""Minimal stand-in for :class:`CohereServingChatV2` used by the
|
||||
router. Each test sets ``self.result`` to either:
|
||||
|
||||
* a :class:`CohereChatV2Response` (non-streaming JSON path);
|
||||
* an async generator yielding SSE frames (streaming path);
|
||||
* an :class:`ErrorResponse` (error envelope path); or
|
||||
* an exception (router-level 500 path).
|
||||
"""
|
||||
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
|
||||
async def create_chat_v2(self, request, raw_request):
|
||||
if isinstance(self.result, Exception):
|
||||
raise self.result
|
||||
return self.result
|
||||
|
||||
|
||||
def _build_app(handler: _Handler | None) -> FastAPI:
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
return app
|
||||
|
||||
|
||||
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
|
||||
"""Build a FastAPI app that mirrors the real vLLM setup by installing
|
||||
``validation_exception_handler`` and ``http_exception_handler``. The
|
||||
:class:`CohereErrorEnvelopeMiddleware` registered by ``attach_router``
|
||||
is expected to translate any resulting vLLM ``ErrorResponse`` body
|
||||
into the ``CohereError`` wire shape.
|
||||
"""
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = handler
|
||||
# ``validation_exception_handler`` reads ``req.app.state.args``; the
|
||||
# real cli builds this via argparse.
|
||||
app.state.args = Namespace(log_error_stack=False)
|
||||
app.exception_handler(RequestValidationError)(validation_exception_handler)
|
||||
app.exception_handler(HTTPException)(http_exception_handler)
|
||||
return app
|
||||
|
||||
|
||||
def _minimal_request_body() -> dict:
|
||||
return {
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Optional-import guard
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestOptionalCohereImport:
|
||||
"""``attach_router`` probes for the SDK once at module load (because
|
||||
the route handler uses types imported from ``cohere``) and stashes
|
||||
the result in ``_SDK_AVAILABLE``. Tests simulate the "SDK missing"
|
||||
state by flipping that flag for the duration of the test.
|
||||
|
||||
``attach_router`` checks the env-var gate before the SDK probe, so
|
||||
the SDK-missing branch is only reachable when the operator opts in
|
||||
via ``VLLM_ENABLE_COHERE_API=1``. The flag-off-and-SDK-missing case
|
||||
below exists to pin down that ordering — the flag-off short-circuits
|
||||
"""
|
||||
|
||||
def test_flag_off_and_sdk_missing_stays_silent_about_sdk(self, monkeypatch, caplog):
|
||||
"""Flag off doesn't do SDK-missing check.
|
||||
|
||||
When the operator hasn't opted in, ``attach_router`` must not
|
||||
warn about the ``cohere`` SDK being missing: they never asked
|
||||
for the endpoint, so surfacing the SDK gap is misleading noise.
|
||||
Only the flag-off DEBUG message should fire.
|
||||
"""
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
|
||||
|
||||
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
# The flag-off short-circuit ran; the SDK check never did.
|
||||
assert not any(
|
||||
"SDK is not installed" in rec.message for rec in caplog.records
|
||||
), "SDK-missing log leaked despite the flag being off"
|
||||
|
||||
def test_flag_on_but_sdk_missing_logs_warning(self, monkeypatch, caplog):
|
||||
"""Misconfiguration path: the operator explicitly opted into the
|
||||
endpoint via ``VLLM_ENABLE_COHERE_API=1`` (already set by the
|
||||
autouse fixture) but forgot to install ``cohere``.
|
||||
"""
|
||||
monkeypatch.setattr(api_router_mod, "_SDK_AVAILABLE", False)
|
||||
|
||||
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
warn_sdk_records = [
|
||||
rec
|
||||
for rec in caplog.records
|
||||
if "VLLM_ENABLE_COHERE_API=1" in rec.message
|
||||
and "SDK is not installed" in rec.message
|
||||
]
|
||||
assert warn_sdk_records, (
|
||||
"expected a WARNING that pairs the opt-in flag with the "
|
||||
"missing SDK so operators notice the misconfiguration"
|
||||
)
|
||||
assert all(rec.levelname == "WARNING" for rec in warn_sdk_records)
|
||||
|
||||
def test_attach_router_registers_route_when_cohere_present(self):
|
||||
app = _build_app(handler=None)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" in paths
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# VLLM_ENABLE_COHERE_API gate
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEnvVarGate:
|
||||
"""The Cohere v2 endpoint is opt-in via ``VLLM_ENABLE_COHERE_API``.
|
||||
|
||||
Even with the SDK installed, :func:`attach_router` must skip route
|
||||
registration and middleware installation unless the env flag is
|
||||
set. The autouse fixture on this module enables the flag by
|
||||
default, so each test here explicitly disables it.
|
||||
"""
|
||||
|
||||
def test_attach_router_noop_when_flag_unset(self, monkeypatch, caplog):
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
|
||||
# The flag-off skip logs at DEBUG on purpose: this is the default
|
||||
# state for every non-Cohere vLLM deployment, so an INFO log on
|
||||
# every server startup would be pointless noise. The test raises
|
||||
# caplog's level accordingly.
|
||||
with caplog.at_level("DEBUG", logger="vllm.entrypoints.cohere.api_router"):
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
debug_flag_records = [
|
||||
rec
|
||||
for rec in caplog.records
|
||||
if "VLLM_ENABLE_COHERE_API is not set" in rec.message
|
||||
]
|
||||
assert debug_flag_records, (
|
||||
"expected a DEBUG message that the cohere flag is off"
|
||||
)
|
||||
assert all(rec.levelname == "DEBUG" for rec in debug_flag_records)
|
||||
|
||||
def test_attach_router_noop_when_flag_zero(self, monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_COHERE_API", "0")
|
||||
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat" not in paths
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Endpoint behavior
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEndpoint:
|
||||
def test_501_when_handler_missing(self):
|
||||
app = _build_app(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
|
||||
body = r.json()
|
||||
assert "does not support" in body["message"]
|
||||
assert "id" not in body # excluded by ``exclude_none=True``
|
||||
|
||||
def test_non_streaming_response_is_json(self):
|
||||
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hello"}])
|
||||
result = CohereChatV2Response(id="r1", finish_reason="COMPLETE", message=msg)
|
||||
app = _build_app(handler=_Handler(result))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("application/json")
|
||||
body = r.json()
|
||||
assert body["id"] == "r1"
|
||||
assert body["finish_reason"] == "COMPLETE"
|
||||
assert body["message"]["content"][0]["text"] == "hello"
|
||||
|
||||
def test_streaming_response_is_sse(self):
|
||||
async def _gen() -> AsyncGenerator[str, None]:
|
||||
yield 'data: {"type":"message-start"}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
app = _build_app(handler=_Handler(_gen()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={**_minimal_request_body(), "stream": True},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("text/event-stream")
|
||||
body = r.text
|
||||
assert "message-start" in body
|
||||
assert body.rstrip().endswith("[DONE]")
|
||||
|
||||
def test_error_response_translated_to_cohere_envelope(self):
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(
|
||||
message="bad request",
|
||||
type="bad_request",
|
||||
code=400,
|
||||
)
|
||||
)
|
||||
app = _build_app(handler=_Handler(err))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
assert body == {"message": "bad request"}
|
||||
|
||||
def test_handler_exception_returns_500_envelope(self):
|
||||
app = _build_app(handler=_Handler(RuntimeError("kaboom")))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
body = r.json()
|
||||
assert body == {"message": "kaboom"}
|
||||
|
||||
def test_non_json_content_type_rejected(self):
|
||||
"""The ``validate_json_request`` dependency raises
|
||||
``RequestValidationError`` (HTTP 422) for non-JSON content
|
||||
types, matching the behavior of the other vLLM API routers.
|
||||
"""
|
||||
app = _build_app(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
content=json.dumps(_minimal_request_body()),
|
||||
headers={"content-type": "text/plain"},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
def test_invalid_body_returns_422(self):
|
||||
# ``model`` is required; omit it to trip Pydantic validation.
|
||||
app = _build_app(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={"messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# CohereErrorEnvelopeMiddleware
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCohereErrorEnvelope:
|
||||
"""When the app installs vLLM's global exception handlers, validation
|
||||
and HTTP errors escape as ``ErrorResponse`` bodies before the route
|
||||
handler runs. The middleware installed by ``attach_router`` must
|
||||
normalise those bodies to the ``CohereError`` shape declared on the
|
||||
endpoint's OpenAPI ``responses`` map so schema-conformance tests
|
||||
(``test_openai_schema.py``) don't see a mismatch on ``/cohere/*``
|
||||
responses.
|
||||
"""
|
||||
|
||||
def test_validation_error_body_is_cohere_shaped(self):
|
||||
# ``model=""`` and ``messages=[]`` trip our custom field
|
||||
# validators, which raise pydantic ValueErrors and are routed
|
||||
# through ``validation_exception_handler`` in the real vLLM
|
||||
# server (producing the ``{"error": {...}}`` shape).
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json={"messages": [], "model": ""})
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
# ``CohereError`` has ``message`` at the top level, not nested
|
||||
# under an ``error`` envelope.
|
||||
assert "error" not in body
|
||||
assert "message" in body
|
||||
assert isinstance(body["message"], str) and body["message"]
|
||||
|
||||
def test_http_error_body_is_cohere_shaped(self):
|
||||
# A raised ``HTTPException`` from anywhere in the request cycle
|
||||
# is routed through ``http_exception_handler`` (producing the
|
||||
# ``ErrorResponse`` shape) and must be translated.
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
|
||||
@app.get("/cohere/v2/boom")
|
||||
async def _boom():
|
||||
raise HTTPException(status_code=418, detail="teapot")
|
||||
|
||||
with TestClient(app) as client:
|
||||
r = client.get("/cohere/v2/boom")
|
||||
assert r.status_code == 418
|
||||
body = r.json()
|
||||
assert body == {"message": "teapot"}
|
||||
|
||||
def test_non_cohere_path_is_not_translated(self):
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
|
||||
@app.get("/v1/other")
|
||||
async def _other():
|
||||
raise HTTPException(status_code=400, detail="nope")
|
||||
|
||||
with TestClient(app) as client:
|
||||
r = client.get("/v1/other")
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
# Non-cohere paths keep the vLLM ``ErrorResponse`` shape.
|
||||
assert "error" in body
|
||||
assert body["error"]["message"] == "nope"
|
||||
|
||||
def test_streaming_response_passes_through(self):
|
||||
# SSE responses have content-type text/event-stream; the
|
||||
# middleware must never buffer these (which would break
|
||||
# streaming) even though they're on ``/cohere/*``.
|
||||
async def _gen() -> AsyncGenerator[str, None]:
|
||||
yield 'data: {"type":"message-start"}\n\n'
|
||||
yield "data: [DONE]\n\n"
|
||||
|
||||
app = _build_app_with_vllm_handlers(handler=_Handler(_gen()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={**_minimal_request_body(), "stream": True},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("text/event-stream")
|
||||
assert "message-start" in r.text
|
||||
assert r.text.rstrip().endswith("[DONE]")
|
||||
|
||||
def test_already_cohere_shaped_body_passes_through(self):
|
||||
# When the handler returns an ``ErrorResponse`` the route
|
||||
# itself translates it to ``CohereError``; the middleware sees
|
||||
# the ``CohereError`` shape and must leave it alone.
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(message="already cohere", type="Bad Request", code=400)
|
||||
)
|
||||
app = _build_app_with_vllm_handlers(handler=_Handler(err))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
# No ``error`` wrapper: the route already emitted the wire shape.
|
||||
assert body == {"message": "already cohere"}
|
||||
|
||||
def test_request_id_preserved_in_translated_body(self):
|
||||
# Client-provided ``X-Request-Id`` should be echoed as
|
||||
# ``CohereError.id`` so callers can correlate failures.
|
||||
app = _build_app_with_vllm_handlers(handler=None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={"messages": [], "model": ""},
|
||||
headers={"X-Request-Id": "req-abc"},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.BAD_REQUEST
|
||||
body = r.json()
|
||||
assert body.get("id") == "req-abc"
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
|
||||
|
||||
These tests spin up a real ``vllm serve`` process via
|
||||
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
|
||||
contract over HTTP. They mirror the pattern used by
|
||||
:mod:`tests.entrypoints.anthropic.test_messages`.
|
||||
|
||||
Two layers of integration are covered:
|
||||
|
||||
1. Raw HTTP via :mod:`httpx` — always runs, verifies the wire contract.
|
||||
2. Cohere SDK (``pip install cohere``) — auto-skipped when the optional
|
||||
dependency isn't installed, verifies SDK-level interop.
|
||||
|
||||
We use a tiny generic chat model (``HuggingFaceTB/SmolLM2-135M-Instruct``,
|
||||
≈135M params) so the fixture stays runnable on CPU-only laptops. The
|
||||
Cohere v2 endpoint is model-agnostic and just performs the v2 ↔ OpenAI
|
||||
chat-completion translation, so the choice of model only matters for
|
||||
response *shape* — the translation logic itself is unit-tested
|
||||
elsewhere.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
# Tiny chat-tuned model so the fixture is cheap to boot on CPU-only
|
||||
# hosts (Mac, CI runners without GPUs). The Cohere v2 translation is
|
||||
# completely independent of which underlying chat model is loaded.
|
||||
MODEL_NAME = "HuggingFaceTB/SmolLM2-135M-Instruct"
|
||||
SERVED_MODEL_NAME = "command-r-plus-08-2024"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
# 1k tokens is plenty for the prompts these tests send; trimming
|
||||
# it from the default keeps the KV-cache footprint tiny.
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--enforce-eager",
|
||||
# Advertise a Cohere model name so Cohere SDK ``model=`` calls
|
||||
# round-trip without the ``model not found`` check.
|
||||
"--served-model-name",
|
||||
SERVED_MODEL_NAME,
|
||||
# Disable the reasoning-model path so the test doesn't require
|
||||
# the conversation to surface a thinking block (SmolLM2 doesn't
|
||||
# emit Cohere-style reasoning tokens out of the box).
|
||||
"--no-cohere-is-reasoning-model",
|
||||
]
|
||||
|
||||
# Cap the CPU KV-cache pool the vLLM CPU backend reserves at
|
||||
# startup. The default (4 GB) trips Mac's RAM check on smaller
|
||||
# machines; 1 GB is more than enough for max_model_len=1024 and
|
||||
# max_num_seqs=4.
|
||||
# ``VLLM_ENABLE_COHERE_API=1`` flips the opt-in gate that
|
||||
# ``vllm.entrypoints.cohere.api_router.attach_router`` reads to
|
||||
# decide whether to register ``POST /cohere/v2/chat`` at server
|
||||
# startup. Without it the fixture would boot a server that returns
|
||||
# 404 for every request the tests below make.
|
||||
env_dict = {
|
||||
"VLLM_CPU_KVCACHE_SPACE": "1",
|
||||
"VLLM_ENABLE_COHERE_API": "1",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Layer 1: raw HTTP contract (no optional deps)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def httpx_client(server):
|
||||
async with httpx.AsyncClient(
|
||||
base_url=server.url_root, timeout=httpx.Timeout(120.0)
|
||||
) as client:
|
||||
yield client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_non_streaming(httpx_client: httpx.AsyncClient):
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Say hi."}],
|
||||
"max_tokens": 16,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
payload = resp.json()
|
||||
# The response envelope follows Cohere v2's schema.
|
||||
assert "message" in payload
|
||||
assert payload["message"]["role"] == "assistant"
|
||||
# ``content`` is a list of content blocks; the synthesized
|
||||
# ``CohereServingChatV2`` should always emit at least one ``text``
|
||||
# block (it falls back to an empty block when the model returned
|
||||
# nothing).
|
||||
content = payload["message"]["content"]
|
||||
assert isinstance(content, list) and len(content) >= 1
|
||||
assert content[0]["type"] == "text"
|
||||
# ``usage`` is always populated by the translator.
|
||||
assert "usage" in payload
|
||||
assert "billed_units" in payload["usage"]
|
||||
assert "tokens" in payload["usage"]
|
||||
# ``finish_reason`` is one of Cohere's enum values.
|
||||
assert payload["finish_reason"] in {
|
||||
"COMPLETE",
|
||||
"MAX_TOKENS",
|
||||
"STOP_SEQUENCE",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_streaming(httpx_client: httpx.AsyncClient):
|
||||
"""Streaming returns SSE frames in the v2 message-lifecycle shape."""
|
||||
events: list[dict] = []
|
||||
async with httpx_client.stream(
|
||||
"POST",
|
||||
"/cohere/v2/chat",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"max_tokens": 8,
|
||||
"stream": True,
|
||||
},
|
||||
) as resp:
|
||||
assert resp.status_code == 200, await resp.aread()
|
||||
assert resp.headers["content-type"].startswith("text/event-stream")
|
||||
async for line in resp.aiter_lines():
|
||||
if not line.startswith("data: "):
|
||||
continue
|
||||
data = line[len("data: ") :]
|
||||
if data == "[DONE]":
|
||||
events.append({"type": "_DONE_"})
|
||||
continue
|
||||
events.append(json.loads(data))
|
||||
|
||||
types = [ev["type"] for ev in events]
|
||||
# The lifecycle always starts with message-start and ends with [DONE]
|
||||
# preceded by message-end.
|
||||
assert types[0] == "message-start"
|
||||
assert types[-1] == "_DONE_"
|
||||
assert types[-2] == "message-end"
|
||||
# message-start must carry the chunk/message id.
|
||||
assert events[0].get("id")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_validation_error_returns_400(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
# Missing required ``model`` field → FastAPI/Pydantic returns 400.
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={"messages": []},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_documents_field_accepted(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""The v2 endpoint forwards ``documents`` into chat_template_kwargs.
|
||||
|
||||
We only assert the request is accepted and produces a 200 response —
|
||||
the renderer-level effect is covered by ``tests/renderers/test_cohere.py``.
|
||||
"""
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Summarize."}],
|
||||
"documents": [{"id": "d1", "data": {"title": "T", "snippet": "S"}}],
|
||||
"max_tokens": 16,
|
||||
"stream": False,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Layer 2: Cohere SDK round-trip (auto-skipped if SDK absent)
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def cohere_async_client(server):
|
||||
cohere = pytest.importorskip("cohere")
|
||||
# The vLLM endpoint is mounted at ``/cohere/v2/chat`` while the
|
||||
# cohere SDK targets ``${base_url}/v2/chat``; point base_url at the
|
||||
# ``/cohere`` prefix so paths line up.
|
||||
client = cohere.AsyncClientV2(
|
||||
api_key="dummy",
|
||||
base_url=server.url_for("cohere"),
|
||||
)
|
||||
try:
|
||||
yield client
|
||||
finally:
|
||||
# ``AsyncClientV2`` exposes a sync close; if a future version
|
||||
# adds aclose we still close cleanly.
|
||||
close = getattr(client, "aclose", None) or getattr(client, "close", None)
|
||||
if close is not None:
|
||||
result = close()
|
||||
if hasattr(result, "__await__"):
|
||||
await result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_sdk_non_streaming(cohere_async_client):
|
||||
resp = await cohere_async_client.chat(
|
||||
model=SERVED_MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Say hi."}],
|
||||
max_tokens=16,
|
||||
)
|
||||
# SDK parses our JSON into typed objects.
|
||||
assert resp.message.role == "assistant"
|
||||
assert resp.message.content is not None
|
||||
assert len(resp.message.content) >= 1
|
||||
assert resp.message.content[0].type == "text"
|
||||
assert resp.finish_reason in {
|
||||
"COMPLETE",
|
||||
"MAX_TOKENS",
|
||||
"STOP_SEQUENCE",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_sdk_streaming(cohere_async_client):
|
||||
events: list[str] = []
|
||||
stream = cohere_async_client.chat_stream(
|
||||
model=SERVED_MODEL_NAME,
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
max_tokens=8,
|
||||
)
|
||||
async for ev in stream:
|
||||
events.append(ev.type)
|
||||
|
||||
assert events, "SDK stream yielded no events"
|
||||
assert events[0] == "message-start"
|
||||
assert events[-1] == "message-end"
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for :mod:`vllm.entrypoints.cohere.cohere_chat_message`.
|
||||
|
||||
The module hosts the citation-carrying subclasses of the OpenAI chat
|
||||
completion protocol (``CohereChatMessage`` / ``CohereDeltaMessage``) plus
|
||||
the shared ``Citation`` / ``CitationSource`` shapes. These tests pin:
|
||||
|
||||
1. Serialization behavior we own (empty-``citations`` cleanup on the
|
||||
subclass ``_serialize``, ``exclude_none`` shape, round-trip).
|
||||
2. The constraint that ``type`` discriminators reject unknown values.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
Citation,
|
||||
CitationSource,
|
||||
CohereChatMessage,
|
||||
CohereDeltaMessage,
|
||||
)
|
||||
|
||||
# ======================================================================
|
||||
# CitationSource
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCitationSource:
|
||||
def test_invalid_type_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
CitationSource(type="other") # type: ignore[arg-type]
|
||||
|
||||
def test_none_fields_excluded_from_dump(self):
|
||||
s = CitationSource(type="document", id="d1")
|
||||
assert s.model_dump(exclude_none=True) == {
|
||||
"type": "document",
|
||||
"id": "d1",
|
||||
}
|
||||
|
||||
def test_resolved_source_survives_json_round_trip(self):
|
||||
# The parser produces fully-resolved sources (see
|
||||
# ``_melody_sources_to_vllm`` in
|
||||
# ``vllm/reasoning/cohere_command_reasoning_parser.py``) which
|
||||
# then flow through the internal streaming pipeline (parser ->
|
||||
# OpenAI stream chunk -> ``_chat_completion_stream_to_v2``).
|
||||
# Every field must round-trip through ``model_dump_json`` at
|
||||
# that layer or the wire event ends up missing pieces.
|
||||
src = CitationSource(
|
||||
type="tool",
|
||||
id="res_a0",
|
||||
tool_output={"id": "res_a0", "text": "r"},
|
||||
)
|
||||
src2 = CitationSource.model_validate_json(src.model_dump_json())
|
||||
assert src2.type == "tool"
|
||||
assert src2.id == "res_a0"
|
||||
assert src2.tool_output == {"id": "res_a0", "text": "r"}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Citation
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCitation:
|
||||
def test_invalid_type_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
Citation(type="OTHER") # type: ignore[arg-type]
|
||||
|
||||
def test_dump_excludes_none_fields(self):
|
||||
c = Citation(start=0, end=5, text="hello")
|
||||
dumped = c.model_dump(exclude_none=True)
|
||||
assert dumped == {
|
||||
"start": 0,
|
||||
"end": 5,
|
||||
"text": "hello",
|
||||
"sources": [],
|
||||
}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereDeltaMessage
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereDeltaMessage:
|
||||
def test_citations_omitted_from_dump_when_none(self):
|
||||
# Non-grounded deltas should never carry ``citations: null`` on
|
||||
# the wire even when routed through the Cohere subclass.
|
||||
d = CohereDeltaMessage(content="hello")
|
||||
dumped = d.model_dump(exclude_none=True)
|
||||
assert "citations" not in dumped
|
||||
|
||||
def test_citations_dump_round_trip(self):
|
||||
d = CohereDeltaMessage(citations=[Citation(start=0, end=5, text="hi")])
|
||||
dumped = d.model_dump(exclude_none=True)
|
||||
assert dumped["citations"][0]["text"] == "hi"
|
||||
# Round-trip through the subclass to confirm the field validates.
|
||||
d2 = CohereDeltaMessage.model_validate(dumped)
|
||||
assert d2.citations[0].text == "hi"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereChatMessage
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereChatMessage:
|
||||
def test_citations_omitted_from_dump_when_none(self):
|
||||
m = CohereChatMessage(role="assistant", content="hi")
|
||||
dumped = m.model_dump(exclude_none=True)
|
||||
assert "citations" not in dumped
|
||||
|
||||
def test_citations_dump_round_trip(self):
|
||||
m = CohereChatMessage(
|
||||
role="assistant",
|
||||
content="hello",
|
||||
citations=[
|
||||
Citation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[CitationSource(type="document", id="d1")],
|
||||
)
|
||||
],
|
||||
)
|
||||
dumped = m.model_dump(exclude_none=True)
|
||||
assert dumped["role"] == "assistant"
|
||||
assert dumped["content"] == "hello"
|
||||
assert dumped["citations"][0]["text"] == "hello"
|
||||
assert dumped["citations"][0]["sources"][0]["id"] == "d1"
|
||||
# Round-trip through the subclass to confirm the field validates.
|
||||
m2 = CohereChatMessage.model_validate(dumped)
|
||||
assert m2.citations[0].text == "hello"
|
||||
assert m2.citations[0].sources[0].id == "d1"
|
||||
@@ -0,0 +1,217 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ``vllm/entrypoints/cohere/protocol.py``.
|
||||
|
||||
The module is mostly a thin wrapper around the official ``cohere`` SDK
|
||||
types plus a few local additions:
|
||||
|
||||
* :class:`CohereError` envelope.
|
||||
* :class:`CohereChatV2Request` (model required, ``max_tokens`` non-negative).
|
||||
* :class:`CohereChatV2Response` plus the usage / logprob helpers.
|
||||
* The streaming event subclasses that bake a wire-format ``type``
|
||||
discriminator into ``model_dump()`` so SSE consumers can demux on it.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
AssistantMessageResponse,
|
||||
CitationEndEvent,
|
||||
CitationStartEvent,
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
CohereUsageTokens,
|
||||
ContentDeltaEvent,
|
||||
ContentEndEvent,
|
||||
ContentStartEvent,
|
||||
MessageEndEvent,
|
||||
MessageStartEvent,
|
||||
ToolCallDeltaEvent,
|
||||
ToolCallEndEvent,
|
||||
ToolCallStartEvent,
|
||||
ToolPlanDeltaEvent,
|
||||
)
|
||||
|
||||
# ======================================================================
|
||||
# CohereError
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereError:
|
||||
def test_model_dump_excludes_none(self):
|
||||
err = CohereError(message="boom")
|
||||
assert err.model_dump(exclude_none=True) == {"message": "boom"}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereChatV2Request
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereChatV2Request:
|
||||
def test_empty_model_rejected(self):
|
||||
with pytest.raises(ValidationError, match="model is required"):
|
||||
CohereChatV2Request(model="", messages=[{"role": "user", "content": "hi"}])
|
||||
|
||||
def test_negative_max_tokens_rejected(self):
|
||||
with pytest.raises(ValidationError, match="non-negative"):
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=-1,
|
||||
)
|
||||
|
||||
def test_zero_max_tokens_allowed(self):
|
||||
# Zero is allowed (the docs allow 0 -> return prompt only).
|
||||
req = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=0,
|
||||
)
|
||||
assert req.max_tokens == 0
|
||||
|
||||
def test_invalid_tool_choice_rejected(self):
|
||||
with pytest.raises(ValidationError):
|
||||
CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tool_choice="ANY", # not REQUIRED/NONE
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Usage / Logprobs
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestUsage:
|
||||
def test_tokens_serialization(self):
|
||||
u = CohereUsageTokens(input_tokens=10, output_tokens=5)
|
||||
assert u.model_dump() == {"input_tokens": 10.0, "output_tokens": 5.0}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# CohereChatV2Response
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCohereChatV2Response:
|
||||
def test_invalid_finish_reason_rejected(self):
|
||||
msg = AssistantMessageResponse(content=[{"type": "text", "text": "hi"}])
|
||||
with pytest.raises(ValidationError):
|
||||
CohereChatV2Response(
|
||||
id="r1",
|
||||
finish_reason="NOT_A_REASON", # type: ignore[arg-type]
|
||||
message=msg,
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Streaming event ``type`` discriminator baked into model_dump()
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestStreamingEventTypeField:
|
||||
"""Each event subclass adds a ``type: Literal[...]`` field with a
|
||||
default so ``model_dump()`` always emits the wire-format discriminator
|
||||
(the parent SDK classes don't declare ``type`` as a Pydantic field).
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cls, expected_type, kwargs",
|
||||
[
|
||||
(
|
||||
MessageStartEvent,
|
||||
"message-start",
|
||||
{"id": "a", "delta": {"message": {"role": "assistant"}}},
|
||||
),
|
||||
(
|
||||
ContentStartEvent,
|
||||
"content-start",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"message": {"content": {"type": "text", "text": ""}}},
|
||||
},
|
||||
),
|
||||
(
|
||||
ContentDeltaEvent,
|
||||
"content-delta",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"message": {"content": {"text": "hi"}}},
|
||||
},
|
||||
),
|
||||
(ContentEndEvent, "content-end", {"index": 0}),
|
||||
(
|
||||
ToolPlanDeltaEvent,
|
||||
"tool-plan-delta",
|
||||
{"delta": {"message": {"tool_plan": "thinking"}}},
|
||||
),
|
||||
(
|
||||
ToolCallStartEvent,
|
||||
"tool-call-start",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {
|
||||
"tool_calls": {
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": ""},
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
ToolCallDeltaEvent,
|
||||
"tool-call-delta",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {"tool_calls": {"function": {"arguments": "{}"}}}
|
||||
},
|
||||
},
|
||||
),
|
||||
(ToolCallEndEvent, "tool-call-end", {"index": 0}),
|
||||
(
|
||||
CitationStartEvent,
|
||||
"citation-start",
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"message": {
|
||||
"citations": {"start": 0, "end": 5, "text": "hello"}
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(CitationEndEvent, "citation-end", {"index": 0}),
|
||||
(
|
||||
MessageEndEvent,
|
||||
"message-end",
|
||||
{"id": "a", "delta": {"finish_reason": "COMPLETE"}},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_type_field_default(self, cls, expected_type, kwargs):
|
||||
ev = cls(**kwargs)
|
||||
# type field is auto-populated from the Literal default.
|
||||
assert ev.type == expected_type
|
||||
# The discriminator must be present in the serialized payload so
|
||||
# clients reading the stream can demux on it.
|
||||
dumped = ev.model_dump(exclude_none=True)
|
||||
assert dumped["type"] == expected_type
|
||||
# Same in JSON form (what ``_emit`` serializes).
|
||||
assert f'"type":"{expected_type}"' in ev.model_dump_json(exclude_none=True)
|
||||
|
||||
def test_type_field_cannot_be_overridden_to_wrong_value(self):
|
||||
# Literal types reject any value other than the bake-in default.
|
||||
with pytest.raises(ValidationError):
|
||||
MessageStartEvent(
|
||||
id="a",
|
||||
delta={"message": {"role": "assistant"}},
|
||||
type="other", # type: ignore[arg-type]
|
||||
)
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sanity tests for the small registry / CLI-arg / config additions made
|
||||
by the Cohere v2 chat API:
|
||||
|
||||
* ``vllm/renderers/registry.py``: new ``"cohere"`` renderer entry.
|
||||
* ``vllm/tokenizers/registry.py``: new ``"cohere"`` tokenizer entry
|
||||
(aliased to the cached HF tokenizer).
|
||||
* ``vllm/entrypoints/openai/cli_args.py``: new
|
||||
``cohere_is_reasoning_model`` field.
|
||||
* ``vllm/config/model.py``: ``"cohere"`` added to ``TokenizerMode``
|
||||
Literal.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
import typing
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.model import TokenizerMode
|
||||
from vllm.entrypoints.openai.cli_args import BaseFrontendArgs, make_arg_parser
|
||||
from vllm.renderers.registry import RENDERER_REGISTRY
|
||||
from vllm.tokenizers.registry import TokenizerRegistry
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class TestRendererRegistry:
|
||||
def test_cohere_renderer_registered(self):
|
||||
# The registry resolves to the importable ``CohereRenderer`` class.
|
||||
cls = RENDERER_REGISTRY.load_renderer_cls("cohere")
|
||||
assert cls.__name__ == "CohereRenderer"
|
||||
# Sanity: class lives in the cohere renderer module.
|
||||
assert cls.__module__ == "vllm.renderers.cohere"
|
||||
|
||||
|
||||
class TestTokenizerRegistry:
|
||||
def test_cohere_aliased_to_cached_hf_tokenizer(self):
|
||||
# ``cohere`` mode uses the standard HF tokenizer; only the
|
||||
# renderer stage is replaced. This test guards against accidental
|
||||
# divergence.
|
||||
cls = TokenizerRegistry.load_tokenizer_cls("cohere")
|
||||
assert cls.__name__ == "CachedHfTokenizer"
|
||||
|
||||
|
||||
class TestTokenizerModeLiteral:
|
||||
def test_cohere_is_a_valid_tokenizer_mode(self):
|
||||
# The Literal must enumerate ``"cohere"`` so engine arg parsing
|
||||
# accepts ``--tokenizer-mode cohere``.
|
||||
modes = typing.get_args(TokenizerMode)
|
||||
assert "cohere" in modes
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# ``--cohere-is-reasoning-model`` CLI flag
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCohereCliArg:
|
||||
"""Verifies the new ``--cohere-is-reasoning-model`` flag end-to-end
|
||||
through ``make_arg_parser`` — mirrors the pattern used in
|
||||
:mod:`tests.entrypoints.openai.test_cli_args`.
|
||||
"""
|
||||
|
||||
def test_default_value_on_dataclass_is_true(self):
|
||||
fields = {f.name: f for f in dataclasses.fields(BaseFrontendArgs)}
|
||||
assert "cohere_is_reasoning_model" in fields
|
||||
field = fields["cohere_is_reasoning_model"]
|
||||
assert field.default is True
|
||||
assert field.type is bool
|
||||
|
||||
@pytest.fixture
|
||||
def serve_parser(self) -> FlexibleArgumentParser:
|
||||
parser = FlexibleArgumentParser()
|
||||
return make_arg_parser(parser)
|
||||
|
||||
def test_default_via_argparse_is_true(self, serve_parser: FlexibleArgumentParser):
|
||||
# No flag supplied → dataclass default (True) wins.
|
||||
args = serve_parser.parse_args(["--model", "m"])
|
||||
assert args.cohere_is_reasoning_model is True
|
||||
|
||||
def test_explicit_false_via_argparse(self, serve_parser: FlexibleArgumentParser):
|
||||
# Boolean dataclass fields are wired up as ``--flag value`` /
|
||||
# ``--no-flag`` pairs by FlexibleArgumentParser.
|
||||
args = serve_parser.parse_args(
|
||||
["--model", "m", "--no-cohere-is-reasoning-model"]
|
||||
)
|
||||
assert args.cohere_is_reasoning_model is False
|
||||
|
||||
def test_explicit_true_via_argparse(self, serve_parser: FlexibleArgumentParser):
|
||||
args = serve_parser.parse_args(["--model", "m", "--cohere-is-reasoning-model"])
|
||||
assert args.cohere_is_reasoning_model is True
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,944 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Cohere v2 SSE stream conversion in
|
||||
``vllm/entrypoints/cohere/serving.py``.
|
||||
|
||||
The stream-translation entry point is
|
||||
:meth:`CohereServingChatV2._chat_completion_stream_to_v2`, which turns an
|
||||
async iterable of OpenAI SSE chunks into Cohere's
|
||||
``message-start → (content|tool-call|citation)* → message-end → [DONE]``
|
||||
event stream.
|
||||
|
||||
We test the helpers (``_StreamState``, ``_handle_*_delta``, etc.) in
|
||||
isolation, plus a handful of end-to-end scenarios that exercise the
|
||||
state machine.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
Citation as VLLMCitation,
|
||||
)
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
CitationSource,
|
||||
)
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
MessageStartEvent,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import (
|
||||
_DONE_FRAME,
|
||||
CohereServingChatV2,
|
||||
ContentBlockType,
|
||||
_emit,
|
||||
_sse,
|
||||
_StreamState,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionStreamResponse,
|
||||
)
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeServing(CohereServingChatV2):
|
||||
"""Lightweight stand-in for :class:`CohereServingChatV2` that skips
|
||||
the heavy ``OpenAIServingChat.__init__`` chain (which would need a
|
||||
real engine client, model registry, render service, etc.).
|
||||
|
||||
Only ``_is_reasoning_model`` is read by the methods under test
|
||||
(``_chat_completion_stream_to_v2`` and the per-delta handlers); the
|
||||
rest is dead weight for unit testing.
|
||||
"""
|
||||
|
||||
def __init__(self, is_reasoning_model: bool = True) -> None:
|
||||
# Intentionally skipping super().__init__ — see class docstring.
|
||||
self._is_reasoning_model = is_reasoning_model
|
||||
|
||||
|
||||
def _serving(is_reasoning_model: bool = True) -> CohereServingChatV2:
|
||||
return _FakeServing(is_reasoning_model=is_reasoning_model)
|
||||
|
||||
|
||||
def _parse_event(frame: str) -> dict[str, Any]:
|
||||
"""Strip the ``data: ... \\n\\n`` wrapper and parse the JSON payload."""
|
||||
assert frame.startswith("data: ")
|
||||
assert frame.endswith("\n\n")
|
||||
return json.loads(frame[len("data: ") : -2])
|
||||
|
||||
|
||||
def _make_chunk(
|
||||
*,
|
||||
chunk_id: str = "chunk_0",
|
||||
role: str | None = None,
|
||||
content: str | None = None,
|
||||
reasoning: str | None = None,
|
||||
tool_calls: list[dict[str, Any]] | None = None,
|
||||
finish_reason: str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
omit_choices: bool = False,
|
||||
citations: list[Any] | None = None,
|
||||
) -> str:
|
||||
"""Build the ``data: {...}\\n\\n`` SSE frame the production code
|
||||
consumes."""
|
||||
delta: dict[str, Any] = {}
|
||||
if role is not None:
|
||||
delta["role"] = role
|
||||
if content is not None:
|
||||
delta["content"] = content
|
||||
if reasoning is not None:
|
||||
delta["reasoning"] = reasoning
|
||||
if tool_calls is not None:
|
||||
delta["tool_calls"] = tool_calls
|
||||
if citations is not None:
|
||||
delta["citations"] = citations
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"id": chunk_id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "m",
|
||||
}
|
||||
if not omit_choices:
|
||||
payload["choices"] = [
|
||||
{"index": 0, "delta": delta, "finish_reason": finish_reason}
|
||||
]
|
||||
else:
|
||||
payload["choices"] = []
|
||||
if usage is not None:
|
||||
payload["usage"] = usage
|
||||
# Use ChatCompletionStreamResponse to normalize the payload shape.
|
||||
chunk = ChatCompletionStreamResponse.model_validate(payload)
|
||||
return f"data: {chunk.model_dump_json(exclude_none=False)}\n\n"
|
||||
|
||||
|
||||
def _make_done() -> str:
|
||||
return "data: [DONE]\n\n"
|
||||
|
||||
|
||||
async def _stream_from(items: list[str]) -> AsyncGenerator[str, None]:
|
||||
for item in items:
|
||||
yield item
|
||||
|
||||
|
||||
async def _drain(serving: CohereServingChatV2, items: list[str]) -> list[str]:
|
||||
"""Drive ``_chat_completion_stream_to_v2`` over ``items`` and collect
|
||||
the emitted SSE frames."""
|
||||
request = CohereChatV2Request(
|
||||
model="m", messages=[{"role": "user", "content": "hi"}], stream=True
|
||||
)
|
||||
gen = serving._chat_completion_stream_to_v2(_stream_from(items), request)
|
||||
return [frame async for frame in gen]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Low-level helpers: _sse, _emit, _DONE_FRAME
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestSSEHelpers:
|
||||
def test_sse_wraps_payload(self):
|
||||
assert _sse("hello") == "data: hello\n\n"
|
||||
|
||||
def test_done_frame_constant(self):
|
||||
# cohere-python and Fern-generated clients key off this exact
|
||||
# sentinel; keep it byte-for-byte stable.
|
||||
assert _DONE_FRAME == "data: [DONE]\n\n"
|
||||
|
||||
def test_emit_serializes_event_with_type_discriminator(self):
|
||||
frame = _emit(
|
||||
MessageStartEvent(id="abc", delta={"message": {"role": "assistant"}})
|
||||
)
|
||||
payload = _parse_event(frame)
|
||||
assert payload["type"] == "message-start"
|
||||
assert payload["id"] == "abc"
|
||||
assert payload["delta"] == {"message": {"role": "assistant"}}
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _StreamState
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestStreamState:
|
||||
def test_content_index_monotonic(self):
|
||||
st = _StreamState()
|
||||
assert st.next_content_index() == 0
|
||||
assert st.next_content_index() == 1
|
||||
assert st.next_content_index() == 2
|
||||
|
||||
def test_citation_index_separate_from_content_index(self):
|
||||
st = _StreamState()
|
||||
st.next_content_index() # 0
|
||||
st.next_content_index() # 1
|
||||
# Citation indexing is independent of content indexing.
|
||||
assert st.next_citation_index() == 0
|
||||
assert st.next_citation_index() == 1
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _close_open_blocks
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestCloseOpenBlocks:
|
||||
def test_no_block_open_emits_nothing(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
assert serving._close_open_blocks(state) == []
|
||||
|
||||
def test_text_block_emits_content_end(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
state.active_block = ContentBlockType.TEXT
|
||||
state.active_block_index = 1
|
||||
out = serving._close_open_blocks(state)
|
||||
assert len(out) == 1
|
||||
payload = _parse_event(out[0])
|
||||
assert payload == {"type": "content-end", "index": 1}
|
||||
assert state.active_block is None
|
||||
assert state.active_block_index is None
|
||||
|
||||
def test_thinking_block_emits_content_end(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
state.active_block = ContentBlockType.THINKING
|
||||
state.active_block_index = 3
|
||||
out = serving._close_open_blocks(state)
|
||||
payload = _parse_event(out[0])
|
||||
assert payload == {"type": "content-end", "index": 3}
|
||||
|
||||
def test_tool_call_block_emits_tool_call_end(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
state.active_block = ContentBlockType.TOOL_CALL
|
||||
state.active_tool_index = 7
|
||||
out = serving._close_open_blocks(state)
|
||||
payload = _parse_event(out[0])
|
||||
assert payload == {"type": "tool-call-end", "index": 7}
|
||||
assert state.active_tool_index is None
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_text_delta
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleTextDelta:
|
||||
def test_opens_block_first_time(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
events = serving._handle_text_delta(state, "Hi")
|
||||
assert len(events) == 2
|
||||
start = _parse_event(events[0])
|
||||
delta = _parse_event(events[1])
|
||||
assert start["type"] == "content-start"
|
||||
assert start["index"] == 0
|
||||
assert start["delta"]["message"]["content"]["type"] == "text"
|
||||
assert delta["type"] == "content-delta"
|
||||
assert delta["index"] == 0
|
||||
assert delta["delta"]["message"]["content"]["text"] == "Hi"
|
||||
assert state.active_block == ContentBlockType.TEXT
|
||||
assert state.active_block_index == 0
|
||||
|
||||
def test_continues_block_with_just_delta(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
serving._handle_text_delta(state, "Hi")
|
||||
events = serving._handle_text_delta(state, " there")
|
||||
# Only a delta event, no new content-start.
|
||||
assert len(events) == 1
|
||||
delta = _parse_event(events[0])
|
||||
assert delta["type"] == "content-delta"
|
||||
assert delta["delta"]["message"]["content"]["text"] == " there"
|
||||
|
||||
def test_switches_from_thinking_block(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
state = _StreamState()
|
||||
# Open a thinking block first, then switch to text.
|
||||
serving._handle_thinking_delta(state, "ponder")
|
||||
events = serving._handle_text_delta(state, "answer")
|
||||
types = [_parse_event(ev)["type"] for ev in events]
|
||||
assert types == ["content-end", "content-start", "content-delta"]
|
||||
# The text block gets a new index (1), distinct from thinking's 0.
|
||||
assert _parse_event(events[1])["index"] == 1
|
||||
assert state.active_block == ContentBlockType.TEXT
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_thinking_delta
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleThinkingDelta:
|
||||
def test_reasoning_model_opens_thinking_block(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
state = _StreamState()
|
||||
events = serving._handle_thinking_delta(state, "thought")
|
||||
assert len(events) == 2
|
||||
start = _parse_event(events[0])
|
||||
delta = _parse_event(events[1])
|
||||
assert start["type"] == "content-start"
|
||||
assert start["delta"]["message"]["content"]["type"] == "thinking"
|
||||
assert delta["type"] == "content-delta"
|
||||
assert delta["delta"]["message"]["content"]["thinking"] == "thought"
|
||||
assert state.active_block == ContentBlockType.THINKING
|
||||
|
||||
def test_reasoning_model_continues_thinking_block(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
state = _StreamState()
|
||||
serving._handle_thinking_delta(state, "first")
|
||||
events = serving._handle_thinking_delta(state, " more")
|
||||
assert len(events) == 1
|
||||
assert _parse_event(events[0])["type"] == "content-delta"
|
||||
|
||||
def test_non_reasoning_model_emits_tool_plan_delta(self):
|
||||
# Older Command models stream reasoning as ``tool_plan`` deltas;
|
||||
# no content-start/end pair is emitted.
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
state = _StreamState()
|
||||
events = serving._handle_thinking_delta(state, "planning")
|
||||
assert len(events) == 1
|
||||
payload = _parse_event(events[0])
|
||||
assert payload["type"] == "tool-plan-delta"
|
||||
assert payload["delta"]["message"]["tool_plan"] == "planning"
|
||||
# ``tool_plan`` deltas don't claim an active content block.
|
||||
assert state.active_block is None
|
||||
|
||||
def test_non_reasoning_model_closes_open_text_block(self):
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
state = _StreamState()
|
||||
# Open a text block first.
|
||||
serving._handle_text_delta(state, "answer")
|
||||
events = serving._handle_thinking_delta(state, "rethink")
|
||||
types = [_parse_event(ev)["type"] for ev in events]
|
||||
assert types == ["content-end", "tool-plan-delta"]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_tool_call_deltas
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleToolCallDeltas:
|
||||
def test_new_tool_call_opens_tool_call_start(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
deltas = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"function": type(
|
||||
"Fn", (), {"name": "calc", "arguments": '{"x":'}
|
||||
)(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
events = serving._handle_tool_call_deltas(state, deltas)
|
||||
assert len(events) == 1
|
||||
payload = _parse_event(events[0])
|
||||
assert payload["type"] == "tool-call-start"
|
||||
assert payload["index"] == 0
|
||||
tc = payload["delta"]["message"]["tool_calls"]
|
||||
assert tc["id"] == "c1"
|
||||
assert tc["function"]["name"] == "calc"
|
||||
assert tc["function"]["arguments"] == '{"x":'
|
||||
assert state.active_block == ContentBlockType.TOOL_CALL
|
||||
assert state.active_tool_index == 0
|
||||
assert 0 in state.tool_calls_seen
|
||||
|
||||
def test_subsequent_arguments_emit_delta(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
# First call: start.
|
||||
first = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"function": type("Fn", (), {"name": "calc", "arguments": ""})(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
serving._handle_tool_call_deltas(state, first)
|
||||
# Second call: same index, additional arguments fragment.
|
||||
more = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": None,
|
||||
"function": type("Fn", (), {"name": None, "arguments": "1}"})(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
events = serving._handle_tool_call_deltas(state, more)
|
||||
assert len(events) == 1
|
||||
payload = _parse_event(events[0])
|
||||
assert payload["type"] == "tool-call-delta"
|
||||
assert payload["index"] == 0
|
||||
assert (
|
||||
payload["delta"]["message"]["tool_calls"]["function"]["arguments"] == "1}"
|
||||
)
|
||||
|
||||
def test_new_tool_call_closes_existing_content_block(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
# Open a text block, then start a tool call.
|
||||
serving._handle_text_delta(state, "I'll call:")
|
||||
deltas = [
|
||||
type(
|
||||
"Delta",
|
||||
(),
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"function": type("Fn", (), {"name": "calc", "arguments": "{}"})(),
|
||||
},
|
||||
)()
|
||||
]
|
||||
events = serving._handle_tool_call_deltas(state, deltas)
|
||||
types = [_parse_event(ev)["type"] for ev in events]
|
||||
assert types == ["content-end", "tool-call-start"]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _handle_citation_deltas
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestHandleCitationDeltas:
|
||||
"""Test the per-delta emitter. Sources are pre-resolved by the
|
||||
reasoning parser via the position map forwarded through
|
||||
``chat_template_kwargs`` (see :func:`_melody_sources_to_vllm` in
|
||||
``vllm/reasoning/cohere_command_reasoning_parser.py``), so the
|
||||
handler only has to coerce to the SDK wire shape, apply the
|
||||
``THINKING_CONTENT`` -> ``PLAN`` rewrite, and drop citations with
|
||||
no attributable sources.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _resolved_source(**overrides: Any) -> CitationSource:
|
||||
base = dict(
|
||||
type="document",
|
||||
id="doc_hamlet",
|
||||
document={"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
|
||||
)
|
||||
base.update(overrides)
|
||||
return CitationSource(**base)
|
||||
|
||||
def test_resolved_citation_emits_start_and_end_with_real_id(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=11,
|
||||
text="Shakespeare",
|
||||
sources=[self._resolved_source()],
|
||||
type="TEXT_CONTENT",
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
assert len(events) == 2
|
||||
start = _parse_event(events[0])
|
||||
end = _parse_event(events[1])
|
||||
assert start["type"] == "citation-start"
|
||||
assert start["index"] == 0
|
||||
cit_payload = start["delta"]["message"]["citations"]
|
||||
assert cit_payload["start"] == 0
|
||||
assert cit_payload["end"] == 11
|
||||
assert cit_payload["text"] == "Shakespeare"
|
||||
assert cit_payload["sources"] == [
|
||||
{
|
||||
"type": "document",
|
||||
"id": "doc_hamlet",
|
||||
"document": {"id": "doc_hamlet", "text": "Hamlet by Shakespeare."},
|
||||
}
|
||||
]
|
||||
assert end == {"type": "citation-end", "index": 0}
|
||||
|
||||
def test_multiple_sources_survive_to_wire(self):
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="works",
|
||||
sources=[
|
||||
self._resolved_source(id="d0", document={"id": "d0"}),
|
||||
self._resolved_source(id="d1", document={"id": "d1"}),
|
||||
],
|
||||
type="TEXT_CONTENT",
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
|
||||
ids = [s["id"] for s in cit_payload["sources"]]
|
||||
assert ids == ["d0", "d1"]
|
||||
|
||||
def test_source_without_id_streamed_with_payload(self):
|
||||
# Match the cohere api: id-less tool sources (from text-only
|
||||
# tool results) still ride through with their ``tool_output``
|
||||
# payload; only the ``id`` field is omitted.
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[CitationSource(type="tool", tool_output={"content": "hi"})],
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
assert len(events) >= 1
|
||||
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
|
||||
assert cit_payload["sources"] == [
|
||||
{"type": "tool", "tool_output": {"content": "hi"}}
|
||||
]
|
||||
assert "id" not in cit_payload["sources"][0]
|
||||
|
||||
def test_citation_with_no_sources_streamed_dropped(self):
|
||||
# Citations whose source list is entirely empty are still
|
||||
# dropped so we don't emit an unattributed anchor.
|
||||
serving = _serving()
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[],
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
assert events == []
|
||||
|
||||
def test_plan_rewrite_on_non_reasoning_model(self):
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
state = _StreamState()
|
||||
citations = [
|
||||
VLLMCitation(
|
||||
start=0,
|
||||
end=5,
|
||||
text="hello",
|
||||
sources=[self._resolved_source()],
|
||||
type="THINKING_CONTENT",
|
||||
)
|
||||
]
|
||||
events = serving._handle_citation_deltas(state, citations)
|
||||
cit_payload = _parse_event(events[0])["delta"]["message"]["citations"]
|
||||
assert cit_payload["type"] == "PLAN"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _build_message_end_event
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestBuildMessageEndEvent:
|
||||
def test_without_usage(self):
|
||||
serving = _serving()
|
||||
frame = serving._build_message_end_event(chunk_id="abc", finish_reason="stop")
|
||||
payload = _parse_event(frame)
|
||||
assert payload["type"] == "message-end"
|
||||
assert payload["id"] == "abc"
|
||||
assert payload["delta"]["finish_reason"] == "COMPLETE"
|
||||
assert "usage" not in payload["delta"]
|
||||
|
||||
def test_with_usage(self):
|
||||
serving = _serving()
|
||||
chunk = ChatCompletionStreamResponse.model_validate(
|
||||
{
|
||||
"id": "abc",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
},
|
||||
}
|
||||
)
|
||||
frame = serving._build_message_end_event(
|
||||
chunk_id="abc", finish_reason="length", usage_chunk=chunk
|
||||
)
|
||||
payload = _parse_event(frame)
|
||||
assert payload["delta"]["finish_reason"] == "MAX_TOKENS"
|
||||
usage = payload["delta"]["usage"]
|
||||
assert usage["billed_units"] == {"input_tokens": 10, "output_tokens": 5}
|
||||
assert usage["tokens"] == {"input_tokens": 10, "output_tokens": 5}
|
||||
assert "cached_tokens" not in usage
|
||||
|
||||
def test_with_cached_tokens(self):
|
||||
serving = _serving()
|
||||
chunk = ChatCompletionStreamResponse.model_validate(
|
||||
{
|
||||
"id": "abc",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 0,
|
||||
"model": "m",
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"total_tokens": 15,
|
||||
"prompt_tokens_details": {"cached_tokens": 3},
|
||||
},
|
||||
}
|
||||
)
|
||||
frame = serving._build_message_end_event(
|
||||
chunk_id="abc", finish_reason="stop", usage_chunk=chunk
|
||||
)
|
||||
payload = _parse_event(frame)
|
||||
assert payload["delta"]["usage"]["cached_tokens"] == 3
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# End-to-end: _chat_completion_stream_to_v2
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestChatCompletionStreamToV2:
|
||||
"""End-to-end stream lifecycle tests."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_only_happy_path_emits_full_lifecycle(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="Hi"),
|
||||
_make_chunk(content=" there"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 4,
|
||||
"completion_tokens": 2,
|
||||
"total_tokens": 6,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
# message-start, content-start, content-delta, content-delta,
|
||||
# content-end, message-end, then [DONE] as the last frame.
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"message-end",
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
# message-end should carry usage stats from the trailing chunk.
|
||||
end_payload = _parse_event(frames[-2])
|
||||
assert end_payload["delta"]["finish_reason"] == "COMPLETE"
|
||||
assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_then_tool_call_closes_text_first(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="planning..."),
|
||||
_make_chunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "calc", "arguments": "{}"},
|
||||
}
|
||||
]
|
||||
),
|
||||
_make_chunk(finish_reason="tool_calls"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
# Text block is opened+delta, then closed before the tool-call
|
||||
# opens, and the final close happens on the usage chunk path.
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"tool-call-start",
|
||||
"tool-call-end",
|
||||
"message-end",
|
||||
]
|
||||
# finish_reason captured from the prior chunk.
|
||||
end = _parse_event(frames[-2])
|
||||
assert end["delta"]["finish_reason"] == "TOOL_CALL"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thinking_then_text_reasoning_model(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(reasoning="thinking..."),
|
||||
_make_chunk(content="answer"),
|
||||
_make_chunk(finish_reason="stop"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
# Thinking block opens with index 0; text block reopens with index 1.
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"message-end",
|
||||
]
|
||||
thinking_start = _parse_event(frames[1])
|
||||
assert thinking_start["delta"]["message"]["content"]["type"] == "thinking"
|
||||
text_start = _parse_event(frames[4])
|
||||
assert text_start["delta"]["message"]["content"]["type"] == "text"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_on_non_reasoning_model_emits_tool_plan_delta(self):
|
||||
serving = _serving(is_reasoning_model=False)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(reasoning="planning"),
|
||||
_make_chunk(
|
||||
tool_calls=[
|
||||
{
|
||||
"index": 0,
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
]
|
||||
),
|
||||
_make_chunk(finish_reason="tool_calls"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
assert types == [
|
||||
"message-start",
|
||||
"tool-plan-delta",
|
||||
"tool-call-start",
|
||||
"tool-call-end",
|
||||
"message-end",
|
||||
]
|
||||
# No thinking content blocks should be present.
|
||||
assert "content-start" not in types
|
||||
assert "content-end" not in types
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_done_marker_in_middle_closes_open_block(self):
|
||||
# Some upstreams send [DONE] without a trailing usage-only chunk.
|
||||
# The translator must still emit message-end before [DONE] so
|
||||
# Cohere clients don't hang.
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="Hi"),
|
||||
_make_done(),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
assert types == [
|
||||
"message-start",
|
||||
"content-start",
|
||||
"content-delta",
|
||||
"content-end",
|
||||
"message-end",
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_empty_and_non_data_lines(self):
|
||||
serving = _serving()
|
||||
items = [
|
||||
"\n",
|
||||
"event: ping\n\n",
|
||||
_make_chunk(role="assistant"),
|
||||
"data: \n\n", # empty data
|
||||
_make_chunk(content="Hi"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
# Should still produce a complete lifecycle.
|
||||
types = [_parse_event(f)["type"] for f in frames[:-1]]
|
||||
assert types[0] == "message-start"
|
||||
assert types[-1] == "message-end"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exception_in_chunk_parsing_emits_error_message_end(self):
|
||||
# An invalid JSON payload after the first chunk triggers the
|
||||
# error path: a synthetic message-end with finish_reason=ERROR
|
||||
# followed by [DONE].
|
||||
serving = _serving()
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
"data: {not valid json}\n\n",
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
# Find the error-shaped message-end.
|
||||
error_end = _parse_event(frames[-2])
|
||||
assert error_end["type"] == "message-end"
|
||||
assert error_end["delta"]["finish_reason"] == "ERROR"
|
||||
assert "error" in error_end["delta"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_error_message_end_is_sanitized(self):
|
||||
# The client-visible ``error`` field on the terminal message-end
|
||||
# must be routed through ``sanitize_message`` so tracebacks,
|
||||
# host filesystem paths, and Python object memory addresses
|
||||
# can't leak from a mid-stream exception into the SSE payload.
|
||||
serving = _serving()
|
||||
|
||||
async def _raises_mid_stream() -> AsyncGenerator[str, None]:
|
||||
yield _make_chunk(role="assistant")
|
||||
raise RuntimeError(
|
||||
"boom <Foo at 0x7fabc0deface> reading /Users/dev/vllm/x.py"
|
||||
)
|
||||
|
||||
request = CohereChatV2Request(
|
||||
model="m",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
stream=True,
|
||||
)
|
||||
frames = [
|
||||
f
|
||||
async for f in serving._chat_completion_stream_to_v2(
|
||||
_raises_mid_stream(), request
|
||||
)
|
||||
]
|
||||
assert frames[-1] == _DONE_FRAME
|
||||
error_end = _parse_event(frames[-2])
|
||||
assert error_end["delta"]["finish_reason"] == "ERROR"
|
||||
err_msg = error_end["delta"]["error"]
|
||||
# Memory address stripped and path replaced with ``<path>``.
|
||||
assert "0x7fabc0deface" not in err_msg
|
||||
assert "/Users/dev/vllm/x.py" not in err_msg
|
||||
assert "<path>" in err_msg
|
||||
# The human-facing prefix should still survive sanitization.
|
||||
assert "boom" in err_msg
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_citations_in_delta_emit_citation_events(self):
|
||||
# End-to-end streaming citation flow. The parser hands us
|
||||
# already-resolved sources (see ``_melody_sources_to_vllm``
|
||||
# in ``vllm/reasoning/cohere_command_reasoning_parser.py``);
|
||||
# the streaming loop just has to coerce and emit.
|
||||
serving = _serving()
|
||||
items = [
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(content="hello"),
|
||||
_make_chunk(
|
||||
citations=[
|
||||
{
|
||||
"start": 0,
|
||||
"end": 5,
|
||||
"text": "hello",
|
||||
"sources": [
|
||||
{
|
||||
"type": "document",
|
||||
"id": "d1",
|
||||
"document": {"id": "d1", "text": "cited"},
|
||||
}
|
||||
],
|
||||
"type": "TEXT_CONTENT",
|
||||
}
|
||||
]
|
||||
),
|
||||
_make_chunk(finish_reason="stop"),
|
||||
_make_chunk(
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
parsed = [_parse_event(f) for f in frames[:-1]]
|
||||
types = [p["type"] for p in parsed]
|
||||
assert "citation-start" in types
|
||||
assert "citation-end" in types
|
||||
cit_start = next(p for p in parsed if p["type"] == "citation-start")
|
||||
assert cit_start["delta"]["message"]["citations"]["sources"] == [
|
||||
{
|
||||
"type": "document",
|
||||
"id": "d1",
|
||||
"document": {"id": "d1", "text": "cited"},
|
||||
}
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_chunk_emits_message_start_with_chunk_id(self):
|
||||
serving = _serving()
|
||||
items = [
|
||||
_make_chunk(chunk_id="my-id", role="assistant"),
|
||||
_make_chunk(chunk_id="my-id", content="hi"),
|
||||
_make_chunk(
|
||||
chunk_id="my-id",
|
||||
omit_choices=True,
|
||||
usage={
|
||||
"prompt_tokens": 1,
|
||||
"completion_tokens": 1,
|
||||
"total_tokens": 2,
|
||||
},
|
||||
),
|
||||
]
|
||||
frames = await _drain(serving, items)
|
||||
ms = _parse_event(frames[0])
|
||||
assert ms["type"] == "message-start"
|
||||
assert ms["id"] == "my-id"
|
||||
assert ms["delta"]["message"]["role"] == "assistant"
|
||||
Reference in New Issue
Block a user