diff --git a/tests/entrypoints/cohere/test_api_router.py b/tests/entrypoints/cohere/test_api_router.py index 41094f774d..9ab39c334c 100644 --- a/tests/entrypoints/cohere/test_api_router.py +++ b/tests/entrypoints/cohere/test_api_router.py @@ -11,6 +11,9 @@ Covers: * The router wiring: response shapes (JSON + SSE), error translation, and the ``cohere_serving_chat_v2 is None`` fallback (501 Not Implemented). +* The render route (``POST /cohere/v2/chat/render``): that the v2 body + reaches :class:`ServingRender` as a converted ``ChatCompletionRequest`` + and that the resulting ``GenerateRequest`` is returned verbatim. """ import json @@ -29,7 +32,9 @@ from vllm.entrypoints.cohere.protocol import ( AssistantMessageResponse, CohereChatV2Response, ) +from vllm.entrypoints.cohere.serving import CohereServingChatV2 from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest from vllm.entrypoints.serve.exception_handling.handlers.http import ( http_exception_handler, ) @@ -40,6 +45,7 @@ from vllm.entrypoints.serve.exception_handling.handlers.vllm_error import ( vllm_error_handler, ) from vllm.exceptions import VLLMError +from vllm.sampling_params import SamplingParams @pytest.fixture(autouse=True) @@ -78,6 +84,49 @@ class _Handler: return self.result +class _RenderChatHandler: + """Stand-in for :class:`CohereServingChatV2` on the render route. + + Only ``to_chat_completion_request`` is reachable from + ``/cohere/v2/chat/render``, and it delegates to the real conversion + classmethod (which needs no engine or tokenizer) so the tests + exercise the actual v2 -> ``ChatCompletionRequest`` translation + rather than a stub of it. + """ + + def to_chat_completion_request(self, request): + return CohereServingChatV2._convert_v2_to_chat_completion(request) + + +class _RenderHandler: + """Minimal stand-in for :class:`ServingRender`. + + Records the ``ChatCompletionRequest`` it was handed so tests can + assert on the conversion, and returns ``self.result`` - either a + :class:`GenerateRequest`, an :class:`ErrorResponse`, or an exception + to raise. + """ + + def __init__(self, result): + self.result = result + self.seen_request = None + + async def render_chat_request(self, request): + self.seen_request = request + if isinstance(self.result, Exception): + raise self.result + return self.result + + +def _generate_request() -> GenerateRequest: + return GenerateRequest( + request_id="chatcmpl-abc", + token_ids=[5, 6, 7], + sampling_params=SamplingParams(temperature=0.25), + model="m", + ) + + def _build_app(handler: _Handler | None) -> FastAPI: app = FastAPI() attach_router(app) @@ -85,6 +134,14 @@ def _build_app(handler: _Handler | None) -> FastAPI: return app +def _build_render_app(chat_handler, render_handler) -> FastAPI: + app = FastAPI() + attach_router(app) + app.state.cohere_serving_chat_v2 = chat_handler + app.state.serving_render = render_handler + return app + + def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI: """Build a FastAPI app that mirrors the real vLLM setup by installing validation, HTTP, and vLLM exception handlers. The @@ -321,6 +378,144 @@ class TestEndpoint: assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY +# ---------------------------------------------------------------------- +# Render endpoint +# ---------------------------------------------------------------------- + + +class TestRenderEndpoint: + """``POST /cohere/v2/chat/render`` is the Cohere counterpart to + ``POST /v1/chat/completions/render``: it converts the v2 body to a + ``ChatCompletionRequest`` and hands it to the shared + ``ServingRender``, returning the resulting ``GenerateRequest`` + instead of running generation. + + The route therefore spans *two* app-state handlers, and these tests + pin both halves: that the conversion preserves the Cohere-specific + request surface, and that the render handler's output (or error) + reaches the client unchanged. + """ + + def test_route_registered(self): + app = _build_render_app(_RenderChatHandler(), _RenderHandler(None)) + paths = [getattr(r, "path", None) for r in app.routes] + assert "/cohere/v2/chat/render" in paths + + def test_route_not_registered_when_flag_unset(self, monkeypatch): + monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False) + app = FastAPI() + attach_router(app) + paths = [getattr(r, "path", None) for r in app.routes] + assert "/cohere/v2/chat/render" not in paths + + def test_returns_generate_request_json(self): + render_handler = _RenderHandler(_generate_request()) + app = _build_render_app(_RenderChatHandler(), render_handler) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.OK + assert r.headers["content-type"].startswith("application/json") + body = r.json() + assert body["request_id"] == "chatcmpl-abc" + assert body["token_ids"] == [5, 6, 7] + assert body["sampling_params"]["temperature"] == 0.25 + # The body must be a wire-valid GenerateRequest so callers can + # feed it straight back to /v1/generate. + assert GenerateRequest.model_validate(body).token_ids == [5, 6, 7] + + def test_cohere_request_surface_survives_conversion(self): + """The point of routing through ``CohereServingChatV2`` rather + than asking callers to pre-convert: Cohere-only fields + (``documents``, ``safety_mode``, ``citation_options``) must land + in ``chat_template_kwargs`` so the rendered prompt matches what + ``/cohere/v2/chat`` would have produced. + """ + render_handler = _RenderHandler(_generate_request()) + app = _build_render_app(_RenderChatHandler(), render_handler) + body = { + **_minimal_request_body(), + "documents": [{"id": "doc-1", "data": {"text": "Paris is in France."}}], + "safety_mode": "STRICT", + "max_tokens": 32, + } + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=body) + assert r.status_code == HTTPStatus.OK + + seen = render_handler.seen_request + assert seen is not None + assert seen.model == "m" + assert seen.messages[0]["content"] == "hi" + # ``max_tokens`` matters because ServingRender feeds it to + # ``get_max_tokens`` when building sampling params. + assert seen.max_completion_tokens == 32 + kwargs = seen.chat_template_kwargs or {} + assert kwargs["documents"][0]["id"] == "doc-1" + # The conversion lower-cases safety_mode for the renderer. + assert kwargs["safety_mode"] == "strict" + + def test_501_when_chat_handler_missing(self): + app = _build_render_app(None, _RenderHandler(_generate_request())) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.NOT_IMPLEMENTED + assert "does not support" in r.json()["message"] + + def test_501_when_render_handler_missing(self): + # A server without ``serving_render`` in app state (e.g. a + # non-generate task set) can't render, even though the Cohere + # chat handler exists. + app = _build_render_app(_RenderChatHandler(), None) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.NOT_IMPLEMENTED + assert "does not support" in r.json()["message"] + + def test_error_response_translated_to_cohere_envelope(self): + err = ErrorResponse( + error=ErrorInfo(message="unknown model", type="NotFound", code=404) + ) + app = _build_render_app(_RenderChatHandler(), _RenderHandler(err)) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.NOT_FOUND + assert r.json() == {"message": "unknown model"} + + def test_render_exception_returns_500_envelope(self): + app = _build_render_app( + _RenderChatHandler(), _RenderHandler(RuntimeError("tokenizer exploded")) + ) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert r.json() == {"message": "tokenizer exploded"} + + def test_conversion_exception_returns_500_envelope(self): + """Conversion runs inside the same ``try`` as the render call, so + a bad v2 body that slips past Pydantic still yields the Cohere + error envelope rather than an unhandled 500 with vLLM's shape. + """ + + class _Boom: + def to_chat_completion_request(self, request): + raise ValueError("unconvertible message") + + app = _build_render_app(_Boom(), _RenderHandler(_generate_request())) + with TestClient(app) as client: + r = client.post("/cohere/v2/chat/render", json=_minimal_request_body()) + assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR + assert r.json() == {"message": "unconvertible message"} + + def test_invalid_body_returns_422(self): + app = _build_render_app(_RenderChatHandler(), _RenderHandler(None)) + with TestClient(app) as client: + r = client.post( + "/cohere/v2/chat/render", + json={"messages": [{"role": "user", "content": "hi"}]}, + ) + assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY + + # ---------------------------------------------------------------------- # CohereErrorEnvelopeMiddleware # ---------------------------------------------------------------------- diff --git a/tests/entrypoints/cohere/test_chat_v2.py b/tests/entrypoints/cohere/test_chat_v2.py index 9e0ec90a74..b8e5cd4d50 100644 --- a/tests/entrypoints/cohere/test_chat_v2.py +++ b/tests/entrypoints/cohere/test_chat_v2.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint. +"""End-to-end integration tests for the Cohere Chat v2 endpoints +(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``). These tests spin up a real ``vllm serve`` process via :class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2 @@ -173,6 +174,87 @@ async def test_cohere_v2_chat_validation_error_returns_400( assert resp.status_code == 400 +@pytest.mark.asyncio +async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient): + """``/cohere/v2/chat/render`` tokenizes without running the model. + + Beyond the ``GenerateRequest`` shape, this pins the cross-layer + contract that makes the endpoint useful: v2 sampling fields + (``temperature``, ``stop_sequences``) survive the v2 -> + ``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can + render once and generate later without re-deriving them. + """ + resp = await httpx_client.post( + "/cohere/v2/chat/render", + json={ + "model": SERVED_MODEL_NAME, + "messages": [{"role": "user", "content": "Say hi."}], + "max_tokens": 16, + "temperature": 0.125, + "stop_sequences": ["END"], + }, + ) + assert resp.status_code == 200, resp.text + data = resp.json() + + token_ids = data["token_ids"] + assert isinstance(token_ids, list) and len(token_ids) > 0 + assert all(isinstance(t, int) and t >= 0 for t in token_ids) + assert data["request_id"] + + sampling_params = data["sampling_params"] + assert sampling_params["temperature"] == 0.125 + assert sampling_params["stop"] == ["END"] + + # No generation happened, so the response carries none of the chat + # envelope the /cohere/v2/chat endpoint would return. + assert "message" not in data + assert "finish_reason" not in data + + +@pytest.mark.asyncio +async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt( + httpx_client: httpx.AsyncClient, +): + """The rendered ids are the real templated prompt, not a debug dump. + + Round-tripping them through ``/detokenize`` and finding the user + message back proves the endpoint applied the chat template and + tokenized it, which is what a caller splitting prompt construction + from generation depends on. + """ + render_resp = await httpx_client.post( + "/cohere/v2/chat/render", + json={ + "model": SERVED_MODEL_NAME, + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "max_tokens": 8, + }, + ) + assert render_resp.status_code == 200, render_resp.text + token_ids = render_resp.json()["token_ids"] + + detok_resp = await httpx_client.post( + "/detokenize", + json={"model": SERVED_MODEL_NAME, "tokens": token_ids}, + ) + assert detok_resp.status_code == 200, detok_resp.text + prompt = detok_resp.json()["prompt"] + assert "What is the capital of France?" in prompt + + +@pytest.mark.asyncio +async def test_cohere_v2_chat_render_validation_error_returns_400( + httpx_client: httpx.AsyncClient, +): + """Bad bodies get the Cohere error envelope, not vLLM's.""" + resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []}) + assert resp.status_code == 400 + body = resp.json() + assert "error" not in body + assert body.get("message") + + @pytest.mark.asyncio async def test_cohere_v2_chat_documents_field_accepted( httpx_client: httpx.AsyncClient, diff --git a/vllm/entrypoints/cohere/api_router.py b/vllm/entrypoints/cohere/api_router.py index 85eee0fd55..2c31442e2a 100644 --- a/vllm/entrypoints/cohere/api_router.py +++ b/vllm/entrypoints/cohere/api_router.py @@ -1,24 +1,30 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``). +"""FastAPI router for the Cohere Chat v2 API. + +Exposes two routes: + +* ``POST /cohere/v2/chat`` - the chat endpoint itself. +* ``POST /cohere/v2/chat/render`` - tokenize a request without running + generation, mirroring ``POST /v1/chat/completions/render``. The Cohere v2 protocol models are sourced from the official ``cohere`` Python SDK (``pip install cohere``). To keep that an *optional* -dependency for vLLM, the SDK-dependent imports - and the route handler -itself - are gated on a one-shot probe at module load. If the SDK isn't -installed, :func:`attach_router` becomes a no-op (with an info log) and -vLLM continues to boot normally. +dependency for vLLM, the SDK-dependent imports - and the route handlers +themselves - are gated on a one-shot probe at module load. If the SDK +isn't installed, :func:`attach_router` becomes a no-op (with an info +log) and vLLM continues to boot normally. Even when the SDK is installed, :func:`attach_router` also requires ``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose -the route. This keeps non-Cohere deployments that pull in the SDK for +the routes. This keeps non-Cohere deployments that pull in the SDK for unrelated reasons (e.g. test dependencies) from accidentally exposing the api. -Note: the handler must live at module scope (not inside +Note: the handlers must live at module scope (not inside ``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves the ``CohereChatV2Request`` body annotation against the module's -globals. Defining it locally inside ``attach_router`` would hide the +globals. Defining them locally inside ``attach_router`` would hide the type from ``get_type_hints``, causing FastAPI to silently degrade the body parameter into a query parameter and reject every request with 422. @@ -61,12 +67,17 @@ if _SDK_AVAILABLE: CohereError, ) from vllm.entrypoints.cohere.serving import CohereServingChatV2 + from vllm.entrypoints.scale_out.render.serving import ServingRender + from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest router = APIRouter() def _serving(request: Request) -> CohereServingChatV2 | None: return getattr(request.app.state, "cohere_serving_chat_v2", None) + def _serving_render(request: Request) -> ServingRender | None: + return getattr(request.app.state, "serving_render", None) + def _request_id(raw_request: Request | None) -> str | None: """Best-effort lookup of the active request id. @@ -143,6 +154,59 @@ if _SDK_AVAILABLE: case _: return StreamingResponse(content=result, media_type="text/event-stream") + @router.post( + "/cohere/v2/chat/render", + dependencies=[Depends(validate_json_request)], + response_model=GenerateRequest, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": CohereError}, + HTTPStatus.NOT_FOUND.value: {"model": CohereError}, + HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError}, + }, + ) + @with_cancellation + async def chat_v2_render(request: CohereChatV2Request, raw_request: Request): + """Tokenize a Cohere v2 chat request without running generation. + + The Cohere counterpart to ``POST /v1/chat/completions/render``. + The v2 body goes through the same conversion + ``/cohere/v2/chat`` uses and is then handed to the shared + :class:`ServingRender`, so the returned ``GenerateRequest`` + carries the prompt tokens and sampling params the chat endpoint + would have sent to the engine. + """ + handler = _serving(raw_request) + render_handler = _serving_render(raw_request) + if handler is None or render_handler is None: + return JSONResponse( + status_code=HTTPStatus.NOT_IMPLEMENTED.value, + content=CohereError( + message=( + "The model does not support the Cohere v2 chat render API." + ), + id=_request_id(raw_request), + ).model_dump(exclude_none=True), + ) + + try: + chat_request = handler.to_chat_completion_request(request) + result = await render_handler.render_chat_request(chat_request) + except Exception as e: # noqa: BLE001 - report as 500 for parity + logger.exception("Error in /cohere/v2/chat/render: %s", e) + return JSONResponse( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, + content=CohereError( + message=sanitize_message(str(e)), + id=_request_id(raw_request), + ).model_dump(exclude_none=True), + ) + + if isinstance(result, ErrorResponse): + return _error_response(result, raw_request) + + return JSONResponse(content=result.model_dump()) + class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware): """Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape. @@ -209,12 +273,12 @@ if _SDK_AVAILABLE: def attach_router(app: FastAPI) -> None: - """Register ``POST /cohere/v2/chat`` on ``app``. + """Register the ``/cohere/v2/chat`` routes on ``app``. No-op when either: * the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The - Cohere v2 endpoint is opt-in because it carries Cohere-specific + Cohere v2 endpoints are opt-in because they carry Cohere-specific request/response semantics (grounding citations, tool_plan, PLAN/THINKING_CONTENT blocks) that are only meaningful when serving a Cohere Command-family model. @@ -223,21 +287,21 @@ def attach_router(app: FastAPI) -> None: The two skip paths log at different levels: an operator who set ``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees - a WARNING (they explicitly asked for the endpoint and it's silently - absent), whereas the default-off skip logs at debug. + a WARNING (they explicitly asked for the endpoints and they're + silently absent), whereas the default-off skip logs at debug. """ enabled = envs.VLLM_ENABLE_COHERE_API if not enabled: logger.debug( - "VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint " - "disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it." + "VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints " + "disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them." ) return if not _SDK_AVAILABLE: logger.warning( "VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not " "installed; /cohere/v2/chat will not be exposed. Install " - "with `pip install cohere` to enable the endpoint." + "with `pip install cohere` to enable the endpoints." ) return app.include_router(router) diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 859ebd2b80..0d5d84a731 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -304,13 +304,7 @@ class CohereServingChatV2(OpenAIServingChat): "Received Cohere v2 chat request %s", request.model_dump_json() ) - chat_req = self._convert_v2_to_chat_completion(request) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "Converted Cohere v2 -> ChatCompletion: %s", - chat_req.model_dump_json(), - ) - + chat_req = self.to_chat_completion_request(request) generator = await self.create_chat_completion(chat_req, raw_request) match generator: @@ -321,6 +315,26 @@ class CohereServingChatV2(OpenAIServingChat): case _: return self._chat_completion_stream_to_v2(generator, request) + def to_chat_completion_request( + self, request: CohereChatV2Request + ) -> ChatCompletionRequest: + """Convert a Cohere v2 request into its ``ChatCompletionRequest``. + + Exposed for callers that need the converted request without + running generation - notably the render endpoint + (``POST /cohere/v2/chat/render``), which hands the result to + :meth:`vllm.entrypoints.scale_out.render.serving.ServingRender.render_chat_request` + so that Cohere requests tokenize through exactly the same path + as ``/v1/chat/completions/render``. + """ + chat_req = self._convert_v2_to_chat_completion(request) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Converted Cohere v2 -> ChatCompletion: %s", + chat_req.model_dump_json(), + ) + return chat_req + def _engine_chat_template_kwargs( self, chat_template_kwargs: dict[str, Any] ) -> dict[str, Any]: diff --git a/vllm/envs.py b/vllm/envs.py index fe9727aea7..16c57e7d7e 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1868,7 +1868,8 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool( int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0")) ), - # If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``. + # If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat`` + # and its render endpoint at ``POST /cohere/v2/chat/render``. # Default off "VLLM_ENABLE_COHERE_API": lambda: bool( int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))