Add Cohere ChatV2 render endpoint (#53219)
This commit is contained in:
@@ -11,6 +11,9 @@ Covers:
|
||||
* The router wiring: response shapes (JSON + SSE), error translation,
|
||||
and the ``cohere_serving_chat_v2 is None`` fallback (501 Not
|
||||
Implemented).
|
||||
* The render route (``POST /cohere/v2/chat/render``): that the v2 body
|
||||
reaches :class:`ServingRender` as a converted ``ChatCompletionRequest``
|
||||
and that the resulting ``GenerateRequest`` is returned verbatim.
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -29,7 +32,9 @@ from vllm.entrypoints.cohere.protocol import (
|
||||
AssistantMessageResponse,
|
||||
CohereChatV2Response,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
from vllm.entrypoints.serve.exception_handling.handlers.http import (
|
||||
http_exception_handler,
|
||||
)
|
||||
@@ -40,6 +45,7 @@ from vllm.entrypoints.serve.exception_handling.handlers.vllm_error import (
|
||||
vllm_error_handler,
|
||||
)
|
||||
from vllm.exceptions import VLLMError
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
@@ -78,6 +84,49 @@ class _Handler:
|
||||
return self.result
|
||||
|
||||
|
||||
class _RenderChatHandler:
|
||||
"""Stand-in for :class:`CohereServingChatV2` on the render route.
|
||||
|
||||
Only ``to_chat_completion_request`` is reachable from
|
||||
``/cohere/v2/chat/render``, and it delegates to the real conversion
|
||||
classmethod (which needs no engine or tokenizer) so the tests
|
||||
exercise the actual v2 -> ``ChatCompletionRequest`` translation
|
||||
rather than a stub of it.
|
||||
"""
|
||||
|
||||
def to_chat_completion_request(self, request):
|
||||
return CohereServingChatV2._convert_v2_to_chat_completion(request)
|
||||
|
||||
|
||||
class _RenderHandler:
|
||||
"""Minimal stand-in for :class:`ServingRender`.
|
||||
|
||||
Records the ``ChatCompletionRequest`` it was handed so tests can
|
||||
assert on the conversion, and returns ``self.result`` - either a
|
||||
:class:`GenerateRequest`, an :class:`ErrorResponse`, or an exception
|
||||
to raise.
|
||||
"""
|
||||
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
self.seen_request = None
|
||||
|
||||
async def render_chat_request(self, request):
|
||||
self.seen_request = request
|
||||
if isinstance(self.result, Exception):
|
||||
raise self.result
|
||||
return self.result
|
||||
|
||||
|
||||
def _generate_request() -> GenerateRequest:
|
||||
return GenerateRequest(
|
||||
request_id="chatcmpl-abc",
|
||||
token_ids=[5, 6, 7],
|
||||
sampling_params=SamplingParams(temperature=0.25),
|
||||
model="m",
|
||||
)
|
||||
|
||||
|
||||
def _build_app(handler: _Handler | None) -> FastAPI:
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
@@ -85,6 +134,14 @@ def _build_app(handler: _Handler | None) -> FastAPI:
|
||||
return app
|
||||
|
||||
|
||||
def _build_render_app(chat_handler, render_handler) -> FastAPI:
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
app.state.cohere_serving_chat_v2 = chat_handler
|
||||
app.state.serving_render = render_handler
|
||||
return app
|
||||
|
||||
|
||||
def _build_app_with_vllm_handlers(handler: _Handler | None) -> FastAPI:
|
||||
"""Build a FastAPI app that mirrors the real vLLM setup by installing
|
||||
validation, HTTP, and vLLM exception handlers. The
|
||||
@@ -321,6 +378,144 @@ class TestEndpoint:
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Render endpoint
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRenderEndpoint:
|
||||
"""``POST /cohere/v2/chat/render`` is the Cohere counterpart to
|
||||
``POST /v1/chat/completions/render``: it converts the v2 body to a
|
||||
``ChatCompletionRequest`` and hands it to the shared
|
||||
``ServingRender``, returning the resulting ``GenerateRequest``
|
||||
instead of running generation.
|
||||
|
||||
The route therefore spans *two* app-state handlers, and these tests
|
||||
pin both halves: that the conversion preserves the Cohere-specific
|
||||
request surface, and that the render handler's output (or error)
|
||||
reaches the client unchanged.
|
||||
"""
|
||||
|
||||
def test_route_registered(self):
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" in paths
|
||||
|
||||
def test_route_not_registered_when_flag_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("VLLM_ENABLE_COHERE_API", raising=False)
|
||||
app = FastAPI()
|
||||
attach_router(app)
|
||||
paths = [getattr(r, "path", None) for r in app.routes]
|
||||
assert "/cohere/v2/chat/render" not in paths
|
||||
|
||||
def test_returns_generate_request_json(self):
|
||||
render_handler = _RenderHandler(_generate_request())
|
||||
app = _build_render_app(_RenderChatHandler(), render_handler)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
assert r.headers["content-type"].startswith("application/json")
|
||||
body = r.json()
|
||||
assert body["request_id"] == "chatcmpl-abc"
|
||||
assert body["token_ids"] == [5, 6, 7]
|
||||
assert body["sampling_params"]["temperature"] == 0.25
|
||||
# The body must be a wire-valid GenerateRequest so callers can
|
||||
# feed it straight back to /v1/generate.
|
||||
assert GenerateRequest.model_validate(body).token_ids == [5, 6, 7]
|
||||
|
||||
def test_cohere_request_surface_survives_conversion(self):
|
||||
"""The point of routing through ``CohereServingChatV2`` rather
|
||||
than asking callers to pre-convert: Cohere-only fields
|
||||
(``documents``, ``safety_mode``, ``citation_options``) must land
|
||||
in ``chat_template_kwargs`` so the rendered prompt matches what
|
||||
``/cohere/v2/chat`` would have produced.
|
||||
"""
|
||||
render_handler = _RenderHandler(_generate_request())
|
||||
app = _build_render_app(_RenderChatHandler(), render_handler)
|
||||
body = {
|
||||
**_minimal_request_body(),
|
||||
"documents": [{"id": "doc-1", "data": {"text": "Paris is in France."}}],
|
||||
"safety_mode": "STRICT",
|
||||
"max_tokens": 32,
|
||||
}
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=body)
|
||||
assert r.status_code == HTTPStatus.OK
|
||||
|
||||
seen = render_handler.seen_request
|
||||
assert seen is not None
|
||||
assert seen.model == "m"
|
||||
assert seen.messages[0]["content"] == "hi"
|
||||
# ``max_tokens`` matters because ServingRender feeds it to
|
||||
# ``get_max_tokens`` when building sampling params.
|
||||
assert seen.max_completion_tokens == 32
|
||||
kwargs = seen.chat_template_kwargs or {}
|
||||
assert kwargs["documents"][0]["id"] == "doc-1"
|
||||
# The conversion lower-cases safety_mode for the renderer.
|
||||
assert kwargs["safety_mode"] == "strict"
|
||||
|
||||
def test_501_when_chat_handler_missing(self):
|
||||
app = _build_render_app(None, _RenderHandler(_generate_request()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
|
||||
assert "does not support" in r.json()["message"]
|
||||
|
||||
def test_501_when_render_handler_missing(self):
|
||||
# A server without ``serving_render`` in app state (e.g. a
|
||||
# non-generate task set) can't render, even though the Cohere
|
||||
# chat handler exists.
|
||||
app = _build_render_app(_RenderChatHandler(), None)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_IMPLEMENTED
|
||||
assert "does not support" in r.json()["message"]
|
||||
|
||||
def test_error_response_translated_to_cohere_envelope(self):
|
||||
err = ErrorResponse(
|
||||
error=ErrorInfo(message="unknown model", type="NotFound", code=404)
|
||||
)
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(err))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.NOT_FOUND
|
||||
assert r.json() == {"message": "unknown model"}
|
||||
|
||||
def test_render_exception_returns_500_envelope(self):
|
||||
app = _build_render_app(
|
||||
_RenderChatHandler(), _RenderHandler(RuntimeError("tokenizer exploded"))
|
||||
)
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
assert r.json() == {"message": "tokenizer exploded"}
|
||||
|
||||
def test_conversion_exception_returns_500_envelope(self):
|
||||
"""Conversion runs inside the same ``try`` as the render call, so
|
||||
a bad v2 body that slips past Pydantic still yields the Cohere
|
||||
error envelope rather than an unhandled 500 with vLLM's shape.
|
||||
"""
|
||||
|
||||
class _Boom:
|
||||
def to_chat_completion_request(self, request):
|
||||
raise ValueError("unconvertible message")
|
||||
|
||||
app = _build_render_app(_Boom(), _RenderHandler(_generate_request()))
|
||||
with TestClient(app) as client:
|
||||
r = client.post("/cohere/v2/chat/render", json=_minimal_request_body())
|
||||
assert r.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
assert r.json() == {"message": "unconvertible message"}
|
||||
|
||||
def test_invalid_body_returns_422(self):
|
||||
app = _build_render_app(_RenderChatHandler(), _RenderHandler(None))
|
||||
with TestClient(app) as client:
|
||||
r = client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={"messages": [{"role": "user", "content": "hi"}]},
|
||||
)
|
||||
assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# CohereErrorEnvelopeMiddleware
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end integration tests for the ``POST /cohere/v2/chat`` endpoint.
|
||||
"""End-to-end integration tests for the Cohere Chat v2 endpoints
|
||||
(``POST /cohere/v2/chat`` and ``POST /cohere/v2/chat/render``).
|
||||
|
||||
These tests spin up a real ``vllm serve`` process via
|
||||
:class:`tests.utils.RemoteOpenAIServer` and exercise the Cohere Chat v2
|
||||
@@ -173,6 +174,87 @@ async def test_cohere_v2_chat_validation_error_returns_400(
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render(httpx_client: httpx.AsyncClient):
|
||||
"""``/cohere/v2/chat/render`` tokenizes without running the model.
|
||||
|
||||
Beyond the ``GenerateRequest`` shape, this pins the cross-layer
|
||||
contract that makes the endpoint useful: v2 sampling fields
|
||||
(``temperature``, ``stop_sequences``) survive the v2 ->
|
||||
``ChatCompletionRequest`` -> ``SamplingParams`` hop, so a caller can
|
||||
render once and generate later without re-deriving them.
|
||||
"""
|
||||
resp = await httpx_client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "Say hi."}],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0.125,
|
||||
"stop_sequences": ["END"],
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
data = resp.json()
|
||||
|
||||
token_ids = data["token_ids"]
|
||||
assert isinstance(token_ids, list) and len(token_ids) > 0
|
||||
assert all(isinstance(t, int) and t >= 0 for t in token_ids)
|
||||
assert data["request_id"]
|
||||
|
||||
sampling_params = data["sampling_params"]
|
||||
assert sampling_params["temperature"] == 0.125
|
||||
assert sampling_params["stop"] == ["END"]
|
||||
|
||||
# No generation happened, so the response carries none of the chat
|
||||
# envelope the /cohere/v2/chat endpoint would return.
|
||||
assert "message" not in data
|
||||
assert "finish_reason" not in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render_token_ids_decode_to_the_prompt(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""The rendered ids are the real templated prompt, not a debug dump.
|
||||
|
||||
Round-tripping them through ``/detokenize`` and finding the user
|
||||
message back proves the endpoint applied the chat template and
|
||||
tokenized it, which is what a caller splitting prompt construction
|
||||
from generation depends on.
|
||||
"""
|
||||
render_resp = await httpx_client.post(
|
||||
"/cohere/v2/chat/render",
|
||||
json={
|
||||
"model": SERVED_MODEL_NAME,
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"max_tokens": 8,
|
||||
},
|
||||
)
|
||||
assert render_resp.status_code == 200, render_resp.text
|
||||
token_ids = render_resp.json()["token_ids"]
|
||||
|
||||
detok_resp = await httpx_client.post(
|
||||
"/detokenize",
|
||||
json={"model": SERVED_MODEL_NAME, "tokens": token_ids},
|
||||
)
|
||||
assert detok_resp.status_code == 200, detok_resp.text
|
||||
prompt = detok_resp.json()["prompt"]
|
||||
assert "What is the capital of France?" in prompt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_render_validation_error_returns_400(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
):
|
||||
"""Bad bodies get the Cohere error envelope, not vLLM's."""
|
||||
resp = await httpx_client.post("/cohere/v2/chat/render", json={"messages": []})
|
||||
assert resp.status_code == 400
|
||||
body = resp.json()
|
||||
assert "error" not in body
|
||||
assert body.get("message")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cohere_v2_chat_documents_field_accepted(
|
||||
httpx_client: httpx.AsyncClient,
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
|
||||
"""FastAPI router for the Cohere Chat v2 API.
|
||||
|
||||
Exposes two routes:
|
||||
|
||||
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
|
||||
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
|
||||
generation, mirroring ``POST /v1/chat/completions/render``.
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handler
|
||||
itself - are gated on a one-shot probe at module load. If the SDK isn't
|
||||
installed, :func:`attach_router` becomes a no-op (with an info log) and
|
||||
vLLM continues to boot normally.
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handlers
|
||||
themselves - are gated on a one-shot probe at module load. If the SDK
|
||||
isn't installed, :func:`attach_router` becomes a no-op (with an info
|
||||
log) and vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the route. This keeps non-Cohere deployments that pull in the SDK for
|
||||
the routes. This keeps non-Cohere deployments that pull in the SDK for
|
||||
unrelated reasons (e.g. test dependencies) from accidentally exposing
|
||||
the api.
|
||||
|
||||
Note: the handler must live at module scope (not inside
|
||||
Note: the handlers must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining it locally inside ``attach_router`` would hide the
|
||||
globals. Defining them locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
@@ -61,12 +67,17 @@ if _SDK_AVAILABLE:
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _serving_render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
@@ -143,6 +154,59 @@ if _SDK_AVAILABLE:
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
|
||||
"""Tokenize a Cohere v2 chat request without running generation.
|
||||
|
||||
The Cohere counterpart to ``POST /v1/chat/completions/render``.
|
||||
The v2 body goes through the same conversion
|
||||
``/cohere/v2/chat`` uses and is then handed to the shared
|
||||
:class:`ServingRender`, so the returned ``GenerateRequest``
|
||||
carries the prompt tokens and sampling params the chat endpoint
|
||||
would have sent to the engine.
|
||||
"""
|
||||
handler = _serving(raw_request)
|
||||
render_handler = _serving_render(raw_request)
|
||||
if handler is None or render_handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message=(
|
||||
"The model does not support the Cohere v2 chat render API."
|
||||
),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
chat_request = handler.to_chat_completion_request(request)
|
||||
result = await render_handler.render_chat_request(chat_request)
|
||||
except Exception as e: # noqa: BLE001 - report as 500 for parity
|
||||
logger.exception("Error in /cohere/v2/chat/render: %s", e)
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
content=CohereError(
|
||||
message=sanitize_message(str(e)),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return _error_response(result, raw_request)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
@@ -209,12 +273,12 @@ if _SDK_AVAILABLE:
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register ``POST /cohere/v2/chat`` on ``app``.
|
||||
"""Register the ``/cohere/v2/chat`` routes on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoint is opt-in because it carries Cohere-specific
|
||||
Cohere v2 endpoints are opt-in because they carry Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
@@ -223,21 +287,21 @@ def attach_router(app: FastAPI) -> None:
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoint and it's silently
|
||||
absent), whereas the default-off skip logs at debug.
|
||||
a WARNING (they explicitly asked for the endpoints and they're
|
||||
silently absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoint."
|
||||
"with `pip install cohere` to enable the endpoints."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
|
||||
@@ -304,13 +304,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
"Received Cohere v2 chat request %s", request.model_dump_json()
|
||||
)
|
||||
|
||||
chat_req = self._convert_v2_to_chat_completion(request)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"Converted Cohere v2 -> ChatCompletion: %s",
|
||||
chat_req.model_dump_json(),
|
||||
)
|
||||
|
||||
chat_req = self.to_chat_completion_request(request)
|
||||
generator = await self.create_chat_completion(chat_req, raw_request)
|
||||
|
||||
match generator:
|
||||
@@ -321,6 +315,26 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
case _:
|
||||
return self._chat_completion_stream_to_v2(generator, request)
|
||||
|
||||
def to_chat_completion_request(
|
||||
self, request: CohereChatV2Request
|
||||
) -> ChatCompletionRequest:
|
||||
"""Convert a Cohere v2 request into its ``ChatCompletionRequest``.
|
||||
|
||||
Exposed for callers that need the converted request without
|
||||
running generation - notably the render endpoint
|
||||
(``POST /cohere/v2/chat/render``), which hands the result to
|
||||
:meth:`vllm.entrypoints.scale_out.render.serving.ServingRender.render_chat_request`
|
||||
so that Cohere requests tokenize through exactly the same path
|
||||
as ``/v1/chat/completions/render``.
|
||||
"""
|
||||
chat_req = self._convert_v2_to_chat_completion(request)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"Converted Cohere v2 -> ChatCompletion: %s",
|
||||
chat_req.model_dump_json(),
|
||||
)
|
||||
return chat_req
|
||||
|
||||
def _engine_chat_template_kwargs(
|
||||
self, chat_template_kwargs: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
|
||||
+2
-1
@@ -1868,7 +1868,8 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
|
||||
),
|
||||
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``.
|
||||
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``
|
||||
# and its render endpoint at ``POST /cohere/v2/chat/render``.
|
||||
# Default off
|
||||
"VLLM_ENABLE_COHERE_API": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
|
||||
|
||||
Reference in New Issue
Block a user