Add Cohere ChatV2 render endpoint (#53219)
This commit is contained in:
@@ -1,24 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
|
||||
"""FastAPI router for the Cohere Chat v2 API.
|
||||
|
||||
Exposes two routes:
|
||||
|
||||
* ``POST /cohere/v2/chat`` - the chat endpoint itself.
|
||||
* ``POST /cohere/v2/chat/render`` - tokenize a request without running
|
||||
generation, mirroring ``POST /v1/chat/completions/render``.
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handler
|
||||
itself - are gated on a one-shot probe at module load. If the SDK isn't
|
||||
installed, :func:`attach_router` becomes a no-op (with an info log) and
|
||||
vLLM continues to boot normally.
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handlers
|
||||
themselves - are gated on a one-shot probe at module load. If the SDK
|
||||
isn't installed, :func:`attach_router` becomes a no-op (with an info
|
||||
log) and vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the route. This keeps non-Cohere deployments that pull in the SDK for
|
||||
the routes. This keeps non-Cohere deployments that pull in the SDK for
|
||||
unrelated reasons (e.g. test dependencies) from accidentally exposing
|
||||
the api.
|
||||
|
||||
Note: the handler must live at module scope (not inside
|
||||
Note: the handlers must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining it locally inside ``attach_router`` would hide the
|
||||
globals. Defining them locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
@@ -61,12 +67,17 @@ if _SDK_AVAILABLE:
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
from vllm.entrypoints.scale_out.render.serving import ServingRender
|
||||
from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _serving_render(request: Request) -> ServingRender | None:
|
||||
return getattr(request.app.state, "serving_render", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
@@ -143,6 +154,59 @@ if _SDK_AVAILABLE:
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat/render",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=GenerateRequest,
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
async def chat_v2_render(request: CohereChatV2Request, raw_request: Request):
|
||||
"""Tokenize a Cohere v2 chat request without running generation.
|
||||
|
||||
The Cohere counterpart to ``POST /v1/chat/completions/render``.
|
||||
The v2 body goes through the same conversion
|
||||
``/cohere/v2/chat`` uses and is then handed to the shared
|
||||
:class:`ServingRender`, so the returned ``GenerateRequest``
|
||||
carries the prompt tokens and sampling params the chat endpoint
|
||||
would have sent to the engine.
|
||||
"""
|
||||
handler = _serving(raw_request)
|
||||
render_handler = _serving_render(raw_request)
|
||||
if handler is None or render_handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message=(
|
||||
"The model does not support the Cohere v2 chat render API."
|
||||
),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
chat_request = handler.to_chat_completion_request(request)
|
||||
result = await render_handler.render_chat_request(chat_request)
|
||||
except Exception as e: # noqa: BLE001 - report as 500 for parity
|
||||
logger.exception("Error in /cohere/v2/chat/render: %s", e)
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
content=CohereError(
|
||||
message=sanitize_message(str(e)),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
if isinstance(result, ErrorResponse):
|
||||
return _error_response(result, raw_request)
|
||||
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
@@ -209,12 +273,12 @@ if _SDK_AVAILABLE:
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register ``POST /cohere/v2/chat`` on ``app``.
|
||||
"""Register the ``/cohere/v2/chat`` routes on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoint is opt-in because it carries Cohere-specific
|
||||
Cohere v2 endpoints are opt-in because they carry Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
@@ -223,21 +287,21 @@ def attach_router(app: FastAPI) -> None:
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoint and it's silently
|
||||
absent), whereas the default-off skip logs at debug.
|
||||
a WARNING (they explicitly asked for the endpoints and they're
|
||||
silently absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoints "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable them."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoint."
|
||||
"with `pip install cohere` to enable the endpoints."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
|
||||
@@ -304,13 +304,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
"Received Cohere v2 chat request %s", request.model_dump_json()
|
||||
)
|
||||
|
||||
chat_req = self._convert_v2_to_chat_completion(request)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"Converted Cohere v2 -> ChatCompletion: %s",
|
||||
chat_req.model_dump_json(),
|
||||
)
|
||||
|
||||
chat_req = self.to_chat_completion_request(request)
|
||||
generator = await self.create_chat_completion(chat_req, raw_request)
|
||||
|
||||
match generator:
|
||||
@@ -321,6 +315,26 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
case _:
|
||||
return self._chat_completion_stream_to_v2(generator, request)
|
||||
|
||||
def to_chat_completion_request(
|
||||
self, request: CohereChatV2Request
|
||||
) -> ChatCompletionRequest:
|
||||
"""Convert a Cohere v2 request into its ``ChatCompletionRequest``.
|
||||
|
||||
Exposed for callers that need the converted request without
|
||||
running generation - notably the render endpoint
|
||||
(``POST /cohere/v2/chat/render``), which hands the result to
|
||||
:meth:`vllm.entrypoints.scale_out.render.serving.ServingRender.render_chat_request`
|
||||
so that Cohere requests tokenize through exactly the same path
|
||||
as ``/v1/chat/completions/render``.
|
||||
"""
|
||||
chat_req = self._convert_v2_to_chat_completion(request)
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(
|
||||
"Converted Cohere v2 -> ChatCompletion: %s",
|
||||
chat_req.model_dump_json(),
|
||||
)
|
||||
return chat_req
|
||||
|
||||
def _engine_chat_template_kwargs(
|
||||
self, chat_template_kwargs: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
|
||||
Reference in New Issue
Block a user