[Frontend] Cohere chat v2 api support (#47189)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
6c91de3689
commit
9c110fa522
@@ -93,6 +93,7 @@ TokenizerMode = Literal[
|
||||
"deepseek_v4",
|
||||
"inkling",
|
||||
"kimi_k3",
|
||||
"cohere",
|
||||
]
|
||||
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
|
||||
LogprobsMode = Literal[
|
||||
@@ -151,6 +152,9 @@ class ModelConfig:
|
||||
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
|
||||
- "kimi_k3" will always use the "hf" tokenizer but render chat prompts
|
||||
with Kimi K3's Python XTML encoding instead of a Jinja template.
|
||||
- "cohere" uses the standard HF tokenizer but renders the chat template
|
||||
via the `cohere_melody` library (cmd3 / cmd4 templates) instead of
|
||||
Jinja, and surfaces grounded-citation metadata on responses.
|
||||
- Other custom values can be supported via plugins.
|
||||
|
||||
To swap the Rust BPE backend that powers HF fast tokenizers for the
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,244 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
|
||||
|
||||
The Cohere v2 protocol models are sourced from the official ``cohere``
|
||||
Python SDK (``pip install cohere``). To keep that an *optional*
|
||||
dependency for vLLM, the SDK-dependent imports - and the route handler
|
||||
itself - are gated on a one-shot probe at module load. If the SDK isn't
|
||||
installed, :func:`attach_router` becomes a no-op (with an info log) and
|
||||
vLLM continues to boot normally.
|
||||
|
||||
Even when the SDK is installed, :func:`attach_router` also requires
|
||||
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
|
||||
the route. This keeps non-Cohere deployments that pull in the SDK for
|
||||
unrelated reasons (e.g. test dependencies) from accidentally exposing
|
||||
the api.
|
||||
|
||||
Note: the handler must live at module scope (not inside
|
||||
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
|
||||
the ``CohereChatV2Request`` body annotation against the module's
|
||||
globals. Defining it locally inside ``attach_router`` would hide the
|
||||
type from ``get_type_hints``, causing FastAPI to silently degrade the
|
||||
body parameter into a query parameter and reject every request with
|
||||
422.
|
||||
"""
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
sanitize_message,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
_COHERE_PATH_PREFIX = "/cohere/"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
import cohere # noqa: F401 -- dependency probe
|
||||
except ImportError:
|
||||
_SDK_AVAILABLE = False
|
||||
else:
|
||||
_SDK_AVAILABLE = True
|
||||
|
||||
|
||||
if _SDK_AVAILABLE:
|
||||
from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereError,
|
||||
)
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
def _serving(request: Request) -> CohereServingChatV2 | None:
|
||||
return getattr(request.app.state, "cohere_serving_chat_v2", None)
|
||||
|
||||
def _request_id(raw_request: Request | None) -> str | None:
|
||||
"""Best-effort lookup of the active request id.
|
||||
|
||||
Prefers the id the underlying chat handler stamped onto
|
||||
``raw_request.state.request_metadata`` (if it got that far before
|
||||
failing), falling back to the ``X-Request-Id`` HTTP header. May
|
||||
return ``None`` if neither is available, in which case the field
|
||||
is omitted from the response.
|
||||
"""
|
||||
if raw_request is None:
|
||||
return None
|
||||
meta = getattr(raw_request.state, "request_metadata", None)
|
||||
if meta is not None and getattr(meta, "request_id", None):
|
||||
return meta.request_id
|
||||
return raw_request.headers.get("X-Request-Id")
|
||||
|
||||
def _error_response(
|
||||
error: ErrorResponse,
|
||||
raw_request: Request | None,
|
||||
*,
|
||||
fallback_status: int = HTTPStatus.BAD_REQUEST,
|
||||
) -> JSONResponse:
|
||||
"""Translate vLLM's internal error envelope into Cohere's shape."""
|
||||
info = error.error
|
||||
status = info.code or fallback_status
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content=CohereError(
|
||||
message=sanitize_message(info.message),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
@router.post(
|
||||
"/cohere/v2/chat",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
|
||||
handler = _serving(raw_request)
|
||||
if handler is None:
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
|
||||
content=CohereError(
|
||||
message="The model does not support the Cohere v2 chat API.",
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
try:
|
||||
result = await handler.create_chat_v2(request, raw_request)
|
||||
except Exception as e: # noqa: BLE001 - report as 500 for parity
|
||||
logger.exception("Error in /cohere/v2/chat: %s", e)
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
|
||||
content=CohereError(
|
||||
message=sanitize_message(str(e)),
|
||||
id=_request_id(raw_request),
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
|
||||
match result:
|
||||
case ErrorResponse():
|
||||
return _error_response(result, raw_request)
|
||||
case CohereChatV2Response():
|
||||
return JSONResponse(content=result.model_dump(exclude_none=True))
|
||||
case _:
|
||||
return StreamingResponse(content=result, media_type="text/event-stream")
|
||||
|
||||
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
|
||||
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
|
||||
|
||||
The endpoint handler above already returns :class:`CohereError` for
|
||||
errors it owns, but globally-registered exception handlers (e.g.
|
||||
:func:`validation_exception_handler` for pydantic body errors,
|
||||
:func:`http_exception_handler`, engine error handlers) fire *before*
|
||||
the handler runs and produce vLLM's internal
|
||||
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
|
||||
shape doesn't match the ``CohereError`` schema advertised on the
|
||||
route's OpenAPI ``responses``, so clients (and schema-conformance
|
||||
tests like ``test_openai_schema.py``) would see a mismatch on
|
||||
those paths. This middleware normalises any error body on
|
||||
``/cohere/*`` responses to :class:`CohereError`.
|
||||
"""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
response = await call_next(request)
|
||||
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
|
||||
return response
|
||||
if response.status_code < 400:
|
||||
return response
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if not content_type.startswith("application/json"):
|
||||
return response
|
||||
|
||||
body = b"".join([chunk async for chunk in response.body_iterator])
|
||||
translated = _translate_vllm_error_body(body, request)
|
||||
if translated is not None:
|
||||
return translated
|
||||
passthrough_headers = {
|
||||
k: v
|
||||
for k, v in response.headers.items()
|
||||
if k.lower() != "content-length"
|
||||
}
|
||||
return Response(
|
||||
content=body,
|
||||
status_code=response.status_code,
|
||||
headers=passthrough_headers,
|
||||
media_type=content_type,
|
||||
)
|
||||
|
||||
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
|
||||
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
|
||||
|
||||
Returns ``None`` if ``raw`` does not match the vLLM error envelope
|
||||
(which signals the middleware to pass the body through unchanged).
|
||||
"""
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
if not (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("error"), dict)
|
||||
and "message" in data["error"]
|
||||
):
|
||||
return None
|
||||
try:
|
||||
err = ErrorResponse.model_validate(data)
|
||||
except Exception: # noqa: BLE001 - malformed envelope; pass through
|
||||
return None
|
||||
return _error_response(err, request)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Register ``POST /cohere/v2/chat`` on ``app``.
|
||||
|
||||
No-op when either:
|
||||
|
||||
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
|
||||
Cohere v2 endpoint is opt-in because it carries Cohere-specific
|
||||
request/response semantics (grounding citations, tool_plan,
|
||||
PLAN/THINKING_CONTENT blocks) that are only meaningful when
|
||||
serving a Cohere Command-family model.
|
||||
* the optional ``cohere`` SDK isn't installed (the v2 protocol
|
||||
models live there)
|
||||
|
||||
The two skip paths log at different levels: an operator who set
|
||||
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
|
||||
a WARNING (they explicitly asked for the endpoint and it's silently
|
||||
absent), whereas the default-off skip logs at debug.
|
||||
"""
|
||||
enabled = envs.VLLM_ENABLE_COHERE_API
|
||||
if not enabled:
|
||||
logger.debug(
|
||||
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
|
||||
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
|
||||
)
|
||||
return
|
||||
if not _SDK_AVAILABLE:
|
||||
logger.warning(
|
||||
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
|
||||
"installed; /cohere/v2/chat will not be exposed. Install "
|
||||
"with `pip install cohere` to enable the endpoint."
|
||||
)
|
||||
return
|
||||
app.include_router(router)
|
||||
app.add_middleware(CohereErrorEnvelopeMiddleware)
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
|
||||
|
||||
This module keeps the OpenAI chat completion protocol classes
|
||||
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
|
||||
It provides two things:
|
||||
|
||||
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
|
||||
representation produced by the Cohere reasoning parser
|
||||
(:mod:`vllm.reasoning.cohere_command_reasoning_parser`) and consumed by
|
||||
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
|
||||
SDK shape -- that conversion happens in the serving layer.
|
||||
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
|
||||
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
|
||||
are only instantiated by the citation-aware serving handler
|
||||
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
|
||||
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
|
||||
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
|
||||
|
||||
The module intentionally does *not* import the ``cohere`` Python SDK so the
|
||||
reasoning parser (which is Cohere-model-specific but ships without the SDK
|
||||
dependency) can import from here freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_serializer
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaMessage,
|
||||
OpenAIBaseModel,
|
||||
)
|
||||
|
||||
|
||||
class CitationSource(OpenAIBaseModel):
|
||||
"""Source attribution for a :class:`Citation`.
|
||||
|
||||
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
|
||||
discriminator (``document`` or ``tool``); ``id`` is the citing
|
||||
document/tool-output identifier; ``document`` and ``tool_output`` carry
|
||||
the original payload that produced the citation.
|
||||
|
||||
Sources are fully resolved by the reasoning parser at emit time
|
||||
(see :func:`_melody_sources_to_vllm` in
|
||||
:mod:`vllm.reasoning.cohere_command_reasoning_parser`) using the
|
||||
``POSITION_TO_SOURCE_KEY`` map forwarded through
|
||||
``chat_template_kwargs`` by
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
|
||||
That means every instance that reaches the serving / wire layer
|
||||
already has ``type`` / ``id`` populated; the serving layer only has
|
||||
to coerce to :class:`cohere.types.Citation` and rewrite
|
||||
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
|
||||
"""
|
||||
|
||||
type: Literal["document", "tool"] | None = None
|
||||
id: str | None = None
|
||||
document: dict[str, Any] | None = None
|
||||
tool_output: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class Citation(OpenAIBaseModel):
|
||||
"""A citation grounding a span of generated text in source material.
|
||||
|
||||
vLLM-internal representation used by the Cohere reasoning parser and
|
||||
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
|
||||
conversion to the SDK's ``cohere.types.Citation`` happens in
|
||||
:mod:`vllm.entrypoints.cohere.serving`.
|
||||
"""
|
||||
|
||||
start: int | None = None
|
||||
"""Start character offset in the surrounding text content."""
|
||||
end: int | None = None
|
||||
"""End character offset (exclusive) in the surrounding text content."""
|
||||
text: str | None = None
|
||||
"""The cited text snippet."""
|
||||
sources: list[CitationSource] = Field(default_factory=list)
|
||||
"""Source documents / tool outputs that ground this citation."""
|
||||
content_index: int | None = None
|
||||
"""Index of the content block this citation refers to (when the message
|
||||
has multiple content blocks)."""
|
||||
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
|
||||
"""Which kind of content block this citation grounds: the user-visible
|
||||
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
|
||||
tool-plan block (``PLAN``). ``None`` means unspecified."""
|
||||
|
||||
|
||||
class CohereChatMessage(ChatMessage):
|
||||
""":class:`ChatMessage` extension carrying grounding citations.
|
||||
|
||||
Only instantiated by :class:`CohereServingChatV2` (and any other
|
||||
:class:`OpenAIServingChat` subclass that opts in by overriding
|
||||
``_create_chat_message``). Regular OpenAI-compatible handlers keep
|
||||
emitting plain :class:`ChatMessage` so their response schema is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
|
||||
so pydantic serializes this subclass with its own schema (including
|
||||
``citations``) when it flows through
|
||||
:class:`ChatCompletionResponseChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# ``mode="wrap"`` fully overrides (rather than chains) the
|
||||
# parent's ``@model_serializer``, so we explicitly delegate via
|
||||
# ``super()._serialize(handler)`` to preserve the parent's
|
||||
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
|
||||
# unset ``citations`` field so the wire matches the OpenAI-style
|
||||
# contract that optional vLLM extensions are omitted rather than
|
||||
# serialized as ``null``.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
|
||||
|
||||
class CohereDeltaMessage(DeltaMessage):
|
||||
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
|
||||
|
||||
Emitted by
|
||||
:class:`vllm.reasoning.cohere_command_reasoning_parser.BaseCohereCommandReasoningParser`
|
||||
on delta events whose payload includes citations. Non-Cohere parsers
|
||||
return plain :class:`DeltaMessage`, so their streamed shape is
|
||||
unchanged.
|
||||
|
||||
The response envelope declares
|
||||
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
|
||||
subclass with its own schema when it flows through
|
||||
:class:`ChatCompletionResponseStreamChoice`.
|
||||
"""
|
||||
|
||||
citations: list[Citation] | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
# See ``CohereChatMessage._serialize`` for why we delegate via
|
||||
# ``super()`` instead of calling ``handler(self)`` directly.
|
||||
data = super()._serialize(handler)
|
||||
if not data.get("citations"):
|
||||
data.pop("citations", None)
|
||||
return data
|
||||
@@ -0,0 +1,362 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere Chat v2 API protocol.
|
||||
|
||||
The bulk of the wire types come straight from the official ``cohere``
|
||||
Python SDK so we stay in sync with the upstream specification and
|
||||
avoid re-declaring the schema. We only own three things locally:
|
||||
|
||||
1. The top-level request body model (the SDK doesn't ship one — its
|
||||
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
|
||||
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
|
||||
2. The non-streaming response envelope (the SDK exposes the message
|
||||
shape via :class:`AssistantMessageResponse` but no full response
|
||||
wrapper).
|
||||
3. The streaming discriminated union (the SDK exports each event type
|
||||
individually but not as a combined ``Annotated[Union[...],
|
||||
discriminator]``).
|
||||
|
||||
Importing this module pulls in the ``cohere`` package. The router that
|
||||
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
|
||||
boots without the SDK installed.
|
||||
|
||||
See https://docs.cohere.com/reference/chat for the upstream spec.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from cohere import types as _sdk
|
||||
from cohere.types import (
|
||||
AssistantChatMessageV2,
|
||||
AssistantMessageResponse,
|
||||
ChatMessageV2,
|
||||
ChatRequestSafetyMode,
|
||||
Citation,
|
||||
CitationOptions,
|
||||
Document,
|
||||
ResponseFormatV2,
|
||||
SystemChatMessageV2,
|
||||
Thinking,
|
||||
ToolCallV2,
|
||||
ToolChatMessageV2,
|
||||
ToolV2,
|
||||
UserChatMessageV2,
|
||||
)
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
# Re-export the SDK wire-format types alongside our local extensions so
|
||||
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
|
||||
# they need from this module.
|
||||
__all__ = [
|
||||
"AssistantChatMessageV2",
|
||||
"AssistantMessageResponse",
|
||||
"ChatMessageV2",
|
||||
"Citation",
|
||||
"CitationEndEvent",
|
||||
"CitationOptions",
|
||||
"CitationStartEvent",
|
||||
"CohereChatV2Request",
|
||||
"CohereChatV2Response",
|
||||
"CohereError",
|
||||
"CohereFinishReason",
|
||||
"CohereLogprobItem",
|
||||
"CohereUsage",
|
||||
"CohereUsageBilledUnits",
|
||||
"CohereUsageTokens",
|
||||
"ContentDeltaEvent",
|
||||
"ContentEndEvent",
|
||||
"ContentStartEvent",
|
||||
"Document",
|
||||
"MessageEndEvent",
|
||||
"MessageStartEvent",
|
||||
"ResponseFormatV2",
|
||||
"SystemChatMessageV2",
|
||||
"Thinking",
|
||||
"ToolCallDeltaEvent",
|
||||
"ToolCallEndEvent",
|
||||
"ToolCallStartEvent",
|
||||
"ToolCallV2",
|
||||
"ToolChatMessageV2",
|
||||
"ToolPlanDeltaEvent",
|
||||
"ToolV2",
|
||||
"UserChatMessageV2",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Errors
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereError(BaseModel):
|
||||
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
|
||||
|
||||
Cohere's documented error schemas are uniform: ``{message, id}``.
|
||||
"""
|
||||
|
||||
message: str
|
||||
id: str | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool choice / finish reasons
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# These literals aren't first-class enums in the SDK but are documented at
|
||||
# https://docs.cohere.com/reference/chat. We declare them here so the
|
||||
# request/response models can validate them.
|
||||
|
||||
CohereToolChoice = Literal["REQUIRED", "NONE"]
|
||||
|
||||
CohereFinishReason = Literal[
|
||||
"COMPLETE",
|
||||
"STOP_SEQUENCE",
|
||||
"MAX_TOKENS",
|
||||
"TOOL_CALL",
|
||||
"ERROR",
|
||||
"TIMEOUT",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Request
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Request(BaseModel):
|
||||
"""Cohere Chat v2 request body.
|
||||
|
||||
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
|
||||
All structured fields delegate to the official SDK types so the body
|
||||
schema stays in sync with the upstream spec.
|
||||
"""
|
||||
|
||||
model: str
|
||||
messages: list[ChatMessageV2]
|
||||
stream: bool | None = False
|
||||
|
||||
# Tooling
|
||||
tools: list[ToolV2] | None = None
|
||||
strict_tools: bool | None = None
|
||||
tool_choice: CohereToolChoice | None = None
|
||||
|
||||
# Grounding
|
||||
documents: list[str | Document] | None = None
|
||||
citation_options: CitationOptions | None = None
|
||||
|
||||
# Output
|
||||
response_format: ResponseFormatV2 | None = None
|
||||
safety_mode: ChatRequestSafetyMode | None = None
|
||||
max_tokens: int | None = None
|
||||
stop_sequences: list[str] | None = None
|
||||
|
||||
# Sampling
|
||||
temperature: float | None = None
|
||||
seed: int | None = None
|
||||
frequency_penalty: float | None = None
|
||||
presence_penalty: float | None = None
|
||||
k: int | None = None
|
||||
p: float | None = None
|
||||
logprobs: bool | None = None
|
||||
|
||||
# Reasoning
|
||||
thinking: Thinking | None = None
|
||||
|
||||
# Scheduling
|
||||
priority: int | None = None
|
||||
|
||||
# vLLM-specific extensions (not in Cohere spec). These mirror what the
|
||||
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
|
||||
# the same engine knobs when needed.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the template."
|
||||
),
|
||||
)
|
||||
|
||||
@field_validator("model")
|
||||
@classmethod
|
||||
def _validate_model(cls, v: str) -> str:
|
||||
if not v:
|
||||
raise ValueError("model is required")
|
||||
return v
|
||||
|
||||
@field_validator("max_tokens")
|
||||
@classmethod
|
||||
def _validate_max_tokens(cls, v: int | None) -> int | None:
|
||||
if v is not None and v < 0:
|
||||
raise ValueError("max_tokens must be non-negative")
|
||||
return v
|
||||
|
||||
@field_validator("messages", mode="before")
|
||||
@classmethod
|
||||
def _normalize_message_roles(cls, v: Any) -> Any:
|
||||
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
|
||||
|
||||
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
|
||||
the four literal roles ``user`` / ``assistant`` / ``system`` /
|
||||
``tool``. OpenAI's ``developer`` role is documented as
|
||||
high-priority system instructions, so we alias it onto
|
||||
``system`` *before* the SDK discriminator runs (otherwise it
|
||||
rejects the message with a ``literal_error`` against each union
|
||||
member). Mirrors ``_role_to_melody`` in the renderer so the
|
||||
same alias is honoured no matter which surface the message
|
||||
arrives through.
|
||||
|
||||
``mode="before"`` is required so the rewrite happens before the
|
||||
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
|
||||
union; a default-mode validator would never see ``developer``
|
||||
because validation would have already failed. On any
|
||||
structural malformation (non-iterable input, items without a
|
||||
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
|
||||
and let Pydantic's normal coercion surface a precise error.
|
||||
"""
|
||||
try:
|
||||
return [
|
||||
{**msg, "role": "system"}
|
||||
if msg.get("role", "").lower() == "developer"
|
||||
else msg
|
||||
for msg in v
|
||||
]
|
||||
except (AttributeError, TypeError):
|
||||
return v
|
||||
|
||||
@field_validator("messages")
|
||||
@classmethod
|
||||
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
|
||||
if not v:
|
||||
raise ValueError("messages must contain at least one message")
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Usage / Logprobs
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
|
||||
# envelope is documented separately in the OpenAPI spec and we declare it
|
||||
# here.
|
||||
|
||||
|
||||
class CohereUsageBilledUnits(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
search_units: float | None = None
|
||||
classifications: float | None = None
|
||||
|
||||
|
||||
class CohereUsageTokens(BaseModel):
|
||||
input_tokens: float | None = None
|
||||
output_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereUsage(BaseModel):
|
||||
billed_units: CohereUsageBilledUnits | None = None
|
||||
tokens: CohereUsageTokens | None = None
|
||||
cached_tokens: float | None = None
|
||||
|
||||
|
||||
class CohereLogprobItem(BaseModel):
|
||||
text: str | None = None
|
||||
token_ids: list[int]
|
||||
logprobs: list[float] | None = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming response
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereChatV2Response(BaseModel):
|
||||
"""Cohere Chat v2 non-streaming response body.
|
||||
|
||||
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
|
||||
the documented v2 response envelope (``id``, ``finish_reason``,
|
||||
``usage``, ``logprobs``). The single constructor in
|
||||
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
|
||||
supplying a non-empty ``id`` (falling back to a synthesized one if
|
||||
the upstream response is missing it) to this model.
|
||||
"""
|
||||
|
||||
id: str
|
||||
finish_reason: CohereFinishReason
|
||||
message: AssistantMessageResponse
|
||||
usage: CohereUsage | None = None
|
||||
logprobs: list[CohereLogprobItem] | None = None
|
||||
|
||||
# vLLM-specific extension.
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming events
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
|
||||
# Sent Events; each event's ``type`` field carries the discriminator. The
|
||||
# SDK exposes a Pydantic model per event but none of them declare ``type``
|
||||
# as a field (the SDK relies on its own deserializer for discrimination),
|
||||
# so a naive ``model_dump_json()`` would silently drop the discriminator
|
||||
# and break clients that demux on ``type``.
|
||||
#
|
||||
# We therefore subclass each SDK event and bake the wire-format ``type``
|
||||
# string in as a ``Literal`` field with a default. ``model_dump()`` now
|
||||
# emits ``type`` for free, and surfaces can simply construct the event
|
||||
# class and serialize it -- no manual ``type`` parameter needed.
|
||||
#
|
||||
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
|
||||
# ``StreamedChatResponseV2`` schema for the wire-format reference.
|
||||
|
||||
|
||||
class MessageStartEvent(_sdk.ChatMessageStartEvent):
|
||||
type: Literal["message-start"] = "message-start"
|
||||
|
||||
|
||||
class ContentStartEvent(_sdk.ChatContentStartEvent):
|
||||
type: Literal["content-start"] = "content-start"
|
||||
|
||||
|
||||
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
|
||||
type: Literal["content-delta"] = "content-delta"
|
||||
|
||||
|
||||
class ContentEndEvent(_sdk.ChatContentEndEvent):
|
||||
type: Literal["content-end"] = "content-end"
|
||||
|
||||
|
||||
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
|
||||
type: Literal["tool-plan-delta"] = "tool-plan-delta"
|
||||
|
||||
|
||||
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
|
||||
type: Literal["tool-call-start"] = "tool-call-start"
|
||||
|
||||
|
||||
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
|
||||
type: Literal["tool-call-delta"] = "tool-call-delta"
|
||||
|
||||
|
||||
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
|
||||
type: Literal["tool-call-end"] = "tool-call-end"
|
||||
|
||||
|
||||
class CitationStartEvent(_sdk.CitationStartEvent):
|
||||
type: Literal["citation-start"] = "citation-start"
|
||||
|
||||
|
||||
class CitationEndEvent(_sdk.CitationEndEvent):
|
||||
type: Literal["citation-end"] = "citation-end"
|
||||
|
||||
|
||||
class MessageEndEvent(_sdk.ChatMessageEndEvent):
|
||||
type: Literal["message-end"] = "message-end"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
@@ -41,6 +43,12 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_anthropic_api_router(app)
|
||||
|
||||
from vllm.entrypoints.cohere.api_router import (
|
||||
attach_router as register_cohere_api_router,
|
||||
)
|
||||
|
||||
register_cohere_api_router(app)
|
||||
|
||||
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||
|
||||
register_generative_scoring_api_router(app)
|
||||
@@ -55,6 +63,23 @@ async def init_generate_state(
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.entrypoints.chat_utils import load_chat_template
|
||||
|
||||
# The Cohere serving handler depends on the optional `cohere` SDK for
|
||||
# its wire-format protocol models, and is additionally gated on the
|
||||
# `VLLM_ENABLE_COHERE_API` env flag (see
|
||||
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
|
||||
# entirely when the endpoint isn't going to be exposed, both because
|
||||
# the SDK may not be installed and because the serving object holds
|
||||
# nontrivial state (chat handler, warmup) that would otherwise be
|
||||
# unused.
|
||||
if envs.VLLM_ENABLE_COHERE_API:
|
||||
try:
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
except ImportError:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
|
||||
from vllm.entrypoints.mcp.tool_server import (
|
||||
DemoToolServer,
|
||||
MCPToolServer,
|
||||
@@ -86,6 +111,19 @@ async def init_generate_state(
|
||||
tool_server = None
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
|
||||
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
|
||||
# default chat-template kwargs. The cohere renderer reads
|
||||
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
|
||||
# rendering; making this a first-class flag keeps the right format
|
||||
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
|
||||
# without forcing them to hand-construct a JSON dict for
|
||||
# ``--default-chat-template-kwargs``. Per-request overrides still
|
||||
# take precedence (see ``merge_kwargs`` in
|
||||
# ``ChatCompletionRequest.build_chat_params``).
|
||||
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
|
||||
if getattr(args, "cohere_format", None):
|
||||
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
|
||||
|
||||
# Render endpoints are always backed by OnlineRenderer so that
|
||||
# /v1/chat/completions/render and /v1/completions/render work on both
|
||||
# generate-mode and render-only servers. Created in init_app_state.
|
||||
@@ -106,7 +144,7 @@ async def init_generate_state(
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
@@ -119,7 +157,7 @@ async def init_generate_state(
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
@@ -169,11 +207,32 @@ async def init_generate_state(
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
default_chat_template_kwargs=args.default_chat_template_kwargs,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.cohere_serving_chat_v2 = (
|
||||
CohereServingChatV2(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
is_reasoning_model=args.cohere_is_reasoning_model,
|
||||
)
|
||||
if CohereServingChatV2 is not None and "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
|
||||
from .generative_scoring.serving import ServingGenerativeScoring
|
||||
|
||||
|
||||
@@ -10,7 +10,13 @@ from openai.types.chat.chat_completion_audio import (
|
||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
|
||||
from pydantic import Field, PrivateAttr, model_serializer, model_validator
|
||||
from pydantic import (
|
||||
Field,
|
||||
PrivateAttr,
|
||||
SerializeAsAny,
|
||||
model_serializer,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -91,7 +97,12 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||
|
||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
message: ChatMessage
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
|
||||
# so that added fields like ``citations`` survive JSON serialization
|
||||
# instead of being stripped down to the base schema. Plain
|
||||
# ``ChatMessage`` instances serialize identically to before.
|
||||
message: SerializeAsAny[ChatMessage]
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
@@ -139,7 +150,11 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
delta: DeltaMessage
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
|
||||
# so streaming ``citations`` survive JSON serialization. Plain
|
||||
# ``DeltaMessage`` instances serialize identically to before.
|
||||
delta: SerializeAsAny[DeltaMessage]
|
||||
logprobs: ChatCompletionLogProbs | None = None
|
||||
finish_reason: str | None = None
|
||||
stop_reason: int | str | None = None
|
||||
|
||||
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
return self.response_role
|
||||
return request.messages[-1]["role"]
|
||||
|
||||
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
|
||||
"""Construct the response :class:`ChatMessage` for the non-streaming path.
|
||||
|
||||
The full-generator calls this at every construction site so
|
||||
subclasses can swap in a specialized :class:`ChatMessage`
|
||||
subclass (e.g. :class:`CohereServingChatV2` returning
|
||||
:class:`CohereChatMessage`) without duplicating the branchy
|
||||
tool-choice / auto-tools logic that decides which fields are
|
||||
populated. The default returns a plain :class:`ChatMessage`.
|
||||
"""
|
||||
return ChatMessage(*args, **kwargs)
|
||||
|
||||
def _finalize_response_message(
|
||||
self,
|
||||
message: ChatMessage,
|
||||
*,
|
||||
parser: Parser | None,
|
||||
) -> ChatMessage:
|
||||
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
|
||||
|
||||
Default is a no-op. Subclasses that need to surface parser-side
|
||||
extras (e.g. :class:`CohereServingChatV2` reading grounding
|
||||
citations off the reasoning parser and populating
|
||||
:class:`CohereChatMessage.citations`) override this to inspect
|
||||
``parser`` and mutate/replace ``message``.
|
||||
"""
|
||||
return message
|
||||
|
||||
async def chat_completion_stream_generator(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
is_required_tool_choice = request.tool_choice == "required"
|
||||
|
||||
# All six construction sites route through ``self._create_chat_message``
|
||||
# so subclasses can swap in a specialized :class:`ChatMessage`
|
||||
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
|
||||
# having to duplicate this branch logic.
|
||||
if (not self.enable_auto_tools or not tool_parser_cls) and (
|
||||
not is_named_tool_choice and not is_required_tool_choice
|
||||
):
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
elif is_named_tool_choice or is_required_tool_choice:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content or "",
|
||||
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
# if the request doesn't use tool choice
|
||||
# OR specifies to not use a tool
|
||||
elif not request.tool_choice or request.tool_choice == "none":
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# handle when there are tools and tool choice is auto
|
||||
elif (
|
||||
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
):
|
||||
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
||||
if tool_calls:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
)
|
||||
|
||||
else:
|
||||
message = ChatMessage(
|
||||
message = self._create_chat_message(
|
||||
role=role,
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
" if tools should be extracted. Returning a standard chat "
|
||||
"completion."
|
||||
)
|
||||
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
||||
message = self._create_chat_message(
|
||||
role=role, reasoning=reasoning, content=content
|
||||
)
|
||||
|
||||
# Subclass hook: enrich the constructed message with any
|
||||
# parser-side extras that don't fit through the plain
|
||||
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
|
||||
# citation-aware handlers use this to surface grounding
|
||||
# metadata cached on the reasoning parser.
|
||||
message = self._finalize_response_message(message, parser=parser)
|
||||
|
||||
# In OpenAI's API, when a tool is called, the finish_reason is:
|
||||
# "tool_calls" for "auto" or "required" tool calls,
|
||||
# and "stop" for named tool calls.
|
||||
|
||||
@@ -148,6 +148,25 @@ class BaseFrontendArgs:
|
||||
"""If set to False, output deltas will not be logged. Relevant only if
|
||||
--enable-log-outputs is set.
|
||||
"""
|
||||
cohere_is_reasoning_model: bool = True
|
||||
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
|
||||
reasoning Command-family model. When True (default), the assistant's
|
||||
chain-of-thought is surfaced as a ``thinking`` content block (or
|
||||
``content-*`` events on the stream). When False, reasoning is
|
||||
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
|
||||
events) whenever the model emits tool calls, matching older non-
|
||||
reasoning Command models. Has no effect on the non-Cohere
|
||||
endpoints."""
|
||||
cohere_format: str = "cmd4"
|
||||
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
|
||||
format to render: ``cmd4`` (current Command A+ models; default) or
|
||||
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
|
||||
wrong format silently produces a prompt the model wasn't trained
|
||||
on, which most commonly manifests as the model emitting text but no
|
||||
citations / tool calls / thinking blocks. Equivalent to passing
|
||||
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
|
||||
explicit request-level ``chat_template_kwargs.cohere_format`` takes
|
||||
priority."""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
|
||||
@@ -39,7 +39,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
|
||||
@@ -240,6 +240,7 @@ if TYPE_CHECKING:
|
||||
VLLM_LOOPBACK_IP: str = ""
|
||||
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
|
||||
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
|
||||
VLLM_ENABLE_COHERE_API: bool = False
|
||||
VLLM_HAS_FLASHINFER_CUBIN: bool = False
|
||||
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
|
||||
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
|
||||
@@ -1758,6 +1759,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
|
||||
),
|
||||
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``.
|
||||
# Default off
|
||||
"VLLM_ENABLE_COHERE_API": lambda: bool(
|
||||
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
|
||||
),
|
||||
# If set, use the fp8 mfma in rocm paged attention.
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
|
||||
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
|
||||
|
||||
@@ -20,6 +20,11 @@ except ImportError as e:
|
||||
) from e
|
||||
|
||||
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import (
|
||||
Citation,
|
||||
CitationSource,
|
||||
CohereDeltaMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
@@ -404,6 +409,83 @@ def _schema_dict_from_structured_outputs(
|
||||
)
|
||||
|
||||
|
||||
def _melody_sources_to_vllm(
|
||||
raw_sources: Any,
|
||||
position_to_source: Mapping[tuple[int, int], CitationSource] | None,
|
||||
) -> list[CitationSource]:
|
||||
"""Convert melody's ``Source`` objects into resolved :class:`CitationSource`.
|
||||
|
||||
Melody's ``Source`` shape is
|
||||
``{tool_call_index, tool_result_indices, document_ids}``. Without
|
||||
:meth:`PyFilterOptions.with_message_history` the ``document_ids``
|
||||
list is empty, so we resolve the numeric address ourselves using
|
||||
``position_to_source``, which is
|
||||
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._build_position_to_source`
|
||||
applied to the inbound request and forwarded through
|
||||
``chat_template_kwargs`` at parser-construction time. Each melody
|
||||
source with ``tool_result_indices=[i, j, ...]`` fans out to one
|
||||
:class:`CitationSource` per resolved position, populating the
|
||||
wire-shape ``type`` / ``id`` / ``document`` / ``tool_output``
|
||||
fields; unresolved positions are silently skipped (the calling
|
||||
citation is dropped at wire-coercion time if every source in it
|
||||
ends up unresolved -- see ``_to_wire_citation`` in
|
||||
:mod:`vllm.entrypoints.cohere.serving`).
|
||||
|
||||
When ``position_to_source`` is ``None`` (parser wired outside of
|
||||
:class:`CohereServingChatV2`, e.g. a plain OpenAI chat completion
|
||||
request that happens to hit a cohere reasoning parser) sources
|
||||
can't be resolved and get dropped; citations then never reach any
|
||||
citation-aware handler downstream.
|
||||
"""
|
||||
if not position_to_source:
|
||||
return []
|
||||
out: list[CitationSource] = []
|
||||
for s in raw_sources or []:
|
||||
bucket = getattr(s, "tool_call_index", None)
|
||||
indices = list(getattr(s, "tool_result_indices", None) or [])
|
||||
if bucket is None or not indices:
|
||||
continue
|
||||
for idx in indices:
|
||||
info = position_to_source.get((bucket, idx))
|
||||
if info is not None:
|
||||
out.append(info)
|
||||
return out
|
||||
|
||||
|
||||
def _melody_citations_to_vllm(
|
||||
raw_citations: Any,
|
||||
position_to_source: Mapping[tuple[int, int], CitationSource] | None,
|
||||
) -> list[Citation] | None:
|
||||
"""Convert melody's ``FilterCitation`` objects into :class:`Citation`.
|
||||
|
||||
Resolves each source in-place via ``position_to_source`` (see
|
||||
:func:`_melody_sources_to_vllm`). Citations whose sources all fail
|
||||
to resolve are still emitted here with ``sources=[]``; the serving
|
||||
layer drops them at wire-coercion time so the fail-closed policy
|
||||
lives in exactly one place.
|
||||
"""
|
||||
if not raw_citations:
|
||||
return None
|
||||
out: list[Citation] = []
|
||||
for c in raw_citations:
|
||||
out.append(
|
||||
Citation(
|
||||
start=getattr(c, "start_index", None),
|
||||
end=getattr(c, "end_index", None),
|
||||
text=getattr(c, "text", None),
|
||||
sources=_melody_sources_to_vllm(
|
||||
getattr(c, "sources", None), position_to_source
|
||||
),
|
||||
type=(
|
||||
"THINKING_CONTENT"
|
||||
if getattr(c, "is_thinking", False)
|
||||
else "TEXT_CONTENT"
|
||||
),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
class BaseCohereCommandReasoningParser(ReasoningParser):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -420,6 +502,32 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
|
||||
self.unary_opts = unary_opts
|
||||
self.melody_unary = PyFilter(unary_opts)
|
||||
self.melody_streaming = PyFilter(streaming_opts)
|
||||
# Citations extracted by the most recent ``extract_reasoning`` call.
|
||||
# Citation-aware handlers (e.g. :class:`CohereServingChatV2` via its
|
||||
# ``_finalize_response_message`` hook) read this back from the
|
||||
# parser instance (which is constructed per-request) and attach
|
||||
# the result to :class:`CohereChatMessage.citations`. ``None`` when
|
||||
# the last parse produced no citations.
|
||||
self.last_unary_citations: list[Citation] | None = None
|
||||
# Request-scoped ``(tool_call_index, tool_result_idx) -> resolved
|
||||
# CitationSource`` map, forwarded from
|
||||
# :meth:`CohereServingChatV2._apply_cohere_template_kwargs` via
|
||||
# ``chat_template_kwargs``. See :func:`_melody_sources_to_vllm`
|
||||
# for how it's consumed. Absent for parser instances that
|
||||
# weren't created by ``CohereServingChatV2`` (e.g. an OpenAI
|
||||
# chat completion request routed through a cohere reasoning
|
||||
# parser); citations from those requests are dropped since we
|
||||
# have no way to attribute their sources.
|
||||
ctk = kwargs.get("chat_template_kwargs") or {}
|
||||
# Lazy import: ``renderers.cohere`` transitively imports
|
||||
# ``cohere_melody`` and other heavy deps; hoisting the constant
|
||||
# to module scope would tie parser importability to that graph.
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
|
||||
raw_map = ctk.get(POSITION_TO_SOURCE_KEY)
|
||||
self._position_to_source: Mapping[tuple[int, int], CitationSource] | None = (
|
||||
raw_map if isinstance(raw_map, Mapping) else None
|
||||
)
|
||||
|
||||
@property
|
||||
def reasoning_start_str(self) -> str | None:
|
||||
@@ -439,9 +547,22 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
|
||||
delta_token_ids: Sequence[int],
|
||||
) -> DeltaMessage | None:
|
||||
r = self.melody_streaming.write_decoded(delta_text)
|
||||
if r.content is None and r.reasoning is None and not r.tool_calls:
|
||||
citations = _melody_citations_to_vllm(
|
||||
getattr(r, "citations", None), self._position_to_source
|
||||
)
|
||||
if (
|
||||
r.content is None
|
||||
and r.reasoning is None
|
||||
and not r.tool_calls
|
||||
and not citations
|
||||
):
|
||||
return None
|
||||
msg = DeltaMessage()
|
||||
# Always emit CohereDeltaMessage so citations reach the wire via
|
||||
# ``SerializeAsAny[DeltaMessage]`` on the streaming choice envelope.
|
||||
# When ``citations`` is unset, ``CohereDeltaMessage._serialize``
|
||||
# drops the field so the emitted shape matches plain
|
||||
# :class:`DeltaMessage`
|
||||
msg = CohereDeltaMessage()
|
||||
if r.content is not None:
|
||||
msg.content = r.content
|
||||
if r.reasoning is not None:
|
||||
@@ -456,12 +577,23 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
|
||||
)
|
||||
for tc in r.tool_calls
|
||||
]
|
||||
if citations:
|
||||
msg.citations = citations
|
||||
return msg
|
||||
|
||||
def extract_reasoning(
|
||||
self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> tuple[str | None, str | None]:
|
||||
result = self.melody_unary.process_full_text(model_output)
|
||||
# Cache citations so citation-aware handlers can surface them
|
||||
# on :class:`CohereChatMessage.citations`. The base
|
||||
# :meth:`ReasoningParser.extract_reasoning` contract only
|
||||
# returns ``(reasoning, content)``, so citations are passed
|
||||
# back via parser-instance state -- safe because the parser is
|
||||
# constructed per-request.
|
||||
self.last_unary_citations = _melody_citations_to_vllm(
|
||||
getattr(result, "citations", None), self._position_to_source
|
||||
)
|
||||
return result.reasoning, result.content
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
@@ -549,12 +681,25 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
|
||||
return request
|
||||
|
||||
|
||||
# melody's streaming filter only buffers a partial ``<co: ...>`` citation
|
||||
# across ``write_decoded`` calls when ``stream_non_grounded_answer`` is
|
||||
# set: otherwise, the moment an opening ``<co`` is seen without a
|
||||
# closing ``</co: ...>`` in the same delta, the filter emits the partial
|
||||
# marker bytes verbatim as plain content. In vLLM's streaming path the
|
||||
# parser is fed one token (1-4 chars) per call, so an unbuffered filter
|
||||
# will leak ``<co: 0>``-style markers into ``delta.content`` and never
|
||||
# emit a ``FilterCitation`` for them. Enabling the flag flips the
|
||||
# partial-match branch in melody's ``parse_citations`` (see
|
||||
# ``src/parsing/citations_filter.rs``) to ``return (None, 0)`` -- i.e.
|
||||
# keep buffering -- which lets a full citation eventually resolve.
|
||||
# Non-streaming (unary) parsing receives the whole output in one call so
|
||||
# the flag is a no-op there and we leave ``unary_opts`` alone.
|
||||
class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
|
||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
*args,
|
||||
streaming_opts=PyFilterOptions().cmd3(),
|
||||
streaming_opts=PyFilterOptions().cmd3().stream_non_grounded_answer(),
|
||||
unary_opts=PyFilterOptions().cmd3().no_tools(),
|
||||
**kwargs,
|
||||
)
|
||||
@@ -565,7 +710,7 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
*args,
|
||||
streaming_opts=PyFilterOptions().cmd4(),
|
||||
streaming_opts=PyFilterOptions().cmd4().stream_non_grounded_answer(),
|
||||
unary_opts=PyFilterOptions().cmd4().no_tools(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,704 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cohere prompt renderer.
|
||||
|
||||
Templates the Cohere Command-family prompt formats (cmd3 / cmd4) using
|
||||
the ``cohere_melody`` Rust bindings instead of Jinja. Enabled by
|
||||
passing ``--tokenizer-mode cohere`` on the engine; tokenization itself
|
||||
still flows through the cached HuggingFace tokenizer.
|
||||
|
||||
The renderer's client surface is ``chat_template_kwargs`` (vLLM's
|
||||
generic passthrough for template-time inputs). It is fed by two
|
||||
different code paths, which speak different vocabularies:
|
||||
|
||||
1. :class:`vllm.entrypoints.cohere.serving.CohereServingChatV2` --
|
||||
translates a native `Cohere Chat v2 request
|
||||
<https://docs.cohere.com/reference/chat>`__ into
|
||||
``chat_template_kwargs``.
|
||||
2. :class:`OpenAIServingChat` -- forwards whatever the client puts in
|
||||
``request.chat_template_kwargs`` directly, without translation.
|
||||
|
||||
Accordingly the renderer accepts keys from both vocabularies.
|
||||
|
||||
**Cohere Chat v2 request fields** (populated automatically by
|
||||
``CohereServingChatV2``; a direct ``chat_template_kwargs`` caller may
|
||||
also pass them):
|
||||
|
||||
* ``documents``: list of documents (v2 ``documents``)
|
||||
* ``tools``: list of tools (v2 ``tools``)
|
||||
* ``safety_mode``: v2 ``safety_mode`` (``CONTEXTUAL`` / ``STRICT`` /
|
||||
``NONE``) -- forwarded to melody's cmd3 ``safety_mode`` slot;
|
||||
lowercased.
|
||||
* ``citation_options``: v2 ``citation_options``. Its ``.mode``
|
||||
(``ENABLED`` / ``DISABLED`` / ``FAST`` / ``ACCURATE`` / ``OFF``) is
|
||||
normalized into melody's cmd3 ``citation_quality`` (``on`` / ``off``)
|
||||
and cmd4 ``grounding`` (``enabled`` / ``disabled`` / ``unknown``)
|
||||
slots. cmd4 has no fast/accurate distinction at the prompt-template
|
||||
layer -- both mean grounding-on.
|
||||
* ``response_format``: v2 ``response_format`` (``json_object`` /
|
||||
``json_schema``) -- mapped onto melody's ``json_mode`` /
|
||||
``json_schema`` config slots.
|
||||
* ``thinking``: v2 ``thinking``; its ``.type`` (``enabled`` /
|
||||
``disabled``) becomes melody's ``reasoning_type``.
|
||||
|
||||
**Melody template-config knobs** (accepted for direct-passthrough
|
||||
callers; not part of the Cohere Chat v2 surface):
|
||||
|
||||
* ``cohere_format``: renderer selector, ``cmd3`` or ``cmd4`` (default
|
||||
``cmd4``).
|
||||
* ``template_id``: pick one of melody's built-in template variants.
|
||||
* ``available_tools``: melody's own name for tools; takes precedence
|
||||
over ``tools`` when both are set.
|
||||
* ``reasoning_type``: direct ``enabled`` / ``disabled`` toggle;
|
||||
overrides derivation from ``thinking.type``.
|
||||
* ``dev_instruction``: developer-instruction override.
|
||||
* ``json_mode`` / ``json_schema``: direct structured-output toggles;
|
||||
override derivation from ``response_format``.
|
||||
* cmd3-only: ``citation_quality`` (``on`` / ``off``), ``skip_preamble``.
|
||||
* cmd4-only: ``grounding`` (``enabled`` / ``disabled`` / ``unknown``;
|
||||
overrides derivation from ``citation_options.mode``),
|
||||
``platform_instruction``.
|
||||
|
||||
**Rejected inputs**: ``template_jinja`` and ``template`` are Cohere's
|
||||
own inlets for raw Jinja template source. They are valid at Cohere's
|
||||
API surface but not at vLLM's -- raw template source must flow through
|
||||
the standard ``chat_template`` request field (guarded by
|
||||
``--trust-request-chat-template``), which this renderer forwards to
|
||||
melody as ``template_jinja``. Setting these keys explicitly in
|
||||
``chat_template_kwargs`` raises ``ValueError`` so client
|
||||
misconfiguration surfaces loudly.
|
||||
|
||||
**Everything else** in ``chat_template_kwargs`` is forwarded verbatim
|
||||
to melody as ``additional_template_fields`` and becomes accessible as
|
||||
Jinja variables inside the template. This matches vLLM's documented
|
||||
contract for ``chat_template_kwargs`` ("kwargs accessible by the
|
||||
template"), so e.g. ``chat_template_kwargs={"reasoning_effort":
|
||||
"low"}`` resolves ``{{ reasoning_effort }}`` inside cmd3 / cmd4
|
||||
templates.
|
||||
|
||||
Citations produced by Cohere models are surfaced through the
|
||||
Cohere-scoped ``CohereChatMessage.citations`` /
|
||||
``CohereDeltaMessage.citations`` fields (see
|
||||
:mod:`vllm.entrypoints.cohere.cohere_chat_message`), populated by the
|
||||
``cohere2`` reasoning parser. The base OpenAI ``ChatMessage`` /
|
||||
``DeltaMessage`` keep their declared schemas unchanged; the response
|
||||
envelope declares them as ``SerializeAsAny[...]`` so the subclass
|
||||
fields survive JSON serialization.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ConversationMessage,
|
||||
parse_chat_messages,
|
||||
parse_chat_messages_async,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.utils.async_utils import make_async
|
||||
|
||||
from .base import BaseRenderer
|
||||
from .inputs import DictPrompt
|
||||
from .inputs.preprocess import parse_dec_only_prompt
|
||||
from .params import ChatParams
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_DEFAULT_FORMAT = "cmd4"
|
||||
_VALID_FORMATS = ("cmd3", "cmd4")
|
||||
|
||||
|
||||
class MelodyContentType(str, Enum):
|
||||
"""Wire-format discriminator for melody content blocks.
|
||||
|
||||
These strings are what the cmd3 / cmd4 Jinja templates check against
|
||||
in ``message.content[0].type`` (e.g. the ``thinking`` branch in
|
||||
``cmd4-v1.jinja``). Keep new values in sync with melody's template
|
||||
schema.
|
||||
"""
|
||||
|
||||
TEXT = "text"
|
||||
THINKING = "thinking"
|
||||
IMAGE = "image"
|
||||
DOCUMENT = "document"
|
||||
|
||||
|
||||
# Reserved ``chat_template_kwargs`` key that carries
|
||||
# ``AssistantChatMessageV2.citations`` from a Cohere v2 request through
|
||||
# to the renderer, keyed by request-message index. Populated by
|
||||
# ``CohereServingChatV2._apply_cohere_template_kwargs`` and consumed by
|
||||
# ``_conversation_to_melody_messages``. The value is a
|
||||
# ``dict[int, list[dict]]`` of melody-shape ``FilterCitation`` dicts.
|
||||
# The leading underscore signals "internal, not part of the public
|
||||
# ``chat_template_kwargs`` surface" -- clients should not set this key
|
||||
# directly.
|
||||
MESSAGES_CITATIONS_KEY = "_messages_citations"
|
||||
|
||||
|
||||
# Reserved ``chat_template_kwargs`` key carrying a request-side
|
||||
# ``dict[tuple[int, int], CitationSource]`` map from melody's numeric
|
||||
# ``(tool_call_index, tool_result_index)`` addressing to fully-resolved
|
||||
# wire-shape sources (type, id, document/tool_output payload).
|
||||
# Populated by :meth:`CohereServingChatV2._apply_cohere_template_kwargs`
|
||||
# and consumed by :class:`BaseCohereCommandReasoningParser` at request
|
||||
# start so the parser can attach real document ids on each melody source
|
||||
# it emits -- eliminating the numeric-coord round-trip through the
|
||||
# internal streaming pipeline. Leading underscore signals "internal,
|
||||
# not part of the public ``chat_template_kwargs`` surface" -- clients
|
||||
# should not set this key directly.
|
||||
POSITION_TO_SOURCE_KEY = "_position_to_source"
|
||||
|
||||
|
||||
# Keys this renderer interprets directly from ``chat_template_kwargs`` and
|
||||
# maps onto typed melody render-config fields. Everything *not* in this
|
||||
# set is forwarded verbatim to melody as ``additional_template_fields``
|
||||
# (i.e. as Jinja template variables), so callers can write
|
||||
# ``chat_template_kwargs = {"my_var": "..."}`` and have ``{{ my_var }}``
|
||||
# resolve inside the template -- matching vLLM's documented contract
|
||||
# for ``chat_template_kwargs``.
|
||||
_RENDERER_CONSUMED_KEYS = frozenset(
|
||||
{
|
||||
"cohere_format",
|
||||
"template_id",
|
||||
# ``template_jinja`` / ``template`` are Cohere-only inputs for
|
||||
# raw Jinja source -- valid at Cohere's own API surface but not
|
||||
# at vLLM's. In vLLM, raw template source must flow through the
|
||||
# standard ``chat_template`` request field, so a non-``None``
|
||||
# value here is rejected in ``_build_render_config``. The keys
|
||||
# stay in the consumed set so an explicit ``None`` (e.g. from
|
||||
# ``.model_dump(exclude_none=False)``) is dropped rather than
|
||||
# surfacing as a stray ``{{ template_jinja }}`` Jinja variable.
|
||||
"template_jinja",
|
||||
"template",
|
||||
"use_jinja",
|
||||
"documents",
|
||||
"available_tools",
|
||||
"tools",
|
||||
"reasoning_type",
|
||||
"thinking",
|
||||
"dev_instruction",
|
||||
"response_format",
|
||||
"json_schema",
|
||||
"json_mode",
|
||||
"safety_mode",
|
||||
"citation_quality",
|
||||
"citation_options",
|
||||
"skip_preamble",
|
||||
"grounding",
|
||||
"platform_instruction",
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _try_import_melody():
|
||||
try:
|
||||
import cohere_melody # type: ignore
|
||||
|
||||
return cohere_melody
|
||||
except ImportError as e: # pragma: no cover - exercised at runtime
|
||||
raise ImportError(
|
||||
"The `cohere` tokenizer/renderer mode requires the "
|
||||
"`cohere_melody` package. Install it via "
|
||||
"`pip install cohere-melody` or build from "
|
||||
"https://github.com/cohere-ai/melody."
|
||||
) from e
|
||||
|
||||
|
||||
_MELODY_ROLES = frozenset({"system", "user", "chatbot", "tool"})
|
||||
|
||||
|
||||
# Cohere v2 ``citation_options.mode`` -> melody cmd4 ``grounding``.
|
||||
# v2 surfaces five modes (``ENABLED`` / ``DISABLED`` / ``FAST`` /
|
||||
# ``ACCURATE`` / ``OFF``) but cmd4's template doesn't differentiate
|
||||
# fast vs accurate at the prompt layer -- both just turn grounding on,
|
||||
# and ``ENABLED`` / ``DISABLED`` are direct aliases. Melody itself
|
||||
# accepts ``unknown`` / ``enabled`` / ``disabled``.
|
||||
_CMD4_GROUNDING_FROM_MODE = {
|
||||
"fast": "enabled",
|
||||
"accurate": "enabled",
|
||||
"on": "enabled",
|
||||
"enabled": "enabled",
|
||||
"off": "disabled",
|
||||
"disabled": "disabled",
|
||||
"unknown": "unknown",
|
||||
}
|
||||
|
||||
|
||||
def _normalize_cmd4_grounding(value: Any) -> str:
|
||||
"""Coerce a user-facing grounding/citation mode to melody's vocab.
|
||||
|
||||
Raises ``ValueError`` rather than letting an unrecognized value
|
||||
slip through to ``render_cmd4`` and surface as a generic
|
||||
``Invalid config: grounding`` from melody.
|
||||
"""
|
||||
out = _CMD4_GROUNDING_FROM_MODE.get(value.lower())
|
||||
if out is None:
|
||||
raise ValueError(
|
||||
f"Unrecognized cmd4 grounding value: {value!r}. Expected one "
|
||||
f"of ENABLED / DISABLED / FAST / ACCURATE / OFF (from "
|
||||
f"citation_options.mode), or the direct melody values "
|
||||
f"enabled / disabled / unknown."
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _role_to_melody(role: str) -> str:
|
||||
"""Map an OpenAI role to the role string melody expects.
|
||||
|
||||
The cmd3 / cmd4 jinja templates only recognize ``system``, ``user``,
|
||||
``assistant``/``chatbot``, and ``tool``: any other role is silently
|
||||
dropped by the template's role-dispatch chain (no fallback branch),
|
||||
which produces a malformed prompt without any error. We therefore
|
||||
refuse unknown roles up front rather than letting them disappear.
|
||||
|
||||
Aliases:
|
||||
|
||||
* ``assistant`` -> ``chatbot`` (Cohere's historical assistant role
|
||||
name used by the templates).
|
||||
* ``developer`` -> ``system`` (OpenAI's ``developer`` role is
|
||||
documented as high-priority instructions, which maps onto the
|
||||
``system`` slot in Cohere's prompt format).
|
||||
"""
|
||||
role = role.lower()
|
||||
if role == "assistant":
|
||||
return "chatbot"
|
||||
if role == "developer":
|
||||
return "system"
|
||||
if role in _MELODY_ROLES:
|
||||
return role
|
||||
raise ValueError(
|
||||
f"Unsupported message role for the cohere renderer: {role!r}. "
|
||||
"Expected one of: system, developer, user, assistant, chatbot, tool."
|
||||
)
|
||||
|
||||
|
||||
def _normalize_tool_call(tc: dict[str, Any] | Any) -> dict[str, Any]:
|
||||
"""Normalize an OpenAI tool call dict into melody's tool call shape.
|
||||
|
||||
melody expects ``{id, name, parameters: <json str>}`` whereas OpenAI
|
||||
delivers ``{id, type, function: {name, arguments: <json|str>}}``.
|
||||
"""
|
||||
# Pydantic objects
|
||||
if hasattr(tc, "model_dump"):
|
||||
tc = tc.model_dump()
|
||||
if not isinstance(tc, dict):
|
||||
raise TypeError(f"Unexpected tool_call value: {tc!r}")
|
||||
|
||||
fn = tc.get("function") or {}
|
||||
name = fn.get("name") or tc.get("name") or ""
|
||||
args = fn.get("arguments")
|
||||
if args is None:
|
||||
args = tc.get("arguments", {})
|
||||
# melody expects a JSON-encoded string
|
||||
if not isinstance(args, str):
|
||||
args = json.dumps(args, ensure_ascii=False)
|
||||
return {
|
||||
"id": tc.get("id") or "",
|
||||
"name": name,
|
||||
"parameters": args,
|
||||
}
|
||||
|
||||
|
||||
def _content_blocks(content: Any) -> list[dict[str, Any]]:
|
||||
"""Convert OpenAI ``ConversationMessage.content`` into melody content blocks.
|
||||
|
||||
The chat_utils ``content_format="openai"`` produces a list of
|
||||
``{type: text, text: ...}`` dicts already; we pass those through with
|
||||
minimal coercion. Plain string content is wrapped in a single text block.
|
||||
Image / multimodal placeholder dicts (``{"type": "image"}``) are
|
||||
forwarded as-is so that templates can reference image placeholders that
|
||||
the upstream tokenizer expands separately.
|
||||
"""
|
||||
if content is None:
|
||||
return []
|
||||
if isinstance(content, str):
|
||||
return [{"type": MelodyContentType.TEXT, "text": content}]
|
||||
blocks: list[dict[str, Any]] = []
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
blocks.append({"type": MelodyContentType.TEXT, "text": part})
|
||||
continue
|
||||
if not isinstance(part, dict):
|
||||
raise TypeError(f"Unexpected content part: {part!r}")
|
||||
part_type = part.get("type", MelodyContentType.TEXT)
|
||||
if part_type in ("text", "input_text", "output_text", "refusal"):
|
||||
blocks.append(
|
||||
{"type": MelodyContentType.TEXT, "text": part.get("text", "")}
|
||||
)
|
||||
elif part_type == MelodyContentType.THINKING:
|
||||
blocks.append(
|
||||
{
|
||||
"type": MelodyContentType.THINKING,
|
||||
"thinking": part.get("thinking", ""),
|
||||
}
|
||||
)
|
||||
elif part_type == MelodyContentType.IMAGE:
|
||||
blocks.append(
|
||||
{
|
||||
"type": MelodyContentType.IMAGE,
|
||||
"image": {
|
||||
"template_placeholder": part.get(
|
||||
"template_placeholder", "<image>"
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
elif part_type == MelodyContentType.DOCUMENT:
|
||||
doc = part.get("document")
|
||||
if isinstance(doc, dict):
|
||||
blocks.append({"type": MelodyContentType.DOCUMENT, "document": doc})
|
||||
else:
|
||||
# Fall back to wrapping arbitrary string as text.
|
||||
blocks.append({"type": MelodyContentType.TEXT, "text": json.dumps(doc)})
|
||||
elif part_type == "tool_reference":
|
||||
# Tool references are rendered by name; emit as text so the
|
||||
# renderer downstream is content-format agnostic.
|
||||
blocks.append(
|
||||
{
|
||||
"type": MelodyContentType.TEXT,
|
||||
"text": part.get("name") or part.get("text", ""),
|
||||
}
|
||||
)
|
||||
else:
|
||||
# Unknown block type: render as text fallback.
|
||||
text = part.get("text") or part.get(part_type) or ""
|
||||
text_str = text if isinstance(text, str) else json.dumps(text)
|
||||
blocks.append({"type": MelodyContentType.TEXT, "text": text_str})
|
||||
return blocks
|
||||
|
||||
|
||||
def _document_to_melody(doc: Any) -> dict[str, Any]:
|
||||
"""Coerce a Cohere v2 document into melody's ``Document`` (dict) shape."""
|
||||
match doc:
|
||||
case str():
|
||||
return {"text": doc}
|
||||
# Cohere v2 wraps documents in {id, data: {...}}; the ``**rest``
|
||||
# capture keeps the outer ``id`` (and any other outer keys) so we
|
||||
# can lift them onto the payload without losing them.
|
||||
case {"data": dict() as data, **rest}:
|
||||
payload = dict(data)
|
||||
if (outer_id := rest.get("id")) is not None and "id" not in payload:
|
||||
payload["id"] = outer_id
|
||||
return payload
|
||||
case dict():
|
||||
return dict(doc)
|
||||
case _:
|
||||
raise TypeError(f"Unsupported document type: {type(doc).__name__}")
|
||||
|
||||
|
||||
def _tool_to_melody(tool: Any) -> dict[str, Any]:
|
||||
"""Coerce a chat completions tool definition into melody's ``Tool`` shape.
|
||||
|
||||
Accepts either the raw OpenAI tool wrapper ``{type:"function", function:
|
||||
{name, description, parameters}}`` or a flat ``{name, description,
|
||||
parameters}`` dict (which is what melody itself expects).
|
||||
"""
|
||||
if hasattr(tool, "model_dump"):
|
||||
tool = tool.model_dump()
|
||||
if not isinstance(tool, dict):
|
||||
raise TypeError(f"Unsupported tool type: {type(tool).__name__}")
|
||||
if "function" in tool and isinstance(tool["function"], dict):
|
||||
fn = tool["function"]
|
||||
else:
|
||||
fn = tool
|
||||
return {
|
||||
"name": fn.get("name", ""),
|
||||
"description": fn.get("description", "") or "",
|
||||
"parameters": fn.get("parameters") or {},
|
||||
}
|
||||
|
||||
|
||||
def _conversation_to_melody_messages(
|
||||
conversation: list[ConversationMessage],
|
||||
messages_citations: dict[int, list[dict[str, Any]]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Convert vLLM ``ConversationMessage``s into melody's message dict shape.
|
||||
|
||||
``messages_citations`` is an optional index-keyed lookup of
|
||||
per-message citations from the request, populated by
|
||||
``CohereServingChatV2`` under the
|
||||
:data:`MESSAGES_CITATIONS_KEY` chat_template_kwargs entry. Values
|
||||
must already be in melody's ``FilterCitation`` dict shape (see
|
||||
``CohereServingChatV2._sdk_citation_to_melody``). Index is the
|
||||
position in the *input* ``conversation`` list, which currently maps
|
||||
1-to-1 with ``CohereChatV2Request.messages`` for text-only Cohere
|
||||
requests. This mapping breaks silently if ``parse_chat_messages``
|
||||
ever expands a single input message into multiple entries (e.g.
|
||||
multi-modal splitting), which is not the case today.
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for i, msg in enumerate(conversation):
|
||||
role = _role_to_melody(msg.get("role", "user"))
|
||||
content_blocks = _content_blocks(msg.get("content"))
|
||||
|
||||
# Treat reasoning content as a thinking block on assistant turns
|
||||
# so multi-turn reasoning is preserved in the rendered prompt.
|
||||
#
|
||||
# Cohere models output thought as either a ``thinking``
|
||||
# block (reasoning models) or a ``tool_plan`` field (older non-
|
||||
# reasoning Command models), but vLLM's ConversationMessage has a
|
||||
# single unified ``reasoning`` field that drops that difference.
|
||||
# The cmd3 / cmd4 jinja templates render ``thinking`` blocks in both
|
||||
# the case of tool calls and regular thinking blocks, so this is fine
|
||||
# from the renderer's perspective.
|
||||
reasoning = msg.get("reasoning") or msg.get("reasoning_content")
|
||||
if role == "chatbot" and reasoning:
|
||||
content_blocks.insert(
|
||||
0,
|
||||
{"type": MelodyContentType.THINKING, "thinking": reasoning},
|
||||
)
|
||||
|
||||
tool_calls = [_normalize_tool_call(tc) for tc in (msg.get("tool_calls") or [])]
|
||||
|
||||
out_msg: dict[str, Any] = {
|
||||
"role": role,
|
||||
"content": content_blocks,
|
||||
"tool_calls": tool_calls,
|
||||
}
|
||||
tool_call_id = msg.get("tool_call_id")
|
||||
if tool_call_id:
|
||||
out_msg["tool_call_id"] = tool_call_id
|
||||
if messages_citations and (cites := messages_citations.get(i)):
|
||||
out_msg["citations"] = cites
|
||||
out.append(out_msg)
|
||||
return out
|
||||
|
||||
|
||||
def _build_render_config(
|
||||
conversation: list[ConversationMessage],
|
||||
chat_template_kwargs: dict[str, Any],
|
||||
chat_template: str | None = None,
|
||||
) -> tuple[str, dict[str, Any]]:
|
||||
"""Build the ``render_cmd3`` / ``render_cmd4`` config dict.
|
||||
|
||||
``chat_template`` is the standard vLLM per-request template override
|
||||
(from ``ChatCompletionRequest.chat_template`` / the server-side
|
||||
``--chat-template`` flag). When non-``None`` it is forwarded to melody
|
||||
as ``template_jinja`` so a single vLLM-native path drives template
|
||||
source through the trust guard.
|
||||
|
||||
Returns ``(format, config_dict)`` where ``format`` is either ``"cmd3"``
|
||||
or ``"cmd4"``.
|
||||
"""
|
||||
fmt = chat_template_kwargs.get("cohere_format", _DEFAULT_FORMAT)
|
||||
if fmt not in _VALID_FORMATS:
|
||||
raise ValueError(
|
||||
f"Invalid cohere_format={fmt!r}; expected one of {_VALID_FORMATS}"
|
||||
)
|
||||
|
||||
# Reject Cohere-only inlets for raw Jinja source. These keys are
|
||||
# valid at Cohere's own API surface but not at vLLM's: callers must
|
||||
# use the standard vLLM ``chat_template`` request field so the
|
||||
# ``--trust-request-chat-template`` guard applies uniformly.
|
||||
for cohere_only_key in ("template_jinja", "template"):
|
||||
if chat_template_kwargs.get(cohere_only_key) is not None:
|
||||
raise ValueError(
|
||||
f"chat_template_kwargs.{cohere_only_key!r} is not accepted "
|
||||
"in vLLM (it is a Cohere-only field); pass the template "
|
||||
"body via the standard 'chat_template' request field "
|
||||
"instead (requires --trust-request-chat-template)."
|
||||
)
|
||||
|
||||
config: dict[str, Any] = {
|
||||
"messages": _conversation_to_melody_messages(
|
||||
conversation,
|
||||
chat_template_kwargs.get(MESSAGES_CITATIONS_KEY),
|
||||
),
|
||||
}
|
||||
|
||||
# ``template_id`` is a selector for one of melody's built-in template
|
||||
# variants (not raw source), so it is safe to accept from the client.
|
||||
if template_id := chat_template_kwargs.get("template_id"):
|
||||
config["template_id"] = template_id
|
||||
# Raw template source flows exclusively through the standard vLLM
|
||||
# ``chat_template`` field so it is subject to the
|
||||
# ``--trust-request-chat-template`` guard in OnlineRenderer.
|
||||
if chat_template:
|
||||
config["template_jinja"] = chat_template
|
||||
# Only support Jinja with vllm
|
||||
config["use_jinja"] = True
|
||||
|
||||
# Documents
|
||||
documents = chat_template_kwargs.get("documents") or []
|
||||
if documents:
|
||||
config["documents"] = [_document_to_melody(d) for d in documents]
|
||||
|
||||
# Tools - prefer explicit ``available_tools``, fall back to OpenAI ``tools``
|
||||
tools = (
|
||||
chat_template_kwargs.get("available_tools")
|
||||
or chat_template_kwargs.get("tools")
|
||||
or []
|
||||
)
|
||||
if tools:
|
||||
config["available_tools"] = [_tool_to_melody(t) for t in tools]
|
||||
|
||||
# Reasoning toggle (cmd3 + cmd4)
|
||||
if (rt := chat_template_kwargs.get("reasoning_type")) is not None:
|
||||
config["reasoning_type"] = str(rt)
|
||||
elif "thinking" in chat_template_kwargs:
|
||||
# Cohere v2 ``thinking: {type: enabled|disabled}`` shorthand
|
||||
thinking = chat_template_kwargs["thinking"]
|
||||
t = thinking.get("type") if isinstance(thinking, dict) else thinking
|
||||
if t in ("enabled", "disabled"):
|
||||
config["reasoning_type"] = t
|
||||
|
||||
if (di := chat_template_kwargs.get("dev_instruction")) is not None:
|
||||
config["dev_instruction"] = str(di)
|
||||
|
||||
# JSON / structured outputs
|
||||
if (rf := chat_template_kwargs.get("response_format")) is not None:
|
||||
rf = rf.model_dump() if hasattr(rf, "model_dump") else dict(rf)
|
||||
rf_type = rf.get("type")
|
||||
if rf_type == "json_object":
|
||||
config["json_mode"] = True
|
||||
elif rf_type in ("json_schema", "json"):
|
||||
schema = rf.get("schema") or rf.get("json_schema")
|
||||
if isinstance(schema, dict) and "schema" in schema:
|
||||
schema = schema["schema"]
|
||||
if schema is not None:
|
||||
config["json_schema"] = (
|
||||
schema if isinstance(schema, str) else json.dumps(schema)
|
||||
)
|
||||
if (js := chat_template_kwargs.get("json_schema")) is not None:
|
||||
config["json_schema"] = js if isinstance(js, str) else json.dumps(js)
|
||||
if "json_mode" in chat_template_kwargs:
|
||||
config["json_mode"] = bool(chat_template_kwargs["json_mode"])
|
||||
|
||||
if fmt == "cmd3":
|
||||
if (sm := chat_template_kwargs.get("safety_mode")) is not None:
|
||||
config["safety_mode"] = str(sm).lower()
|
||||
# citation_quality: ``on`` / ``off``
|
||||
cq = chat_template_kwargs.get("citation_quality")
|
||||
if cq is None and (co := chat_template_kwargs.get("citation_options")):
|
||||
mode = co.get("mode") if isinstance(co, dict) else None
|
||||
if mode is not None:
|
||||
cq = "on" if str(mode).lower() != "off" else "off"
|
||||
if cq is not None:
|
||||
config["citation_quality"] = str(cq).lower()
|
||||
if "skip_preamble" in chat_template_kwargs:
|
||||
config["skip_preamble"] = bool(chat_template_kwargs["skip_preamble"])
|
||||
else: # cmd4
|
||||
# cmd4 uses ``grounding`` rather than safety_mode/citation_quality.
|
||||
# melody's cmd4 only accepts ``unknown`` / ``enabled`` / ``disabled``,
|
||||
# so the Cohere v2 ``citation_options.mode`` values
|
||||
# (``FAST`` / ``ACCURATE`` / ``OFF``) have to be normalized -- a
|
||||
# raw lowercased passthrough would raise from
|
||||
# ``render_cmd4`` (cmd4 has no fast/accurate distinction at the
|
||||
# prompt-template layer; both request grounding-on).
|
||||
if (gr := chat_template_kwargs.get("grounding")) is not None:
|
||||
config["grounding"] = _normalize_cmd4_grounding(gr)
|
||||
elif co := chat_template_kwargs.get("citation_options"):
|
||||
mode = co.get("mode") if isinstance(co, dict) else None
|
||||
if mode is not None:
|
||||
config["grounding"] = _normalize_cmd4_grounding(mode)
|
||||
if (pi := chat_template_kwargs.get("platform_instruction")) is not None:
|
||||
config["platform_instruction"] = str(pi)
|
||||
|
||||
# Anything we didn't explicitly interpret above is forwarded to melody
|
||||
# as a Jinja template variable. This matches vLLM's documented contract
|
||||
# for ``chat_template_kwargs`` ("kwargs accessible by the template")
|
||||
# and lets callers write e.g. ``{"reasoning_effort": "low"}`` directly
|
||||
# without a nested ``additional_template_fields`` wrapper.
|
||||
extra = {
|
||||
k: v
|
||||
for k, v in chat_template_kwargs.items()
|
||||
if k not in _RENDERER_CONSUMED_KEYS
|
||||
}
|
||||
if extra:
|
||||
config["additional_template_fields"] = extra
|
||||
|
||||
return fmt, config
|
||||
|
||||
|
||||
class CohereRenderer(BaseRenderer[HfTokenizer]):
|
||||
"""Renderer that templates Cohere prompts via the melody Rust bindings.
|
||||
|
||||
Tokenization is delegated to the standard HF tokenizer; only the
|
||||
chat-template step is replaced with ``cohere_melody.render_cmd3`` /
|
||||
``render_cmd4``. Enabled via ``--tokenizer-mode cohere``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: VllmConfig,
|
||||
tokenizer: HfTokenizer | None,
|
||||
) -> None:
|
||||
# Match HfRenderer in not mutating the cached tokenizer instance
|
||||
tokenizer = copy.copy(tokenizer)
|
||||
super().__init__(config, tokenizer)
|
||||
|
||||
# Lazy import to keep `cohere_melody` an optional dependency
|
||||
self._melody = _try_import_melody()
|
||||
# ``render_cmd3`` / ``render_cmd4`` are pure CPU work; cache the
|
||||
# thread-pool wrapper once so the async path doesn't allocate a new
|
||||
# adapter on every request.
|
||||
self._render_async = make_async(self._render, executor=self._executor)
|
||||
|
||||
def _render(self, fmt: str, config_dict: dict[str, Any]) -> str:
|
||||
if fmt == "cmd3":
|
||||
return self._melody.render_cmd3(config_dict)
|
||||
return self._melody.render_cmd4(config_dict)
|
||||
|
||||
def render_messages(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
conversation, mm_data, mm_uuids = parse_chat_messages(
|
||||
messages,
|
||||
self.model_config,
|
||||
content_format="openai",
|
||||
media_io_kwargs=params.media_io_kwargs,
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
chat_template_kwargs = dict(params.chat_template_kwargs)
|
||||
fmt, config_dict = _build_render_config(
|
||||
conversation, chat_template_kwargs, params.chat_template
|
||||
)
|
||||
prompt_text = self._render(fmt, config_dict)
|
||||
prompt = parse_dec_only_prompt(prompt_text)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt
|
||||
|
||||
async def render_messages_async(
|
||||
self,
|
||||
messages: list[ChatCompletionMessageParam],
|
||||
params: ChatParams,
|
||||
) -> tuple[list[ConversationMessage], DictPrompt]:
|
||||
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
|
||||
messages,
|
||||
self.model_config,
|
||||
content_format="openai",
|
||||
media_io_kwargs=params.media_io_kwargs,
|
||||
mm_processor_kwargs=params.mm_processor_kwargs,
|
||||
)
|
||||
|
||||
chat_template_kwargs = dict(params.chat_template_kwargs)
|
||||
fmt, config_dict = _build_render_config(
|
||||
conversation, chat_template_kwargs, params.chat_template
|
||||
)
|
||||
prompt_text = await self._render_async(fmt, config_dict)
|
||||
prompt = parse_dec_only_prompt(prompt_text)
|
||||
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return conversation, prompt
|
||||
@@ -20,6 +20,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
_VLLM_RENDERERS = {
|
||||
"cohere": ("cohere", "CohereRenderer"),
|
||||
"deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"),
|
||||
"deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"),
|
||||
"hf": ("hf", "HfRenderer"),
|
||||
|
||||
@@ -40,6 +40,9 @@ _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {
|
||||
}
|
||||
|
||||
_VLLM_TOKENIZERS = {
|
||||
# ``cohere`` mode uses the standard cached HF tokenizer; only the
|
||||
# renderer (template stage) is replaced with a melody-based one.
|
||||
"cohere": ("hf", "CachedHfTokenizer"),
|
||||
"deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
|
||||
"deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"),
|
||||
"hf": ("hf", "CachedHfTokenizer"),
|
||||
|
||||
Reference in New Issue
Block a user