[Frontend] Cohere chat v2 api support (#47189)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
andrewbcohere
2026-07-31 22:53:54 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 6c91de3689
commit 9c110fa522
31 changed files with 8770 additions and 20 deletions
+4
View File
@@ -93,6 +93,7 @@ TokenizerMode = Literal[
"deepseek_v4",
"inkling",
"kimi_k3",
"cohere",
]
ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
LogprobsMode = Literal[
@@ -151,6 +152,9 @@ class ModelConfig:
- "deepseek_v4" will always use the tokenizer from `deepseek_v4`.
- "kimi_k3" will always use the "hf" tokenizer but render chat prompts
with Kimi K3's Python XTML encoding instead of a Jinja template.
- "cohere" uses the standard HF tokenizer but renders the chat template
via the `cohere_melody` library (cmd3 / cmd4 templates) instead of
Jinja, and surfaces grounded-citation metadata on responses.
- Other custom values can be supported via plugins.
To swap the Rust BPE backend that powers HF fast tokenizers for the
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+244
View File
@@ -0,0 +1,244 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FastAPI router for the Cohere Chat v2 API (``POST /cohere/v2/chat``).
The Cohere v2 protocol models are sourced from the official ``cohere``
Python SDK (``pip install cohere``). To keep that an *optional*
dependency for vLLM, the SDK-dependent imports - and the route handler
itself - are gated on a one-shot probe at module load. If the SDK isn't
installed, :func:`attach_router` becomes a no-op (with an info log) and
vLLM continues to boot normally.
Even when the SDK is installed, :func:`attach_router` also requires
``VLLM_ENABLE_COHERE_API=1`` in the environment before it will expose
the route. This keeps non-Cohere deployments that pull in the SDK for
unrelated reasons (e.g. test dependencies) from accidentally exposing
the api.
Note: the handler must live at module scope (not inside
``attach_router``) so that FastAPI's ``typing.get_type_hints`` resolves
the ``CohereChatV2Request`` body annotation against the module's
globals. Defining it locally inside ``attach_router`` would hide the
type from ``get_type_hints``, causing FastAPI to silently degrade the
body parameter into a query parameter and reject every request with
422.
"""
import json
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from starlette.middleware.base import BaseHTTPMiddleware
import vllm.envs as envs
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
sanitize_message,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
_COHERE_PATH_PREFIX = "/cohere/"
logger = init_logger(__name__)
try:
import cohere # noqa: F401 -- dependency probe
except ImportError:
_SDK_AVAILABLE = False
else:
_SDK_AVAILABLE = True
if _SDK_AVAILABLE:
from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
CohereChatV2Response,
CohereError,
)
from vllm.entrypoints.cohere.serving import CohereServingChatV2
router = APIRouter()
def _serving(request: Request) -> CohereServingChatV2 | None:
return getattr(request.app.state, "cohere_serving_chat_v2", None)
def _request_id(raw_request: Request | None) -> str | None:
"""Best-effort lookup of the active request id.
Prefers the id the underlying chat handler stamped onto
``raw_request.state.request_metadata`` (if it got that far before
failing), falling back to the ``X-Request-Id`` HTTP header. May
return ``None`` if neither is available, in which case the field
is omitted from the response.
"""
if raw_request is None:
return None
meta = getattr(raw_request.state, "request_metadata", None)
if meta is not None and getattr(meta, "request_id", None):
return meta.request_id
return raw_request.headers.get("X-Request-Id")
def _error_response(
error: ErrorResponse,
raw_request: Request | None,
*,
fallback_status: int = HTTPStatus.BAD_REQUEST,
) -> JSONResponse:
"""Translate vLLM's internal error envelope into Cohere's shape."""
info = error.error
status = info.code or fallback_status
return JSONResponse(
status_code=status,
content=CohereError(
message=sanitize_message(info.message),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
@router.post(
"/cohere/v2/chat",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
HTTPStatus.BAD_REQUEST.value: {"model": CohereError},
HTTPStatus.NOT_FOUND.value: {"model": CohereError},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": CohereError},
},
)
@with_cancellation
@load_aware_call
async def chat_v2(request: CohereChatV2Request, raw_request: Request):
handler = _serving(raw_request)
if handler is None:
return JSONResponse(
status_code=HTTPStatus.NOT_IMPLEMENTED.value,
content=CohereError(
message="The model does not support the Cohere v2 chat API.",
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
try:
result = await handler.create_chat_v2(request, raw_request)
except Exception as e: # noqa: BLE001 - report as 500 for parity
logger.exception("Error in /cohere/v2/chat: %s", e)
return JSONResponse(
status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value,
content=CohereError(
message=sanitize_message(str(e)),
id=_request_id(raw_request),
).model_dump(exclude_none=True),
)
match result:
case ErrorResponse():
return _error_response(result, raw_request)
case CohereChatV2Response():
return JSONResponse(content=result.model_dump(exclude_none=True))
case _:
return StreamingResponse(content=result, media_type="text/event-stream")
class CohereErrorEnvelopeMiddleware(BaseHTTPMiddleware):
"""Rewrite vLLM error bodies into the Cohere ``{message, id}`` shape.
The endpoint handler above already returns :class:`CohereError` for
errors it owns, but globally-registered exception handlers (e.g.
:func:`validation_exception_handler` for pydantic body errors,
:func:`http_exception_handler`, engine error handlers) fire *before*
the handler runs and produce vLLM's internal
``ErrorResponse`` shape (``{"error": {"message": ...}}``). That
shape doesn't match the ``CohereError`` schema advertised on the
route's OpenAPI ``responses``, so clients (and schema-conformance
tests like ``test_openai_schema.py``) would see a mismatch on
those paths. This middleware normalises any error body on
``/cohere/*`` responses to :class:`CohereError`.
"""
async def dispatch(self, request: Request, call_next):
response = await call_next(request)
if not request.url.path.startswith(_COHERE_PATH_PREFIX):
return response
if response.status_code < 400:
return response
content_type = response.headers.get("content-type", "")
if not content_type.startswith("application/json"):
return response
body = b"".join([chunk async for chunk in response.body_iterator])
translated = _translate_vllm_error_body(body, request)
if translated is not None:
return translated
passthrough_headers = {
k: v
for k, v in response.headers.items()
if k.lower() != "content-length"
}
return Response(
content=body,
status_code=response.status_code,
headers=passthrough_headers,
media_type=content_type,
)
def _translate_vllm_error_body(raw: bytes, request: Request) -> JSONResponse | None:
"""Translate a vLLM ``ErrorResponse`` body to a ``CohereError`` body.
Returns ``None`` if ``raw`` does not match the vLLM error envelope
(which signals the middleware to pass the body through unchanged).
"""
try:
data = json.loads(raw)
except (json.JSONDecodeError, TypeError, ValueError):
return None
if not (
isinstance(data, dict)
and isinstance(data.get("error"), dict)
and "message" in data["error"]
):
return None
try:
err = ErrorResponse.model_validate(data)
except Exception: # noqa: BLE001 - malformed envelope; pass through
return None
return _error_response(err, request)
def attach_router(app: FastAPI) -> None:
"""Register ``POST /cohere/v2/chat`` on ``app``.
No-op when either:
* the ``VLLM_ENABLE_COHERE_API`` env var isn't set to ``1``. The
Cohere v2 endpoint is opt-in because it carries Cohere-specific
request/response semantics (grounding citations, tool_plan,
PLAN/THINKING_CONTENT blocks) that are only meaningful when
serving a Cohere Command-family model.
* the optional ``cohere`` SDK isn't installed (the v2 protocol
models live there)
The two skip paths log at different levels: an operator who set
``VLLM_ENABLE_COHERE_API=1`` but forgot to install ``cohere`` sees
a WARNING (they explicitly asked for the endpoint and it's silently
absent), whereas the default-off skip logs at debug.
"""
enabled = envs.VLLM_ENABLE_COHERE_API
if not enabled:
logger.debug(
"VLLM_ENABLE_COHERE_API is not set; /cohere/v2/chat endpoint "
"disabled. Set VLLM_ENABLE_COHERE_API=1 to enable it."
)
return
if not _SDK_AVAILABLE:
logger.warning(
"VLLM_ENABLE_COHERE_API=1 but the `cohere` SDK is not "
"installed; /cohere/v2/chat will not be exposed. Install "
"with `pip install cohere` to enable the endpoint."
)
return
app.include_router(router)
app.add_middleware(CohereErrorEnvelopeMiddleware)
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM chat-protocol extensions for grounded (citation-carrying) models.
This module keeps the OpenAI chat completion protocol classes
(:class:`ChatMessage` / :class:`DeltaMessage`) free of Cohere-specific fields.
It provides two things:
* :class:`Citation` / :class:`CitationSource`: the vLLM-internal citation
representation produced by the Cohere reasoning parser
(:mod:`vllm.reasoning.cohere_command_reasoning_parser`) and consumed by
:mod:`vllm.entrypoints.cohere.serving`. This is *not* the on-wire Cohere
SDK shape -- that conversion happens in the serving layer.
* :class:`CohereChatMessage` / :class:`CohereDeltaMessage`: :class:`ChatMessage`
/ :class:`DeltaMessage` subclasses that add a ``citations`` field. They
are only instantiated by the citation-aware serving handler
(:class:`vllm.entrypoints.cohere.serving.CohereServingChatV2`) and by the
Cohere reasoning parser's streaming path. Non-Cohere code paths continue
to construct plain :class:`ChatMessage` / :class:`DeltaMessage`.
The module intentionally does *not* import the ``cohere`` Python SDK so the
reasoning parser (which is Cohere-model-specific but ships without the SDK
dependency) can import from here freely.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import Field, model_serializer
from vllm.entrypoints.openai.chat_completion.protocol import ChatMessage
from vllm.entrypoints.openai.engine.protocol import (
DeltaMessage,
OpenAIBaseModel,
)
class CitationSource(OpenAIBaseModel):
"""Source attribution for a :class:`Citation`.
Mirrors the shape used by Cohere's Chat v2 API. ``type`` is the source
discriminator (``document`` or ``tool``); ``id`` is the citing
document/tool-output identifier; ``document`` and ``tool_output`` carry
the original payload that produced the citation.
Sources are fully resolved by the reasoning parser at emit time
(see :func:`_melody_sources_to_vllm` in
:mod:`vllm.reasoning.cohere_command_reasoning_parser`) using the
``POSITION_TO_SOURCE_KEY`` map forwarded through
``chat_template_kwargs`` by
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._apply_cohere_template_kwargs`.
That means every instance that reaches the serving / wire layer
already has ``type`` / ``id`` populated; the serving layer only has
to coerce to :class:`cohere.types.Citation` and rewrite
``THINKING_CONTENT`` -> ``PLAN`` for non-reasoning models.
"""
type: Literal["document", "tool"] | None = None
id: str | None = None
document: dict[str, Any] | None = None
tool_output: dict[str, Any] | None = None
class Citation(OpenAIBaseModel):
"""A citation grounding a span of generated text in source material.
vLLM-internal representation used by the Cohere reasoning parser and
the Cohere v2 serving layer. This is not the on-wire Cohere SDK shape:
conversion to the SDK's ``cohere.types.Citation`` happens in
:mod:`vllm.entrypoints.cohere.serving`.
"""
start: int | None = None
"""Start character offset in the surrounding text content."""
end: int | None = None
"""End character offset (exclusive) in the surrounding text content."""
text: str | None = None
"""The cited text snippet."""
sources: list[CitationSource] = Field(default_factory=list)
"""Source documents / tool outputs that ground this citation."""
content_index: int | None = None
"""Index of the content block this citation refers to (when the message
has multiple content blocks)."""
type: Literal["TEXT_CONTENT", "THINKING_CONTENT", "PLAN"] | None = None
"""Which kind of content block this citation grounds: the user-visible
text (``TEXT_CONTENT``), a thinking block (``THINKING_CONTENT``), or a
tool-plan block (``PLAN``). ``None`` means unspecified."""
class CohereChatMessage(ChatMessage):
""":class:`ChatMessage` extension carrying grounding citations.
Only instantiated by :class:`CohereServingChatV2` (and any other
:class:`OpenAIServingChat` subclass that opts in by overriding
``_create_chat_message``). Regular OpenAI-compatible handlers keep
emitting plain :class:`ChatMessage` so their response schema is
unchanged.
The response envelope declares ``message: SerializeAsAny[ChatMessage]``,
so pydantic serializes this subclass with its own schema (including
``citations``) when it flows through
:class:`ChatCompletionResponseChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# ``mode="wrap"`` fully overrides (rather than chains) the
# parent's ``@model_serializer``, so we explicitly delegate via
# ``super()._serialize(handler)`` to preserve the parent's
# cleanup (e.g. stripping empty ``tool_calls``). Then we drop an
# unset ``citations`` field so the wire matches the OpenAI-style
# contract that optional vLLM extensions are omitted rather than
# serialized as ``null``.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
class CohereDeltaMessage(DeltaMessage):
""":class:`DeltaMessage` extension carrying grounding citations for streaming.
Emitted by
:class:`vllm.reasoning.cohere_command_reasoning_parser.BaseCohereCommandReasoningParser`
on delta events whose payload includes citations. Non-Cohere parsers
return plain :class:`DeltaMessage`, so their streamed shape is
unchanged.
The response envelope declares
``delta: SerializeAsAny[DeltaMessage]``, so pydantic serializes this
subclass with its own schema when it flows through
:class:`ChatCompletionResponseStreamChoice`.
"""
citations: list[Citation] | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
# See ``CohereChatMessage._serialize`` for why we delegate via
# ``super()`` instead of calling ``handler(self)`` directly.
data = super()._serialize(handler)
if not data.get("citations"):
data.pop("citations", None)
return data
+362
View File
@@ -0,0 +1,362 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cohere Chat v2 API protocol.
The bulk of the wire types come straight from the official ``cohere``
Python SDK so we stay in sync with the upstream specification and
avoid re-declaring the schema. We only own three things locally:
1. The top-level request body model (the SDK doesn't ship one — its
``ClientV2.chat`` takes the body as kwargs), with vLLM-specific
extensions (``kv_transfer_params`` / ``chat_template_kwargs``).
2. The non-streaming response envelope (the SDK exposes the message
shape via :class:`AssistantMessageResponse` but no full response
wrapper).
3. The streaming discriminated union (the SDK exports each event type
individually but not as a combined ``Annotated[Union[...],
discriminator]``).
Importing this module pulls in the ``cohere`` package. The router that
mounts ``POST /cohere/v2/chat`` guards on that import succeeding so vLLM still
boots without the SDK installed.
See https://docs.cohere.com/reference/chat for the upstream spec.
"""
from __future__ import annotations
from typing import Any, Literal
from cohere import types as _sdk
from cohere.types import (
AssistantChatMessageV2,
AssistantMessageResponse,
ChatMessageV2,
ChatRequestSafetyMode,
Citation,
CitationOptions,
Document,
ResponseFormatV2,
SystemChatMessageV2,
Thinking,
ToolCallV2,
ToolChatMessageV2,
ToolV2,
UserChatMessageV2,
)
from pydantic import BaseModel, Field, field_validator
# Re-export the SDK wire-format types alongside our local extensions so
# ``vllm.entrypoints.cohere.serving`` and friends can import everything
# they need from this module.
__all__ = [
"AssistantChatMessageV2",
"AssistantMessageResponse",
"ChatMessageV2",
"Citation",
"CitationEndEvent",
"CitationOptions",
"CitationStartEvent",
"CohereChatV2Request",
"CohereChatV2Response",
"CohereError",
"CohereFinishReason",
"CohereLogprobItem",
"CohereUsage",
"CohereUsageBilledUnits",
"CohereUsageTokens",
"ContentDeltaEvent",
"ContentEndEvent",
"ContentStartEvent",
"Document",
"MessageEndEvent",
"MessageStartEvent",
"ResponseFormatV2",
"SystemChatMessageV2",
"Thinking",
"ToolCallDeltaEvent",
"ToolCallEndEvent",
"ToolCallStartEvent",
"ToolCallV2",
"ToolChatMessageV2",
"ToolPlanDeltaEvent",
"ToolV2",
"UserChatMessageV2",
]
# ---------------------------------------------------------------------------
# Errors
# ---------------------------------------------------------------------------
class CohereError(BaseModel):
"""Top-level error body returned by ``/cohere/v2/chat`` error responses.
Cohere's documented error schemas are uniform: ``{message, id}``.
"""
message: str
id: str | None = None
# ---------------------------------------------------------------------------
# Tool choice / finish reasons
# ---------------------------------------------------------------------------
#
# These literals aren't first-class enums in the SDK but are documented at
# https://docs.cohere.com/reference/chat. We declare them here so the
# request/response models can validate them.
CohereToolChoice = Literal["REQUIRED", "NONE"]
CohereFinishReason = Literal[
"COMPLETE",
"STOP_SEQUENCE",
"MAX_TOKENS",
"TOOL_CALL",
"ERROR",
"TIMEOUT",
]
# ---------------------------------------------------------------------------
# Request
# ---------------------------------------------------------------------------
class CohereChatV2Request(BaseModel):
"""Cohere Chat v2 request body.
Mirrors the schema documented at https://docs.cohere.com/reference/chat.
All structured fields delegate to the official SDK types so the body
schema stays in sync with the upstream spec.
"""
model: str
messages: list[ChatMessageV2]
stream: bool | None = False
# Tooling
tools: list[ToolV2] | None = None
strict_tools: bool | None = None
tool_choice: CohereToolChoice | None = None
# Grounding
documents: list[str | Document] | None = None
citation_options: CitationOptions | None = None
# Output
response_format: ResponseFormatV2 | None = None
safety_mode: ChatRequestSafetyMode | None = None
max_tokens: int | None = None
stop_sequences: list[str] | None = None
# Sampling
temperature: float | None = None
seed: int | None = None
frequency_penalty: float | None = None
presence_penalty: float | None = None
k: int | None = None
p: float | None = None
logprobs: bool | None = None
# Reasoning
thinking: Thinking | None = None
# Scheduling
priority: int | None = None
# vLLM-specific extensions (not in Cohere spec). These mirror what the
# Anthropic and OpenAI surfaces already expose so V2 callers can reach
# the same engine knobs when needed.
kv_transfer_params: dict[str, Any] | None = Field(
default=None,
description="KVTransfer parameters used for disaggregated serving.",
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the template."
),
)
@field_validator("model")
@classmethod
def _validate_model(cls, v: str) -> str:
if not v:
raise ValueError("model is required")
return v
@field_validator("max_tokens")
@classmethod
def _validate_max_tokens(cls, v: int | None) -> int | None:
if v is not None and v < 0:
raise ValueError("max_tokens must be non-negative")
return v
@field_validator("messages", mode="before")
@classmethod
def _normalize_message_roles(cls, v: Any) -> Any:
"""Rewrite OpenAI-style ``developer`` roles to ``system``.
Cohere's v2 ``ChatMessageV2`` discriminated union only admits
the four literal roles ``user`` / ``assistant`` / ``system`` /
``tool``. OpenAI's ``developer`` role is documented as
high-priority system instructions, so we alias it onto
``system`` *before* the SDK discriminator runs (otherwise it
rejects the message with a ``literal_error`` against each union
member). Mirrors ``_role_to_melody`` in the renderer so the
same alias is honoured no matter which surface the message
arrives through.
``mode="before"`` is required so the rewrite happens before the
``list[ChatMessageV2]`` coercion runs the SDK's discriminated
union; a default-mode validator would never see ``developer``
because validation would have already failed. On any
structural malformation (non-iterable input, items without a
dict-shaped ``role`` field, etc.) we hand ``v`` back unchanged
and let Pydantic's normal coercion surface a precise error.
"""
try:
return [
{**msg, "role": "system"}
if msg.get("role", "").lower() == "developer"
else msg
for msg in v
]
except (AttributeError, TypeError):
return v
@field_validator("messages")
@classmethod
def _validate_messages(cls, v: list[ChatMessageV2]) -> list[ChatMessageV2]:
if not v:
raise ValueError("messages must contain at least one message")
return v
# ---------------------------------------------------------------------------
# Usage / Logprobs
# ---------------------------------------------------------------------------
#
# The Cohere SDK only exposes a v1 ``ApiMetaBilledUnits``. The v2 usage
# envelope is documented separately in the OpenAPI spec and we declare it
# here.
class CohereUsageBilledUnits(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
search_units: float | None = None
classifications: float | None = None
class CohereUsageTokens(BaseModel):
input_tokens: float | None = None
output_tokens: float | None = None
class CohereUsage(BaseModel):
billed_units: CohereUsageBilledUnits | None = None
tokens: CohereUsageTokens | None = None
cached_tokens: float | None = None
class CohereLogprobItem(BaseModel):
text: str | None = None
token_ids: list[int]
logprobs: list[float] | None = None
# ---------------------------------------------------------------------------
# Non-streaming response
# ---------------------------------------------------------------------------
class CohereChatV2Response(BaseModel):
"""Cohere Chat v2 non-streaming response body.
Wraps the SDK :class:`AssistantMessageResponse` (the message shape) in
the documented v2 response envelope (``id``, ``finish_reason``,
``usage``, ``logprobs``). The single constructor in
:class:`CohereServingChatV2._chat_completion_to_v2` is responsible for
supplying a non-empty ``id`` (falling back to a synthesized one if
the upstream response is missing it) to this model.
"""
id: str
finish_reason: CohereFinishReason
message: AssistantMessageResponse
usage: CohereUsage | None = None
logprobs: list[CohereLogprobItem] | None = None
# vLLM-specific extension.
kv_transfer_params: dict[str, Any] | None = Field(
default=None, description="KVTransfer parameters."
)
# ---------------------------------------------------------------------------
# Streaming events
# ---------------------------------------------------------------------------
#
# Cohere V2 streams a sequence of typed JSON events delivered as Server-
# Sent Events; each event's ``type`` field carries the discriminator. The
# SDK exposes a Pydantic model per event but none of them declare ``type``
# as a field (the SDK relies on its own deserializer for discrimination),
# so a naive ``model_dump_json()`` would silently drop the discriminator
# and break clients that demux on ``type``.
#
# We therefore subclass each SDK event and bake the wire-format ``type``
# string in as a ``Literal`` field with a default. ``model_dump()`` now
# emits ``type`` for free, and surfaces can simply construct the event
# class and serialize it -- no manual ``type`` parameter needed.
#
# See https://docs.cohere.com/v2/docs/streaming and the OpenAPI
# ``StreamedChatResponseV2`` schema for the wire-format reference.
class MessageStartEvent(_sdk.ChatMessageStartEvent):
type: Literal["message-start"] = "message-start"
class ContentStartEvent(_sdk.ChatContentStartEvent):
type: Literal["content-start"] = "content-start"
class ContentDeltaEvent(_sdk.ChatContentDeltaEvent):
type: Literal["content-delta"] = "content-delta"
class ContentEndEvent(_sdk.ChatContentEndEvent):
type: Literal["content-end"] = "content-end"
class ToolPlanDeltaEvent(_sdk.ChatToolPlanDeltaEvent):
type: Literal["tool-plan-delta"] = "tool-plan-delta"
class ToolCallStartEvent(_sdk.ChatToolCallStartEvent):
type: Literal["tool-call-start"] = "tool-call-start"
class ToolCallDeltaEvent(_sdk.ChatToolCallDeltaEvent):
type: Literal["tool-call-delta"] = "tool-call-delta"
class ToolCallEndEvent(_sdk.ChatToolCallEndEvent):
type: Literal["tool-call-end"] = "tool-call-end"
class CitationStartEvent(_sdk.CitationStartEvent):
type: Literal["citation-start"] = "citation-start"
class CitationEndEvent(_sdk.CitationEndEvent):
type: Literal["citation-end"] = "citation-end"
class MessageEndEvent(_sdk.ChatMessageEndEvent):
type: Literal["message-end"] = "message-end"
File diff suppressed because it is too large Load Diff
+62 -3
View File
@@ -4,6 +4,8 @@ from typing import TYPE_CHECKING
from fastapi import FastAPI
import vllm.envs as envs
if TYPE_CHECKING:
from argparse import Namespace
@@ -41,6 +43,12 @@ def register_generate_api_routers(app: FastAPI):
register_anthropic_api_router(app)
from vllm.entrypoints.cohere.api_router import (
attach_router as register_cohere_api_router,
)
register_cohere_api_router(app)
from .generative_scoring.api_router import register_generative_scoring_api_router
register_generative_scoring_api_router(app)
@@ -55,6 +63,23 @@ async def init_generate_state(
):
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
from vllm.entrypoints.chat_utils import load_chat_template
# The Cohere serving handler depends on the optional `cohere` SDK for
# its wire-format protocol models, and is additionally gated on the
# `VLLM_ENABLE_COHERE_API` env flag (see
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
# entirely when the endpoint isn't going to be exposed, both because
# the SDK may not be installed and because the serving object holds
# nontrivial state (chat handler, warmup) that would otherwise be
# unused.
if envs.VLLM_ENABLE_COHERE_API:
try:
from vllm.entrypoints.cohere.serving import CohereServingChatV2
except ImportError:
CohereServingChatV2 = None # type: ignore[assignment,misc]
else:
CohereServingChatV2 = None # type: ignore[assignment,misc]
from vllm.entrypoints.mcp.tool_server import (
DemoToolServer,
MCPToolServer,
@@ -86,6 +111,19 @@ async def init_generate_state(
tool_server = None
resolved_chat_template = load_chat_template(args.chat_template)
# Fold the dedicated ``--cohere-format`` CLI flag into the renderer's
# default chat-template kwargs. The cohere renderer reads
# ``chat_template_kwargs["cohere_format"]`` to pick cmd3 vs cmd4
# rendering; making this a first-class flag keeps the right format
# discoverable for ``vllm serve --tokenizer-mode cohere`` users
# without forcing them to hand-construct a JSON dict for
# ``--default-chat-template-kwargs``. Per-request overrides still
# take precedence (see ``merge_kwargs`` in
# ``ChatCompletionRequest.build_chat_params``).
default_chat_template_kwargs = dict(args.default_chat_template_kwargs or {})
if getattr(args, "cohere_format", None):
default_chat_template_kwargs.setdefault("cohere_format", args.cohere_format)
# Render endpoints are always backed by OnlineRenderer so that
# /v1/chat/completions/render and /v1/completions/render work on both
# generate-mode and render-only servers. Created in init_app_state.
@@ -106,7 +144,7 @@ async def init_generate_state(
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
enable_log_outputs=args.enable_log_outputs,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
)
if "generate" in supported_tasks
else None
@@ -119,7 +157,7 @@ async def init_generate_state(
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
trust_request_chat_template=args.trust_request_chat_template,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
@@ -169,11 +207,32 @@ async def init_generate_state(
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
default_chat_template_kwargs=args.default_chat_template_kwargs,
default_chat_template_kwargs=default_chat_template_kwargs,
)
if "generate" in supported_tasks
else None
)
state.cohere_serving_chat_v2 = (
CohereServingChatV2(
engine_client,
state.openai_serving_models,
args.response_role,
online_renderer=state.online_renderer,
request_logger=request_logger,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
enable_auto_tools=args.enable_auto_tool_choice,
tool_parser=args.tool_call_parser,
reasoning_parser=args.structured_outputs_config.reasoning_parser,
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
enable_force_include_usage=args.enable_force_include_usage,
default_chat_template_kwargs=default_chat_template_kwargs,
is_reasoning_model=args.cohere_is_reasoning_model,
)
if CohereServingChatV2 is not None and "generate" in supported_tasks
else None
)
from .generative_scoring.serving import ServingGenerativeScoring
@@ -10,7 +10,13 @@ from openai.types.chat.chat_completion_audio import (
ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
from pydantic import Field, PrivateAttr, model_serializer, model_validator
from pydantic import (
Field,
PrivateAttr,
SerializeAsAny,
model_serializer,
model_validator,
)
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import (
@@ -91,7 +97,12 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
class ChatCompletionResponseChoice(OpenAIBaseModel):
index: int
message: ChatMessage
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
# so that added fields like ``citations`` survive JSON serialization
# instead of being stripped down to the base schema. Plain
# ``ChatMessage`` instances serialize identically to before.
message: SerializeAsAny[ChatMessage]
logprobs: ChatCompletionLogProbs | None = None
# per OpenAI spec this is the default
finish_reason: str | None = "stop"
@@ -139,7 +150,11 @@ class ChatCompletionResponse(OpenAIBaseModel):
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
index: int
delta: DeltaMessage
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
# so streaming ``citations`` survive JSON serialization. Plain
# ``DeltaMessage`` instances serialize identically to before.
delta: SerializeAsAny[DeltaMessage]
logprobs: ChatCompletionLogProbs | None = None
finish_reason: str | None = None
stop_reason: int | str | None = None
@@ -391,6 +391,34 @@ class OpenAIServingChat(GenerateBaseServing):
return self.response_role
return request.messages[-1]["role"]
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
"""Construct the response :class:`ChatMessage` for the non-streaming path.
The full-generator calls this at every construction site so
subclasses can swap in a specialized :class:`ChatMessage`
subclass (e.g. :class:`CohereServingChatV2` returning
:class:`CohereChatMessage`) without duplicating the branchy
tool-choice / auto-tools logic that decides which fields are
populated. The default returns a plain :class:`ChatMessage`.
"""
return ChatMessage(*args, **kwargs)
def _finalize_response_message(
self,
message: ChatMessage,
*,
parser: Parser | None,
) -> ChatMessage:
"""Subclass hook to enrich a fully-constructed :class:`ChatMessage`.
Default is a no-op. Subclasses that need to surface parser-side
extras (e.g. :class:`CohereServingChatV2` reading grounding
citations off the reasoning parser and populating
:class:`CohereChatMessage.citations`) override this to inspect
``parser`` and mutate/replace ``message``.
"""
return message
async def chat_completion_stream_generator(
self,
request: ChatCompletionRequest,
@@ -894,13 +922,19 @@ class OpenAIServingChat(GenerateBaseServing):
)
is_required_tool_choice = request.tool_choice == "required"
# All six construction sites route through ``self._create_chat_message``
# so subclasses can swap in a specialized :class:`ChatMessage`
# (e.g. the Cohere v2 handler's ``CohereChatMessage``) without
# having to duplicate this branch logic.
if (not self.enable_auto_tools or not tool_parser_cls) and (
not is_named_tool_choice and not is_required_tool_choice
):
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
elif is_named_tool_choice or is_required_tool_choice:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content or "",
@@ -913,7 +947,9 @@ class OpenAIServingChat(GenerateBaseServing):
# if the request doesn't use tool choice
# OR specifies to not use a tool
elif not request.tool_choice or request.tool_choice == "none":
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
# handle when there are tools and tool choice is auto
elif (
@@ -924,7 +960,7 @@ class OpenAIServingChat(GenerateBaseServing):
):
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
if tool_calls:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content,
@@ -935,7 +971,7 @@ class OpenAIServingChat(GenerateBaseServing):
)
else:
message = ChatMessage(
message = self._create_chat_message(
role=role,
reasoning=reasoning,
content=content,
@@ -948,7 +984,17 @@ class OpenAIServingChat(GenerateBaseServing):
" if tools should be extracted. Returning a standard chat "
"completion."
)
message = ChatMessage(role=role, reasoning=reasoning, content=content)
message = self._create_chat_message(
role=role, reasoning=reasoning, content=content
)
# Subclass hook: enrich the constructed message with any
# parser-side extras that don't fit through the plain
# ``(reasoning, content, tool_calls)`` tuple. Base is a no-op;
# citation-aware handlers use this to surface grounding
# metadata cached on the reasoning parser.
message = self._finalize_response_message(message, parser=parser)
# In OpenAI's API, when a tool is called, the finish_reason is:
# "tool_calls" for "auto" or "required" tool calls,
# and "stop" for named tool calls.
+19
View File
@@ -148,6 +148,25 @@ class BaseFrontendArgs:
"""If set to False, output deltas will not be logged. Relevant only if
--enable-log-outputs is set.
"""
cohere_is_reasoning_model: bool = True
"""Cohere ``/cohere/v2/chat`` only. Whether the served model is a
reasoning Command-family model. When True (default), the assistant's
chain-of-thought is surfaced as a ``thinking`` content block (or
``content-*`` events on the stream). When False, reasoning is
surfaced as Cohere's ``tool_plan`` field (or ``tool-plan-delta``
events) whenever the model emits tool calls, matching older non-
reasoning Command models. Has no effect on the non-Cohere
endpoints."""
cohere_format: str = "cmd4"
"""Cohere ``--tokenizer-mode cohere`` only. Which Cohere prompt
format to render: ``cmd4`` (current Command A+ models; default) or
``cmd3`` (earlier Cmd-A and Cmd-A reasoning models). Selecting the
wrong format silently produces a prompt the model wasn't trained
on, which most commonly manifests as the model emitting text but no
citations / tool calls / thinking blocks. Equivalent to passing
``--default-chat-template-kwargs '{"cohere_format": "..."}'`` -- any
explicit request-level ``chat_template_kwargs.cohere_format`` takes
priority."""
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
"""If set to True, log the stack trace of error responses"""
tokens_only: bool = False
+1 -1
View File
@@ -39,7 +39,7 @@ from vllm.v1.engine.exceptions import EngineDeadError, EngineGenerateError
logger = init_logger("vllm.entrypoints.openai.server_utils")
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
class AuthenticationMiddleware:
+6
View File
@@ -240,6 +240,7 @@ if TYPE_CHECKING:
VLLM_LOOPBACK_IP: str = ""
VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
VLLM_ENABLE_RESPONSES_API_STORE: bool = False
VLLM_ENABLE_COHERE_API: bool = False
VLLM_HAS_FLASHINFER_CUBIN: bool = False
VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
@@ -1758,6 +1759,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENABLE_RESPONSES_API_STORE": lambda: bool(
int(os.getenv("VLLM_ENABLE_RESPONSES_API_STORE", "0"))
),
# If set to 1, expose the Cohere Chat v2 API at ``POST /cohere/v2/chat``.
# Default off
"VLLM_ENABLE_COHERE_API": lambda: bool(
int(os.getenv("VLLM_ENABLE_COHERE_API", "0"))
),
# If set, use the fp8 mfma in rocm paged attention.
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": lambda: bool(
int(os.getenv("VLLM_ROCM_FP8_MFMA_PAGE_ATTN", "0"))
@@ -20,6 +20,11 @@ except ImportError as e:
) from e
from vllm.entrypoints.cohere.cohere_chat_message import (
Citation,
CitationSource,
CohereDeltaMessage,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
@@ -404,6 +409,83 @@ def _schema_dict_from_structured_outputs(
)
def _melody_sources_to_vllm(
raw_sources: Any,
position_to_source: Mapping[tuple[int, int], CitationSource] | None,
) -> list[CitationSource]:
"""Convert melody's ``Source`` objects into resolved :class:`CitationSource`.
Melody's ``Source`` shape is
``{tool_call_index, tool_result_indices, document_ids}``. Without
:meth:`PyFilterOptions.with_message_history` the ``document_ids``
list is empty, so we resolve the numeric address ourselves using
``position_to_source``, which is
:meth:`vllm.entrypoints.cohere.serving.CohereServingChatV2._build_position_to_source`
applied to the inbound request and forwarded through
``chat_template_kwargs`` at parser-construction time. Each melody
source with ``tool_result_indices=[i, j, ...]`` fans out to one
:class:`CitationSource` per resolved position, populating the
wire-shape ``type`` / ``id`` / ``document`` / ``tool_output``
fields; unresolved positions are silently skipped (the calling
citation is dropped at wire-coercion time if every source in it
ends up unresolved -- see ``_to_wire_citation`` in
:mod:`vllm.entrypoints.cohere.serving`).
When ``position_to_source`` is ``None`` (parser wired outside of
:class:`CohereServingChatV2`, e.g. a plain OpenAI chat completion
request that happens to hit a cohere reasoning parser) sources
can't be resolved and get dropped; citations then never reach any
citation-aware handler downstream.
"""
if not position_to_source:
return []
out: list[CitationSource] = []
for s in raw_sources or []:
bucket = getattr(s, "tool_call_index", None)
indices = list(getattr(s, "tool_result_indices", None) or [])
if bucket is None or not indices:
continue
for idx in indices:
info = position_to_source.get((bucket, idx))
if info is not None:
out.append(info)
return out
def _melody_citations_to_vllm(
raw_citations: Any,
position_to_source: Mapping[tuple[int, int], CitationSource] | None,
) -> list[Citation] | None:
"""Convert melody's ``FilterCitation`` objects into :class:`Citation`.
Resolves each source in-place via ``position_to_source`` (see
:func:`_melody_sources_to_vllm`). Citations whose sources all fail
to resolve are still emitted here with ``sources=[]``; the serving
layer drops them at wire-coercion time so the fail-closed policy
lives in exactly one place.
"""
if not raw_citations:
return None
out: list[Citation] = []
for c in raw_citations:
out.append(
Citation(
start=getattr(c, "start_index", None),
end=getattr(c, "end_index", None),
text=getattr(c, "text", None),
sources=_melody_sources_to_vllm(
getattr(c, "sources", None), position_to_source
),
type=(
"THINKING_CONTENT"
if getattr(c, "is_thinking", False)
else "TEXT_CONTENT"
),
)
)
return out
class BaseCohereCommandReasoningParser(ReasoningParser):
def __init__(
self,
@@ -420,6 +502,32 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
self.unary_opts = unary_opts
self.melody_unary = PyFilter(unary_opts)
self.melody_streaming = PyFilter(streaming_opts)
# Citations extracted by the most recent ``extract_reasoning`` call.
# Citation-aware handlers (e.g. :class:`CohereServingChatV2` via its
# ``_finalize_response_message`` hook) read this back from the
# parser instance (which is constructed per-request) and attach
# the result to :class:`CohereChatMessage.citations`. ``None`` when
# the last parse produced no citations.
self.last_unary_citations: list[Citation] | None = None
# Request-scoped ``(tool_call_index, tool_result_idx) -> resolved
# CitationSource`` map, forwarded from
# :meth:`CohereServingChatV2._apply_cohere_template_kwargs` via
# ``chat_template_kwargs``. See :func:`_melody_sources_to_vllm`
# for how it's consumed. Absent for parser instances that
# weren't created by ``CohereServingChatV2`` (e.g. an OpenAI
# chat completion request routed through a cohere reasoning
# parser); citations from those requests are dropped since we
# have no way to attribute their sources.
ctk = kwargs.get("chat_template_kwargs") or {}
# Lazy import: ``renderers.cohere`` transitively imports
# ``cohere_melody`` and other heavy deps; hoisting the constant
# to module scope would tie parser importability to that graph.
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
raw_map = ctk.get(POSITION_TO_SOURCE_KEY)
self._position_to_source: Mapping[tuple[int, int], CitationSource] | None = (
raw_map if isinstance(raw_map, Mapping) else None
)
@property
def reasoning_start_str(self) -> str | None:
@@ -439,9 +547,22 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
delta_token_ids: Sequence[int],
) -> DeltaMessage | None:
r = self.melody_streaming.write_decoded(delta_text)
if r.content is None and r.reasoning is None and not r.tool_calls:
citations = _melody_citations_to_vllm(
getattr(r, "citations", None), self._position_to_source
)
if (
r.content is None
and r.reasoning is None
and not r.tool_calls
and not citations
):
return None
msg = DeltaMessage()
# Always emit CohereDeltaMessage so citations reach the wire via
# ``SerializeAsAny[DeltaMessage]`` on the streaming choice envelope.
# When ``citations`` is unset, ``CohereDeltaMessage._serialize``
# drops the field so the emitted shape matches plain
# :class:`DeltaMessage`
msg = CohereDeltaMessage()
if r.content is not None:
msg.content = r.content
if r.reasoning is not None:
@@ -456,12 +577,23 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
)
for tc in r.tool_calls
]
if citations:
msg.citations = citations
return msg
def extract_reasoning(
self, model_output: str, request: ChatCompletionRequest | ResponsesRequest
) -> tuple[str | None, str | None]:
result = self.melody_unary.process_full_text(model_output)
# Cache citations so citation-aware handlers can surface them
# on :class:`CohereChatMessage.citations`. The base
# :meth:`ReasoningParser.extract_reasoning` contract only
# returns ``(reasoning, content)``, so citations are passed
# back via parser-instance state -- safe because the parser is
# constructed per-request.
self.last_unary_citations = _melody_citations_to_vllm(
getattr(result, "citations", None), self._position_to_source
)
return result.reasoning, result.content
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
@@ -549,12 +681,25 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
return request
# melody's streaming filter only buffers a partial ``<co: ...>`` citation
# across ``write_decoded`` calls when ``stream_non_grounded_answer`` is
# set: otherwise, the moment an opening ``<co`` is seen without a
# closing ``</co: ...>`` in the same delta, the filter emits the partial
# marker bytes verbatim as plain content. In vLLM's streaming path the
# parser is fed one token (1-4 chars) per call, so an unbuffered filter
# will leak ``<co: 0>``-style markers into ``delta.content`` and never
# emit a ``FilterCitation`` for them. Enabling the flag flips the
# partial-match branch in melody's ``parse_citations`` (see
# ``src/parsing/citations_filter.rs``) to ``return (None, 0)`` -- i.e.
# keep buffering -- which lets a full citation eventually resolve.
# Non-streaming (unary) parsing receives the whole output in one call so
# the flag is a no-op there and we leave ``unary_opts`` alone.
class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
super().__init__(
tokenizer,
*args,
streaming_opts=PyFilterOptions().cmd3(),
streaming_opts=PyFilterOptions().cmd3().stream_non_grounded_answer(),
unary_opts=PyFilterOptions().cmd3().no_tools(),
**kwargs,
)
@@ -565,7 +710,7 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
super().__init__(
tokenizer,
*args,
streaming_opts=PyFilterOptions().cmd4(),
streaming_opts=PyFilterOptions().cmd4().stream_non_grounded_answer(),
unary_opts=PyFilterOptions().cmd4().no_tools(),
**kwargs,
)
+704
View File
@@ -0,0 +1,704 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cohere prompt renderer.
Templates the Cohere Command-family prompt formats (cmd3 / cmd4) using
the ``cohere_melody`` Rust bindings instead of Jinja. Enabled by
passing ``--tokenizer-mode cohere`` on the engine; tokenization itself
still flows through the cached HuggingFace tokenizer.
The renderer's client surface is ``chat_template_kwargs`` (vLLM's
generic passthrough for template-time inputs). It is fed by two
different code paths, which speak different vocabularies:
1. :class:`vllm.entrypoints.cohere.serving.CohereServingChatV2` --
translates a native `Cohere Chat v2 request
<https://docs.cohere.com/reference/chat>`__ into
``chat_template_kwargs``.
2. :class:`OpenAIServingChat` -- forwards whatever the client puts in
``request.chat_template_kwargs`` directly, without translation.
Accordingly the renderer accepts keys from both vocabularies.
**Cohere Chat v2 request fields** (populated automatically by
``CohereServingChatV2``; a direct ``chat_template_kwargs`` caller may
also pass them):
* ``documents``: list of documents (v2 ``documents``)
* ``tools``: list of tools (v2 ``tools``)
* ``safety_mode``: v2 ``safety_mode`` (``CONTEXTUAL`` / ``STRICT`` /
``NONE``) -- forwarded to melody's cmd3 ``safety_mode`` slot;
lowercased.
* ``citation_options``: v2 ``citation_options``. Its ``.mode``
(``ENABLED`` / ``DISABLED`` / ``FAST`` / ``ACCURATE`` / ``OFF``) is
normalized into melody's cmd3 ``citation_quality`` (``on`` / ``off``)
and cmd4 ``grounding`` (``enabled`` / ``disabled`` / ``unknown``)
slots. cmd4 has no fast/accurate distinction at the prompt-template
layer -- both mean grounding-on.
* ``response_format``: v2 ``response_format`` (``json_object`` /
``json_schema``) -- mapped onto melody's ``json_mode`` /
``json_schema`` config slots.
* ``thinking``: v2 ``thinking``; its ``.type`` (``enabled`` /
``disabled``) becomes melody's ``reasoning_type``.
**Melody template-config knobs** (accepted for direct-passthrough
callers; not part of the Cohere Chat v2 surface):
* ``cohere_format``: renderer selector, ``cmd3`` or ``cmd4`` (default
``cmd4``).
* ``template_id``: pick one of melody's built-in template variants.
* ``available_tools``: melody's own name for tools; takes precedence
over ``tools`` when both are set.
* ``reasoning_type``: direct ``enabled`` / ``disabled`` toggle;
overrides derivation from ``thinking.type``.
* ``dev_instruction``: developer-instruction override.
* ``json_mode`` / ``json_schema``: direct structured-output toggles;
override derivation from ``response_format``.
* cmd3-only: ``citation_quality`` (``on`` / ``off``), ``skip_preamble``.
* cmd4-only: ``grounding`` (``enabled`` / ``disabled`` / ``unknown``;
overrides derivation from ``citation_options.mode``),
``platform_instruction``.
**Rejected inputs**: ``template_jinja`` and ``template`` are Cohere's
own inlets for raw Jinja template source. They are valid at Cohere's
API surface but not at vLLM's -- raw template source must flow through
the standard ``chat_template`` request field (guarded by
``--trust-request-chat-template``), which this renderer forwards to
melody as ``template_jinja``. Setting these keys explicitly in
``chat_template_kwargs`` raises ``ValueError`` so client
misconfiguration surfaces loudly.
**Everything else** in ``chat_template_kwargs`` is forwarded verbatim
to melody as ``additional_template_fields`` and becomes accessible as
Jinja variables inside the template. This matches vLLM's documented
contract for ``chat_template_kwargs`` ("kwargs accessible by the
template"), so e.g. ``chat_template_kwargs={"reasoning_effort":
"low"}`` resolves ``{{ reasoning_effort }}`` inside cmd3 / cmd4
templates.
Citations produced by Cohere models are surfaced through the
Cohere-scoped ``CohereChatMessage.citations`` /
``CohereDeltaMessage.citations`` fields (see
:mod:`vllm.entrypoints.cohere.cohere_chat_message`), populated by the
``cohere2`` reasoning parser. The base OpenAI ``ChatMessage`` /
``DeltaMessage`` keep their declared schemas unchanged; the response
envelope declares them as ``SerializeAsAny[...]`` so the subclass
fields survive JSON serialization.
"""
from __future__ import annotations
import copy
import json
from enum import Enum
from typing import Any
from vllm.config import VllmConfig
from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ConversationMessage,
parse_chat_messages,
parse_chat_messages_async,
)
from vllm.logger import init_logger
from vllm.tokenizers.hf import HfTokenizer
from vllm.utils.async_utils import make_async
from .base import BaseRenderer
from .inputs import DictPrompt
from .inputs.preprocess import parse_dec_only_prompt
from .params import ChatParams
logger = init_logger(__name__)
_DEFAULT_FORMAT = "cmd4"
_VALID_FORMATS = ("cmd3", "cmd4")
class MelodyContentType(str, Enum):
"""Wire-format discriminator for melody content blocks.
These strings are what the cmd3 / cmd4 Jinja templates check against
in ``message.content[0].type`` (e.g. the ``thinking`` branch in
``cmd4-v1.jinja``). Keep new values in sync with melody's template
schema.
"""
TEXT = "text"
THINKING = "thinking"
IMAGE = "image"
DOCUMENT = "document"
# Reserved ``chat_template_kwargs`` key that carries
# ``AssistantChatMessageV2.citations`` from a Cohere v2 request through
# to the renderer, keyed by request-message index. Populated by
# ``CohereServingChatV2._apply_cohere_template_kwargs`` and consumed by
# ``_conversation_to_melody_messages``. The value is a
# ``dict[int, list[dict]]`` of melody-shape ``FilterCitation`` dicts.
# The leading underscore signals "internal, not part of the public
# ``chat_template_kwargs`` surface" -- clients should not set this key
# directly.
MESSAGES_CITATIONS_KEY = "_messages_citations"
# Reserved ``chat_template_kwargs`` key carrying a request-side
# ``dict[tuple[int, int], CitationSource]`` map from melody's numeric
# ``(tool_call_index, tool_result_index)`` addressing to fully-resolved
# wire-shape sources (type, id, document/tool_output payload).
# Populated by :meth:`CohereServingChatV2._apply_cohere_template_kwargs`
# and consumed by :class:`BaseCohereCommandReasoningParser` at request
# start so the parser can attach real document ids on each melody source
# it emits -- eliminating the numeric-coord round-trip through the
# internal streaming pipeline. Leading underscore signals "internal,
# not part of the public ``chat_template_kwargs`` surface" -- clients
# should not set this key directly.
POSITION_TO_SOURCE_KEY = "_position_to_source"
# Keys this renderer interprets directly from ``chat_template_kwargs`` and
# maps onto typed melody render-config fields. Everything *not* in this
# set is forwarded verbatim to melody as ``additional_template_fields``
# (i.e. as Jinja template variables), so callers can write
# ``chat_template_kwargs = {"my_var": "..."}`` and have ``{{ my_var }}``
# resolve inside the template -- matching vLLM's documented contract
# for ``chat_template_kwargs``.
_RENDERER_CONSUMED_KEYS = frozenset(
{
"cohere_format",
"template_id",
# ``template_jinja`` / ``template`` are Cohere-only inputs for
# raw Jinja source -- valid at Cohere's own API surface but not
# at vLLM's. In vLLM, raw template source must flow through the
# standard ``chat_template`` request field, so a non-``None``
# value here is rejected in ``_build_render_config``. The keys
# stay in the consumed set so an explicit ``None`` (e.g. from
# ``.model_dump(exclude_none=False)``) is dropped rather than
# surfacing as a stray ``{{ template_jinja }}`` Jinja variable.
"template_jinja",
"template",
"use_jinja",
"documents",
"available_tools",
"tools",
"reasoning_type",
"thinking",
"dev_instruction",
"response_format",
"json_schema",
"json_mode",
"safety_mode",
"citation_quality",
"citation_options",
"skip_preamble",
"grounding",
"platform_instruction",
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
}
)
def _try_import_melody():
try:
import cohere_melody # type: ignore
return cohere_melody
except ImportError as e: # pragma: no cover - exercised at runtime
raise ImportError(
"The `cohere` tokenizer/renderer mode requires the "
"`cohere_melody` package. Install it via "
"`pip install cohere-melody` or build from "
"https://github.com/cohere-ai/melody."
) from e
_MELODY_ROLES = frozenset({"system", "user", "chatbot", "tool"})
# Cohere v2 ``citation_options.mode`` -> melody cmd4 ``grounding``.
# v2 surfaces five modes (``ENABLED`` / ``DISABLED`` / ``FAST`` /
# ``ACCURATE`` / ``OFF``) but cmd4's template doesn't differentiate
# fast vs accurate at the prompt layer -- both just turn grounding on,
# and ``ENABLED`` / ``DISABLED`` are direct aliases. Melody itself
# accepts ``unknown`` / ``enabled`` / ``disabled``.
_CMD4_GROUNDING_FROM_MODE = {
"fast": "enabled",
"accurate": "enabled",
"on": "enabled",
"enabled": "enabled",
"off": "disabled",
"disabled": "disabled",
"unknown": "unknown",
}
def _normalize_cmd4_grounding(value: Any) -> str:
"""Coerce a user-facing grounding/citation mode to melody's vocab.
Raises ``ValueError`` rather than letting an unrecognized value
slip through to ``render_cmd4`` and surface as a generic
``Invalid config: grounding`` from melody.
"""
out = _CMD4_GROUNDING_FROM_MODE.get(value.lower())
if out is None:
raise ValueError(
f"Unrecognized cmd4 grounding value: {value!r}. Expected one "
f"of ENABLED / DISABLED / FAST / ACCURATE / OFF (from "
f"citation_options.mode), or the direct melody values "
f"enabled / disabled / unknown."
)
return out
def _role_to_melody(role: str) -> str:
"""Map an OpenAI role to the role string melody expects.
The cmd3 / cmd4 jinja templates only recognize ``system``, ``user``,
``assistant``/``chatbot``, and ``tool``: any other role is silently
dropped by the template's role-dispatch chain (no fallback branch),
which produces a malformed prompt without any error. We therefore
refuse unknown roles up front rather than letting them disappear.
Aliases:
* ``assistant`` -> ``chatbot`` (Cohere's historical assistant role
name used by the templates).
* ``developer`` -> ``system`` (OpenAI's ``developer`` role is
documented as high-priority instructions, which maps onto the
``system`` slot in Cohere's prompt format).
"""
role = role.lower()
if role == "assistant":
return "chatbot"
if role == "developer":
return "system"
if role in _MELODY_ROLES:
return role
raise ValueError(
f"Unsupported message role for the cohere renderer: {role!r}. "
"Expected one of: system, developer, user, assistant, chatbot, tool."
)
def _normalize_tool_call(tc: dict[str, Any] | Any) -> dict[str, Any]:
"""Normalize an OpenAI tool call dict into melody's tool call shape.
melody expects ``{id, name, parameters: <json str>}`` whereas OpenAI
delivers ``{id, type, function: {name, arguments: <json|str>}}``.
"""
# Pydantic objects
if hasattr(tc, "model_dump"):
tc = tc.model_dump()
if not isinstance(tc, dict):
raise TypeError(f"Unexpected tool_call value: {tc!r}")
fn = tc.get("function") or {}
name = fn.get("name") or tc.get("name") or ""
args = fn.get("arguments")
if args is None:
args = tc.get("arguments", {})
# melody expects a JSON-encoded string
if not isinstance(args, str):
args = json.dumps(args, ensure_ascii=False)
return {
"id": tc.get("id") or "",
"name": name,
"parameters": args,
}
def _content_blocks(content: Any) -> list[dict[str, Any]]:
"""Convert OpenAI ``ConversationMessage.content`` into melody content blocks.
The chat_utils ``content_format="openai"`` produces a list of
``{type: text, text: ...}`` dicts already; we pass those through with
minimal coercion. Plain string content is wrapped in a single text block.
Image / multimodal placeholder dicts (``{"type": "image"}``) are
forwarded as-is so that templates can reference image placeholders that
the upstream tokenizer expands separately.
"""
if content is None:
return []
if isinstance(content, str):
return [{"type": MelodyContentType.TEXT, "text": content}]
blocks: list[dict[str, Any]] = []
for part in content:
if isinstance(part, str):
blocks.append({"type": MelodyContentType.TEXT, "text": part})
continue
if not isinstance(part, dict):
raise TypeError(f"Unexpected content part: {part!r}")
part_type = part.get("type", MelodyContentType.TEXT)
if part_type in ("text", "input_text", "output_text", "refusal"):
blocks.append(
{"type": MelodyContentType.TEXT, "text": part.get("text", "")}
)
elif part_type == MelodyContentType.THINKING:
blocks.append(
{
"type": MelodyContentType.THINKING,
"thinking": part.get("thinking", ""),
}
)
elif part_type == MelodyContentType.IMAGE:
blocks.append(
{
"type": MelodyContentType.IMAGE,
"image": {
"template_placeholder": part.get(
"template_placeholder", "<image>"
),
},
}
)
elif part_type == MelodyContentType.DOCUMENT:
doc = part.get("document")
if isinstance(doc, dict):
blocks.append({"type": MelodyContentType.DOCUMENT, "document": doc})
else:
# Fall back to wrapping arbitrary string as text.
blocks.append({"type": MelodyContentType.TEXT, "text": json.dumps(doc)})
elif part_type == "tool_reference":
# Tool references are rendered by name; emit as text so the
# renderer downstream is content-format agnostic.
blocks.append(
{
"type": MelodyContentType.TEXT,
"text": part.get("name") or part.get("text", ""),
}
)
else:
# Unknown block type: render as text fallback.
text = part.get("text") or part.get(part_type) or ""
text_str = text if isinstance(text, str) else json.dumps(text)
blocks.append({"type": MelodyContentType.TEXT, "text": text_str})
return blocks
def _document_to_melody(doc: Any) -> dict[str, Any]:
"""Coerce a Cohere v2 document into melody's ``Document`` (dict) shape."""
match doc:
case str():
return {"text": doc}
# Cohere v2 wraps documents in {id, data: {...}}; the ``**rest``
# capture keeps the outer ``id`` (and any other outer keys) so we
# can lift them onto the payload without losing them.
case {"data": dict() as data, **rest}:
payload = dict(data)
if (outer_id := rest.get("id")) is not None and "id" not in payload:
payload["id"] = outer_id
return payload
case dict():
return dict(doc)
case _:
raise TypeError(f"Unsupported document type: {type(doc).__name__}")
def _tool_to_melody(tool: Any) -> dict[str, Any]:
"""Coerce a chat completions tool definition into melody's ``Tool`` shape.
Accepts either the raw OpenAI tool wrapper ``{type:"function", function:
{name, description, parameters}}`` or a flat ``{name, description,
parameters}`` dict (which is what melody itself expects).
"""
if hasattr(tool, "model_dump"):
tool = tool.model_dump()
if not isinstance(tool, dict):
raise TypeError(f"Unsupported tool type: {type(tool).__name__}")
if "function" in tool and isinstance(tool["function"], dict):
fn = tool["function"]
else:
fn = tool
return {
"name": fn.get("name", ""),
"description": fn.get("description", "") or "",
"parameters": fn.get("parameters") or {},
}
def _conversation_to_melody_messages(
conversation: list[ConversationMessage],
messages_citations: dict[int, list[dict[str, Any]]] | None = None,
) -> list[dict[str, Any]]:
"""Convert vLLM ``ConversationMessage``s into melody's message dict shape.
``messages_citations`` is an optional index-keyed lookup of
per-message citations from the request, populated by
``CohereServingChatV2`` under the
:data:`MESSAGES_CITATIONS_KEY` chat_template_kwargs entry. Values
must already be in melody's ``FilterCitation`` dict shape (see
``CohereServingChatV2._sdk_citation_to_melody``). Index is the
position in the *input* ``conversation`` list, which currently maps
1-to-1 with ``CohereChatV2Request.messages`` for text-only Cohere
requests. This mapping breaks silently if ``parse_chat_messages``
ever expands a single input message into multiple entries (e.g.
multi-modal splitting), which is not the case today.
"""
out: list[dict[str, Any]] = []
for i, msg in enumerate(conversation):
role = _role_to_melody(msg.get("role", "user"))
content_blocks = _content_blocks(msg.get("content"))
# Treat reasoning content as a thinking block on assistant turns
# so multi-turn reasoning is preserved in the rendered prompt.
#
# Cohere models output thought as either a ``thinking``
# block (reasoning models) or a ``tool_plan`` field (older non-
# reasoning Command models), but vLLM's ConversationMessage has a
# single unified ``reasoning`` field that drops that difference.
# The cmd3 / cmd4 jinja templates render ``thinking`` blocks in both
# the case of tool calls and regular thinking blocks, so this is fine
# from the renderer's perspective.
reasoning = msg.get("reasoning") or msg.get("reasoning_content")
if role == "chatbot" and reasoning:
content_blocks.insert(
0,
{"type": MelodyContentType.THINKING, "thinking": reasoning},
)
tool_calls = [_normalize_tool_call(tc) for tc in (msg.get("tool_calls") or [])]
out_msg: dict[str, Any] = {
"role": role,
"content": content_blocks,
"tool_calls": tool_calls,
}
tool_call_id = msg.get("tool_call_id")
if tool_call_id:
out_msg["tool_call_id"] = tool_call_id
if messages_citations and (cites := messages_citations.get(i)):
out_msg["citations"] = cites
out.append(out_msg)
return out
def _build_render_config(
conversation: list[ConversationMessage],
chat_template_kwargs: dict[str, Any],
chat_template: str | None = None,
) -> tuple[str, dict[str, Any]]:
"""Build the ``render_cmd3`` / ``render_cmd4`` config dict.
``chat_template`` is the standard vLLM per-request template override
(from ``ChatCompletionRequest.chat_template`` / the server-side
``--chat-template`` flag). When non-``None`` it is forwarded to melody
as ``template_jinja`` so a single vLLM-native path drives template
source through the trust guard.
Returns ``(format, config_dict)`` where ``format`` is either ``"cmd3"``
or ``"cmd4"``.
"""
fmt = chat_template_kwargs.get("cohere_format", _DEFAULT_FORMAT)
if fmt not in _VALID_FORMATS:
raise ValueError(
f"Invalid cohere_format={fmt!r}; expected one of {_VALID_FORMATS}"
)
# Reject Cohere-only inlets for raw Jinja source. These keys are
# valid at Cohere's own API surface but not at vLLM's: callers must
# use the standard vLLM ``chat_template`` request field so the
# ``--trust-request-chat-template`` guard applies uniformly.
for cohere_only_key in ("template_jinja", "template"):
if chat_template_kwargs.get(cohere_only_key) is not None:
raise ValueError(
f"chat_template_kwargs.{cohere_only_key!r} is not accepted "
"in vLLM (it is a Cohere-only field); pass the template "
"body via the standard 'chat_template' request field "
"instead (requires --trust-request-chat-template)."
)
config: dict[str, Any] = {
"messages": _conversation_to_melody_messages(
conversation,
chat_template_kwargs.get(MESSAGES_CITATIONS_KEY),
),
}
# ``template_id`` is a selector for one of melody's built-in template
# variants (not raw source), so it is safe to accept from the client.
if template_id := chat_template_kwargs.get("template_id"):
config["template_id"] = template_id
# Raw template source flows exclusively through the standard vLLM
# ``chat_template`` field so it is subject to the
# ``--trust-request-chat-template`` guard in OnlineRenderer.
if chat_template:
config["template_jinja"] = chat_template
# Only support Jinja with vllm
config["use_jinja"] = True
# Documents
documents = chat_template_kwargs.get("documents") or []
if documents:
config["documents"] = [_document_to_melody(d) for d in documents]
# Tools - prefer explicit ``available_tools``, fall back to OpenAI ``tools``
tools = (
chat_template_kwargs.get("available_tools")
or chat_template_kwargs.get("tools")
or []
)
if tools:
config["available_tools"] = [_tool_to_melody(t) for t in tools]
# Reasoning toggle (cmd3 + cmd4)
if (rt := chat_template_kwargs.get("reasoning_type")) is not None:
config["reasoning_type"] = str(rt)
elif "thinking" in chat_template_kwargs:
# Cohere v2 ``thinking: {type: enabled|disabled}`` shorthand
thinking = chat_template_kwargs["thinking"]
t = thinking.get("type") if isinstance(thinking, dict) else thinking
if t in ("enabled", "disabled"):
config["reasoning_type"] = t
if (di := chat_template_kwargs.get("dev_instruction")) is not None:
config["dev_instruction"] = str(di)
# JSON / structured outputs
if (rf := chat_template_kwargs.get("response_format")) is not None:
rf = rf.model_dump() if hasattr(rf, "model_dump") else dict(rf)
rf_type = rf.get("type")
if rf_type == "json_object":
config["json_mode"] = True
elif rf_type in ("json_schema", "json"):
schema = rf.get("schema") or rf.get("json_schema")
if isinstance(schema, dict) and "schema" in schema:
schema = schema["schema"]
if schema is not None:
config["json_schema"] = (
schema if isinstance(schema, str) else json.dumps(schema)
)
if (js := chat_template_kwargs.get("json_schema")) is not None:
config["json_schema"] = js if isinstance(js, str) else json.dumps(js)
if "json_mode" in chat_template_kwargs:
config["json_mode"] = bool(chat_template_kwargs["json_mode"])
if fmt == "cmd3":
if (sm := chat_template_kwargs.get("safety_mode")) is not None:
config["safety_mode"] = str(sm).lower()
# citation_quality: ``on`` / ``off``
cq = chat_template_kwargs.get("citation_quality")
if cq is None and (co := chat_template_kwargs.get("citation_options")):
mode = co.get("mode") if isinstance(co, dict) else None
if mode is not None:
cq = "on" if str(mode).lower() != "off" else "off"
if cq is not None:
config["citation_quality"] = str(cq).lower()
if "skip_preamble" in chat_template_kwargs:
config["skip_preamble"] = bool(chat_template_kwargs["skip_preamble"])
else: # cmd4
# cmd4 uses ``grounding`` rather than safety_mode/citation_quality.
# melody's cmd4 only accepts ``unknown`` / ``enabled`` / ``disabled``,
# so the Cohere v2 ``citation_options.mode`` values
# (``FAST`` / ``ACCURATE`` / ``OFF``) have to be normalized -- a
# raw lowercased passthrough would raise from
# ``render_cmd4`` (cmd4 has no fast/accurate distinction at the
# prompt-template layer; both request grounding-on).
if (gr := chat_template_kwargs.get("grounding")) is not None:
config["grounding"] = _normalize_cmd4_grounding(gr)
elif co := chat_template_kwargs.get("citation_options"):
mode = co.get("mode") if isinstance(co, dict) else None
if mode is not None:
config["grounding"] = _normalize_cmd4_grounding(mode)
if (pi := chat_template_kwargs.get("platform_instruction")) is not None:
config["platform_instruction"] = str(pi)
# Anything we didn't explicitly interpret above is forwarded to melody
# as a Jinja template variable. This matches vLLM's documented contract
# for ``chat_template_kwargs`` ("kwargs accessible by the template")
# and lets callers write e.g. ``{"reasoning_effort": "low"}`` directly
# without a nested ``additional_template_fields`` wrapper.
extra = {
k: v
for k, v in chat_template_kwargs.items()
if k not in _RENDERER_CONSUMED_KEYS
}
if extra:
config["additional_template_fields"] = extra
return fmt, config
class CohereRenderer(BaseRenderer[HfTokenizer]):
"""Renderer that templates Cohere prompts via the melody Rust bindings.
Tokenization is delegated to the standard HF tokenizer; only the
chat-template step is replaced with ``cohere_melody.render_cmd3`` /
``render_cmd4``. Enabled via ``--tokenizer-mode cohere``.
"""
def __init__(
self,
config: VllmConfig,
tokenizer: HfTokenizer | None,
) -> None:
# Match HfRenderer in not mutating the cached tokenizer instance
tokenizer = copy.copy(tokenizer)
super().__init__(config, tokenizer)
# Lazy import to keep `cohere_melody` an optional dependency
self._melody = _try_import_melody()
# ``render_cmd3`` / ``render_cmd4`` are pure CPU work; cache the
# thread-pool wrapper once so the async path doesn't allocate a new
# adapter on every request.
self._render_async = make_async(self._render, executor=self._executor)
def _render(self, fmt: str, config_dict: dict[str, Any]) -> str:
if fmt == "cmd3":
return self._melody.render_cmd3(config_dict)
return self._melody.render_cmd4(config_dict)
def render_messages(
self,
messages: list[ChatCompletionMessageParam],
params: ChatParams,
) -> tuple[list[ConversationMessage], DictPrompt]:
conversation, mm_data, mm_uuids = parse_chat_messages(
messages,
self.model_config,
content_format="openai",
media_io_kwargs=params.media_io_kwargs,
mm_processor_kwargs=params.mm_processor_kwargs,
)
chat_template_kwargs = dict(params.chat_template_kwargs)
fmt, config_dict = _build_render_config(
conversation, chat_template_kwargs, params.chat_template
)
prompt_text = self._render(fmt, config_dict)
prompt = parse_dec_only_prompt(prompt_text)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids
return conversation, prompt
async def render_messages_async(
self,
messages: list[ChatCompletionMessageParam],
params: ChatParams,
) -> tuple[list[ConversationMessage], DictPrompt]:
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
messages,
self.model_config,
content_format="openai",
media_io_kwargs=params.media_io_kwargs,
mm_processor_kwargs=params.mm_processor_kwargs,
)
chat_template_kwargs = dict(params.chat_template_kwargs)
fmt, config_dict = _build_render_config(
conversation, chat_template_kwargs, params.chat_template
)
prompt_text = await self._render_async(fmt, config_dict)
prompt = parse_dec_only_prompt(prompt_text)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids
return conversation, prompt
+1
View File
@@ -20,6 +20,7 @@ logger = init_logger(__name__)
_VLLM_RENDERERS = {
"cohere": ("cohere", "CohereRenderer"),
"deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"),
"deepseek_v4": ("deepseek_v4", "DeepseekV4Renderer"),
"hf": ("hf", "HfRenderer"),
+3
View File
@@ -40,6 +40,9 @@ _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: set[str] = {
}
_VLLM_TOKENIZERS = {
# ``cohere`` mode uses the standard cached HF tokenizer; only the
# renderer (template stage) is replaced with a melody-based one.
"cohere": ("hf", "CachedHfTokenizer"),
"deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
"deepseek_v4": ("deepseek_v4", "DeepseekV4Tokenizer"),
"hf": ("hf", "CachedHfTokenizer"),