Fix Cohere ChatV2 citation and tool handling issues (#52175)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
andrewbcohere
2026-08-21 15:23:17 -07:00
committed by GitHub
parent f37d5868b9
commit f94dcde5c5
7 changed files with 1024 additions and 68 deletions
@@ -11,6 +11,7 @@ a real engine client, model registry, etc.) — the same pattern used in
``test_serving_streaming.py``.
"""
import json
from typing import Any
import pytest
@@ -34,11 +35,61 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.renderers.cohere import (
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
)
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
# A full grounded tool round-trip: a tool call, a tool result carrying both
# a document and a bare text block, and an assistant turn citing it. Between
# them these populate every reserved ``_``-prefixed chat_template_kwargs key.
_TOOL_ROUND_TRIP_MESSAGES: list[dict[str, Any]] = [
{"role": "user", "content": "What is the capital?"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "search", "arguments": '{"q": "capital"}'},
}
],
},
{
"role": "tool",
"tool_call_id": "call-1",
"content": [
{
"type": "document",
"document": {"id": "doc-9", "data": {"text": "Paris"}},
},
{"type": "text", "text": "Paris is the capital."},
],
},
{
"role": "assistant",
"content": "Paris.",
"citations": [
{
"start": 0,
"end": 5,
"text": "Paris",
"sources": [
{"type": "tool", "id": "doc-9", "tool_output": {"text": "Paris"}}
],
}
],
},
{"role": "user", "content": "Are you sure?"},
]
def _make_request(**kwargs) -> CohereChatV2Request:
kwargs.setdefault("model", "m")
@@ -447,9 +498,15 @@ class TestConvertToolMessage:
"content": "result text",
}
def test_text_only_list_flattened_to_newline_string(self):
# Text-only tool results are flattened to a single newline-joined
# string for compatibility with standard chat templates.
def test_text_only_list_preserves_per_block_text_parts(self):
# A list of text blocks maps 1-to-1 onto ``text`` content
# parts, mirroring the shape ``_convert_user_message`` emits
# for multi-part user content. vLLM's
# ``_parse_chat_message_content`` will collapse this back to
# ``"\n".join(...)`` for the ``ConversationMessage`` handed to
# renderers (see the tool-role branch in
# ``vllm/entrypoints/chat_utils.py``), so downstream chat
# templates that expect string tool content still work.
req = self._request_with_tool_message(
[
{"type": "text", "text": "line 1"},
@@ -458,11 +515,27 @@ class TestConvertToolMessage:
)
result = _convert(req)
tool_msg = result.messages[-1]
assert tool_msg["content"] == "line 1\nline 2"
assert tool_msg["content"] == [
{"type": "text", "text": "line 1"},
{"type": "text", "text": "line 2"},
]
def test_with_document_preserves_structured_content(self):
# When documents appear in the tool result, we keep the list shape
# so the Cohere renderer can lift them into grounding sources.
def test_document_blocks_serialized_as_text_parts(self):
"""Document blocks must not survive on the OpenAI-shaped tool
message.
vLLM's shared ``_parse_chat_message_content_parts`` runs a
Pydantic ``ValidatorIterator`` over the OpenAI content-part
union, which only knows ``text`` / ``image_url`` / etc. and
rejects ``{"type": "document", ...}`` blocks outright. We
therefore JSON-serialize each document into its own ``text``
content part so the request survives OpenAI validation; the
cohere renderer still resolves outbound citations against
the original cohere v2 messages via ``POSITION_TO_SOURCE_KEY``
and reconstructs structured melody ``document`` blocks from the
original v2 content forwarded via
``TOOL_MESSAGE_V2_CONTENT_KEY``.
"""
req = self._request_with_tool_message(
[
{"type": "text", "text": "see attachment"},
@@ -475,11 +548,75 @@ class TestConvertToolMessage:
result = _convert(req)
tool_msg = result.messages[-1]
assert isinstance(tool_msg["content"], list)
assert len(tool_msg["content"]) == 2
# First block passes through unchanged.
assert tool_msg["content"][0] == {"type": "text", "text": "see attachment"}
assert tool_msg["content"][1] == {
"type": "document",
"document": {"data": {"text": "doc text"}, "id": "d1"},
}
# Second block: document JSON-serialized into a ``text`` part.
second = tool_msg["content"][1]
assert second["type"] == "text"
assert json.loads(second["text"]) == {"data": {"text": "doc text"}, "id": "d1"}
def test_document_only_content_serialized_as_text_part(self):
"""Document-only tool results must also produce a text-typed
content part (no ``text`` block above them), so a single-doc
result still passes the OpenAI content-parts validator.
"""
req = self._request_with_tool_message(
[
{
"type": "document",
"document": {"data": {"result": "K2 is 8611m."}},
}
]
)
result = _convert(req)
tool_msg = result.messages[-1]
assert isinstance(tool_msg["content"], list)
(part,) = tool_msg["content"]
assert part["type"] == "text"
assert json.loads(part["text"]) == {"data": {"result": "K2 is 8611m."}}
def test_converted_tool_message_content_survives_openai_validation(self):
"""Regression for the reported crash.
``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts``
runs a Pydantic ``ValidatorIterator`` over
``list[ChatCompletionContentPartParam]``, which is a
discriminated union that only accepts ``text`` / ``image_url``
/ ``input_audio`` / ``refusal`` / ``file`` types. Previously
the converted tool message carried ``{"type": "document",
"document": {...}}`` blocks, which the validator rejected with:
2 validation errors for ValidatorIterator
0.text Field required
0.type Input should be 'text', got 'document'
The fix rewrites every ``DocumentToolContent`` into a
JSON-serialized ``text`` part. This test asserts the invariant
that keeps the validator happy: on the OpenAI-shaped request,
every content entry a tool message carries must be a ``text``
part (or a bare string) -- so no future refactor accidentally
re-introduces a ``document``-typed content part.
"""
req = self._request_with_tool_message(
[
{
"type": "document",
"document": {
"data": {"result": "The second tallest mountain is K2."}
},
}
]
)
result = _convert(req)
tool_msg = result.messages[-1]
content = tool_msg["content"]
assert isinstance(content, (str, list))
if isinstance(content, list):
for part in content:
assert isinstance(part, dict)
assert part.get("type") == "text"
# ======================================================================
@@ -808,6 +945,276 @@ class TestApplyCohereTemplateKwargs:
assert result.chat_template_kwargs is None
# ======================================================================
# Engine-core hop (_engine_chat_template_kwargs)
# ======================================================================
class TestEngineChatTemplateKwargs:
"""The internal ``_``-prefixed entries must not reach the engine core.
``OpenAIServingChat`` uses one ``chat_template_kwargs`` dict twice:
to build the API-server-side parser, and to populate
``EngineCoreRequest.reasoning_parser_kwargs``, which crosses ZMQ as
msgpack. This handler's reserved keys are only meaningful to the
former, and ``_position_to_source`` holds ``CitationSource`` models
that msgpack cannot encode at all -- forwarding them fails the
request with a ``TypeError``. The
``_engine_chat_template_kwargs`` override drops them.
"""
@staticmethod
def _engine_kwargs(request: CohereChatV2Request) -> dict[str, Any]:
full = _convert(request).chat_template_kwargs
assert full, "expected the request to populate chat_template_kwargs"
return _FakeServing()._engine_chat_template_kwargs(full)
def test_grounded_request_drops_internal_keys_keeps_public_ones(self):
request = _make_request(
documents=[{"id": "d1", "data": {"text": "hi"}}],
citation_options={"mode": "accurate"},
)
full = _convert(request).chat_template_kwargs
# Sanity: the key we expect to be dropped is actually present.
assert POSITION_TO_SOURCE_KEY in full
engine_kwargs = self._engine_kwargs(request)
assert POSITION_TO_SOURCE_KEY not in engine_kwargs
# Public template inputs the engine-side parser may legitimately
# read must survive.
assert engine_kwargs["citation_options"] == {"mode": "accurate"}
assert engine_kwargs["documents"] == [{"id": "d1", "data": {"text": "hi"}}]
def test_all_internal_keys_dropped(self):
"""Exercises all three reserved keys at once: the tool-result
position map, the inbound assistant citations, and the raw v2 tool
content blocks.
"""
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
full = _convert(request).chat_template_kwargs
internal = {k for k in full if k.startswith("_")}
# Sanity: this request really does populate all three.
assert internal == {
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
}
engine_kwargs = self._engine_kwargs(request)
assert not [k for k in engine_kwargs if k.startswith("_")]
def test_engine_kwargs_are_msgpack_encodable(self):
"""The real failure mode: the full dict raises, the filtered one
doesn't. Pins that the filter is what makes the engine hop work.
"""
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
full = _convert(request).chat_template_kwargs
with pytest.raises(TypeError, match="is not serializable"):
MsgpackEncoder().encode({"chat_template_kwargs": full})
engine_kwargs = self._engine_kwargs(request)
encoder = MsgpackEncoder()
decoder = MsgpackDecoder(dict[str, Any])
round_tripped = decoder.decode(
encoder.encode({"chat_template_kwargs": engine_kwargs})[0]
)
assert round_tripped["chat_template_kwargs"] == engine_kwargs
def test_original_dict_not_mutated(self):
"""The caller still passes the full dict to the in-process parser,
so the filter must return a copy.
"""
request = _make_request(documents=[{"id": "d1", "data": {"text": "hi"}}])
full = _convert(request).chat_template_kwargs
before = set(full)
self._engine_kwargs(request)
assert set(full) == before
def test_base_implementation_is_a_passthrough(self):
"""Only the Cohere subclass filters; plain chat completions must
keep forwarding whatever the request asked for.
"""
kwargs = {
"documents": [{"id": "d1"}],
POSITION_TO_SOURCE_KEY: {(0, 0): object()},
}
passed_through = OpenAIServingChat._engine_chat_template_kwargs(
_FakeServing(), kwargs
)
assert passed_through == kwargs
# ======================================================================
# Tool-message content forwarding
# (chat_template_kwargs["_tool_message_v2_content"])
# ======================================================================
class TestToolMessageV2ContentForwarding:
"""The cohere renderer needs the original v2 tool-message content
(text / document blocks) to reconstruct melody's ``document``-block
shape, but the OpenAI content-parts validator on the way in only
accepts text / image / audio / etc. blocks. So
:meth:`CohereServingChatV2._convert_tool_message` rewrites each
content block into a plain ``text`` part on the request that
survives validation, and
:meth:`CohereServingChatV2._collect_v2_tool_message_content`
forwards the *original* v2 content blocks -- as ``model_dump``'d
dicts -- to the renderer under
:data:`TOOL_MESSAGE_V2_CONTENT_KEY`. These tests pin the
forwarding contract (presence, dict-shape, index-alignment); the
v2 → melody mapping the renderer applies to these blocks lives in
``tests/renderers/test_cohere.py``.
"""
def _request_with_tool_message(self, content: Any) -> CohereChatV2Request:
return _make_request(
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "c1", "content": content},
]
)
def test_absent_when_no_tool_messages(self):
result = _convert(_make_request())
# ``chat_template_kwargs`` may be ``None`` here (no cohere-side
# forwarding happened at all) or a dict without the key; both
# mean the same thing to the renderer.
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
def test_absent_for_string_tool_content(self):
# Bare-string tool content works through the renderer's default
# ``_content_blocks`` path, so no forwarded blocks are needed.
req = self._request_with_tool_message("plain text")
result = _convert(req)
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
def test_forwarded_blocks_are_plain_v2_shape_dicts(self):
# The forwarded map must contain the original v2 content
# blocks verbatim (as plain dicts, not Pydantic instances) so
# the renderer can drive the v2 → melody mapping and the map
# stays trivially serializable.
req = self._request_with_tool_message(
[
{"type": "text", "text": "K2 is 8611m."},
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
},
]
)
blocks_by_index = _forwarded_v2_content(req)
# Index 2 = the tool message (user=0, assistant=1, tool=2).
assert list(blocks_by_index.keys()) == [2]
assert blocks_by_index[2] == [
{"type": "text", "text": "K2 is 8611m."},
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
},
]
# Guard against a regression where Pydantic instances leak into
# the forwarded map -- msgpack can't serialize them, and even
# for keys filtered out of the engine-core payload we want the
# renderer to see plain data.
for block in blocks_by_index[2]:
assert isinstance(block, dict)
def test_forwarded_blocks_preserve_input_order(self):
# Melody's tool-result rendering keys result slots by their
# position within a tool message's content list, and the
# outbound citation source map (built by
# ``_walk_citable_positions``) uses that same positional index.
# Reordering would drop citations pointing at slot ``1`` onto
# the wrong payload.
req = self._request_with_tool_message(
[
{"type": "text", "text": "leading note"},
{
"type": "document",
"document": {"data": {"body": "doc body"}, "id": "d0"},
},
{"type": "text", "text": '{"parsed": "yes"}'},
]
)
blocks = _forwarded_v2_content(req)[2]
assert [b["type"] for b in blocks] == ["text", "document", "text"]
def test_index_matches_original_request_position(self):
# Entries are keyed by *request-message* index so the renderer,
# walking the OpenAI-shaped conversation with the same
# enumeration, resolves the right slot. If ``_convert_messages``
# ever splits or merges messages this invariant breaks, so pin
# the current 1-to-1 behavior with a two-tool-message request.
req = _make_request(
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "g", "arguments": "{}"},
},
],
},
{
"role": "tool",
"tool_call_id": "c1",
"content": [{"type": "text", "text": "first result"}],
},
{
"role": "tool",
"tool_call_id": "c2",
"content": [{"type": "text", "text": "second result"}],
},
]
)
blocks_by_index = _forwarded_v2_content(req)
assert sorted(blocks_by_index.keys()) == [2, 3]
assert blocks_by_index[2] == [{"type": "text", "text": "first result"}]
assert blocks_by_index[3] == [{"type": "text", "text": "second result"}]
def _forwarded_v2_content(
request: CohereChatV2Request,
) -> dict[int, list[dict[str, Any]]]:
"""Extract the forwarded v2 tool-message content from a converted
request, or fail the test if the key isn't present.
"""
result = _convert(request)
kwargs = result.chat_template_kwargs or {}
assert TOOL_MESSAGE_V2_CONTENT_KEY in kwargs, (
f"expected {TOOL_MESSAGE_V2_CONTENT_KEY} in chat_template_kwargs, "
f"got keys {list(kwargs)}"
)
return kwargs[TOOL_MESSAGE_V2_CONTENT_KEY]
# ======================================================================
# Message-citation forwarding (chat_template_kwargs["_messages_citations"])
# ======================================================================
@@ -655,3 +655,123 @@ class TestMelodySourceResolution:
assert out is not None
assert out[0].type == "THINKING_CONTENT"
assert out[0].sources[0].id == "d0"
class TestParserStreamingEndToEnd:
"""End-to-end streaming shape: raw model output text is tokenized
with the mock byte-level tokenizer and fed one token at a time
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
We collect the ordered sequence of ``DeltaMessage``s the parser
hands back -- i.e. what the client would receive on the wire --
and pin the exact shape.
Combines the three flavors that reach ``delta``: a thinking block
(populates ``reasoning``), a text block (populates ``content``),
and a resolved citation (populates ``citations``) referencing
``(start, end)`` offsets into the already-streamed content.
"""
_MODEL_OUTPUT = (
"<|START_THINKING|>Let me check.<|END_THINKING|>"
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
)
@staticmethod
def _drive_parser(
parser: CohereCommand3ReasoningParser, text: str, tok: MockCohereTokenizer
) -> list[Any]:
"""Feed ``text`` through the parser one token at a time
(byte-level) and return the ordered list of non-``None``
``DeltaMessage``s.
"""
deltas: list[Any] = []
previous_text = ""
previous_token_ids: list[int] = []
for token_str in _token_deltas(tok, text):
current_text = previous_text + token_str
current_token_ids = previous_token_ids + [0]
delta = parser.extract_reasoning_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=token_str,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=[0],
)
if delta is not None:
deltas.append(delta)
previous_text = current_text
previous_token_ids = current_token_ids
return deltas
def test_full_wire_stream_shape(self, tokenizer):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
# installs on a request whose tool_call_index=0 tool result at
# index 1 is the "France" document.
position_to_source = {
(0, 1): CitationSource(
type="document",
id="doc-paris",
document={"id": "doc-paris", "title": "France"},
),
}
parser = CohereCommand3ReasoningParser(
tokenizer,
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
)
deltas = self._drive_parser(parser, self._MODEL_OUTPUT, tokenizer)
# -- 1. Reasoning and content stream separately and reach
# the wire only as their real payload. Framing tokens
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
# state machine and never produce a delta.
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
content_stream = [d.content for d in deltas if d.content is not None]
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
assert "".join(reasoning_stream) == "Let me check."
assert "".join(content_stream) == "The capital is Paris."
# -- 2. Exactly one citation reaches the wire, resolved to the
# source from ``position_to_source`` (not the raw
# ``(bucket, idx)`` coordinates).
assert len(citation_deltas) == 1
(cite_delta,) = citation_deltas
assert cite_delta.citations is not None
assert len(cite_delta.citations) == 1
cite = cite_delta.citations[0]
assert cite.text == "Paris"
# Char offsets are into the accumulated ``content``: "The
# capital is " is 15 chars, "Paris" is 5.
assert cite.start == 15
assert cite.end == 20
assert len(cite.sources) == 1
assert cite.sources[0].id == "doc-paris"
assert cite.sources[0].type == "document"
# -- 3. The citation's ``(start, end)`` offsets point into
# bytes that were already emitted as content-deltas earlier
# in the stream -- i.e. the citation is anchored to real
# already-shipped content.
cite_delta_idx = deltas.index(cite_delta)
earlier_content = "".join(
d.content for d in deltas[:cite_delta_idx] if d.content is not None
)
assert earlier_content[cite.start : cite.end] == "Paris"
# Everything after the citation delta is the trailing period.
later_content = "".join(
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
)
assert later_content == "."
# -- 4. Reasoning and content are never mixed on the same
# delta: the parser flips modes on the ``<|END_THINKING|>``
# boundary and never emits a delta with both fields set.
for d in deltas:
assert not (d.reasoning is not None and d.content is not None)
+202
View File
@@ -30,6 +30,7 @@ from vllm.renderers.cohere import (
_normalize_tool_call,
_role_to_melody,
_tool_to_melody,
_v2_tool_content_to_melody_block,
)
from vllm.tokenizers.hf import HfTokenizer
@@ -304,6 +305,108 @@ class TestToolToMelody:
_tool_to_melody(42) # type: ignore[arg-type]
# ======================================================================
# _v2_tool_content_to_melody_block
# ======================================================================
class TestV2ToolContentToMelodyBlock:
"""Pins the v2 → melody mapping applied to every tool-message
content block. The serving layer forwards the original v2 blocks
verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer
owns this mapping so cmd3 / cmd4 render each result field as its
own entry rather than a single stringified-JSON blob.
"""
def test_document_block_unwraps_data_into_melody_document(self):
# ``document.data`` becomes the melody ``document`` payload
# directly. Keeping the wrapping ``data`` key would produce a
# redundant ``{"data": {...}}`` nesting at render time.
# ``document.id`` is dropped here on purpose: outbound
# citation binding tracks IDs through the separate
# ``POSITION_TO_SOURCE_KEY`` map.
result = _v2_tool_content_to_melody_block(
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
}
)
assert result == {
"type": "document",
"document": {"result": "The second tallest mountain is K2."},
}
def test_document_block_with_missing_data_yields_empty_payload(self):
# A document block whose ``data`` isn't a mapping (e.g.
# missing entirely) still produces a document block so
# per-slot numbering stays aligned with the outbound citation
# walk in ``CohereServingChatV2._walk_citable_positions``.
result = _v2_tool_content_to_melody_block(
{"type": "document", "document": {"id": "d1"}}
)
assert result == {"type": "document", "document": {}}
def test_json_parseable_text_becomes_document_dict(self):
# A ``text`` block whose payload parses to a JSON object lands
# directly under ``document`` (no ``content`` wrapper) so
# cmd3/cmd4 render each field individually.
result = _v2_tool_content_to_melody_block(
{"type": "text", "text": '{"temperature": "20C"}'}
)
assert result == {"type": "document", "document": {"temperature": "20C"}}
def test_non_json_text_wrapped_under_content_key(self):
# Plain-text tool output gets wrapped under a ``content`` key
# so cmd3/cmd4 templates have a stable field to look up.
result = _v2_tool_content_to_melody_block(
{"type": "text", "text": "K2 is 8611m."}
)
assert result == {"type": "document", "document": {"content": "K2 is 8611m."}}
def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self):
# A ``text`` block whose payload parses as a JSON *array* or
# *scalar* isn't a valid melody document (which must be a
# mapping). Fall back to the ``{"content": <text>}`` wrapper so
# the raw text still reaches the prompt.
result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"})
assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}}
def test_empty_text_still_produces_content_block(self):
# Even an empty text block must produce an entry so melody's
# per-content-block numbering stays aligned with the
# citation-side walk in ``_walk_citable_positions``.
result = _v2_tool_content_to_melody_block({"type": "text", "text": ""})
assert result == {"type": "document", "document": {"content": ""}}
def test_missing_text_field_treated_as_empty(self):
# ``model_dump(exclude_none=True)`` on a ``TextToolContent``
# with an explicitly empty text still emits ``text: ""``, but
# guard against a future v2 shape that omits it entirely.
result = _v2_tool_content_to_melody_block({"type": "text"})
assert result == {"type": "document", "document": {"content": ""}}
def test_unknown_block_type_returns_none(self):
# Callers use ``None`` to know they should skip the slot; a
# future v2 addition shouldn't silently render as an empty
# document.
assert _v2_tool_content_to_melody_block({"type": "audio"}) is None
assert _v2_tool_content_to_melody_block({"not": "a block"}) is None
def test_document_block_with_non_dict_document_returns_none(self):
# Malformed ``document`` field -- e.g. someone forwarding a
# raw string rather than the ``{id, data}`` wrapper -- is
# unusable, so return ``None`` so the caller skips it.
assert (
_v2_tool_content_to_melody_block(
{"type": "document", "document": "not a dict"}
)
is None
)
# ======================================================================
# _conversation_to_melody_messages
# ======================================================================
@@ -444,6 +547,71 @@ class TestConversationToMelody:
out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type]
assert "citations" not in out[0]
def test_tool_message_v2_content_replaces_default_content(self):
# When ``tool_message_v2_content`` has an entry for a message
# index, the renderer maps those v2 blocks through
# :func:`_v2_tool_content_to_melody_block` and uses the result
# instead of the block list ``_content_blocks`` would derive
# from ``msg["content"]``.
conv = [
{"role": "user", "content": "q"},
{
"role": "tool",
"tool_call_id": "c1",
# ``_convert_tool_message`` on the serving side flattens
# the original ``document`` block to this text part.
# The renderer should ignore this flattened content
# because the forwarded map carries the structural
# truth.
"content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]',
},
]
v2_content = {
1: [
{
"type": "document",
"document": {"id": "d1", "data": {"result": "K2"}},
}
]
}
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=v2_content,
)
# User message (no forwarded entry) still goes through the
# default ``_content_blocks`` path.
assert out[0]["content"] == [{"type": "text", "text": "q"}]
# Tool message picks up the mapped melody document block.
assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}]
assert out[1]["tool_call_id"] == "c1"
def test_tool_message_v2_content_skips_unrecognized_blocks(self):
# ``_v2_tool_content_to_melody_block`` returns ``None`` for
# shapes it doesn't recognise; the renderer must skip those
# rather than let a ``None`` reach melody.
conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}]
v2_content = {
0: [
{"type": "text", "text": "keep me"},
{"type": "mystery", "unexpected": True},
]
}
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=v2_content,
)
assert out[0]["content"] == [
{"type": "document", "document": {"content": "keep me"}}
]
def test_tool_message_v2_content_none_falls_back_to_default(self):
conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}]
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=None,
)
assert out[0]["content"] == [{"type": "text", "text": "plain text"}]
# ======================================================================
# _build_render_config
@@ -762,6 +930,40 @@ class TestBuildRenderConfig:
) # type: ignore[arg-type]
assert "template_jinja" not in cfg
def test_tool_message_v2_content_key_picked_up_from_kwargs(self):
# ``_build_render_config`` must pull
# ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of
# ``chat_template_kwargs`` and hand it to
# ``_conversation_to_melody_messages`` so the renderer's
# v2 → melody mapping runs over the original v2 blocks rather
# than the flattened string content on the OpenAI-shape tool
# message.
from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY
conv = [
{"role": "user", "content": "q"},
{"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'},
]
kwargs = {
TOOL_MESSAGE_V2_CONTENT_KEY: {
1: [
{
"type": "document",
"document": {"id": "d1", "data": {"r": "k"}},
}
]
}
}
_, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type]
tool_msg = cfg["messages"][1]
assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}]
# And the key must be *consumed* -- not surfaced as a stray
# Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but
# pin it here so a removal from that set surfaces immediately).
assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get(
"additional_template_fields", {}
)
# ======================================================================
# End-to-end async rendering (mirrors ``test_mistral.py``)
+130 -31
View File
@@ -89,7 +89,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.parser.abstract_parser import Parser
from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY
from vllm.renderers.cohere import (
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
)
if TYPE_CHECKING:
from vllm.renderers.online_renderer import OnlineRenderer
@@ -97,6 +101,19 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
# Reserved ``chat_template_kwargs`` entries this handler populates for
# the API-server-side parser and renderer only. Stripped before the dict
# is forwarded to the engine core -- see
# :meth:`CohereServingChatV2._engine_chat_template_kwargs`.
_API_SERVER_ONLY_TEMPLATE_KWARGS = frozenset(
{
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
}
)
# ---------------------------------------------------------------------------
# SSE helpers
# ---------------------------------------------------------------------------
@@ -304,6 +321,28 @@ class CohereServingChatV2(OpenAIServingChat):
case _:
return self._chat_completion_stream_to_v2(generator, request)
def _engine_chat_template_kwargs(
self, chat_template_kwargs: dict[str, Any]
) -> dict[str, Any]:
"""Keep this handler's internal entries out of the engine hop.
Overrides :meth:`OpenAIServingChat._engine_chat_template_kwargs`.
The reserved keys in :data:`_API_SERVER_ONLY_TEMPLATE_KWARGS`
exist only to hand request-side state to the API-server-side
parser and renderer; the engine-core-side reasoning parser (used
for structured-output reasoning-end gating) never reads them. They
also can't survive the trip: ``POSITION_TO_SOURCE_KEY`` holds
:class:`CitationSource` models, and msgpack -- which encodes
``reasoning_parser_kwargs`` on the way to the engine core -- has
no native encoding for Pydantic models, so forwarding them fails
the request with a ``TypeError``.
"""
return {
k: v
for k, v in chat_template_kwargs.items()
if k not in _API_SERVER_ONLY_TEMPLATE_KWARGS
}
# ==================================================================
# Request conversion: Cohere V2 -> ChatCompletionRequest
# ==================================================================
@@ -438,46 +477,53 @@ class CohereServingChatV2(OpenAIServingChat):
"content": msg.content,
}
# When the tool result is text-only, flatten to a string for
# maximum compatibility with standard chat templates. When it
# includes documents, preserve them as structured content parts
# so the cohere renderer can surface them as grounding sources
# (the :class:`CohereRenderer` understands
# ``{type: document, document: {...}}`` blocks). Non-cohere
# renderers may not honor document blocks, but that matches
# the broader "documents are no-op for OSS models" contract
# documented on this endpoint.
# Map every content block to a plain ``{"type": "text", "text":
# ...}`` part, mirroring the shape :meth:`_convert_user_message`
# emits for multi-part user content. vLLM's
# ``_parse_chat_message_content`` collapses a text-only tool
# part list back into ``"\n".join(...)`` on the way to the
# renderer, so simple chat templates that expect string tool
# content still work; keeping the list here just preserves the
# per-block structure at the OpenAI-shape request level for
# symmetry with the user-message conversion.
#
# Tool message content uses ``ToolMessageV2Content``, which is a
# union of ``TextToolContent`` and ``DocumentToolContent`` -
# distinct from the user-message text/image union. We discriminate
# on the ``type`` literal so we don't have to import each variant.
has_documents = any(block.type == "document" for block in msg.content)
if not has_documents:
text = "\n".join(
block.text for block in msg.content if block.type == "text"
)
return {
"role": "tool",
"tool_call_id": msg.tool_call_id,
"content": text,
}
parts: list[dict[str, Any]] = []
# Tool message content uses ``ToolMessageV2Content``, a union of
# ``TextToolContent`` and ``DocumentToolContent``. The latter
# produces ``{"type": "document", "document": {...}}`` blocks,
# which vLLM's shared ``_parse_chat_message_content_parts`` in
# ``vllm/entrypoints/chat_utils.py`` rejects because the OpenAI
# ``ChatCompletionContentPartParam`` union only knows text /
# image / audio / etc. Emitting them anyway triggers a Pydantic
# ``ValidatorIterator`` failure at chat-parsing time, so we
# JSON-serialize each document into a text part instead.
#
# The cohere renderer doesn't consume this flattened content:
# :meth:`_collect_v2_tool_message_content` forwards the
# original v2 content blocks under
# :data:`TOOL_MESSAGE_V2_CONTENT_KEY`, and the renderer applies
# :func:`vllm.renderers.cohere._v2_tool_content_to_melody_block`
# to produce the melody-shape ``document`` blocks cmd3/cmd4
# tool-result rendering expects. Outbound citation binding is
# unaffected either way: sources resolve through the
# ``POSITION_TO_SOURCE_KEY`` map (built from the original v2
# messages, not the converted OpenAI messages).
content_parts: list[dict[str, Any]] = []
for block in msg.content:
if block.type == "text":
parts.append({"type": "text", "text": block.text})
content_parts.append({"type": "text", "text": block.text})
elif block.type == "document":
parts.append(
content_parts.append(
{
"type": "document",
"document": block.document.model_dump(exclude_none=True),
"type": "text",
"text": json.dumps(
block.document.model_dump(exclude_none=True)
),
}
)
return {
"role": "tool",
"tool_call_id": msg.tool_call_id,
"content": parts,
"content": content_parts,
}
@classmethod
@@ -660,6 +706,16 @@ class CohereServingChatV2(OpenAIServingChat):
if position_to_source:
kwargs.setdefault(POSITION_TO_SOURCE_KEY, position_to_source)
# Forward the original v2 tool-message content blocks so the
# renderer can apply its v2 → melody mapping over them (the
# renderer owns the melody-shape rules, this handler only
# ships the raw structural info that OpenAI validation would
# otherwise destroy -- see the flatten comment in
# :meth:`_convert_tool_message`).
v2_tool_content = cls._collect_v2_tool_message_content(request)
if v2_tool_content:
kwargs.setdefault(TOOL_MESSAGE_V2_CONTENT_KEY, v2_tool_content)
if kwargs:
chat_req.chat_template_kwargs = kwargs
@@ -1262,6 +1318,49 @@ class CohereServingChatV2(OpenAIServingChat):
)
return None
@classmethod
def _collect_v2_tool_message_content(
cls, request: CohereChatV2Request
) -> dict[int, list[dict[str, Any]]]:
"""Collect the original v2 tool-message content blocks so the
renderer can apply its v2 → melody mapping.
Builds the map that
:meth:`_apply_cohere_template_kwargs` stashes under
:data:`TOOL_MESSAGE_V2_CONTENT_KEY` for the renderer to
consume (see the constant definition in
:mod:`vllm.renderers.cohere` for why the map exists). Walks
``request.messages`` -- not the OpenAI-shape messages built by
:meth:`_convert_messages` -- because that flattening step
already threw away the structural distinction between text and
document blocks. Each surviving block is
:meth:`~pydantic.BaseModel.model_dump`'d to a plain dict so the
forwarded map carries no Pydantic instances (msgpack- /
general-serialization-safe on principle even though internal
underscore-prefixed keys are filtered before crossing the
engine-core boundary).
Only emits an entry for a tool message whose content is a
non-empty list of blocks; a bare-string tool message renders
fine through the default text-block path in
:func:`vllm.renderers.cohere._content_blocks`, so no forwarded
blocks are needed there. Returns an empty dict when there is
nothing to forward, so the caller can use truthiness to decide
whether to stash the key at all.
"""
blocks_by_index: dict[int, list[dict[str, Any]]] = {}
for i, msg in enumerate(request.messages):
if not isinstance(msg, ToolChatMessageV2):
continue
if isinstance(msg.content, str) or not msg.content:
continue
v2_blocks: list[dict[str, Any]] = []
for block in msg.content:
v2_blocks.append(block.model_dump(exclude_none=True))
if v2_blocks:
blocks_by_index[i] = v2_blocks
return blocks_by_index
@staticmethod
def _build_usage(response: ChatCompletionResponse) -> CohereUsage | None:
if response.usage is None:
@@ -196,6 +196,24 @@ class OpenAIServingChat(GenerateBaseServing):
.chat_template_kwargs
)
def _engine_chat_template_kwargs(
self, chat_template_kwargs: dict[str, Any]
) -> dict[str, Any]:
"""Subclass hook to narrow ``chat_template_kwargs`` for the engine.
The same dict is used twice: to build the API-server-side parser
instance, and to populate
``EngineCoreRequest.reasoning_parser_kwargs`` for the engine-core
side. The latter crosses ZMQ as msgpack, so a handler that stashes
request-scoped state only the API-server-side parser needs (values
msgpack can't encode, or payloads not worth shipping) can drop
those entries here without affecting the in-process parser.
Must not mutate the argument -- the caller still needs the full
dict. The default forwards it unchanged.
"""
return chat_template_kwargs
async def render_chat_request(
self,
request: ChatCompletionRequest,
@@ -360,7 +378,9 @@ class OpenAIServingChat(GenerateBaseServing):
session_id=session_id,
reasoning_ended=reasoning_ended,
reasoning_parser_kwargs={
"chat_template_kwargs": chat_template_kwargs,
"chat_template_kwargs": self._engine_chat_template_kwargs(
chat_template_kwargs
),
}
if parser is not None and parser.reasoning_parser is not None
else None,
@@ -681,27 +681,12 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
return request
# melody's streaming filter only buffers a partial ``<co: ...>`` citation
# across ``write_decoded`` calls when ``stream_non_grounded_answer`` is
# set: otherwise, the moment an opening ``<co`` is seen without a
# closing ``</co: ...>`` in the same delta, the filter emits the partial
# marker bytes verbatim as plain content. In vLLM's streaming path the
# parser is fed one token (1-4 chars) per call, so an unbuffered filter
# will leak ``<co: 0>``-style markers into ``delta.content`` and never
# emit a ``FilterCitation`` for them. Enabling the flag flips the
# partial-match branch in melody's ``parse_citations`` (see
# ``src/parsing/citations_filter.rs``) to ``return (None, 0)`` -- i.e.
# keep buffering -- which lets a full citation eventually resolve.
# Non-streaming (unary) parsing receives the whole output in one call so
# the flag is a no-op there and we leave ``unary_opts`` alone.
class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
super().__init__(
tokenizer,
*args,
streaming_opts=(
PyFilterOptions().cmd3().no_tools().stream_non_grounded_answer()
),
streaming_opts=PyFilterOptions().cmd3().no_tools(),
unary_opts=PyFilterOptions().cmd3().no_tools(),
**kwargs,
)
@@ -712,9 +697,7 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
super().__init__(
tokenizer,
*args,
streaming_opts=(
PyFilterOptions().cmd4().no_tools().stream_non_grounded_answer()
),
streaming_opts=PyFilterOptions().cmd4().no_tools(),
unary_opts=PyFilterOptions().cmd4().no_tools(),
**kwargs,
)
+131 -6
View File
@@ -157,6 +157,40 @@ MESSAGES_CITATIONS_KEY = "_messages_citations"
POSITION_TO_SOURCE_KEY = "_position_to_source"
# Reserved ``chat_template_kwargs`` key carrying the original Cohere v2
# tool-message content, keyed by request-message index. Populated by
# :meth:`CohereServingChatV2._apply_cohere_template_kwargs` and consumed
# by :func:`_conversation_to_melody_messages`, which runs each block
# through :func:`_v2_tool_content_to_melody_block` to produce the
# melody-shape content list the renderer emits.
#
# Motivation: OpenAI's ``ChatCompletionContentPartParam`` union (in
# ``vllm/entrypoints/chat_utils.py``) only allows ``text`` / ``image_url``
# / ``audio`` / ``video`` / ``file`` content parts on tool messages, so
# ``{"type": "document", "document": {...}}`` blocks -- valid in Cohere
# v2 -- get rejected by vLLM's shared chat validator before they can
# reach the renderer.
# ``CohereServingChatV2._convert_tool_message`` works around that by
# rewriting each content block into a plain ``text`` part (with document
# blocks JSON-serialized inline), but that flattening loses the
# structural distinction the cohere renderer needs to shape tool-result
# payloads for cmd3/cmd4. The serving layer therefore forwards the
# original v2 content blocks verbatim (as plain dicts) under this key,
# and the renderer owns the v2-shape → melody-shape mapping.
#
# Value type: ``dict[int, list[dict[str, Any]]]``. Each list entry is a
# ``model_dump``'d ``TextToolContent`` / ``DocumentToolContent`` block
# (e.g. ``{"type": "text", "text": ...}`` or ``{"type": "document",
# "document": {"id": ..., "data": {...}}}``). Index is the position of
# the tool message in the request's ``messages`` list, which -- since
# ``_convert_messages`` is 1-to-1 -- also matches its position in the
# ``ConversationMessage`` list handed to the renderer. Leading
# underscore signals "internal, not part of the public
# ``chat_template_kwargs`` surface" -- clients should not set this key
# directly.
TOOL_MESSAGE_V2_CONTENT_KEY = "_tool_message_v2_content"
# Keys this renderer interprets directly from ``chat_template_kwargs`` and
# maps onto typed melody render-config fields. Everything *not* in this
# set is forwarded verbatim to melody as ``additional_template_fields``
@@ -196,6 +230,7 @@ _RENDERER_CONSUMED_KEYS = frozenset(
"platform_instruction",
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
}
)
@@ -377,6 +412,69 @@ def _content_blocks(content: Any) -> list[dict[str, Any]]:
return blocks
def _v2_tool_content_to_melody_block(block: dict[str, Any]) -> dict[str, Any] | None:
"""Convert one Cohere v2 tool-message content block into a melody
``document`` content block.
Cohere's chat v2 admits two shapes for tool-message content parts:
* ``TextToolContent`` -- ``{"type": "text", "text": "..."}``.
* ``DocumentToolContent`` -- ``{"type": "document", "document":
{"id"?: str, "data": {...}}}``.
Both are emitted as melody ``document`` blocks, mirroring the cohere
api, which routes every tool content part through the same
"tool output as document" conversion. The rules:
* Text that parses to a JSON object becomes that object directly
(so structured tool output flows through as first-class fields).
* Text that doesn't parse as an object -- empty, non-JSON,
or JSON scalar/array -- gets wrapped as ``{"content": <text>}``
so templates have a stable field name to look up.
* ``document`` blocks unwrap ``document.data`` (a flat map) into
the melody payload; ``document.id`` is intentionally dropped
here because outbound citation binding tracks it separately via
``POSITION_TO_SOURCE_KEY``.
Only the JSON-object case actually changes what gets rendered. The
cmd3/cmd4 templates already wrap a bare ``text`` content block as
``{"content": <text>}`` themselves, so routing plain text through
here is a no-op on the prompt -- it is the JSON-object unwrap that
promotes structured tool output from an escaped string blob into
individually addressable (and citable) result fields.
Note the payload is the *flat* field map: melody's ``Content``
carries ``document`` as a plain map, so re-nesting it under a
``data`` key would render a spurious ``{"data": {...}}`` wrapper
into the tool-result block.
Returns ``None`` for unrecognised shapes; the caller should skip
those blocks (the citation-numbering walk in
``CohereServingChatV2._walk_citable_positions`` advances its own
cursor over unknown blocks, so slot alignment is preserved).
"""
block_type = block.get("type")
if block_type == "text":
text = block.get("text") or ""
try:
parsed = json.loads(text)
except (json.JSONDecodeError, TypeError):
parsed = None
if isinstance(parsed, dict):
payload: dict[str, Any] = dict(parsed)
else:
payload = {"content": text}
return {"type": MelodyContentType.DOCUMENT, "document": payload}
if block_type == "document":
doc = block.get("document")
if not isinstance(doc, dict):
return None
data = doc.get("data")
payload = dict(data) if isinstance(data, dict) else {}
return {"type": MelodyContentType.DOCUMENT, "document": payload}
return None
def _document_to_melody(doc: Any) -> dict[str, Any]:
"""Coerce a Cohere v2 document into melody's ``Document`` (dict) shape."""
match doc:
@@ -421,6 +519,7 @@ def _tool_to_melody(tool: Any) -> dict[str, Any]:
def _conversation_to_melody_messages(
conversation: list[ConversationMessage],
messages_citations: dict[int, list[dict[str, Any]]] | None = None,
tool_message_v2_content: dict[int, list[dict[str, Any]]] | None = None,
) -> list[dict[str, Any]]:
"""Convert vLLM ``ConversationMessage``s into melody's message dict shape.
@@ -429,17 +528,42 @@ def _conversation_to_melody_messages(
``CohereServingChatV2`` under the
:data:`MESSAGES_CITATIONS_KEY` chat_template_kwargs entry. Values
must already be in melody's ``FilterCitation`` dict shape (see
``CohereServingChatV2._sdk_citation_to_melody``). Index is the
position in the *input* ``conversation`` list, which currently maps
1-to-1 with ``CohereChatV2Request.messages`` for text-only Cohere
requests. This mapping breaks silently if ``parse_chat_messages``
ever expands a single input message into multiple entries (e.g.
``CohereServingChatV2._sdk_citation_to_melody``).
``tool_message_v2_content`` is an optional index-keyed lookup of
the original Cohere v2 tool-message content blocks (as plain
dicts), populated by ``CohereServingChatV2`` under the
:data:`TOOL_MESSAGE_V2_CONTENT_KEY` entry. When set for a given
index, its blocks -- run through
:func:`_v2_tool_content_to_melody_block` to apply the v2 → melody
mapping -- replace the block list this function would otherwise
derive from ``msg["content"]``. This lets the renderer own the
v2-shape → melody-shape conversion for the structured ``document``
blocks cohere v2 allows on tool messages (which don't survive
OpenAI's content-parts validator on the way in, so
``_convert_tool_message`` rewrites them into ``text`` parts).
Both index lookups key by position in the *input* ``conversation``
list, which currently maps 1-to-1 with
``CohereChatV2Request.messages`` for text-only Cohere requests.
This mapping breaks silently if ``parse_chat_messages`` ever
expands a single input message into multiple entries (e.g.
multi-modal splitting), which is not the case today.
"""
out: list[dict[str, Any]] = []
for i, msg in enumerate(conversation):
role = _role_to_melody(msg.get("role", "user"))
content_blocks = _content_blocks(msg.get("content"))
if (
tool_message_v2_content is not None
and (v2_blocks := tool_message_v2_content.get(i)) is not None
):
content_blocks = []
for v2_block in v2_blocks:
mapped = _v2_tool_content_to_melody_block(v2_block)
if mapped is not None:
content_blocks.append(mapped)
else:
content_blocks = _content_blocks(msg.get("content"))
# Treat reasoning content as a thinking block on assistant turns
# so multi-turn reasoning is preserved in the rendered prompt.
@@ -513,6 +637,7 @@ def _build_render_config(
"messages": _conversation_to_melody_messages(
conversation,
chat_template_kwargs.get(MESSAGES_CITATIONS_KEY),
chat_template_kwargs.get(TOOL_MESSAGE_V2_CONTENT_KEY),
),
}