Fix Cohere ChatV2 citation and tool handling issues (#52175)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
andrewbcohere
2026-08-21 15:23:17 -07:00
committed by GitHub
parent f37d5868b9
commit f94dcde5c5
7 changed files with 1024 additions and 68 deletions
@@ -11,6 +11,7 @@ a real engine client, model registry, etc.) — the same pattern used in
``test_serving_streaming.py``.
"""
import json
from typing import Any
import pytest
@@ -34,11 +35,61 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.renderers.cohere import (
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
)
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
# A full grounded tool round-trip: a tool call, a tool result carrying both
# a document and a bare text block, and an assistant turn citing it. Between
# them these populate every reserved ``_``-prefixed chat_template_kwargs key.
_TOOL_ROUND_TRIP_MESSAGES: list[dict[str, Any]] = [
{"role": "user", "content": "What is the capital?"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "search", "arguments": '{"q": "capital"}'},
}
],
},
{
"role": "tool",
"tool_call_id": "call-1",
"content": [
{
"type": "document",
"document": {"id": "doc-9", "data": {"text": "Paris"}},
},
{"type": "text", "text": "Paris is the capital."},
],
},
{
"role": "assistant",
"content": "Paris.",
"citations": [
{
"start": 0,
"end": 5,
"text": "Paris",
"sources": [
{"type": "tool", "id": "doc-9", "tool_output": {"text": "Paris"}}
],
}
],
},
{"role": "user", "content": "Are you sure?"},
]
def _make_request(**kwargs) -> CohereChatV2Request:
kwargs.setdefault("model", "m")
@@ -447,9 +498,15 @@ class TestConvertToolMessage:
"content": "result text",
}
def test_text_only_list_flattened_to_newline_string(self):
# Text-only tool results are flattened to a single newline-joined
# string for compatibility with standard chat templates.
def test_text_only_list_preserves_per_block_text_parts(self):
# A list of text blocks maps 1-to-1 onto ``text`` content
# parts, mirroring the shape ``_convert_user_message`` emits
# for multi-part user content. vLLM's
# ``_parse_chat_message_content`` will collapse this back to
# ``"\n".join(...)`` for the ``ConversationMessage`` handed to
# renderers (see the tool-role branch in
# ``vllm/entrypoints/chat_utils.py``), so downstream chat
# templates that expect string tool content still work.
req = self._request_with_tool_message(
[
{"type": "text", "text": "line 1"},
@@ -458,11 +515,27 @@ class TestConvertToolMessage:
)
result = _convert(req)
tool_msg = result.messages[-1]
assert tool_msg["content"] == "line 1\nline 2"
assert tool_msg["content"] == [
{"type": "text", "text": "line 1"},
{"type": "text", "text": "line 2"},
]
def test_with_document_preserves_structured_content(self):
# When documents appear in the tool result, we keep the list shape
# so the Cohere renderer can lift them into grounding sources.
def test_document_blocks_serialized_as_text_parts(self):
"""Document blocks must not survive on the OpenAI-shaped tool
message.
vLLM's shared ``_parse_chat_message_content_parts`` runs a
Pydantic ``ValidatorIterator`` over the OpenAI content-part
union, which only knows ``text`` / ``image_url`` / etc. and
rejects ``{"type": "document", ...}`` blocks outright. We
therefore JSON-serialize each document into its own ``text``
content part so the request survives OpenAI validation; the
cohere renderer still resolves outbound citations against
the original cohere v2 messages via ``POSITION_TO_SOURCE_KEY``
and reconstructs structured melody ``document`` blocks from the
original v2 content forwarded via
``TOOL_MESSAGE_V2_CONTENT_KEY``.
"""
req = self._request_with_tool_message(
[
{"type": "text", "text": "see attachment"},
@@ -475,11 +548,75 @@ class TestConvertToolMessage:
result = _convert(req)
tool_msg = result.messages[-1]
assert isinstance(tool_msg["content"], list)
assert len(tool_msg["content"]) == 2
# First block passes through unchanged.
assert tool_msg["content"][0] == {"type": "text", "text": "see attachment"}
assert tool_msg["content"][1] == {
"type": "document",
"document": {"data": {"text": "doc text"}, "id": "d1"},
}
# Second block: document JSON-serialized into a ``text`` part.
second = tool_msg["content"][1]
assert second["type"] == "text"
assert json.loads(second["text"]) == {"data": {"text": "doc text"}, "id": "d1"}
def test_document_only_content_serialized_as_text_part(self):
"""Document-only tool results must also produce a text-typed
content part (no ``text`` block above them), so a single-doc
result still passes the OpenAI content-parts validator.
"""
req = self._request_with_tool_message(
[
{
"type": "document",
"document": {"data": {"result": "K2 is 8611m."}},
}
]
)
result = _convert(req)
tool_msg = result.messages[-1]
assert isinstance(tool_msg["content"], list)
(part,) = tool_msg["content"]
assert part["type"] == "text"
assert json.loads(part["text"]) == {"data": {"result": "K2 is 8611m."}}
def test_converted_tool_message_content_survives_openai_validation(self):
"""Regression for the reported crash.
``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts``
runs a Pydantic ``ValidatorIterator`` over
``list[ChatCompletionContentPartParam]``, which is a
discriminated union that only accepts ``text`` / ``image_url``
/ ``input_audio`` / ``refusal`` / ``file`` types. Previously
the converted tool message carried ``{"type": "document",
"document": {...}}`` blocks, which the validator rejected with:
2 validation errors for ValidatorIterator
0.text Field required
0.type Input should be 'text', got 'document'
The fix rewrites every ``DocumentToolContent`` into a
JSON-serialized ``text`` part. This test asserts the invariant
that keeps the validator happy: on the OpenAI-shaped request,
every content entry a tool message carries must be a ``text``
part (or a bare string) -- so no future refactor accidentally
re-introduces a ``document``-typed content part.
"""
req = self._request_with_tool_message(
[
{
"type": "document",
"document": {
"data": {"result": "The second tallest mountain is K2."}
},
}
]
)
result = _convert(req)
tool_msg = result.messages[-1]
content = tool_msg["content"]
assert isinstance(content, (str, list))
if isinstance(content, list):
for part in content:
assert isinstance(part, dict)
assert part.get("type") == "text"
# ======================================================================
@@ -808,6 +945,276 @@ class TestApplyCohereTemplateKwargs:
assert result.chat_template_kwargs is None
# ======================================================================
# Engine-core hop (_engine_chat_template_kwargs)
# ======================================================================
class TestEngineChatTemplateKwargs:
"""The internal ``_``-prefixed entries must not reach the engine core.
``OpenAIServingChat`` uses one ``chat_template_kwargs`` dict twice:
to build the API-server-side parser, and to populate
``EngineCoreRequest.reasoning_parser_kwargs``, which crosses ZMQ as
msgpack. This handler's reserved keys are only meaningful to the
former, and ``_position_to_source`` holds ``CitationSource`` models
that msgpack cannot encode at all -- forwarding them fails the
request with a ``TypeError``. The
``_engine_chat_template_kwargs`` override drops them.
"""
@staticmethod
def _engine_kwargs(request: CohereChatV2Request) -> dict[str, Any]:
full = _convert(request).chat_template_kwargs
assert full, "expected the request to populate chat_template_kwargs"
return _FakeServing()._engine_chat_template_kwargs(full)
def test_grounded_request_drops_internal_keys_keeps_public_ones(self):
request = _make_request(
documents=[{"id": "d1", "data": {"text": "hi"}}],
citation_options={"mode": "accurate"},
)
full = _convert(request).chat_template_kwargs
# Sanity: the key we expect to be dropped is actually present.
assert POSITION_TO_SOURCE_KEY in full
engine_kwargs = self._engine_kwargs(request)
assert POSITION_TO_SOURCE_KEY not in engine_kwargs
# Public template inputs the engine-side parser may legitimately
# read must survive.
assert engine_kwargs["citation_options"] == {"mode": "accurate"}
assert engine_kwargs["documents"] == [{"id": "d1", "data": {"text": "hi"}}]
def test_all_internal_keys_dropped(self):
"""Exercises all three reserved keys at once: the tool-result
position map, the inbound assistant citations, and the raw v2 tool
content blocks.
"""
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
full = _convert(request).chat_template_kwargs
internal = {k for k in full if k.startswith("_")}
# Sanity: this request really does populate all three.
assert internal == {
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
}
engine_kwargs = self._engine_kwargs(request)
assert not [k for k in engine_kwargs if k.startswith("_")]
def test_engine_kwargs_are_msgpack_encodable(self):
"""The real failure mode: the full dict raises, the filtered one
doesn't. Pins that the filter is what makes the engine hop work.
"""
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
full = _convert(request).chat_template_kwargs
with pytest.raises(TypeError, match="is not serializable"):
MsgpackEncoder().encode({"chat_template_kwargs": full})
engine_kwargs = self._engine_kwargs(request)
encoder = MsgpackEncoder()
decoder = MsgpackDecoder(dict[str, Any])
round_tripped = decoder.decode(
encoder.encode({"chat_template_kwargs": engine_kwargs})[0]
)
assert round_tripped["chat_template_kwargs"] == engine_kwargs
def test_original_dict_not_mutated(self):
"""The caller still passes the full dict to the in-process parser,
so the filter must return a copy.
"""
request = _make_request(documents=[{"id": "d1", "data": {"text": "hi"}}])
full = _convert(request).chat_template_kwargs
before = set(full)
self._engine_kwargs(request)
assert set(full) == before
def test_base_implementation_is_a_passthrough(self):
"""Only the Cohere subclass filters; plain chat completions must
keep forwarding whatever the request asked for.
"""
kwargs = {
"documents": [{"id": "d1"}],
POSITION_TO_SOURCE_KEY: {(0, 0): object()},
}
passed_through = OpenAIServingChat._engine_chat_template_kwargs(
_FakeServing(), kwargs
)
assert passed_through == kwargs
# ======================================================================
# Tool-message content forwarding
# (chat_template_kwargs["_tool_message_v2_content"])
# ======================================================================
class TestToolMessageV2ContentForwarding:
"""The cohere renderer needs the original v2 tool-message content
(text / document blocks) to reconstruct melody's ``document``-block
shape, but the OpenAI content-parts validator on the way in only
accepts text / image / audio / etc. blocks. So
:meth:`CohereServingChatV2._convert_tool_message` rewrites each
content block into a plain ``text`` part on the request that
survives validation, and
:meth:`CohereServingChatV2._collect_v2_tool_message_content`
forwards the *original* v2 content blocks -- as ``model_dump``'d
dicts -- to the renderer under
:data:`TOOL_MESSAGE_V2_CONTENT_KEY`. These tests pin the
forwarding contract (presence, dict-shape, index-alignment); the
v2 → melody mapping the renderer applies to these blocks lives in
``tests/renderers/test_cohere.py``.
"""
def _request_with_tool_message(self, content: Any) -> CohereChatV2Request:
return _make_request(
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "c1", "content": content},
]
)
def test_absent_when_no_tool_messages(self):
result = _convert(_make_request())
# ``chat_template_kwargs`` may be ``None`` here (no cohere-side
# forwarding happened at all) or a dict without the key; both
# mean the same thing to the renderer.
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
def test_absent_for_string_tool_content(self):
# Bare-string tool content works through the renderer's default
# ``_content_blocks`` path, so no forwarded blocks are needed.
req = self._request_with_tool_message("plain text")
result = _convert(req)
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
def test_forwarded_blocks_are_plain_v2_shape_dicts(self):
# The forwarded map must contain the original v2 content
# blocks verbatim (as plain dicts, not Pydantic instances) so
# the renderer can drive the v2 → melody mapping and the map
# stays trivially serializable.
req = self._request_with_tool_message(
[
{"type": "text", "text": "K2 is 8611m."},
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
},
]
)
blocks_by_index = _forwarded_v2_content(req)
# Index 2 = the tool message (user=0, assistant=1, tool=2).
assert list(blocks_by_index.keys()) == [2]
assert blocks_by_index[2] == [
{"type": "text", "text": "K2 is 8611m."},
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
},
]
# Guard against a regression where Pydantic instances leak into
# the forwarded map -- msgpack can't serialize them, and even
# for keys filtered out of the engine-core payload we want the
# renderer to see plain data.
for block in blocks_by_index[2]:
assert isinstance(block, dict)
def test_forwarded_blocks_preserve_input_order(self):
# Melody's tool-result rendering keys result slots by their
# position within a tool message's content list, and the
# outbound citation source map (built by
# ``_walk_citable_positions``) uses that same positional index.
# Reordering would drop citations pointing at slot ``1`` onto
# the wrong payload.
req = self._request_with_tool_message(
[
{"type": "text", "text": "leading note"},
{
"type": "document",
"document": {"data": {"body": "doc body"}, "id": "d0"},
},
{"type": "text", "text": '{"parsed": "yes"}'},
]
)
blocks = _forwarded_v2_content(req)[2]
assert [b["type"] for b in blocks] == ["text", "document", "text"]
def test_index_matches_original_request_position(self):
# Entries are keyed by *request-message* index so the renderer,
# walking the OpenAI-shaped conversation with the same
# enumeration, resolves the right slot. If ``_convert_messages``
# ever splits or merges messages this invariant breaks, so pin
# the current 1-to-1 behavior with a two-tool-message request.
req = _make_request(
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "g", "arguments": "{}"},
},
],
},
{
"role": "tool",
"tool_call_id": "c1",
"content": [{"type": "text", "text": "first result"}],
},
{
"role": "tool",
"tool_call_id": "c2",
"content": [{"type": "text", "text": "second result"}],
},
]
)
blocks_by_index = _forwarded_v2_content(req)
assert sorted(blocks_by_index.keys()) == [2, 3]
assert blocks_by_index[2] == [{"type": "text", "text": "first result"}]
assert blocks_by_index[3] == [{"type": "text", "text": "second result"}]
def _forwarded_v2_content(
request: CohereChatV2Request,
) -> dict[int, list[dict[str, Any]]]:
"""Extract the forwarded v2 tool-message content from a converted
request, or fail the test if the key isn't present.
"""
result = _convert(request)
kwargs = result.chat_template_kwargs or {}
assert TOOL_MESSAGE_V2_CONTENT_KEY in kwargs, (
f"expected {TOOL_MESSAGE_V2_CONTENT_KEY} in chat_template_kwargs, "
f"got keys {list(kwargs)}"
)
return kwargs[TOOL_MESSAGE_V2_CONTENT_KEY]
# ======================================================================
# Message-citation forwarding (chat_template_kwargs["_messages_citations"])
# ======================================================================
@@ -655,3 +655,123 @@ class TestMelodySourceResolution:
assert out is not None
assert out[0].type == "THINKING_CONTENT"
assert out[0].sources[0].id == "d0"
class TestParserStreamingEndToEnd:
"""End-to-end streaming shape: raw model output text is tokenized
with the mock byte-level tokenizer and fed one token at a time
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
We collect the ordered sequence of ``DeltaMessage``s the parser
hands back -- i.e. what the client would receive on the wire --
and pin the exact shape.
Combines the three flavors that reach ``delta``: a thinking block
(populates ``reasoning``), a text block (populates ``content``),
and a resolved citation (populates ``citations``) referencing
``(start, end)`` offsets into the already-streamed content.
"""
_MODEL_OUTPUT = (
"<|START_THINKING|>Let me check.<|END_THINKING|>"
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
)
@staticmethod
def _drive_parser(
parser: CohereCommand3ReasoningParser, text: str, tok: MockCohereTokenizer
) -> list[Any]:
"""Feed ``text`` through the parser one token at a time
(byte-level) and return the ordered list of non-``None``
``DeltaMessage``s.
"""
deltas: list[Any] = []
previous_text = ""
previous_token_ids: list[int] = []
for token_str in _token_deltas(tok, text):
current_text = previous_text + token_str
current_token_ids = previous_token_ids + [0]
delta = parser.extract_reasoning_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=token_str,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=[0],
)
if delta is not None:
deltas.append(delta)
previous_text = current_text
previous_token_ids = current_token_ids
return deltas
def test_full_wire_stream_shape(self, tokenizer):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
# installs on a request whose tool_call_index=0 tool result at
# index 1 is the "France" document.
position_to_source = {
(0, 1): CitationSource(
type="document",
id="doc-paris",
document={"id": "doc-paris", "title": "France"},
),
}
parser = CohereCommand3ReasoningParser(
tokenizer,
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
)
deltas = self._drive_parser(parser, self._MODEL_OUTPUT, tokenizer)
# -- 1. Reasoning and content stream separately and reach
# the wire only as their real payload. Framing tokens
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
# state machine and never produce a delta.
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
content_stream = [d.content for d in deltas if d.content is not None]
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
assert "".join(reasoning_stream) == "Let me check."
assert "".join(content_stream) == "The capital is Paris."
# -- 2. Exactly one citation reaches the wire, resolved to the
# source from ``position_to_source`` (not the raw
# ``(bucket, idx)`` coordinates).
assert len(citation_deltas) == 1
(cite_delta,) = citation_deltas
assert cite_delta.citations is not None
assert len(cite_delta.citations) == 1
cite = cite_delta.citations[0]
assert cite.text == "Paris"
# Char offsets are into the accumulated ``content``: "The
# capital is " is 15 chars, "Paris" is 5.
assert cite.start == 15
assert cite.end == 20
assert len(cite.sources) == 1
assert cite.sources[0].id == "doc-paris"
assert cite.sources[0].type == "document"
# -- 3. The citation's ``(start, end)`` offsets point into
# bytes that were already emitted as content-deltas earlier
# in the stream -- i.e. the citation is anchored to real
# already-shipped content.
cite_delta_idx = deltas.index(cite_delta)
earlier_content = "".join(
d.content for d in deltas[:cite_delta_idx] if d.content is not None
)
assert earlier_content[cite.start : cite.end] == "Paris"
# Everything after the citation delta is the trailing period.
later_content = "".join(
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
)
assert later_content == "."
# -- 4. Reasoning and content are never mixed on the same
# delta: the parser flips modes on the ``<|END_THINKING|>``
# boundary and never emits a delta with both fields set.
for d in deltas:
assert not (d.reasoning is not None and d.content is not None)
+202
View File
@@ -30,6 +30,7 @@ from vllm.renderers.cohere import (
_normalize_tool_call,
_role_to_melody,
_tool_to_melody,
_v2_tool_content_to_melody_block,
)
from vllm.tokenizers.hf import HfTokenizer
@@ -304,6 +305,108 @@ class TestToolToMelody:
_tool_to_melody(42) # type: ignore[arg-type]
# ======================================================================
# _v2_tool_content_to_melody_block
# ======================================================================
class TestV2ToolContentToMelodyBlock:
"""Pins the v2 → melody mapping applied to every tool-message
content block. The serving layer forwards the original v2 blocks
verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer
owns this mapping so cmd3 / cmd4 render each result field as its
own entry rather than a single stringified-JSON blob.
"""
def test_document_block_unwraps_data_into_melody_document(self):
# ``document.data`` becomes the melody ``document`` payload
# directly. Keeping the wrapping ``data`` key would produce a
# redundant ``{"data": {...}}`` nesting at render time.
# ``document.id`` is dropped here on purpose: outbound
# citation binding tracks IDs through the separate
# ``POSITION_TO_SOURCE_KEY`` map.
result = _v2_tool_content_to_melody_block(
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
}
)
assert result == {
"type": "document",
"document": {"result": "The second tallest mountain is K2."},
}
def test_document_block_with_missing_data_yields_empty_payload(self):
# A document block whose ``data`` isn't a mapping (e.g.
# missing entirely) still produces a document block so
# per-slot numbering stays aligned with the outbound citation
# walk in ``CohereServingChatV2._walk_citable_positions``.
result = _v2_tool_content_to_melody_block(
{"type": "document", "document": {"id": "d1"}}
)
assert result == {"type": "document", "document": {}}
def test_json_parseable_text_becomes_document_dict(self):
# A ``text`` block whose payload parses to a JSON object lands
# directly under ``document`` (no ``content`` wrapper) so
# cmd3/cmd4 render each field individually.
result = _v2_tool_content_to_melody_block(
{"type": "text", "text": '{"temperature": "20C"}'}
)
assert result == {"type": "document", "document": {"temperature": "20C"}}
def test_non_json_text_wrapped_under_content_key(self):
# Plain-text tool output gets wrapped under a ``content`` key
# so cmd3/cmd4 templates have a stable field to look up.
result = _v2_tool_content_to_melody_block(
{"type": "text", "text": "K2 is 8611m."}
)
assert result == {"type": "document", "document": {"content": "K2 is 8611m."}}
def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self):
# A ``text`` block whose payload parses as a JSON *array* or
# *scalar* isn't a valid melody document (which must be a
# mapping). Fall back to the ``{"content": <text>}`` wrapper so
# the raw text still reaches the prompt.
result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"})
assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}}
def test_empty_text_still_produces_content_block(self):
# Even an empty text block must produce an entry so melody's
# per-content-block numbering stays aligned with the
# citation-side walk in ``_walk_citable_positions``.
result = _v2_tool_content_to_melody_block({"type": "text", "text": ""})
assert result == {"type": "document", "document": {"content": ""}}
def test_missing_text_field_treated_as_empty(self):
# ``model_dump(exclude_none=True)`` on a ``TextToolContent``
# with an explicitly empty text still emits ``text: ""``, but
# guard against a future v2 shape that omits it entirely.
result = _v2_tool_content_to_melody_block({"type": "text"})
assert result == {"type": "document", "document": {"content": ""}}
def test_unknown_block_type_returns_none(self):
# Callers use ``None`` to know they should skip the slot; a
# future v2 addition shouldn't silently render as an empty
# document.
assert _v2_tool_content_to_melody_block({"type": "audio"}) is None
assert _v2_tool_content_to_melody_block({"not": "a block"}) is None
def test_document_block_with_non_dict_document_returns_none(self):
# Malformed ``document`` field -- e.g. someone forwarding a
# raw string rather than the ``{id, data}`` wrapper -- is
# unusable, so return ``None`` so the caller skips it.
assert (
_v2_tool_content_to_melody_block(
{"type": "document", "document": "not a dict"}
)
is None
)
# ======================================================================
# _conversation_to_melody_messages
# ======================================================================
@@ -444,6 +547,71 @@ class TestConversationToMelody:
out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type]
assert "citations" not in out[0]
def test_tool_message_v2_content_replaces_default_content(self):
# When ``tool_message_v2_content`` has an entry for a message
# index, the renderer maps those v2 blocks through
# :func:`_v2_tool_content_to_melody_block` and uses the result
# instead of the block list ``_content_blocks`` would derive
# from ``msg["content"]``.
conv = [
{"role": "user", "content": "q"},
{
"role": "tool",
"tool_call_id": "c1",
# ``_convert_tool_message`` on the serving side flattens
# the original ``document`` block to this text part.
# The renderer should ignore this flattened content
# because the forwarded map carries the structural
# truth.
"content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]',
},
]
v2_content = {
1: [
{
"type": "document",
"document": {"id": "d1", "data": {"result": "K2"}},
}
]
}
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=v2_content,
)
# User message (no forwarded entry) still goes through the
# default ``_content_blocks`` path.
assert out[0]["content"] == [{"type": "text", "text": "q"}]
# Tool message picks up the mapped melody document block.
assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}]
assert out[1]["tool_call_id"] == "c1"
def test_tool_message_v2_content_skips_unrecognized_blocks(self):
# ``_v2_tool_content_to_melody_block`` returns ``None`` for
# shapes it doesn't recognise; the renderer must skip those
# rather than let a ``None`` reach melody.
conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}]
v2_content = {
0: [
{"type": "text", "text": "keep me"},
{"type": "mystery", "unexpected": True},
]
}
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=v2_content,
)
assert out[0]["content"] == [
{"type": "document", "document": {"content": "keep me"}}
]
def test_tool_message_v2_content_none_falls_back_to_default(self):
conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}]
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=None,
)
assert out[0]["content"] == [{"type": "text", "text": "plain text"}]
# ======================================================================
# _build_render_config
@@ -762,6 +930,40 @@ class TestBuildRenderConfig:
) # type: ignore[arg-type]
assert "template_jinja" not in cfg
def test_tool_message_v2_content_key_picked_up_from_kwargs(self):
# ``_build_render_config`` must pull
# ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of
# ``chat_template_kwargs`` and hand it to
# ``_conversation_to_melody_messages`` so the renderer's
# v2 → melody mapping runs over the original v2 blocks rather
# than the flattened string content on the OpenAI-shape tool
# message.
from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY
conv = [
{"role": "user", "content": "q"},
{"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'},
]
kwargs = {
TOOL_MESSAGE_V2_CONTENT_KEY: {
1: [
{
"type": "document",
"document": {"id": "d1", "data": {"r": "k"}},
}
]
}
}
_, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type]
tool_msg = cfg["messages"][1]
assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}]
# And the key must be *consumed* -- not surfaced as a stray
# Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but
# pin it here so a removal from that set surfaces immediately).
assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get(
"additional_template_fields", {}
)
# ======================================================================
# End-to-end async rendering (mirrors ``test_mistral.py``)