Fix Cohere ChatV2 citation and tool handling issues (#52175)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
andrewbcohere
2026-08-21 15:23:17 -07:00
committed by GitHub
parent f37d5868b9
commit f94dcde5c5
7 changed files with 1024 additions and 68 deletions
@@ -11,6 +11,7 @@ a real engine client, model registry, etc.) — the same pattern used in
``test_serving_streaming.py``.
"""
import json
from typing import Any
import pytest
@@ -34,11 +35,61 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
from vllm.renderers.cohere import (
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
)
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------
# A full grounded tool round-trip: a tool call, a tool result carrying both
# a document and a bare text block, and an assistant turn citing it. Between
# them these populate every reserved ``_``-prefixed chat_template_kwargs key.
_TOOL_ROUND_TRIP_MESSAGES: list[dict[str, Any]] = [
{"role": "user", "content": "What is the capital?"},
{
"role": "assistant",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "search", "arguments": '{"q": "capital"}'},
}
],
},
{
"role": "tool",
"tool_call_id": "call-1",
"content": [
{
"type": "document",
"document": {"id": "doc-9", "data": {"text": "Paris"}},
},
{"type": "text", "text": "Paris is the capital."},
],
},
{
"role": "assistant",
"content": "Paris.",
"citations": [
{
"start": 0,
"end": 5,
"text": "Paris",
"sources": [
{"type": "tool", "id": "doc-9", "tool_output": {"text": "Paris"}}
],
}
],
},
{"role": "user", "content": "Are you sure?"},
]
def _make_request(**kwargs) -> CohereChatV2Request:
kwargs.setdefault("model", "m")
@@ -447,9 +498,15 @@ class TestConvertToolMessage:
"content": "result text",
}
def test_text_only_list_flattened_to_newline_string(self):
# Text-only tool results are flattened to a single newline-joined
# string for compatibility with standard chat templates.
def test_text_only_list_preserves_per_block_text_parts(self):
# A list of text blocks maps 1-to-1 onto ``text`` content
# parts, mirroring the shape ``_convert_user_message`` emits
# for multi-part user content. vLLM's
# ``_parse_chat_message_content`` will collapse this back to
# ``"\n".join(...)`` for the ``ConversationMessage`` handed to
# renderers (see the tool-role branch in
# ``vllm/entrypoints/chat_utils.py``), so downstream chat
# templates that expect string tool content still work.
req = self._request_with_tool_message(
[
{"type": "text", "text": "line 1"},
@@ -458,11 +515,27 @@ class TestConvertToolMessage:
)
result = _convert(req)
tool_msg = result.messages[-1]
assert tool_msg["content"] == "line 1\nline 2"
assert tool_msg["content"] == [
{"type": "text", "text": "line 1"},
{"type": "text", "text": "line 2"},
]
def test_with_document_preserves_structured_content(self):
# When documents appear in the tool result, we keep the list shape
# so the Cohere renderer can lift them into grounding sources.
def test_document_blocks_serialized_as_text_parts(self):
"""Document blocks must not survive on the OpenAI-shaped tool
message.
vLLM's shared ``_parse_chat_message_content_parts`` runs a
Pydantic ``ValidatorIterator`` over the OpenAI content-part
union, which only knows ``text`` / ``image_url`` / etc. and
rejects ``{"type": "document", ...}`` blocks outright. We
therefore JSON-serialize each document into its own ``text``
content part so the request survives OpenAI validation; the
cohere renderer still resolves outbound citations against
the original cohere v2 messages via ``POSITION_TO_SOURCE_KEY``
and reconstructs structured melody ``document`` blocks from the
original v2 content forwarded via
``TOOL_MESSAGE_V2_CONTENT_KEY``.
"""
req = self._request_with_tool_message(
[
{"type": "text", "text": "see attachment"},
@@ -475,11 +548,75 @@ class TestConvertToolMessage:
result = _convert(req)
tool_msg = result.messages[-1]
assert isinstance(tool_msg["content"], list)
assert len(tool_msg["content"]) == 2
# First block passes through unchanged.
assert tool_msg["content"][0] == {"type": "text", "text": "see attachment"}
assert tool_msg["content"][1] == {
"type": "document",
"document": {"data": {"text": "doc text"}, "id": "d1"},
}
# Second block: document JSON-serialized into a ``text`` part.
second = tool_msg["content"][1]
assert second["type"] == "text"
assert json.loads(second["text"]) == {"data": {"text": "doc text"}, "id": "d1"}
def test_document_only_content_serialized_as_text_part(self):
"""Document-only tool results must also produce a text-typed
content part (no ``text`` block above them), so a single-doc
result still passes the OpenAI content-parts validator.
"""
req = self._request_with_tool_message(
[
{
"type": "document",
"document": {"data": {"result": "K2 is 8611m."}},
}
]
)
result = _convert(req)
tool_msg = result.messages[-1]
assert isinstance(tool_msg["content"], list)
(part,) = tool_msg["content"]
assert part["type"] == "text"
assert json.loads(part["text"]) == {"data": {"result": "K2 is 8611m."}}
def test_converted_tool_message_content_survives_openai_validation(self):
"""Regression for the reported crash.
``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts``
runs a Pydantic ``ValidatorIterator`` over
``list[ChatCompletionContentPartParam]``, which is a
discriminated union that only accepts ``text`` / ``image_url``
/ ``input_audio`` / ``refusal`` / ``file`` types. Previously
the converted tool message carried ``{"type": "document",
"document": {...}}`` blocks, which the validator rejected with:
2 validation errors for ValidatorIterator
0.text Field required
0.type Input should be 'text', got 'document'
The fix rewrites every ``DocumentToolContent`` into a
JSON-serialized ``text`` part. This test asserts the invariant
that keeps the validator happy: on the OpenAI-shaped request,
every content entry a tool message carries must be a ``text``
part (or a bare string) -- so no future refactor accidentally
re-introduces a ``document``-typed content part.
"""
req = self._request_with_tool_message(
[
{
"type": "document",
"document": {
"data": {"result": "The second tallest mountain is K2."}
},
}
]
)
result = _convert(req)
tool_msg = result.messages[-1]
content = tool_msg["content"]
assert isinstance(content, (str, list))
if isinstance(content, list):
for part in content:
assert isinstance(part, dict)
assert part.get("type") == "text"
# ======================================================================
@@ -808,6 +945,276 @@ class TestApplyCohereTemplateKwargs:
assert result.chat_template_kwargs is None
# ======================================================================
# Engine-core hop (_engine_chat_template_kwargs)
# ======================================================================
class TestEngineChatTemplateKwargs:
"""The internal ``_``-prefixed entries must not reach the engine core.
``OpenAIServingChat`` uses one ``chat_template_kwargs`` dict twice:
to build the API-server-side parser, and to populate
``EngineCoreRequest.reasoning_parser_kwargs``, which crosses ZMQ as
msgpack. This handler's reserved keys are only meaningful to the
former, and ``_position_to_source`` holds ``CitationSource`` models
that msgpack cannot encode at all -- forwarding them fails the
request with a ``TypeError``. The
``_engine_chat_template_kwargs`` override drops them.
"""
@staticmethod
def _engine_kwargs(request: CohereChatV2Request) -> dict[str, Any]:
full = _convert(request).chat_template_kwargs
assert full, "expected the request to populate chat_template_kwargs"
return _FakeServing()._engine_chat_template_kwargs(full)
def test_grounded_request_drops_internal_keys_keeps_public_ones(self):
request = _make_request(
documents=[{"id": "d1", "data": {"text": "hi"}}],
citation_options={"mode": "accurate"},
)
full = _convert(request).chat_template_kwargs
# Sanity: the key we expect to be dropped is actually present.
assert POSITION_TO_SOURCE_KEY in full
engine_kwargs = self._engine_kwargs(request)
assert POSITION_TO_SOURCE_KEY not in engine_kwargs
# Public template inputs the engine-side parser may legitimately
# read must survive.
assert engine_kwargs["citation_options"] == {"mode": "accurate"}
assert engine_kwargs["documents"] == [{"id": "d1", "data": {"text": "hi"}}]
def test_all_internal_keys_dropped(self):
"""Exercises all three reserved keys at once: the tool-result
position map, the inbound assistant citations, and the raw v2 tool
content blocks.
"""
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
full = _convert(request).chat_template_kwargs
internal = {k for k in full if k.startswith("_")}
# Sanity: this request really does populate all three.
assert internal == {
MESSAGES_CITATIONS_KEY,
POSITION_TO_SOURCE_KEY,
TOOL_MESSAGE_V2_CONTENT_KEY,
}
engine_kwargs = self._engine_kwargs(request)
assert not [k for k in engine_kwargs if k.startswith("_")]
def test_engine_kwargs_are_msgpack_encodable(self):
"""The real failure mode: the full dict raises, the filtered one
doesn't. Pins that the filter is what makes the engine hop work.
"""
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
full = _convert(request).chat_template_kwargs
with pytest.raises(TypeError, match="is not serializable"):
MsgpackEncoder().encode({"chat_template_kwargs": full})
engine_kwargs = self._engine_kwargs(request)
encoder = MsgpackEncoder()
decoder = MsgpackDecoder(dict[str, Any])
round_tripped = decoder.decode(
encoder.encode({"chat_template_kwargs": engine_kwargs})[0]
)
assert round_tripped["chat_template_kwargs"] == engine_kwargs
def test_original_dict_not_mutated(self):
"""The caller still passes the full dict to the in-process parser,
so the filter must return a copy.
"""
request = _make_request(documents=[{"id": "d1", "data": {"text": "hi"}}])
full = _convert(request).chat_template_kwargs
before = set(full)
self._engine_kwargs(request)
assert set(full) == before
def test_base_implementation_is_a_passthrough(self):
"""Only the Cohere subclass filters; plain chat completions must
keep forwarding whatever the request asked for.
"""
kwargs = {
"documents": [{"id": "d1"}],
POSITION_TO_SOURCE_KEY: {(0, 0): object()},
}
passed_through = OpenAIServingChat._engine_chat_template_kwargs(
_FakeServing(), kwargs
)
assert passed_through == kwargs
# ======================================================================
# Tool-message content forwarding
# (chat_template_kwargs["_tool_message_v2_content"])
# ======================================================================
class TestToolMessageV2ContentForwarding:
"""The cohere renderer needs the original v2 tool-message content
(text / document blocks) to reconstruct melody's ``document``-block
shape, but the OpenAI content-parts validator on the way in only
accepts text / image / audio / etc. blocks. So
:meth:`CohereServingChatV2._convert_tool_message` rewrites each
content block into a plain ``text`` part on the request that
survives validation, and
:meth:`CohereServingChatV2._collect_v2_tool_message_content`
forwards the *original* v2 content blocks -- as ``model_dump``'d
dicts -- to the renderer under
:data:`TOOL_MESSAGE_V2_CONTENT_KEY`. These tests pin the
forwarding contract (presence, dict-shape, index-alignment); the
v2 → melody mapping the renderer applies to these blocks lives in
``tests/renderers/test_cohere.py``.
"""
def _request_with_tool_message(self, content: Any) -> CohereChatV2Request:
return _make_request(
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "c1", "content": content},
]
)
def test_absent_when_no_tool_messages(self):
result = _convert(_make_request())
# ``chat_template_kwargs`` may be ``None`` here (no cohere-side
# forwarding happened at all) or a dict without the key; both
# mean the same thing to the renderer.
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
def test_absent_for_string_tool_content(self):
# Bare-string tool content works through the renderer's default
# ``_content_blocks`` path, so no forwarded blocks are needed.
req = self._request_with_tool_message("plain text")
result = _convert(req)
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
def test_forwarded_blocks_are_plain_v2_shape_dicts(self):
# The forwarded map must contain the original v2 content
# blocks verbatim (as plain dicts, not Pydantic instances) so
# the renderer can drive the v2 → melody mapping and the map
# stays trivially serializable.
req = self._request_with_tool_message(
[
{"type": "text", "text": "K2 is 8611m."},
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
},
]
)
blocks_by_index = _forwarded_v2_content(req)
# Index 2 = the tool message (user=0, assistant=1, tool=2).
assert list(blocks_by_index.keys()) == [2]
assert blocks_by_index[2] == [
{"type": "text", "text": "K2 is 8611m."},
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
},
]
# Guard against a regression where Pydantic instances leak into
# the forwarded map -- msgpack can't serialize them, and even
# for keys filtered out of the engine-core payload we want the
# renderer to see plain data.
for block in blocks_by_index[2]:
assert isinstance(block, dict)
def test_forwarded_blocks_preserve_input_order(self):
# Melody's tool-result rendering keys result slots by their
# position within a tool message's content list, and the
# outbound citation source map (built by
# ``_walk_citable_positions``) uses that same positional index.
# Reordering would drop citations pointing at slot ``1`` onto
# the wrong payload.
req = self._request_with_tool_message(
[
{"type": "text", "text": "leading note"},
{
"type": "document",
"document": {"data": {"body": "doc body"}, "id": "d0"},
},
{"type": "text", "text": '{"parsed": "yes"}'},
]
)
blocks = _forwarded_v2_content(req)[2]
assert [b["type"] for b in blocks] == ["text", "document", "text"]
def test_index_matches_original_request_position(self):
# Entries are keyed by *request-message* index so the renderer,
# walking the OpenAI-shaped conversation with the same
# enumeration, resolves the right slot. If ``_convert_messages``
# ever splits or merges messages this invariant breaks, so pin
# the current 1-to-1 behavior with a two-tool-message request.
req = _make_request(
messages=[
{"role": "user", "content": "hi"},
{
"role": "assistant",
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "f", "arguments": "{}"},
},
{
"id": "c2",
"type": "function",
"function": {"name": "g", "arguments": "{}"},
},
],
},
{
"role": "tool",
"tool_call_id": "c1",
"content": [{"type": "text", "text": "first result"}],
},
{
"role": "tool",
"tool_call_id": "c2",
"content": [{"type": "text", "text": "second result"}],
},
]
)
blocks_by_index = _forwarded_v2_content(req)
assert sorted(blocks_by_index.keys()) == [2, 3]
assert blocks_by_index[2] == [{"type": "text", "text": "first result"}]
assert blocks_by_index[3] == [{"type": "text", "text": "second result"}]
def _forwarded_v2_content(
request: CohereChatV2Request,
) -> dict[int, list[dict[str, Any]]]:
"""Extract the forwarded v2 tool-message content from a converted
request, or fail the test if the key isn't present.
"""
result = _convert(request)
kwargs = result.chat_template_kwargs or {}
assert TOOL_MESSAGE_V2_CONTENT_KEY in kwargs, (
f"expected {TOOL_MESSAGE_V2_CONTENT_KEY} in chat_template_kwargs, "
f"got keys {list(kwargs)}"
)
return kwargs[TOOL_MESSAGE_V2_CONTENT_KEY]
# ======================================================================
# Message-citation forwarding (chat_template_kwargs["_messages_citations"])
# ======================================================================