Fix Cohere ChatV2 citation and tool handling issues (#52175)

Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
andrewbcohere
2026-08-21 15:23:17 -07:00
committed by GitHub
parent f37d5868b9
commit f94dcde5c5
7 changed files with 1024 additions and 68 deletions
+202
View File
@@ -30,6 +30,7 @@ from vllm.renderers.cohere import (
_normalize_tool_call,
_role_to_melody,
_tool_to_melody,
_v2_tool_content_to_melody_block,
)
from vllm.tokenizers.hf import HfTokenizer
@@ -304,6 +305,108 @@ class TestToolToMelody:
_tool_to_melody(42) # type: ignore[arg-type]
# ======================================================================
# _v2_tool_content_to_melody_block
# ======================================================================
class TestV2ToolContentToMelodyBlock:
"""Pins the v2 → melody mapping applied to every tool-message
content block. The serving layer forwards the original v2 blocks
verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer
owns this mapping so cmd3 / cmd4 render each result field as its
own entry rather than a single stringified-JSON blob.
"""
def test_document_block_unwraps_data_into_melody_document(self):
# ``document.data`` becomes the melody ``document`` payload
# directly. Keeping the wrapping ``data`` key would produce a
# redundant ``{"data": {...}}`` nesting at render time.
# ``document.id`` is dropped here on purpose: outbound
# citation binding tracks IDs through the separate
# ``POSITION_TO_SOURCE_KEY`` map.
result = _v2_tool_content_to_melody_block(
{
"type": "document",
"document": {
"id": "d1",
"data": {"result": "The second tallest mountain is K2."},
},
}
)
assert result == {
"type": "document",
"document": {"result": "The second tallest mountain is K2."},
}
def test_document_block_with_missing_data_yields_empty_payload(self):
# A document block whose ``data`` isn't a mapping (e.g.
# missing entirely) still produces a document block so
# per-slot numbering stays aligned with the outbound citation
# walk in ``CohereServingChatV2._walk_citable_positions``.
result = _v2_tool_content_to_melody_block(
{"type": "document", "document": {"id": "d1"}}
)
assert result == {"type": "document", "document": {}}
def test_json_parseable_text_becomes_document_dict(self):
# A ``text`` block whose payload parses to a JSON object lands
# directly under ``document`` (no ``content`` wrapper) so
# cmd3/cmd4 render each field individually.
result = _v2_tool_content_to_melody_block(
{"type": "text", "text": '{"temperature": "20C"}'}
)
assert result == {"type": "document", "document": {"temperature": "20C"}}
def test_non_json_text_wrapped_under_content_key(self):
# Plain-text tool output gets wrapped under a ``content`` key
# so cmd3/cmd4 templates have a stable field to look up.
result = _v2_tool_content_to_melody_block(
{"type": "text", "text": "K2 is 8611m."}
)
assert result == {"type": "document", "document": {"content": "K2 is 8611m."}}
def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self):
# A ``text`` block whose payload parses as a JSON *array* or
# *scalar* isn't a valid melody document (which must be a
# mapping). Fall back to the ``{"content": <text>}`` wrapper so
# the raw text still reaches the prompt.
result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"})
assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}}
def test_empty_text_still_produces_content_block(self):
# Even an empty text block must produce an entry so melody's
# per-content-block numbering stays aligned with the
# citation-side walk in ``_walk_citable_positions``.
result = _v2_tool_content_to_melody_block({"type": "text", "text": ""})
assert result == {"type": "document", "document": {"content": ""}}
def test_missing_text_field_treated_as_empty(self):
# ``model_dump(exclude_none=True)`` on a ``TextToolContent``
# with an explicitly empty text still emits ``text: ""``, but
# guard against a future v2 shape that omits it entirely.
result = _v2_tool_content_to_melody_block({"type": "text"})
assert result == {"type": "document", "document": {"content": ""}}
def test_unknown_block_type_returns_none(self):
# Callers use ``None`` to know they should skip the slot; a
# future v2 addition shouldn't silently render as an empty
# document.
assert _v2_tool_content_to_melody_block({"type": "audio"}) is None
assert _v2_tool_content_to_melody_block({"not": "a block"}) is None
def test_document_block_with_non_dict_document_returns_none(self):
# Malformed ``document`` field -- e.g. someone forwarding a
# raw string rather than the ``{id, data}`` wrapper -- is
# unusable, so return ``None`` so the caller skips it.
assert (
_v2_tool_content_to_melody_block(
{"type": "document", "document": "not a dict"}
)
is None
)
# ======================================================================
# _conversation_to_melody_messages
# ======================================================================
@@ -444,6 +547,71 @@ class TestConversationToMelody:
out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type]
assert "citations" not in out[0]
def test_tool_message_v2_content_replaces_default_content(self):
# When ``tool_message_v2_content`` has an entry for a message
# index, the renderer maps those v2 blocks through
# :func:`_v2_tool_content_to_melody_block` and uses the result
# instead of the block list ``_content_blocks`` would derive
# from ``msg["content"]``.
conv = [
{"role": "user", "content": "q"},
{
"role": "tool",
"tool_call_id": "c1",
# ``_convert_tool_message`` on the serving side flattens
# the original ``document`` block to this text part.
# The renderer should ignore this flattened content
# because the forwarded map carries the structural
# truth.
"content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]',
},
]
v2_content = {
1: [
{
"type": "document",
"document": {"id": "d1", "data": {"result": "K2"}},
}
]
}
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=v2_content,
)
# User message (no forwarded entry) still goes through the
# default ``_content_blocks`` path.
assert out[0]["content"] == [{"type": "text", "text": "q"}]
# Tool message picks up the mapped melody document block.
assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}]
assert out[1]["tool_call_id"] == "c1"
def test_tool_message_v2_content_skips_unrecognized_blocks(self):
# ``_v2_tool_content_to_melody_block`` returns ``None`` for
# shapes it doesn't recognise; the renderer must skip those
# rather than let a ``None`` reach melody.
conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}]
v2_content = {
0: [
{"type": "text", "text": "keep me"},
{"type": "mystery", "unexpected": True},
]
}
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=v2_content,
)
assert out[0]["content"] == [
{"type": "document", "document": {"content": "keep me"}}
]
def test_tool_message_v2_content_none_falls_back_to_default(self):
conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}]
out = _conversation_to_melody_messages(
conv, # type: ignore[arg-type]
tool_message_v2_content=None,
)
assert out[0]["content"] == [{"type": "text", "text": "plain text"}]
# ======================================================================
# _build_render_config
@@ -762,6 +930,40 @@ class TestBuildRenderConfig:
) # type: ignore[arg-type]
assert "template_jinja" not in cfg
def test_tool_message_v2_content_key_picked_up_from_kwargs(self):
# ``_build_render_config`` must pull
# ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of
# ``chat_template_kwargs`` and hand it to
# ``_conversation_to_melody_messages`` so the renderer's
# v2 → melody mapping runs over the original v2 blocks rather
# than the flattened string content on the OpenAI-shape tool
# message.
from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY
conv = [
{"role": "user", "content": "q"},
{"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'},
]
kwargs = {
TOOL_MESSAGE_V2_CONTENT_KEY: {
1: [
{
"type": "document",
"document": {"id": "d1", "data": {"r": "k"}},
}
]
}
}
_, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type]
tool_msg = cfg["messages"][1]
assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}]
# And the key must be *consumed* -- not surfaced as a stray
# Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but
# pin it here so a removal from that set surfaces immediately).
assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get(
"additional_template_fields", {}
)
# ======================================================================
# End-to-end async rendering (mirrors ``test_mistral.py``)