Fix Cohere ChatV2 citation and tool handling issues (#52175)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from vllm.renderers.cohere import (
|
||||
_normalize_tool_call,
|
||||
_role_to_melody,
|
||||
_tool_to_melody,
|
||||
_v2_tool_content_to_melody_block,
|
||||
)
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
@@ -304,6 +305,108 @@ class TestToolToMelody:
|
||||
_tool_to_melody(42) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _v2_tool_content_to_melody_block
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestV2ToolContentToMelodyBlock:
|
||||
"""Pins the v2 → melody mapping applied to every tool-message
|
||||
content block. The serving layer forwards the original v2 blocks
|
||||
verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer
|
||||
owns this mapping so cmd3 / cmd4 render each result field as its
|
||||
own entry rather than a single stringified-JSON blob.
|
||||
"""
|
||||
|
||||
def test_document_block_unwraps_data_into_melody_document(self):
|
||||
# ``document.data`` becomes the melody ``document`` payload
|
||||
# directly. Keeping the wrapping ``data`` key would produce a
|
||||
# redundant ``{"data": {...}}`` nesting at render time.
|
||||
# ``document.id`` is dropped here on purpose: outbound
|
||||
# citation binding tracks IDs through the separate
|
||||
# ``POSITION_TO_SOURCE_KEY`` map.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert result == {
|
||||
"type": "document",
|
||||
"document": {"result": "The second tallest mountain is K2."},
|
||||
}
|
||||
|
||||
def test_document_block_with_missing_data_yields_empty_payload(self):
|
||||
# A document block whose ``data`` isn't a mapping (e.g.
|
||||
# missing entirely) still produces a document block so
|
||||
# per-slot numbering stays aligned with the outbound citation
|
||||
# walk in ``CohereServingChatV2._walk_citable_positions``.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "document", "document": {"id": "d1"}}
|
||||
)
|
||||
assert result == {"type": "document", "document": {}}
|
||||
|
||||
def test_json_parseable_text_becomes_document_dict(self):
|
||||
# A ``text`` block whose payload parses to a JSON object lands
|
||||
# directly under ``document`` (no ``content`` wrapper) so
|
||||
# cmd3/cmd4 render each field individually.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "text", "text": '{"temperature": "20C"}'}
|
||||
)
|
||||
assert result == {"type": "document", "document": {"temperature": "20C"}}
|
||||
|
||||
def test_non_json_text_wrapped_under_content_key(self):
|
||||
# Plain-text tool output gets wrapped under a ``content`` key
|
||||
# so cmd3/cmd4 templates have a stable field to look up.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "text", "text": "K2 is 8611m."}
|
||||
)
|
||||
assert result == {"type": "document", "document": {"content": "K2 is 8611m."}}
|
||||
|
||||
def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self):
|
||||
# A ``text`` block whose payload parses as a JSON *array* or
|
||||
# *scalar* isn't a valid melody document (which must be a
|
||||
# mapping). Fall back to the ``{"content": <text>}`` wrapper so
|
||||
# the raw text still reaches the prompt.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"})
|
||||
assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}}
|
||||
|
||||
def test_empty_text_still_produces_content_block(self):
|
||||
# Even an empty text block must produce an entry so melody's
|
||||
# per-content-block numbering stays aligned with the
|
||||
# citation-side walk in ``_walk_citable_positions``.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text", "text": ""})
|
||||
assert result == {"type": "document", "document": {"content": ""}}
|
||||
|
||||
def test_missing_text_field_treated_as_empty(self):
|
||||
# ``model_dump(exclude_none=True)`` on a ``TextToolContent``
|
||||
# with an explicitly empty text still emits ``text: ""``, but
|
||||
# guard against a future v2 shape that omits it entirely.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text"})
|
||||
assert result == {"type": "document", "document": {"content": ""}}
|
||||
|
||||
def test_unknown_block_type_returns_none(self):
|
||||
# Callers use ``None`` to know they should skip the slot; a
|
||||
# future v2 addition shouldn't silently render as an empty
|
||||
# document.
|
||||
assert _v2_tool_content_to_melody_block({"type": "audio"}) is None
|
||||
assert _v2_tool_content_to_melody_block({"not": "a block"}) is None
|
||||
|
||||
def test_document_block_with_non_dict_document_returns_none(self):
|
||||
# Malformed ``document`` field -- e.g. someone forwarding a
|
||||
# raw string rather than the ``{id, data}`` wrapper -- is
|
||||
# unusable, so return ``None`` so the caller skips it.
|
||||
assert (
|
||||
_v2_tool_content_to_melody_block(
|
||||
{"type": "document", "document": "not a dict"}
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _conversation_to_melody_messages
|
||||
# ======================================================================
|
||||
@@ -444,6 +547,71 @@ class TestConversationToMelody:
|
||||
out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type]
|
||||
assert "citations" not in out[0]
|
||||
|
||||
def test_tool_message_v2_content_replaces_default_content(self):
|
||||
# When ``tool_message_v2_content`` has an entry for a message
|
||||
# index, the renderer maps those v2 blocks through
|
||||
# :func:`_v2_tool_content_to_melody_block` and uses the result
|
||||
# instead of the block list ``_content_blocks`` would derive
|
||||
# from ``msg["content"]``.
|
||||
conv = [
|
||||
{"role": "user", "content": "q"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
# ``_convert_tool_message`` on the serving side flattens
|
||||
# the original ``document`` block to this text part.
|
||||
# The renderer should ignore this flattened content
|
||||
# because the forwarded map carries the structural
|
||||
# truth.
|
||||
"content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]',
|
||||
},
|
||||
]
|
||||
v2_content = {
|
||||
1: [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "d1", "data": {"result": "K2"}},
|
||||
}
|
||||
]
|
||||
}
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=v2_content,
|
||||
)
|
||||
# User message (no forwarded entry) still goes through the
|
||||
# default ``_content_blocks`` path.
|
||||
assert out[0]["content"] == [{"type": "text", "text": "q"}]
|
||||
# Tool message picks up the mapped melody document block.
|
||||
assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}]
|
||||
assert out[1]["tool_call_id"] == "c1"
|
||||
|
||||
def test_tool_message_v2_content_skips_unrecognized_blocks(self):
|
||||
# ``_v2_tool_content_to_melody_block`` returns ``None`` for
|
||||
# shapes it doesn't recognise; the renderer must skip those
|
||||
# rather than let a ``None`` reach melody.
|
||||
conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}]
|
||||
v2_content = {
|
||||
0: [
|
||||
{"type": "text", "text": "keep me"},
|
||||
{"type": "mystery", "unexpected": True},
|
||||
]
|
||||
}
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=v2_content,
|
||||
)
|
||||
assert out[0]["content"] == [
|
||||
{"type": "document", "document": {"content": "keep me"}}
|
||||
]
|
||||
|
||||
def test_tool_message_v2_content_none_falls_back_to_default(self):
|
||||
conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}]
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=None,
|
||||
)
|
||||
assert out[0]["content"] == [{"type": "text", "text": "plain text"}]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _build_render_config
|
||||
@@ -762,6 +930,40 @@ class TestBuildRenderConfig:
|
||||
) # type: ignore[arg-type]
|
||||
assert "template_jinja" not in cfg
|
||||
|
||||
def test_tool_message_v2_content_key_picked_up_from_kwargs(self):
|
||||
# ``_build_render_config`` must pull
|
||||
# ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of
|
||||
# ``chat_template_kwargs`` and hand it to
|
||||
# ``_conversation_to_melody_messages`` so the renderer's
|
||||
# v2 → melody mapping runs over the original v2 blocks rather
|
||||
# than the flattened string content on the OpenAI-shape tool
|
||||
# message.
|
||||
from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY
|
||||
|
||||
conv = [
|
||||
{"role": "user", "content": "q"},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'},
|
||||
]
|
||||
kwargs = {
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY: {
|
||||
1: [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "d1", "data": {"r": "k"}},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
_, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type]
|
||||
tool_msg = cfg["messages"][1]
|
||||
assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}]
|
||||
# And the key must be *consumed* -- not surfaced as a stray
|
||||
# Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but
|
||||
# pin it here so a removal from that set surfaces immediately).
|
||||
assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get(
|
||||
"additional_template_fields", {}
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# End-to-end async rendering (mirrors ``test_mistral.py``)
|
||||
|
||||
Reference in New Issue
Block a user