Fix Cohere ChatV2 citation and tool handling issues (#52175)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
@@ -11,6 +11,7 @@ a real engine client, model registry, etc.) — the same pattern used in
|
||||
``test_serving_streaming.py``.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -34,11 +35,61 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.renderers.cohere import (
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
)
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# A full grounded tool round-trip: a tool call, a tool result carrying both
|
||||
# a document and a bare text block, and an assistant turn citing it. Between
|
||||
# them these populate every reserved ``_``-prefixed chat_template_kwargs key.
|
||||
_TOOL_ROUND_TRIP_MESSAGES: list[dict[str, Any]] = [
|
||||
{"role": "user", "content": "What is the capital?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": '{"q": "capital"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call-1",
|
||||
"content": [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "doc-9", "data": {"text": "Paris"}},
|
||||
},
|
||||
{"type": "text", "text": "Paris is the capital."},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Paris.",
|
||||
"citations": [
|
||||
{
|
||||
"start": 0,
|
||||
"end": 5,
|
||||
"text": "Paris",
|
||||
"sources": [
|
||||
{"type": "tool", "id": "doc-9", "tool_output": {"text": "Paris"}}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Are you sure?"},
|
||||
]
|
||||
|
||||
|
||||
def _make_request(**kwargs) -> CohereChatV2Request:
|
||||
kwargs.setdefault("model", "m")
|
||||
@@ -447,9 +498,15 @@ class TestConvertToolMessage:
|
||||
"content": "result text",
|
||||
}
|
||||
|
||||
def test_text_only_list_flattened_to_newline_string(self):
|
||||
# Text-only tool results are flattened to a single newline-joined
|
||||
# string for compatibility with standard chat templates.
|
||||
def test_text_only_list_preserves_per_block_text_parts(self):
|
||||
# A list of text blocks maps 1-to-1 onto ``text`` content
|
||||
# parts, mirroring the shape ``_convert_user_message`` emits
|
||||
# for multi-part user content. vLLM's
|
||||
# ``_parse_chat_message_content`` will collapse this back to
|
||||
# ``"\n".join(...)`` for the ``ConversationMessage`` handed to
|
||||
# renderers (see the tool-role branch in
|
||||
# ``vllm/entrypoints/chat_utils.py``), so downstream chat
|
||||
# templates that expect string tool content still work.
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "line 1"},
|
||||
@@ -458,11 +515,27 @@ class TestConvertToolMessage:
|
||||
)
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
assert tool_msg["content"] == "line 1\nline 2"
|
||||
assert tool_msg["content"] == [
|
||||
{"type": "text", "text": "line 1"},
|
||||
{"type": "text", "text": "line 2"},
|
||||
]
|
||||
|
||||
def test_with_document_preserves_structured_content(self):
|
||||
# When documents appear in the tool result, we keep the list shape
|
||||
# so the Cohere renderer can lift them into grounding sources.
|
||||
def test_document_blocks_serialized_as_text_parts(self):
|
||||
"""Document blocks must not survive on the OpenAI-shaped tool
|
||||
message.
|
||||
|
||||
vLLM's shared ``_parse_chat_message_content_parts`` runs a
|
||||
Pydantic ``ValidatorIterator`` over the OpenAI content-part
|
||||
union, which only knows ``text`` / ``image_url`` / etc. and
|
||||
rejects ``{"type": "document", ...}`` blocks outright. We
|
||||
therefore JSON-serialize each document into its own ``text``
|
||||
content part so the request survives OpenAI validation; the
|
||||
cohere renderer still resolves outbound citations against
|
||||
the original cohere v2 messages via ``POSITION_TO_SOURCE_KEY``
|
||||
and reconstructs structured melody ``document`` blocks from the
|
||||
original v2 content forwarded via
|
||||
``TOOL_MESSAGE_V2_CONTENT_KEY``.
|
||||
"""
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "see attachment"},
|
||||
@@ -475,11 +548,75 @@ class TestConvertToolMessage:
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
assert isinstance(tool_msg["content"], list)
|
||||
assert len(tool_msg["content"]) == 2
|
||||
# First block passes through unchanged.
|
||||
assert tool_msg["content"][0] == {"type": "text", "text": "see attachment"}
|
||||
assert tool_msg["content"][1] == {
|
||||
"type": "document",
|
||||
"document": {"data": {"text": "doc text"}, "id": "d1"},
|
||||
}
|
||||
# Second block: document JSON-serialized into a ``text`` part.
|
||||
second = tool_msg["content"][1]
|
||||
assert second["type"] == "text"
|
||||
assert json.loads(second["text"]) == {"data": {"text": "doc text"}, "id": "d1"}
|
||||
|
||||
def test_document_only_content_serialized_as_text_part(self):
|
||||
"""Document-only tool results must also produce a text-typed
|
||||
content part (no ``text`` block above them), so a single-doc
|
||||
result still passes the OpenAI content-parts validator.
|
||||
"""
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"data": {"result": "K2 is 8611m."}},
|
||||
}
|
||||
]
|
||||
)
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
assert isinstance(tool_msg["content"], list)
|
||||
(part,) = tool_msg["content"]
|
||||
assert part["type"] == "text"
|
||||
assert json.loads(part["text"]) == {"data": {"result": "K2 is 8611m."}}
|
||||
|
||||
def test_converted_tool_message_content_survives_openai_validation(self):
|
||||
"""Regression for the reported crash.
|
||||
|
||||
``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts``
|
||||
runs a Pydantic ``ValidatorIterator`` over
|
||||
``list[ChatCompletionContentPartParam]``, which is a
|
||||
discriminated union that only accepts ``text`` / ``image_url``
|
||||
/ ``input_audio`` / ``refusal`` / ``file`` types. Previously
|
||||
the converted tool message carried ``{"type": "document",
|
||||
"document": {...}}`` blocks, which the validator rejected with:
|
||||
|
||||
2 validation errors for ValidatorIterator
|
||||
0.text Field required
|
||||
0.type Input should be 'text', got 'document'
|
||||
|
||||
The fix rewrites every ``DocumentToolContent`` into a
|
||||
JSON-serialized ``text`` part. This test asserts the invariant
|
||||
that keeps the validator happy: on the OpenAI-shaped request,
|
||||
every content entry a tool message carries must be a ``text``
|
||||
part (or a bare string) -- so no future refactor accidentally
|
||||
re-introduces a ``document``-typed content part.
|
||||
"""
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"data": {"result": "The second tallest mountain is K2."}
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
|
||||
content = tool_msg["content"]
|
||||
assert isinstance(content, (str, list))
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
assert isinstance(part, dict)
|
||||
assert part.get("type") == "text"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
@@ -808,6 +945,276 @@ class TestApplyCohereTemplateKwargs:
|
||||
assert result.chat_template_kwargs is None
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Engine-core hop (_engine_chat_template_kwargs)
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestEngineChatTemplateKwargs:
|
||||
"""The internal ``_``-prefixed entries must not reach the engine core.
|
||||
|
||||
``OpenAIServingChat`` uses one ``chat_template_kwargs`` dict twice:
|
||||
to build the API-server-side parser, and to populate
|
||||
``EngineCoreRequest.reasoning_parser_kwargs``, which crosses ZMQ as
|
||||
msgpack. This handler's reserved keys are only meaningful to the
|
||||
former, and ``_position_to_source`` holds ``CitationSource`` models
|
||||
that msgpack cannot encode at all -- forwarding them fails the
|
||||
request with a ``TypeError``. The
|
||||
``_engine_chat_template_kwargs`` override drops them.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _engine_kwargs(request: CohereChatV2Request) -> dict[str, Any]:
|
||||
full = _convert(request).chat_template_kwargs
|
||||
assert full, "expected the request to populate chat_template_kwargs"
|
||||
return _FakeServing()._engine_chat_template_kwargs(full)
|
||||
|
||||
def test_grounded_request_drops_internal_keys_keeps_public_ones(self):
|
||||
request = _make_request(
|
||||
documents=[{"id": "d1", "data": {"text": "hi"}}],
|
||||
citation_options={"mode": "accurate"},
|
||||
)
|
||||
full = _convert(request).chat_template_kwargs
|
||||
# Sanity: the key we expect to be dropped is actually present.
|
||||
assert POSITION_TO_SOURCE_KEY in full
|
||||
|
||||
engine_kwargs = self._engine_kwargs(request)
|
||||
assert POSITION_TO_SOURCE_KEY not in engine_kwargs
|
||||
# Public template inputs the engine-side parser may legitimately
|
||||
# read must survive.
|
||||
assert engine_kwargs["citation_options"] == {"mode": "accurate"}
|
||||
assert engine_kwargs["documents"] == [{"id": "d1", "data": {"text": "hi"}}]
|
||||
|
||||
def test_all_internal_keys_dropped(self):
|
||||
"""Exercises all three reserved keys at once: the tool-result
|
||||
position map, the inbound assistant citations, and the raw v2 tool
|
||||
content blocks.
|
||||
"""
|
||||
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
|
||||
full = _convert(request).chat_template_kwargs
|
||||
internal = {k for k in full if k.startswith("_")}
|
||||
# Sanity: this request really does populate all three.
|
||||
assert internal == {
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
}
|
||||
|
||||
engine_kwargs = self._engine_kwargs(request)
|
||||
assert not [k for k in engine_kwargs if k.startswith("_")]
|
||||
|
||||
def test_engine_kwargs_are_msgpack_encodable(self):
|
||||
"""The real failure mode: the full dict raises, the filtered one
|
||||
doesn't. Pins that the filter is what makes the engine hop work.
|
||||
"""
|
||||
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
|
||||
full = _convert(request).chat_template_kwargs
|
||||
|
||||
with pytest.raises(TypeError, match="is not serializable"):
|
||||
MsgpackEncoder().encode({"chat_template_kwargs": full})
|
||||
|
||||
engine_kwargs = self._engine_kwargs(request)
|
||||
encoder = MsgpackEncoder()
|
||||
decoder = MsgpackDecoder(dict[str, Any])
|
||||
round_tripped = decoder.decode(
|
||||
encoder.encode({"chat_template_kwargs": engine_kwargs})[0]
|
||||
)
|
||||
assert round_tripped["chat_template_kwargs"] == engine_kwargs
|
||||
|
||||
def test_original_dict_not_mutated(self):
|
||||
"""The caller still passes the full dict to the in-process parser,
|
||||
so the filter must return a copy.
|
||||
"""
|
||||
request = _make_request(documents=[{"id": "d1", "data": {"text": "hi"}}])
|
||||
full = _convert(request).chat_template_kwargs
|
||||
before = set(full)
|
||||
|
||||
self._engine_kwargs(request)
|
||||
assert set(full) == before
|
||||
|
||||
def test_base_implementation_is_a_passthrough(self):
|
||||
"""Only the Cohere subclass filters; plain chat completions must
|
||||
keep forwarding whatever the request asked for.
|
||||
"""
|
||||
kwargs = {
|
||||
"documents": [{"id": "d1"}],
|
||||
POSITION_TO_SOURCE_KEY: {(0, 0): object()},
|
||||
}
|
||||
passed_through = OpenAIServingChat._engine_chat_template_kwargs(
|
||||
_FakeServing(), kwargs
|
||||
)
|
||||
assert passed_through == kwargs
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Tool-message content forwarding
|
||||
# (chat_template_kwargs["_tool_message_v2_content"])
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestToolMessageV2ContentForwarding:
|
||||
"""The cohere renderer needs the original v2 tool-message content
|
||||
(text / document blocks) to reconstruct melody's ``document``-block
|
||||
shape, but the OpenAI content-parts validator on the way in only
|
||||
accepts text / image / audio / etc. blocks. So
|
||||
:meth:`CohereServingChatV2._convert_tool_message` rewrites each
|
||||
content block into a plain ``text`` part on the request that
|
||||
survives validation, and
|
||||
:meth:`CohereServingChatV2._collect_v2_tool_message_content`
|
||||
forwards the *original* v2 content blocks -- as ``model_dump``'d
|
||||
dicts -- to the renderer under
|
||||
:data:`TOOL_MESSAGE_V2_CONTENT_KEY`. These tests pin the
|
||||
forwarding contract (presence, dict-shape, index-alignment); the
|
||||
v2 → melody mapping the renderer applies to these blocks lives in
|
||||
``tests/renderers/test_cohere.py``.
|
||||
"""
|
||||
|
||||
def _request_with_tool_message(self, content: Any) -> CohereChatV2Request:
|
||||
return _make_request(
|
||||
messages=[
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": content},
|
||||
]
|
||||
)
|
||||
|
||||
def test_absent_when_no_tool_messages(self):
|
||||
result = _convert(_make_request())
|
||||
# ``chat_template_kwargs`` may be ``None`` here (no cohere-side
|
||||
# forwarding happened at all) or a dict without the key; both
|
||||
# mean the same thing to the renderer.
|
||||
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
|
||||
|
||||
def test_absent_for_string_tool_content(self):
|
||||
# Bare-string tool content works through the renderer's default
|
||||
# ``_content_blocks`` path, so no forwarded blocks are needed.
|
||||
req = self._request_with_tool_message("plain text")
|
||||
result = _convert(req)
|
||||
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
|
||||
|
||||
def test_forwarded_blocks_are_plain_v2_shape_dicts(self):
|
||||
# The forwarded map must contain the original v2 content
|
||||
# blocks verbatim (as plain dicts, not Pydantic instances) so
|
||||
# the renderer can drive the v2 → melody mapping and the map
|
||||
# stays trivially serializable.
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "K2 is 8611m."},
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
blocks_by_index = _forwarded_v2_content(req)
|
||||
# Index 2 = the tool message (user=0, assistant=1, tool=2).
|
||||
assert list(blocks_by_index.keys()) == [2]
|
||||
assert blocks_by_index[2] == [
|
||||
{"type": "text", "text": "K2 is 8611m."},
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
},
|
||||
]
|
||||
# Guard against a regression where Pydantic instances leak into
|
||||
# the forwarded map -- msgpack can't serialize them, and even
|
||||
# for keys filtered out of the engine-core payload we want the
|
||||
# renderer to see plain data.
|
||||
for block in blocks_by_index[2]:
|
||||
assert isinstance(block, dict)
|
||||
|
||||
def test_forwarded_blocks_preserve_input_order(self):
|
||||
# Melody's tool-result rendering keys result slots by their
|
||||
# position within a tool message's content list, and the
|
||||
# outbound citation source map (built by
|
||||
# ``_walk_citable_positions``) uses that same positional index.
|
||||
# Reordering would drop citations pointing at slot ``1`` onto
|
||||
# the wrong payload.
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "leading note"},
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"data": {"body": "doc body"}, "id": "d0"},
|
||||
},
|
||||
{"type": "text", "text": '{"parsed": "yes"}'},
|
||||
]
|
||||
)
|
||||
blocks = _forwarded_v2_content(req)[2]
|
||||
assert [b["type"] for b in blocks] == ["text", "document", "text"]
|
||||
|
||||
def test_index_matches_original_request_position(self):
|
||||
# Entries are keyed by *request-message* index so the renderer,
|
||||
# walking the OpenAI-shaped conversation with the same
|
||||
# enumeration, resolves the right slot. If ``_convert_messages``
|
||||
# ever splits or merges messages this invariant breaks, so pin
|
||||
# the current 1-to-1 behavior with a two-tool-message request.
|
||||
req = _make_request(
|
||||
messages=[
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "c2",
|
||||
"type": "function",
|
||||
"function": {"name": "g", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
"content": [{"type": "text", "text": "first result"}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c2",
|
||||
"content": [{"type": "text", "text": "second result"}],
|
||||
},
|
||||
]
|
||||
)
|
||||
blocks_by_index = _forwarded_v2_content(req)
|
||||
assert sorted(blocks_by_index.keys()) == [2, 3]
|
||||
assert blocks_by_index[2] == [{"type": "text", "text": "first result"}]
|
||||
assert blocks_by_index[3] == [{"type": "text", "text": "second result"}]
|
||||
|
||||
|
||||
def _forwarded_v2_content(
|
||||
request: CohereChatV2Request,
|
||||
) -> dict[int, list[dict[str, Any]]]:
|
||||
"""Extract the forwarded v2 tool-message content from a converted
|
||||
request, or fail the test if the key isn't present.
|
||||
"""
|
||||
result = _convert(request)
|
||||
kwargs = result.chat_template_kwargs or {}
|
||||
assert TOOL_MESSAGE_V2_CONTENT_KEY in kwargs, (
|
||||
f"expected {TOOL_MESSAGE_V2_CONTENT_KEY} in chat_template_kwargs, "
|
||||
f"got keys {list(kwargs)}"
|
||||
)
|
||||
return kwargs[TOOL_MESSAGE_V2_CONTENT_KEY]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Message-citation forwarding (chat_template_kwargs["_messages_citations"])
|
||||
# ======================================================================
|
||||
|
||||
@@ -655,3 +655,123 @@ class TestMelodySourceResolution:
|
||||
assert out is not None
|
||||
assert out[0].type == "THINKING_CONTENT"
|
||||
assert out[0].sources[0].id == "d0"
|
||||
|
||||
|
||||
class TestParserStreamingEndToEnd:
|
||||
"""End-to-end streaming shape: raw model output text is tokenized
|
||||
with the mock byte-level tokenizer and fed one token at a time
|
||||
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
|
||||
We collect the ordered sequence of ``DeltaMessage``s the parser
|
||||
hands back -- i.e. what the client would receive on the wire --
|
||||
and pin the exact shape.
|
||||
|
||||
Combines the three flavors that reach ``delta``: a thinking block
|
||||
(populates ``reasoning``), a text block (populates ``content``),
|
||||
and a resolved citation (populates ``citations``) referencing
|
||||
``(start, end)`` offsets into the already-streamed content.
|
||||
"""
|
||||
|
||||
_MODEL_OUTPUT = (
|
||||
"<|START_THINKING|>Let me check.<|END_THINKING|>"
|
||||
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _drive_parser(
|
||||
parser: CohereCommand3ReasoningParser, text: str, tok: MockCohereTokenizer
|
||||
) -> list[Any]:
|
||||
"""Feed ``text`` through the parser one token at a time
|
||||
(byte-level) and return the ordered list of non-``None``
|
||||
``DeltaMessage``s.
|
||||
"""
|
||||
deltas: list[Any] = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
for token_str in _token_deltas(tok, text):
|
||||
current_text = previous_text + token_str
|
||||
current_token_ids = previous_token_ids + [0]
|
||||
delta = parser.extract_reasoning_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=token_str,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=[0],
|
||||
)
|
||||
if delta is not None:
|
||||
deltas.append(delta)
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
return deltas
|
||||
|
||||
def test_full_wire_stream_shape(self, tokenizer):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
|
||||
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
|
||||
# installs on a request whose tool_call_index=0 tool result at
|
||||
# index 1 is the "France" document.
|
||||
position_to_source = {
|
||||
(0, 1): CitationSource(
|
||||
type="document",
|
||||
id="doc-paris",
|
||||
document={"id": "doc-paris", "title": "France"},
|
||||
),
|
||||
}
|
||||
|
||||
parser = CohereCommand3ReasoningParser(
|
||||
tokenizer,
|
||||
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
|
||||
)
|
||||
|
||||
deltas = self._drive_parser(parser, self._MODEL_OUTPUT, tokenizer)
|
||||
|
||||
# -- 1. Reasoning and content stream separately and reach
|
||||
# the wire only as their real payload. Framing tokens
|
||||
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
|
||||
# state machine and never produce a delta.
|
||||
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
|
||||
content_stream = [d.content for d in deltas if d.content is not None]
|
||||
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
|
||||
|
||||
assert "".join(reasoning_stream) == "Let me check."
|
||||
assert "".join(content_stream) == "The capital is Paris."
|
||||
|
||||
# -- 2. Exactly one citation reaches the wire, resolved to the
|
||||
# source from ``position_to_source`` (not the raw
|
||||
# ``(bucket, idx)`` coordinates).
|
||||
assert len(citation_deltas) == 1
|
||||
(cite_delta,) = citation_deltas
|
||||
assert cite_delta.citations is not None
|
||||
assert len(cite_delta.citations) == 1
|
||||
cite = cite_delta.citations[0]
|
||||
assert cite.text == "Paris"
|
||||
# Char offsets are into the accumulated ``content``: "The
|
||||
# capital is " is 15 chars, "Paris" is 5.
|
||||
assert cite.start == 15
|
||||
assert cite.end == 20
|
||||
assert len(cite.sources) == 1
|
||||
assert cite.sources[0].id == "doc-paris"
|
||||
assert cite.sources[0].type == "document"
|
||||
|
||||
# -- 3. The citation's ``(start, end)`` offsets point into
|
||||
# bytes that were already emitted as content-deltas earlier
|
||||
# in the stream -- i.e. the citation is anchored to real
|
||||
# already-shipped content.
|
||||
cite_delta_idx = deltas.index(cite_delta)
|
||||
earlier_content = "".join(
|
||||
d.content for d in deltas[:cite_delta_idx] if d.content is not None
|
||||
)
|
||||
assert earlier_content[cite.start : cite.end] == "Paris"
|
||||
|
||||
# Everything after the citation delta is the trailing period.
|
||||
later_content = "".join(
|
||||
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
|
||||
)
|
||||
assert later_content == "."
|
||||
|
||||
# -- 4. Reasoning and content are never mixed on the same
|
||||
# delta: the parser flips modes on the ``<|END_THINKING|>``
|
||||
# boundary and never emits a delta with both fields set.
|
||||
for d in deltas:
|
||||
assert not (d.reasoning is not None and d.content is not None)
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.renderers.cohere import (
|
||||
_normalize_tool_call,
|
||||
_role_to_melody,
|
||||
_tool_to_melody,
|
||||
_v2_tool_content_to_melody_block,
|
||||
)
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
@@ -304,6 +305,108 @@ class TestToolToMelody:
|
||||
_tool_to_melody(42) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _v2_tool_content_to_melody_block
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestV2ToolContentToMelodyBlock:
|
||||
"""Pins the v2 → melody mapping applied to every tool-message
|
||||
content block. The serving layer forwards the original v2 blocks
|
||||
verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer
|
||||
owns this mapping so cmd3 / cmd4 render each result field as its
|
||||
own entry rather than a single stringified-JSON blob.
|
||||
"""
|
||||
|
||||
def test_document_block_unwraps_data_into_melody_document(self):
|
||||
# ``document.data`` becomes the melody ``document`` payload
|
||||
# directly. Keeping the wrapping ``data`` key would produce a
|
||||
# redundant ``{"data": {...}}`` nesting at render time.
|
||||
# ``document.id`` is dropped here on purpose: outbound
|
||||
# citation binding tracks IDs through the separate
|
||||
# ``POSITION_TO_SOURCE_KEY`` map.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert result == {
|
||||
"type": "document",
|
||||
"document": {"result": "The second tallest mountain is K2."},
|
||||
}
|
||||
|
||||
def test_document_block_with_missing_data_yields_empty_payload(self):
|
||||
# A document block whose ``data`` isn't a mapping (e.g.
|
||||
# missing entirely) still produces a document block so
|
||||
# per-slot numbering stays aligned with the outbound citation
|
||||
# walk in ``CohereServingChatV2._walk_citable_positions``.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "document", "document": {"id": "d1"}}
|
||||
)
|
||||
assert result == {"type": "document", "document": {}}
|
||||
|
||||
def test_json_parseable_text_becomes_document_dict(self):
|
||||
# A ``text`` block whose payload parses to a JSON object lands
|
||||
# directly under ``document`` (no ``content`` wrapper) so
|
||||
# cmd3/cmd4 render each field individually.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "text", "text": '{"temperature": "20C"}'}
|
||||
)
|
||||
assert result == {"type": "document", "document": {"temperature": "20C"}}
|
||||
|
||||
def test_non_json_text_wrapped_under_content_key(self):
|
||||
# Plain-text tool output gets wrapped under a ``content`` key
|
||||
# so cmd3/cmd4 templates have a stable field to look up.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "text", "text": "K2 is 8611m."}
|
||||
)
|
||||
assert result == {"type": "document", "document": {"content": "K2 is 8611m."}}
|
||||
|
||||
def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self):
|
||||
# A ``text`` block whose payload parses as a JSON *array* or
|
||||
# *scalar* isn't a valid melody document (which must be a
|
||||
# mapping). Fall back to the ``{"content": <text>}`` wrapper so
|
||||
# the raw text still reaches the prompt.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"})
|
||||
assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}}
|
||||
|
||||
def test_empty_text_still_produces_content_block(self):
|
||||
# Even an empty text block must produce an entry so melody's
|
||||
# per-content-block numbering stays aligned with the
|
||||
# citation-side walk in ``_walk_citable_positions``.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text", "text": ""})
|
||||
assert result == {"type": "document", "document": {"content": ""}}
|
||||
|
||||
def test_missing_text_field_treated_as_empty(self):
|
||||
# ``model_dump(exclude_none=True)`` on a ``TextToolContent``
|
||||
# with an explicitly empty text still emits ``text: ""``, but
|
||||
# guard against a future v2 shape that omits it entirely.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text"})
|
||||
assert result == {"type": "document", "document": {"content": ""}}
|
||||
|
||||
def test_unknown_block_type_returns_none(self):
|
||||
# Callers use ``None`` to know they should skip the slot; a
|
||||
# future v2 addition shouldn't silently render as an empty
|
||||
# document.
|
||||
assert _v2_tool_content_to_melody_block({"type": "audio"}) is None
|
||||
assert _v2_tool_content_to_melody_block({"not": "a block"}) is None
|
||||
|
||||
def test_document_block_with_non_dict_document_returns_none(self):
|
||||
# Malformed ``document`` field -- e.g. someone forwarding a
|
||||
# raw string rather than the ``{id, data}`` wrapper -- is
|
||||
# unusable, so return ``None`` so the caller skips it.
|
||||
assert (
|
||||
_v2_tool_content_to_melody_block(
|
||||
{"type": "document", "document": "not a dict"}
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _conversation_to_melody_messages
|
||||
# ======================================================================
|
||||
@@ -444,6 +547,71 @@ class TestConversationToMelody:
|
||||
out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type]
|
||||
assert "citations" not in out[0]
|
||||
|
||||
def test_tool_message_v2_content_replaces_default_content(self):
|
||||
# When ``tool_message_v2_content`` has an entry for a message
|
||||
# index, the renderer maps those v2 blocks through
|
||||
# :func:`_v2_tool_content_to_melody_block` and uses the result
|
||||
# instead of the block list ``_content_blocks`` would derive
|
||||
# from ``msg["content"]``.
|
||||
conv = [
|
||||
{"role": "user", "content": "q"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
# ``_convert_tool_message`` on the serving side flattens
|
||||
# the original ``document`` block to this text part.
|
||||
# The renderer should ignore this flattened content
|
||||
# because the forwarded map carries the structural
|
||||
# truth.
|
||||
"content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]',
|
||||
},
|
||||
]
|
||||
v2_content = {
|
||||
1: [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "d1", "data": {"result": "K2"}},
|
||||
}
|
||||
]
|
||||
}
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=v2_content,
|
||||
)
|
||||
# User message (no forwarded entry) still goes through the
|
||||
# default ``_content_blocks`` path.
|
||||
assert out[0]["content"] == [{"type": "text", "text": "q"}]
|
||||
# Tool message picks up the mapped melody document block.
|
||||
assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}]
|
||||
assert out[1]["tool_call_id"] == "c1"
|
||||
|
||||
def test_tool_message_v2_content_skips_unrecognized_blocks(self):
|
||||
# ``_v2_tool_content_to_melody_block`` returns ``None`` for
|
||||
# shapes it doesn't recognise; the renderer must skip those
|
||||
# rather than let a ``None`` reach melody.
|
||||
conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}]
|
||||
v2_content = {
|
||||
0: [
|
||||
{"type": "text", "text": "keep me"},
|
||||
{"type": "mystery", "unexpected": True},
|
||||
]
|
||||
}
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=v2_content,
|
||||
)
|
||||
assert out[0]["content"] == [
|
||||
{"type": "document", "document": {"content": "keep me"}}
|
||||
]
|
||||
|
||||
def test_tool_message_v2_content_none_falls_back_to_default(self):
|
||||
conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}]
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=None,
|
||||
)
|
||||
assert out[0]["content"] == [{"type": "text", "text": "plain text"}]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _build_render_config
|
||||
@@ -762,6 +930,40 @@ class TestBuildRenderConfig:
|
||||
) # type: ignore[arg-type]
|
||||
assert "template_jinja" not in cfg
|
||||
|
||||
def test_tool_message_v2_content_key_picked_up_from_kwargs(self):
|
||||
# ``_build_render_config`` must pull
|
||||
# ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of
|
||||
# ``chat_template_kwargs`` and hand it to
|
||||
# ``_conversation_to_melody_messages`` so the renderer's
|
||||
# v2 → melody mapping runs over the original v2 blocks rather
|
||||
# than the flattened string content on the OpenAI-shape tool
|
||||
# message.
|
||||
from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY
|
||||
|
||||
conv = [
|
||||
{"role": "user", "content": "q"},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'},
|
||||
]
|
||||
kwargs = {
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY: {
|
||||
1: [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "d1", "data": {"r": "k"}},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
_, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type]
|
||||
tool_msg = cfg["messages"][1]
|
||||
assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}]
|
||||
# And the key must be *consumed* -- not surfaced as a stray
|
||||
# Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but
|
||||
# pin it here so a removal from that set surfaces immediately).
|
||||
assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get(
|
||||
"additional_template_fields", {}
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# End-to-end async rendering (mirrors ``test_mistral.py``)
|
||||
|
||||
Reference in New Issue
Block a user