diff --git a/tests/entrypoints/cohere/test_serving_conversion.py b/tests/entrypoints/cohere/test_serving_conversion.py index a456d113ac..5cbe13df02 100644 --- a/tests/entrypoints/cohere/test_serving_conversion.py +++ b/tests/entrypoints/cohere/test_serving_conversion.py @@ -11,6 +11,7 @@ a real engine client, model registry, etc.) — the same pattern used in ``test_serving_streaming.py``. """ +import json from typing import Any import pytest @@ -34,11 +35,61 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, ) +from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat +from vllm.renderers.cohere import ( + MESSAGES_CITATIONS_KEY, + POSITION_TO_SOURCE_KEY, + TOOL_MESSAGE_V2_CONTENT_KEY, +) +from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder # ---------------------------------------------------------------------- # Helpers # ---------------------------------------------------------------------- +# A full grounded tool round-trip: a tool call, a tool result carrying both +# a document and a bare text block, and an assistant turn citing it. Between +# them these populate every reserved ``_``-prefixed chat_template_kwargs key. +_TOOL_ROUND_TRIP_MESSAGES: list[dict[str, Any]] = [ + {"role": "user", "content": "What is the capital?"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "search", "arguments": '{"q": "capital"}'}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call-1", + "content": [ + { + "type": "document", + "document": {"id": "doc-9", "data": {"text": "Paris"}}, + }, + {"type": "text", "text": "Paris is the capital."}, + ], + }, + { + "role": "assistant", + "content": "Paris.", + "citations": [ + { + "start": 0, + "end": 5, + "text": "Paris", + "sources": [ + {"type": "tool", "id": "doc-9", "tool_output": {"text": "Paris"}} + ], + } + ], + }, + {"role": "user", "content": "Are you sure?"}, +] + def _make_request(**kwargs) -> CohereChatV2Request: kwargs.setdefault("model", "m") @@ -447,9 +498,15 @@ class TestConvertToolMessage: "content": "result text", } - def test_text_only_list_flattened_to_newline_string(self): - # Text-only tool results are flattened to a single newline-joined - # string for compatibility with standard chat templates. + def test_text_only_list_preserves_per_block_text_parts(self): + # A list of text blocks maps 1-to-1 onto ``text`` content + # parts, mirroring the shape ``_convert_user_message`` emits + # for multi-part user content. vLLM's + # ``_parse_chat_message_content`` will collapse this back to + # ``"\n".join(...)`` for the ``ConversationMessage`` handed to + # renderers (see the tool-role branch in + # ``vllm/entrypoints/chat_utils.py``), so downstream chat + # templates that expect string tool content still work. req = self._request_with_tool_message( [ {"type": "text", "text": "line 1"}, @@ -458,11 +515,27 @@ class TestConvertToolMessage: ) result = _convert(req) tool_msg = result.messages[-1] - assert tool_msg["content"] == "line 1\nline 2" + assert tool_msg["content"] == [ + {"type": "text", "text": "line 1"}, + {"type": "text", "text": "line 2"}, + ] - def test_with_document_preserves_structured_content(self): - # When documents appear in the tool result, we keep the list shape - # so the Cohere renderer can lift them into grounding sources. + def test_document_blocks_serialized_as_text_parts(self): + """Document blocks must not survive on the OpenAI-shaped tool + message. + + vLLM's shared ``_parse_chat_message_content_parts`` runs a + Pydantic ``ValidatorIterator`` over the OpenAI content-part + union, which only knows ``text`` / ``image_url`` / etc. and + rejects ``{"type": "document", ...}`` blocks outright. We + therefore JSON-serialize each document into its own ``text`` + content part so the request survives OpenAI validation; the + cohere renderer still resolves outbound citations against + the original cohere v2 messages via ``POSITION_TO_SOURCE_KEY`` + and reconstructs structured melody ``document`` blocks from the + original v2 content forwarded via + ``TOOL_MESSAGE_V2_CONTENT_KEY``. + """ req = self._request_with_tool_message( [ {"type": "text", "text": "see attachment"}, @@ -475,11 +548,75 @@ class TestConvertToolMessage: result = _convert(req) tool_msg = result.messages[-1] assert isinstance(tool_msg["content"], list) + assert len(tool_msg["content"]) == 2 + # First block passes through unchanged. assert tool_msg["content"][0] == {"type": "text", "text": "see attachment"} - assert tool_msg["content"][1] == { - "type": "document", - "document": {"data": {"text": "doc text"}, "id": "d1"}, - } + # Second block: document JSON-serialized into a ``text`` part. + second = tool_msg["content"][1] + assert second["type"] == "text" + assert json.loads(second["text"]) == {"data": {"text": "doc text"}, "id": "d1"} + + def test_document_only_content_serialized_as_text_part(self): + """Document-only tool results must also produce a text-typed + content part (no ``text`` block above them), so a single-doc + result still passes the OpenAI content-parts validator. + """ + req = self._request_with_tool_message( + [ + { + "type": "document", + "document": {"data": {"result": "K2 is 8611m."}}, + } + ] + ) + result = _convert(req) + tool_msg = result.messages[-1] + assert isinstance(tool_msg["content"], list) + (part,) = tool_msg["content"] + assert part["type"] == "text" + assert json.loads(part["text"]) == {"data": {"result": "K2 is 8611m."}} + + def test_converted_tool_message_content_survives_openai_validation(self): + """Regression for the reported crash. + + ``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts`` + runs a Pydantic ``ValidatorIterator`` over + ``list[ChatCompletionContentPartParam]``, which is a + discriminated union that only accepts ``text`` / ``image_url`` + / ``input_audio`` / ``refusal`` / ``file`` types. Previously + the converted tool message carried ``{"type": "document", + "document": {...}}`` blocks, which the validator rejected with: + + 2 validation errors for ValidatorIterator + 0.text Field required + 0.type Input should be 'text', got 'document' + + The fix rewrites every ``DocumentToolContent`` into a + JSON-serialized ``text`` part. This test asserts the invariant + that keeps the validator happy: on the OpenAI-shaped request, + every content entry a tool message carries must be a ``text`` + part (or a bare string) -- so no future refactor accidentally + re-introduces a ``document``-typed content part. + """ + req = self._request_with_tool_message( + [ + { + "type": "document", + "document": { + "data": {"result": "The second tallest mountain is K2."} + }, + } + ] + ) + result = _convert(req) + tool_msg = result.messages[-1] + + content = tool_msg["content"] + assert isinstance(content, (str, list)) + if isinstance(content, list): + for part in content: + assert isinstance(part, dict) + assert part.get("type") == "text" # ====================================================================== @@ -808,6 +945,276 @@ class TestApplyCohereTemplateKwargs: assert result.chat_template_kwargs is None +# ====================================================================== +# Engine-core hop (_engine_chat_template_kwargs) +# ====================================================================== + + +class TestEngineChatTemplateKwargs: + """The internal ``_``-prefixed entries must not reach the engine core. + + ``OpenAIServingChat`` uses one ``chat_template_kwargs`` dict twice: + to build the API-server-side parser, and to populate + ``EngineCoreRequest.reasoning_parser_kwargs``, which crosses ZMQ as + msgpack. This handler's reserved keys are only meaningful to the + former, and ``_position_to_source`` holds ``CitationSource`` models + that msgpack cannot encode at all -- forwarding them fails the + request with a ``TypeError``. The + ``_engine_chat_template_kwargs`` override drops them. + """ + + @staticmethod + def _engine_kwargs(request: CohereChatV2Request) -> dict[str, Any]: + full = _convert(request).chat_template_kwargs + assert full, "expected the request to populate chat_template_kwargs" + return _FakeServing()._engine_chat_template_kwargs(full) + + def test_grounded_request_drops_internal_keys_keeps_public_ones(self): + request = _make_request( + documents=[{"id": "d1", "data": {"text": "hi"}}], + citation_options={"mode": "accurate"}, + ) + full = _convert(request).chat_template_kwargs + # Sanity: the key we expect to be dropped is actually present. + assert POSITION_TO_SOURCE_KEY in full + + engine_kwargs = self._engine_kwargs(request) + assert POSITION_TO_SOURCE_KEY not in engine_kwargs + # Public template inputs the engine-side parser may legitimately + # read must survive. + assert engine_kwargs["citation_options"] == {"mode": "accurate"} + assert engine_kwargs["documents"] == [{"id": "d1", "data": {"text": "hi"}}] + + def test_all_internal_keys_dropped(self): + """Exercises all three reserved keys at once: the tool-result + position map, the inbound assistant citations, and the raw v2 tool + content blocks. + """ + request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES) + full = _convert(request).chat_template_kwargs + internal = {k for k in full if k.startswith("_")} + # Sanity: this request really does populate all three. + assert internal == { + MESSAGES_CITATIONS_KEY, + POSITION_TO_SOURCE_KEY, + TOOL_MESSAGE_V2_CONTENT_KEY, + } + + engine_kwargs = self._engine_kwargs(request) + assert not [k for k in engine_kwargs if k.startswith("_")] + + def test_engine_kwargs_are_msgpack_encodable(self): + """The real failure mode: the full dict raises, the filtered one + doesn't. Pins that the filter is what makes the engine hop work. + """ + request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES) + full = _convert(request).chat_template_kwargs + + with pytest.raises(TypeError, match="is not serializable"): + MsgpackEncoder().encode({"chat_template_kwargs": full}) + + engine_kwargs = self._engine_kwargs(request) + encoder = MsgpackEncoder() + decoder = MsgpackDecoder(dict[str, Any]) + round_tripped = decoder.decode( + encoder.encode({"chat_template_kwargs": engine_kwargs})[0] + ) + assert round_tripped["chat_template_kwargs"] == engine_kwargs + + def test_original_dict_not_mutated(self): + """The caller still passes the full dict to the in-process parser, + so the filter must return a copy. + """ + request = _make_request(documents=[{"id": "d1", "data": {"text": "hi"}}]) + full = _convert(request).chat_template_kwargs + before = set(full) + + self._engine_kwargs(request) + assert set(full) == before + + def test_base_implementation_is_a_passthrough(self): + """Only the Cohere subclass filters; plain chat completions must + keep forwarding whatever the request asked for. + """ + kwargs = { + "documents": [{"id": "d1"}], + POSITION_TO_SOURCE_KEY: {(0, 0): object()}, + } + passed_through = OpenAIServingChat._engine_chat_template_kwargs( + _FakeServing(), kwargs + ) + assert passed_through == kwargs + + +# ====================================================================== +# Tool-message content forwarding +# (chat_template_kwargs["_tool_message_v2_content"]) +# ====================================================================== + + +class TestToolMessageV2ContentForwarding: + """The cohere renderer needs the original v2 tool-message content + (text / document blocks) to reconstruct melody's ``document``-block + shape, but the OpenAI content-parts validator on the way in only + accepts text / image / audio / etc. blocks. So + :meth:`CohereServingChatV2._convert_tool_message` rewrites each + content block into a plain ``text`` part on the request that + survives validation, and + :meth:`CohereServingChatV2._collect_v2_tool_message_content` + forwards the *original* v2 content blocks -- as ``model_dump``'d + dicts -- to the renderer under + :data:`TOOL_MESSAGE_V2_CONTENT_KEY`. These tests pin the + forwarding contract (presence, dict-shape, index-alignment); the + v2 → melody mapping the renderer applies to these blocks lives in + ``tests/renderers/test_cohere.py``. + """ + + def _request_with_tool_message(self, content: Any) -> CohereChatV2Request: + return _make_request( + messages=[ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + } + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": content}, + ] + ) + + def test_absent_when_no_tool_messages(self): + result = _convert(_make_request()) + # ``chat_template_kwargs`` may be ``None`` here (no cohere-side + # forwarding happened at all) or a dict without the key; both + # mean the same thing to the renderer. + assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY) + + def test_absent_for_string_tool_content(self): + # Bare-string tool content works through the renderer's default + # ``_content_blocks`` path, so no forwarded blocks are needed. + req = self._request_with_tool_message("plain text") + result = _convert(req) + assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY) + + def test_forwarded_blocks_are_plain_v2_shape_dicts(self): + # The forwarded map must contain the original v2 content + # blocks verbatim (as plain dicts, not Pydantic instances) so + # the renderer can drive the v2 → melody mapping and the map + # stays trivially serializable. + req = self._request_with_tool_message( + [ + {"type": "text", "text": "K2 is 8611m."}, + { + "type": "document", + "document": { + "id": "d1", + "data": {"result": "The second tallest mountain is K2."}, + }, + }, + ] + ) + blocks_by_index = _forwarded_v2_content(req) + # Index 2 = the tool message (user=0, assistant=1, tool=2). + assert list(blocks_by_index.keys()) == [2] + assert blocks_by_index[2] == [ + {"type": "text", "text": "K2 is 8611m."}, + { + "type": "document", + "document": { + "id": "d1", + "data": {"result": "The second tallest mountain is K2."}, + }, + }, + ] + # Guard against a regression where Pydantic instances leak into + # the forwarded map -- msgpack can't serialize them, and even + # for keys filtered out of the engine-core payload we want the + # renderer to see plain data. + for block in blocks_by_index[2]: + assert isinstance(block, dict) + + def test_forwarded_blocks_preserve_input_order(self): + # Melody's tool-result rendering keys result slots by their + # position within a tool message's content list, and the + # outbound citation source map (built by + # ``_walk_citable_positions``) uses that same positional index. + # Reordering would drop citations pointing at slot ``1`` onto + # the wrong payload. + req = self._request_with_tool_message( + [ + {"type": "text", "text": "leading note"}, + { + "type": "document", + "document": {"data": {"body": "doc body"}, "id": "d0"}, + }, + {"type": "text", "text": '{"parsed": "yes"}'}, + ] + ) + blocks = _forwarded_v2_content(req)[2] + assert [b["type"] for b in blocks] == ["text", "document", "text"] + + def test_index_matches_original_request_position(self): + # Entries are keyed by *request-message* index so the renderer, + # walking the OpenAI-shaped conversation with the same + # enumeration, resolves the right slot. If ``_convert_messages`` + # ever splits or merges messages this invariant breaks, so pin + # the current 1-to-1 behavior with a two-tool-message request. + req = _make_request( + messages=[ + {"role": "user", "content": "hi"}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "c1", + "type": "function", + "function": {"name": "f", "arguments": "{}"}, + }, + { + "id": "c2", + "type": "function", + "function": {"name": "g", "arguments": "{}"}, + }, + ], + }, + { + "role": "tool", + "tool_call_id": "c1", + "content": [{"type": "text", "text": "first result"}], + }, + { + "role": "tool", + "tool_call_id": "c2", + "content": [{"type": "text", "text": "second result"}], + }, + ] + ) + blocks_by_index = _forwarded_v2_content(req) + assert sorted(blocks_by_index.keys()) == [2, 3] + assert blocks_by_index[2] == [{"type": "text", "text": "first result"}] + assert blocks_by_index[3] == [{"type": "text", "text": "second result"}] + + +def _forwarded_v2_content( + request: CohereChatV2Request, +) -> dict[int, list[dict[str, Any]]]: + """Extract the forwarded v2 tool-message content from a converted + request, or fail the test if the key isn't present. + """ + result = _convert(request) + kwargs = result.chat_template_kwargs or {} + assert TOOL_MESSAGE_V2_CONTENT_KEY in kwargs, ( + f"expected {TOOL_MESSAGE_V2_CONTENT_KEY} in chat_template_kwargs, " + f"got keys {list(kwargs)}" + ) + return kwargs[TOOL_MESSAGE_V2_CONTENT_KEY] + + # ====================================================================== # Message-citation forwarding (chat_template_kwargs["_messages_citations"]) # ====================================================================== diff --git a/tests/reasoning/test_cohere_command_reasoning_parser.py b/tests/reasoning/test_cohere_command_reasoning_parser.py index 0c1fc89009..eeac95319d 100644 --- a/tests/reasoning/test_cohere_command_reasoning_parser.py +++ b/tests/reasoning/test_cohere_command_reasoning_parser.py @@ -655,3 +655,123 @@ class TestMelodySourceResolution: assert out is not None assert out[0].type == "THINKING_CONTENT" assert out[0].sources[0].id == "d0" + + +class TestParserStreamingEndToEnd: + """End-to-end streaming shape: raw model output text is tokenized + with the mock byte-level tokenizer and fed one token at a time + through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`. + We collect the ordered sequence of ``DeltaMessage``s the parser + hands back -- i.e. what the client would receive on the wire -- + and pin the exact shape. + + Combines the three flavors that reach ``delta``: a thinking block + (populates ``reasoning``), a text block (populates ``content``), + and a resolved citation (populates ``citations``) referencing + ``(start, end)`` offsets into the already-streamed content. + """ + + _MODEL_OUTPUT = ( + "<|START_THINKING|>Let me check.<|END_THINKING|>" + "<|START_RESPONSE|>The capital is Paris.<|END_RESPONSE|>" + ) + + @staticmethod + def _drive_parser( + parser: CohereCommand3ReasoningParser, text: str, tok: MockCohereTokenizer + ) -> list[Any]: + """Feed ``text`` through the parser one token at a time + (byte-level) and return the ordered list of non-``None`` + ``DeltaMessage``s. + """ + deltas: list[Any] = [] + previous_text = "" + previous_token_ids: list[int] = [] + for token_str in _token_deltas(tok, text): + current_text = previous_text + token_str + current_token_ids = previous_token_ids + [0] + delta = parser.extract_reasoning_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=token_str, + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=[0], + ) + if delta is not None: + deltas.append(delta) + previous_text = current_text + previous_token_ids = current_token_ids + return deltas + + def test_full_wire_stream_shape(self, tokenizer): + from vllm.entrypoints.cohere.cohere_chat_message import CitationSource + from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY + + # Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs`` + # installs on a request whose tool_call_index=0 tool result at + # index 1 is the "France" document. + position_to_source = { + (0, 1): CitationSource( + type="document", + id="doc-paris", + document={"id": "doc-paris", "title": "France"}, + ), + } + + parser = CohereCommand3ReasoningParser( + tokenizer, + chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source}, + ) + + deltas = self._drive_parser(parser, self._MODEL_OUTPUT, tokenizer) + + # -- 1. Reasoning and content stream separately and reach + # the wire only as their real payload. Framing tokens + # (``<|START_THINKING|>`` etc.) are consumed by the parser's + # state machine and never produce a delta. + reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None] + content_stream = [d.content for d in deltas if d.content is not None] + citation_deltas = [d for d in deltas if getattr(d, "citations", None)] + + assert "".join(reasoning_stream) == "Let me check." + assert "".join(content_stream) == "The capital is Paris." + + # -- 2. Exactly one citation reaches the wire, resolved to the + # source from ``position_to_source`` (not the raw + # ``(bucket, idx)`` coordinates). + assert len(citation_deltas) == 1 + (cite_delta,) = citation_deltas + assert cite_delta.citations is not None + assert len(cite_delta.citations) == 1 + cite = cite_delta.citations[0] + assert cite.text == "Paris" + # Char offsets are into the accumulated ``content``: "The + # capital is " is 15 chars, "Paris" is 5. + assert cite.start == 15 + assert cite.end == 20 + assert len(cite.sources) == 1 + assert cite.sources[0].id == "doc-paris" + assert cite.sources[0].type == "document" + + # -- 3. The citation's ``(start, end)`` offsets point into + # bytes that were already emitted as content-deltas earlier + # in the stream -- i.e. the citation is anchored to real + # already-shipped content. + cite_delta_idx = deltas.index(cite_delta) + earlier_content = "".join( + d.content for d in deltas[:cite_delta_idx] if d.content is not None + ) + assert earlier_content[cite.start : cite.end] == "Paris" + + # Everything after the citation delta is the trailing period. + later_content = "".join( + d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None + ) + assert later_content == "." + + # -- 4. Reasoning and content are never mixed on the same + # delta: the parser flips modes on the ``<|END_THINKING|>`` + # boundary and never emits a delta with both fields set. + for d in deltas: + assert not (d.reasoning is not None and d.content is not None) diff --git a/tests/renderers/test_cohere.py b/tests/renderers/test_cohere.py index 0ec2156d71..76b38734c2 100644 --- a/tests/renderers/test_cohere.py +++ b/tests/renderers/test_cohere.py @@ -30,6 +30,7 @@ from vllm.renderers.cohere import ( _normalize_tool_call, _role_to_melody, _tool_to_melody, + _v2_tool_content_to_melody_block, ) from vllm.tokenizers.hf import HfTokenizer @@ -304,6 +305,108 @@ class TestToolToMelody: _tool_to_melody(42) # type: ignore[arg-type] +# ====================================================================== +# _v2_tool_content_to_melody_block +# ====================================================================== + + +class TestV2ToolContentToMelodyBlock: + """Pins the v2 → melody mapping applied to every tool-message + content block. The serving layer forwards the original v2 blocks + verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer + owns this mapping so cmd3 / cmd4 render each result field as its + own entry rather than a single stringified-JSON blob. + """ + + def test_document_block_unwraps_data_into_melody_document(self): + # ``document.data`` becomes the melody ``document`` payload + # directly. Keeping the wrapping ``data`` key would produce a + # redundant ``{"data": {...}}`` nesting at render time. + # ``document.id`` is dropped here on purpose: outbound + # citation binding tracks IDs through the separate + # ``POSITION_TO_SOURCE_KEY`` map. + result = _v2_tool_content_to_melody_block( + { + "type": "document", + "document": { + "id": "d1", + "data": {"result": "The second tallest mountain is K2."}, + }, + } + ) + assert result == { + "type": "document", + "document": {"result": "The second tallest mountain is K2."}, + } + + def test_document_block_with_missing_data_yields_empty_payload(self): + # A document block whose ``data`` isn't a mapping (e.g. + # missing entirely) still produces a document block so + # per-slot numbering stays aligned with the outbound citation + # walk in ``CohereServingChatV2._walk_citable_positions``. + result = _v2_tool_content_to_melody_block( + {"type": "document", "document": {"id": "d1"}} + ) + assert result == {"type": "document", "document": {}} + + def test_json_parseable_text_becomes_document_dict(self): + # A ``text`` block whose payload parses to a JSON object lands + # directly under ``document`` (no ``content`` wrapper) so + # cmd3/cmd4 render each field individually. + result = _v2_tool_content_to_melody_block( + {"type": "text", "text": '{"temperature": "20C"}'} + ) + assert result == {"type": "document", "document": {"temperature": "20C"}} + + def test_non_json_text_wrapped_under_content_key(self): + # Plain-text tool output gets wrapped under a ``content`` key + # so cmd3/cmd4 templates have a stable field to look up. + result = _v2_tool_content_to_melody_block( + {"type": "text", "text": "K2 is 8611m."} + ) + assert result == {"type": "document", "document": {"content": "K2 is 8611m."}} + + def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self): + # A ``text`` block whose payload parses as a JSON *array* or + # *scalar* isn't a valid melody document (which must be a + # mapping). Fall back to the ``{"content": }`` wrapper so + # the raw text still reaches the prompt. + result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"}) + assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}} + + def test_empty_text_still_produces_content_block(self): + # Even an empty text block must produce an entry so melody's + # per-content-block numbering stays aligned with the + # citation-side walk in ``_walk_citable_positions``. + result = _v2_tool_content_to_melody_block({"type": "text", "text": ""}) + assert result == {"type": "document", "document": {"content": ""}} + + def test_missing_text_field_treated_as_empty(self): + # ``model_dump(exclude_none=True)`` on a ``TextToolContent`` + # with an explicitly empty text still emits ``text: ""``, but + # guard against a future v2 shape that omits it entirely. + result = _v2_tool_content_to_melody_block({"type": "text"}) + assert result == {"type": "document", "document": {"content": ""}} + + def test_unknown_block_type_returns_none(self): + # Callers use ``None`` to know they should skip the slot; a + # future v2 addition shouldn't silently render as an empty + # document. + assert _v2_tool_content_to_melody_block({"type": "audio"}) is None + assert _v2_tool_content_to_melody_block({"not": "a block"}) is None + + def test_document_block_with_non_dict_document_returns_none(self): + # Malformed ``document`` field -- e.g. someone forwarding a + # raw string rather than the ``{id, data}`` wrapper -- is + # unusable, so return ``None`` so the caller skips it. + assert ( + _v2_tool_content_to_melody_block( + {"type": "document", "document": "not a dict"} + ) + is None + ) + + # ====================================================================== # _conversation_to_melody_messages # ====================================================================== @@ -444,6 +547,71 @@ class TestConversationToMelody: out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type] assert "citations" not in out[0] + def test_tool_message_v2_content_replaces_default_content(self): + # When ``tool_message_v2_content`` has an entry for a message + # index, the renderer maps those v2 blocks through + # :func:`_v2_tool_content_to_melody_block` and uses the result + # instead of the block list ``_content_blocks`` would derive + # from ``msg["content"]``. + conv = [ + {"role": "user", "content": "q"}, + { + "role": "tool", + "tool_call_id": "c1", + # ``_convert_tool_message`` on the serving side flattens + # the original ``document`` block to this text part. + # The renderer should ignore this flattened content + # because the forwarded map carries the structural + # truth. + "content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]', + }, + ] + v2_content = { + 1: [ + { + "type": "document", + "document": {"id": "d1", "data": {"result": "K2"}}, + } + ] + } + out = _conversation_to_melody_messages( + conv, # type: ignore[arg-type] + tool_message_v2_content=v2_content, + ) + # User message (no forwarded entry) still goes through the + # default ``_content_blocks`` path. + assert out[0]["content"] == [{"type": "text", "text": "q"}] + # Tool message picks up the mapped melody document block. + assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}] + assert out[1]["tool_call_id"] == "c1" + + def test_tool_message_v2_content_skips_unrecognized_blocks(self): + # ``_v2_tool_content_to_melody_block`` returns ``None`` for + # shapes it doesn't recognise; the renderer must skip those + # rather than let a ``None`` reach melody. + conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}] + v2_content = { + 0: [ + {"type": "text", "text": "keep me"}, + {"type": "mystery", "unexpected": True}, + ] + } + out = _conversation_to_melody_messages( + conv, # type: ignore[arg-type] + tool_message_v2_content=v2_content, + ) + assert out[0]["content"] == [ + {"type": "document", "document": {"content": "keep me"}} + ] + + def test_tool_message_v2_content_none_falls_back_to_default(self): + conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}] + out = _conversation_to_melody_messages( + conv, # type: ignore[arg-type] + tool_message_v2_content=None, + ) + assert out[0]["content"] == [{"type": "text", "text": "plain text"}] + # ====================================================================== # _build_render_config @@ -762,6 +930,40 @@ class TestBuildRenderConfig: ) # type: ignore[arg-type] assert "template_jinja" not in cfg + def test_tool_message_v2_content_key_picked_up_from_kwargs(self): + # ``_build_render_config`` must pull + # ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of + # ``chat_template_kwargs`` and hand it to + # ``_conversation_to_melody_messages`` so the renderer's + # v2 → melody mapping runs over the original v2 blocks rather + # than the flattened string content on the OpenAI-shape tool + # message. + from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY + + conv = [ + {"role": "user", "content": "q"}, + {"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'}, + ] + kwargs = { + TOOL_MESSAGE_V2_CONTENT_KEY: { + 1: [ + { + "type": "document", + "document": {"id": "d1", "data": {"r": "k"}}, + } + ] + } + } + _, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type] + tool_msg = cfg["messages"][1] + assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}] + # And the key must be *consumed* -- not surfaced as a stray + # Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but + # pin it here so a removal from that set surfaces immediately). + assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get( + "additional_template_fields", {} + ) + # ====================================================================== # End-to-end async rendering (mirrors ``test_mistral.py``) diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index b285319e7e..859ebd2b80 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -89,7 +89,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.parser.abstract_parser import Parser -from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY +from vllm.renderers.cohere import ( + MESSAGES_CITATIONS_KEY, + POSITION_TO_SOURCE_KEY, + TOOL_MESSAGE_V2_CONTENT_KEY, +) if TYPE_CHECKING: from vllm.renderers.online_renderer import OnlineRenderer @@ -97,6 +101,19 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +# Reserved ``chat_template_kwargs`` entries this handler populates for +# the API-server-side parser and renderer only. Stripped before the dict +# is forwarded to the engine core -- see +# :meth:`CohereServingChatV2._engine_chat_template_kwargs`. +_API_SERVER_ONLY_TEMPLATE_KWARGS = frozenset( + { + MESSAGES_CITATIONS_KEY, + POSITION_TO_SOURCE_KEY, + TOOL_MESSAGE_V2_CONTENT_KEY, + } +) + + # --------------------------------------------------------------------------- # SSE helpers # --------------------------------------------------------------------------- @@ -304,6 +321,28 @@ class CohereServingChatV2(OpenAIServingChat): case _: return self._chat_completion_stream_to_v2(generator, request) + def _engine_chat_template_kwargs( + self, chat_template_kwargs: dict[str, Any] + ) -> dict[str, Any]: + """Keep this handler's internal entries out of the engine hop. + + Overrides :meth:`OpenAIServingChat._engine_chat_template_kwargs`. + The reserved keys in :data:`_API_SERVER_ONLY_TEMPLATE_KWARGS` + exist only to hand request-side state to the API-server-side + parser and renderer; the engine-core-side reasoning parser (used + for structured-output reasoning-end gating) never reads them. They + also can't survive the trip: ``POSITION_TO_SOURCE_KEY`` holds + :class:`CitationSource` models, and msgpack -- which encodes + ``reasoning_parser_kwargs`` on the way to the engine core -- has + no native encoding for Pydantic models, so forwarding them fails + the request with a ``TypeError``. + """ + return { + k: v + for k, v in chat_template_kwargs.items() + if k not in _API_SERVER_ONLY_TEMPLATE_KWARGS + } + # ================================================================== # Request conversion: Cohere V2 -> ChatCompletionRequest # ================================================================== @@ -438,46 +477,53 @@ class CohereServingChatV2(OpenAIServingChat): "content": msg.content, } - # When the tool result is text-only, flatten to a string for - # maximum compatibility with standard chat templates. When it - # includes documents, preserve them as structured content parts - # so the cohere renderer can surface them as grounding sources - # (the :class:`CohereRenderer` understands - # ``{type: document, document: {...}}`` blocks). Non-cohere - # renderers may not honor document blocks, but that matches - # the broader "documents are no-op for OSS models" contract - # documented on this endpoint. + # Map every content block to a plain ``{"type": "text", "text": + # ...}`` part, mirroring the shape :meth:`_convert_user_message` + # emits for multi-part user content. vLLM's + # ``_parse_chat_message_content`` collapses a text-only tool + # part list back into ``"\n".join(...)`` on the way to the + # renderer, so simple chat templates that expect string tool + # content still work; keeping the list here just preserves the + # per-block structure at the OpenAI-shape request level for + # symmetry with the user-message conversion. # - # Tool message content uses ``ToolMessageV2Content``, which is a - # union of ``TextToolContent`` and ``DocumentToolContent`` - - # distinct from the user-message text/image union. We discriminate - # on the ``type`` literal so we don't have to import each variant. - has_documents = any(block.type == "document" for block in msg.content) - if not has_documents: - text = "\n".join( - block.text for block in msg.content if block.type == "text" - ) - return { - "role": "tool", - "tool_call_id": msg.tool_call_id, - "content": text, - } - - parts: list[dict[str, Any]] = [] + # Tool message content uses ``ToolMessageV2Content``, a union of + # ``TextToolContent`` and ``DocumentToolContent``. The latter + # produces ``{"type": "document", "document": {...}}`` blocks, + # which vLLM's shared ``_parse_chat_message_content_parts`` in + # ``vllm/entrypoints/chat_utils.py`` rejects because the OpenAI + # ``ChatCompletionContentPartParam`` union only knows text / + # image / audio / etc. Emitting them anyway triggers a Pydantic + # ``ValidatorIterator`` failure at chat-parsing time, so we + # JSON-serialize each document into a text part instead. + # + # The cohere renderer doesn't consume this flattened content: + # :meth:`_collect_v2_tool_message_content` forwards the + # original v2 content blocks under + # :data:`TOOL_MESSAGE_V2_CONTENT_KEY`, and the renderer applies + # :func:`vllm.renderers.cohere._v2_tool_content_to_melody_block` + # to produce the melody-shape ``document`` blocks cmd3/cmd4 + # tool-result rendering expects. Outbound citation binding is + # unaffected either way: sources resolve through the + # ``POSITION_TO_SOURCE_KEY`` map (built from the original v2 + # messages, not the converted OpenAI messages). + content_parts: list[dict[str, Any]] = [] for block in msg.content: if block.type == "text": - parts.append({"type": "text", "text": block.text}) + content_parts.append({"type": "text", "text": block.text}) elif block.type == "document": - parts.append( + content_parts.append( { - "type": "document", - "document": block.document.model_dump(exclude_none=True), + "type": "text", + "text": json.dumps( + block.document.model_dump(exclude_none=True) + ), } ) return { "role": "tool", "tool_call_id": msg.tool_call_id, - "content": parts, + "content": content_parts, } @classmethod @@ -660,6 +706,16 @@ class CohereServingChatV2(OpenAIServingChat): if position_to_source: kwargs.setdefault(POSITION_TO_SOURCE_KEY, position_to_source) + # Forward the original v2 tool-message content blocks so the + # renderer can apply its v2 → melody mapping over them (the + # renderer owns the melody-shape rules, this handler only + # ships the raw structural info that OpenAI validation would + # otherwise destroy -- see the flatten comment in + # :meth:`_convert_tool_message`). + v2_tool_content = cls._collect_v2_tool_message_content(request) + if v2_tool_content: + kwargs.setdefault(TOOL_MESSAGE_V2_CONTENT_KEY, v2_tool_content) + if kwargs: chat_req.chat_template_kwargs = kwargs @@ -1262,6 +1318,49 @@ class CohereServingChatV2(OpenAIServingChat): ) return None + @classmethod + def _collect_v2_tool_message_content( + cls, request: CohereChatV2Request + ) -> dict[int, list[dict[str, Any]]]: + """Collect the original v2 tool-message content blocks so the + renderer can apply its v2 → melody mapping. + + Builds the map that + :meth:`_apply_cohere_template_kwargs` stashes under + :data:`TOOL_MESSAGE_V2_CONTENT_KEY` for the renderer to + consume (see the constant definition in + :mod:`vllm.renderers.cohere` for why the map exists). Walks + ``request.messages`` -- not the OpenAI-shape messages built by + :meth:`_convert_messages` -- because that flattening step + already threw away the structural distinction between text and + document blocks. Each surviving block is + :meth:`~pydantic.BaseModel.model_dump`'d to a plain dict so the + forwarded map carries no Pydantic instances (msgpack- / + general-serialization-safe on principle even though internal + underscore-prefixed keys are filtered before crossing the + engine-core boundary). + + Only emits an entry for a tool message whose content is a + non-empty list of blocks; a bare-string tool message renders + fine through the default text-block path in + :func:`vllm.renderers.cohere._content_blocks`, so no forwarded + blocks are needed there. Returns an empty dict when there is + nothing to forward, so the caller can use truthiness to decide + whether to stash the key at all. + """ + blocks_by_index: dict[int, list[dict[str, Any]]] = {} + for i, msg in enumerate(request.messages): + if not isinstance(msg, ToolChatMessageV2): + continue + if isinstance(msg.content, str) or not msg.content: + continue + v2_blocks: list[dict[str, Any]] = [] + for block in msg.content: + v2_blocks.append(block.model_dump(exclude_none=True)) + if v2_blocks: + blocks_by_index[i] = v2_blocks + return blocks_by_index + @staticmethod def _build_usage(response: ChatCompletionResponse) -> CohereUsage | None: if response.usage is None: diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 13932aac2d..8259ef65f4 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -196,6 +196,24 @@ class OpenAIServingChat(GenerateBaseServing): .chat_template_kwargs ) + def _engine_chat_template_kwargs( + self, chat_template_kwargs: dict[str, Any] + ) -> dict[str, Any]: + """Subclass hook to narrow ``chat_template_kwargs`` for the engine. + + The same dict is used twice: to build the API-server-side parser + instance, and to populate + ``EngineCoreRequest.reasoning_parser_kwargs`` for the engine-core + side. The latter crosses ZMQ as msgpack, so a handler that stashes + request-scoped state only the API-server-side parser needs (values + msgpack can't encode, or payloads not worth shipping) can drop + those entries here without affecting the in-process parser. + + Must not mutate the argument -- the caller still needs the full + dict. The default forwards it unchanged. + """ + return chat_template_kwargs + async def render_chat_request( self, request: ChatCompletionRequest, @@ -360,7 +378,9 @@ class OpenAIServingChat(GenerateBaseServing): session_id=session_id, reasoning_ended=reasoning_ended, reasoning_parser_kwargs={ - "chat_template_kwargs": chat_template_kwargs, + "chat_template_kwargs": self._engine_chat_template_kwargs( + chat_template_kwargs + ), } if parser is not None and parser.reasoning_parser is not None else None, diff --git a/vllm/reasoning/cohere_command_reasoning_parser.py b/vllm/reasoning/cohere_command_reasoning_parser.py index dffea49731..d2ee958cfa 100644 --- a/vllm/reasoning/cohere_command_reasoning_parser.py +++ b/vllm/reasoning/cohere_command_reasoning_parser.py @@ -681,27 +681,12 @@ class BaseCohereCommandReasoningParser(ReasoningParser): return request -# melody's streaming filter only buffers a partial ```` citation -# across ``write_decoded`` calls when ``stream_non_grounded_answer`` is -# set: otherwise, the moment an opening ```` in the same delta, the filter emits the partial -# marker bytes verbatim as plain content. In vLLM's streaming path the -# parser is fed one token (1-4 chars) per call, so an unbuffered filter -# will leak ````-style markers into ``delta.content`` and never -# emit a ``FilterCitation`` for them. Enabling the flag flips the -# partial-match branch in melody's ``parse_citations`` (see -# ``src/parsing/citations_filter.rs``) to ``return (None, 0)`` -- i.e. -# keep buffering -- which lets a full citation eventually resolve. -# Non-streaming (unary) parsing receives the whole output in one call so -# the flag is a no-op there and we leave ``unary_opts`` alone. class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser): def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): super().__init__( tokenizer, *args, - streaming_opts=( - PyFilterOptions().cmd3().no_tools().stream_non_grounded_answer() - ), + streaming_opts=PyFilterOptions().cmd3().no_tools(), unary_opts=PyFilterOptions().cmd3().no_tools(), **kwargs, ) @@ -712,9 +697,7 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser): super().__init__( tokenizer, *args, - streaming_opts=( - PyFilterOptions().cmd4().no_tools().stream_non_grounded_answer() - ), + streaming_opts=PyFilterOptions().cmd4().no_tools(), unary_opts=PyFilterOptions().cmd4().no_tools(), **kwargs, ) diff --git a/vllm/renderers/cohere.py b/vllm/renderers/cohere.py index d3bf9dbfd8..8e6c4a5ac8 100644 --- a/vllm/renderers/cohere.py +++ b/vllm/renderers/cohere.py @@ -157,6 +157,40 @@ MESSAGES_CITATIONS_KEY = "_messages_citations" POSITION_TO_SOURCE_KEY = "_position_to_source" +# Reserved ``chat_template_kwargs`` key carrying the original Cohere v2 +# tool-message content, keyed by request-message index. Populated by +# :meth:`CohereServingChatV2._apply_cohere_template_kwargs` and consumed +# by :func:`_conversation_to_melody_messages`, which runs each block +# through :func:`_v2_tool_content_to_melody_block` to produce the +# melody-shape content list the renderer emits. +# +# Motivation: OpenAI's ``ChatCompletionContentPartParam`` union (in +# ``vllm/entrypoints/chat_utils.py``) only allows ``text`` / ``image_url`` +# / ``audio`` / ``video`` / ``file`` content parts on tool messages, so +# ``{"type": "document", "document": {...}}`` blocks -- valid in Cohere +# v2 -- get rejected by vLLM's shared chat validator before they can +# reach the renderer. +# ``CohereServingChatV2._convert_tool_message`` works around that by +# rewriting each content block into a plain ``text`` part (with document +# blocks JSON-serialized inline), but that flattening loses the +# structural distinction the cohere renderer needs to shape tool-result +# payloads for cmd3/cmd4. The serving layer therefore forwards the +# original v2 content blocks verbatim (as plain dicts) under this key, +# and the renderer owns the v2-shape → melody-shape mapping. +# +# Value type: ``dict[int, list[dict[str, Any]]]``. Each list entry is a +# ``model_dump``'d ``TextToolContent`` / ``DocumentToolContent`` block +# (e.g. ``{"type": "text", "text": ...}`` or ``{"type": "document", +# "document": {"id": ..., "data": {...}}}``). Index is the position of +# the tool message in the request's ``messages`` list, which -- since +# ``_convert_messages`` is 1-to-1 -- also matches its position in the +# ``ConversationMessage`` list handed to the renderer. Leading +# underscore signals "internal, not part of the public +# ``chat_template_kwargs`` surface" -- clients should not set this key +# directly. +TOOL_MESSAGE_V2_CONTENT_KEY = "_tool_message_v2_content" + + # Keys this renderer interprets directly from ``chat_template_kwargs`` and # maps onto typed melody render-config fields. Everything *not* in this # set is forwarded verbatim to melody as ``additional_template_fields`` @@ -196,6 +230,7 @@ _RENDERER_CONSUMED_KEYS = frozenset( "platform_instruction", MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY, + TOOL_MESSAGE_V2_CONTENT_KEY, } ) @@ -377,6 +412,69 @@ def _content_blocks(content: Any) -> list[dict[str, Any]]: return blocks +def _v2_tool_content_to_melody_block(block: dict[str, Any]) -> dict[str, Any] | None: + """Convert one Cohere v2 tool-message content block into a melody + ``document`` content block. + + Cohere's chat v2 admits two shapes for tool-message content parts: + + * ``TextToolContent`` -- ``{"type": "text", "text": "..."}``. + * ``DocumentToolContent`` -- ``{"type": "document", "document": + {"id"?: str, "data": {...}}}``. + + Both are emitted as melody ``document`` blocks, mirroring the cohere + api, which routes every tool content part through the same + "tool output as document" conversion. The rules: + + * Text that parses to a JSON object becomes that object directly + (so structured tool output flows through as first-class fields). + * Text that doesn't parse as an object -- empty, non-JSON, + or JSON scalar/array -- gets wrapped as ``{"content": }`` + so templates have a stable field name to look up. + * ``document`` blocks unwrap ``document.data`` (a flat map) into + the melody payload; ``document.id`` is intentionally dropped + here because outbound citation binding tracks it separately via + ``POSITION_TO_SOURCE_KEY``. + + Only the JSON-object case actually changes what gets rendered. The + cmd3/cmd4 templates already wrap a bare ``text`` content block as + ``{"content": }`` themselves, so routing plain text through + here is a no-op on the prompt -- it is the JSON-object unwrap that + promotes structured tool output from an escaped string blob into + individually addressable (and citable) result fields. + + Note the payload is the *flat* field map: melody's ``Content`` + carries ``document`` as a plain map, so re-nesting it under a + ``data`` key would render a spurious ``{"data": {...}}`` wrapper + into the tool-result block. + + Returns ``None`` for unrecognised shapes; the caller should skip + those blocks (the citation-numbering walk in + ``CohereServingChatV2._walk_citable_positions`` advances its own + cursor over unknown blocks, so slot alignment is preserved). + """ + block_type = block.get("type") + if block_type == "text": + text = block.get("text") or "" + try: + parsed = json.loads(text) + except (json.JSONDecodeError, TypeError): + parsed = None + if isinstance(parsed, dict): + payload: dict[str, Any] = dict(parsed) + else: + payload = {"content": text} + return {"type": MelodyContentType.DOCUMENT, "document": payload} + if block_type == "document": + doc = block.get("document") + if not isinstance(doc, dict): + return None + data = doc.get("data") + payload = dict(data) if isinstance(data, dict) else {} + return {"type": MelodyContentType.DOCUMENT, "document": payload} + return None + + def _document_to_melody(doc: Any) -> dict[str, Any]: """Coerce a Cohere v2 document into melody's ``Document`` (dict) shape.""" match doc: @@ -421,6 +519,7 @@ def _tool_to_melody(tool: Any) -> dict[str, Any]: def _conversation_to_melody_messages( conversation: list[ConversationMessage], messages_citations: dict[int, list[dict[str, Any]]] | None = None, + tool_message_v2_content: dict[int, list[dict[str, Any]]] | None = None, ) -> list[dict[str, Any]]: """Convert vLLM ``ConversationMessage``s into melody's message dict shape. @@ -429,17 +528,42 @@ def _conversation_to_melody_messages( ``CohereServingChatV2`` under the :data:`MESSAGES_CITATIONS_KEY` chat_template_kwargs entry. Values must already be in melody's ``FilterCitation`` dict shape (see - ``CohereServingChatV2._sdk_citation_to_melody``). Index is the - position in the *input* ``conversation`` list, which currently maps - 1-to-1 with ``CohereChatV2Request.messages`` for text-only Cohere - requests. This mapping breaks silently if ``parse_chat_messages`` - ever expands a single input message into multiple entries (e.g. + ``CohereServingChatV2._sdk_citation_to_melody``). + + ``tool_message_v2_content`` is an optional index-keyed lookup of + the original Cohere v2 tool-message content blocks (as plain + dicts), populated by ``CohereServingChatV2`` under the + :data:`TOOL_MESSAGE_V2_CONTENT_KEY` entry. When set for a given + index, its blocks -- run through + :func:`_v2_tool_content_to_melody_block` to apply the v2 → melody + mapping -- replace the block list this function would otherwise + derive from ``msg["content"]``. This lets the renderer own the + v2-shape → melody-shape conversion for the structured ``document`` + blocks cohere v2 allows on tool messages (which don't survive + OpenAI's content-parts validator on the way in, so + ``_convert_tool_message`` rewrites them into ``text`` parts). + + Both index lookups key by position in the *input* ``conversation`` + list, which currently maps 1-to-1 with + ``CohereChatV2Request.messages`` for text-only Cohere requests. + This mapping breaks silently if ``parse_chat_messages`` ever + expands a single input message into multiple entries (e.g. multi-modal splitting), which is not the case today. """ out: list[dict[str, Any]] = [] for i, msg in enumerate(conversation): role = _role_to_melody(msg.get("role", "user")) - content_blocks = _content_blocks(msg.get("content")) + if ( + tool_message_v2_content is not None + and (v2_blocks := tool_message_v2_content.get(i)) is not None + ): + content_blocks = [] + for v2_block in v2_blocks: + mapped = _v2_tool_content_to_melody_block(v2_block) + if mapped is not None: + content_blocks.append(mapped) + else: + content_blocks = _content_blocks(msg.get("content")) # Treat reasoning content as a thinking block on assistant turns # so multi-turn reasoning is preserved in the rendered prompt. @@ -513,6 +637,7 @@ def _build_render_config( "messages": _conversation_to_melody_messages( conversation, chat_template_kwargs.get(MESSAGES_CITATIONS_KEY), + chat_template_kwargs.get(TOOL_MESSAGE_V2_CONTENT_KEY), ), }