Fix Cohere ChatV2 citation and tool handling issues (#52175)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
@@ -11,6 +11,7 @@ a real engine client, model registry, etc.) — the same pattern used in
|
||||
``test_serving_streaming.py``.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -34,11 +35,61 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
|
||||
from vllm.renderers.cohere import (
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
)
|
||||
from vllm.v1.serial_utils import MsgpackDecoder, MsgpackEncoder
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# A full grounded tool round-trip: a tool call, a tool result carrying both
|
||||
# a document and a bare text block, and an assistant turn citing it. Between
|
||||
# them these populate every reserved ``_``-prefixed chat_template_kwargs key.
|
||||
_TOOL_ROUND_TRIP_MESSAGES: list[dict[str, Any]] = [
|
||||
{"role": "user", "content": "What is the capital?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "search", "arguments": '{"q": "capital"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call-1",
|
||||
"content": [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "doc-9", "data": {"text": "Paris"}},
|
||||
},
|
||||
{"type": "text", "text": "Paris is the capital."},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Paris.",
|
||||
"citations": [
|
||||
{
|
||||
"start": 0,
|
||||
"end": 5,
|
||||
"text": "Paris",
|
||||
"sources": [
|
||||
{"type": "tool", "id": "doc-9", "tool_output": {"text": "Paris"}}
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Are you sure?"},
|
||||
]
|
||||
|
||||
|
||||
def _make_request(**kwargs) -> CohereChatV2Request:
|
||||
kwargs.setdefault("model", "m")
|
||||
@@ -447,9 +498,15 @@ class TestConvertToolMessage:
|
||||
"content": "result text",
|
||||
}
|
||||
|
||||
def test_text_only_list_flattened_to_newline_string(self):
|
||||
# Text-only tool results are flattened to a single newline-joined
|
||||
# string for compatibility with standard chat templates.
|
||||
def test_text_only_list_preserves_per_block_text_parts(self):
|
||||
# A list of text blocks maps 1-to-1 onto ``text`` content
|
||||
# parts, mirroring the shape ``_convert_user_message`` emits
|
||||
# for multi-part user content. vLLM's
|
||||
# ``_parse_chat_message_content`` will collapse this back to
|
||||
# ``"\n".join(...)`` for the ``ConversationMessage`` handed to
|
||||
# renderers (see the tool-role branch in
|
||||
# ``vllm/entrypoints/chat_utils.py``), so downstream chat
|
||||
# templates that expect string tool content still work.
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "line 1"},
|
||||
@@ -458,11 +515,27 @@ class TestConvertToolMessage:
|
||||
)
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
assert tool_msg["content"] == "line 1\nline 2"
|
||||
assert tool_msg["content"] == [
|
||||
{"type": "text", "text": "line 1"},
|
||||
{"type": "text", "text": "line 2"},
|
||||
]
|
||||
|
||||
def test_with_document_preserves_structured_content(self):
|
||||
# When documents appear in the tool result, we keep the list shape
|
||||
# so the Cohere renderer can lift them into grounding sources.
|
||||
def test_document_blocks_serialized_as_text_parts(self):
|
||||
"""Document blocks must not survive on the OpenAI-shaped tool
|
||||
message.
|
||||
|
||||
vLLM's shared ``_parse_chat_message_content_parts`` runs a
|
||||
Pydantic ``ValidatorIterator`` over the OpenAI content-part
|
||||
union, which only knows ``text`` / ``image_url`` / etc. and
|
||||
rejects ``{"type": "document", ...}`` blocks outright. We
|
||||
therefore JSON-serialize each document into its own ``text``
|
||||
content part so the request survives OpenAI validation; the
|
||||
cohere renderer still resolves outbound citations against
|
||||
the original cohere v2 messages via ``POSITION_TO_SOURCE_KEY``
|
||||
and reconstructs structured melody ``document`` blocks from the
|
||||
original v2 content forwarded via
|
||||
``TOOL_MESSAGE_V2_CONTENT_KEY``.
|
||||
"""
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "see attachment"},
|
||||
@@ -475,11 +548,75 @@ class TestConvertToolMessage:
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
assert isinstance(tool_msg["content"], list)
|
||||
assert len(tool_msg["content"]) == 2
|
||||
# First block passes through unchanged.
|
||||
assert tool_msg["content"][0] == {"type": "text", "text": "see attachment"}
|
||||
assert tool_msg["content"][1] == {
|
||||
"type": "document",
|
||||
"document": {"data": {"text": "doc text"}, "id": "d1"},
|
||||
}
|
||||
# Second block: document JSON-serialized into a ``text`` part.
|
||||
second = tool_msg["content"][1]
|
||||
assert second["type"] == "text"
|
||||
assert json.loads(second["text"]) == {"data": {"text": "doc text"}, "id": "d1"}
|
||||
|
||||
def test_document_only_content_serialized_as_text_part(self):
|
||||
"""Document-only tool results must also produce a text-typed
|
||||
content part (no ``text`` block above them), so a single-doc
|
||||
result still passes the OpenAI content-parts validator.
|
||||
"""
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"data": {"result": "K2 is 8611m."}},
|
||||
}
|
||||
]
|
||||
)
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
assert isinstance(tool_msg["content"], list)
|
||||
(part,) = tool_msg["content"]
|
||||
assert part["type"] == "text"
|
||||
assert json.loads(part["text"]) == {"data": {"result": "K2 is 8611m."}}
|
||||
|
||||
def test_converted_tool_message_content_survives_openai_validation(self):
|
||||
"""Regression for the reported crash.
|
||||
|
||||
``vllm/entrypoints/chat_utils.py:_parse_chat_message_content_parts``
|
||||
runs a Pydantic ``ValidatorIterator`` over
|
||||
``list[ChatCompletionContentPartParam]``, which is a
|
||||
discriminated union that only accepts ``text`` / ``image_url``
|
||||
/ ``input_audio`` / ``refusal`` / ``file`` types. Previously
|
||||
the converted tool message carried ``{"type": "document",
|
||||
"document": {...}}`` blocks, which the validator rejected with:
|
||||
|
||||
2 validation errors for ValidatorIterator
|
||||
0.text Field required
|
||||
0.type Input should be 'text', got 'document'
|
||||
|
||||
The fix rewrites every ``DocumentToolContent`` into a
|
||||
JSON-serialized ``text`` part. This test asserts the invariant
|
||||
that keeps the validator happy: on the OpenAI-shaped request,
|
||||
every content entry a tool message carries must be a ``text``
|
||||
part (or a bare string) -- so no future refactor accidentally
|
||||
re-introduces a ``document``-typed content part.
|
||||
"""
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"data": {"result": "The second tallest mountain is K2."}
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
result = _convert(req)
|
||||
tool_msg = result.messages[-1]
|
||||
|
||||
content = tool_msg["content"]
|
||||
assert isinstance(content, (str, list))
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
assert isinstance(part, dict)
|
||||
assert part.get("type") == "text"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
@@ -808,6 +945,276 @@ class TestApplyCohereTemplateKwargs:
|
||||
assert result.chat_template_kwargs is None
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Engine-core hop (_engine_chat_template_kwargs)
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestEngineChatTemplateKwargs:
|
||||
"""The internal ``_``-prefixed entries must not reach the engine core.
|
||||
|
||||
``OpenAIServingChat`` uses one ``chat_template_kwargs`` dict twice:
|
||||
to build the API-server-side parser, and to populate
|
||||
``EngineCoreRequest.reasoning_parser_kwargs``, which crosses ZMQ as
|
||||
msgpack. This handler's reserved keys are only meaningful to the
|
||||
former, and ``_position_to_source`` holds ``CitationSource`` models
|
||||
that msgpack cannot encode at all -- forwarding them fails the
|
||||
request with a ``TypeError``. The
|
||||
``_engine_chat_template_kwargs`` override drops them.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _engine_kwargs(request: CohereChatV2Request) -> dict[str, Any]:
|
||||
full = _convert(request).chat_template_kwargs
|
||||
assert full, "expected the request to populate chat_template_kwargs"
|
||||
return _FakeServing()._engine_chat_template_kwargs(full)
|
||||
|
||||
def test_grounded_request_drops_internal_keys_keeps_public_ones(self):
|
||||
request = _make_request(
|
||||
documents=[{"id": "d1", "data": {"text": "hi"}}],
|
||||
citation_options={"mode": "accurate"},
|
||||
)
|
||||
full = _convert(request).chat_template_kwargs
|
||||
# Sanity: the key we expect to be dropped is actually present.
|
||||
assert POSITION_TO_SOURCE_KEY in full
|
||||
|
||||
engine_kwargs = self._engine_kwargs(request)
|
||||
assert POSITION_TO_SOURCE_KEY not in engine_kwargs
|
||||
# Public template inputs the engine-side parser may legitimately
|
||||
# read must survive.
|
||||
assert engine_kwargs["citation_options"] == {"mode": "accurate"}
|
||||
assert engine_kwargs["documents"] == [{"id": "d1", "data": {"text": "hi"}}]
|
||||
|
||||
def test_all_internal_keys_dropped(self):
|
||||
"""Exercises all three reserved keys at once: the tool-result
|
||||
position map, the inbound assistant citations, and the raw v2 tool
|
||||
content blocks.
|
||||
"""
|
||||
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
|
||||
full = _convert(request).chat_template_kwargs
|
||||
internal = {k for k in full if k.startswith("_")}
|
||||
# Sanity: this request really does populate all three.
|
||||
assert internal == {
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
}
|
||||
|
||||
engine_kwargs = self._engine_kwargs(request)
|
||||
assert not [k for k in engine_kwargs if k.startswith("_")]
|
||||
|
||||
def test_engine_kwargs_are_msgpack_encodable(self):
|
||||
"""The real failure mode: the full dict raises, the filtered one
|
||||
doesn't. Pins that the filter is what makes the engine hop work.
|
||||
"""
|
||||
request = _make_request(messages=_TOOL_ROUND_TRIP_MESSAGES)
|
||||
full = _convert(request).chat_template_kwargs
|
||||
|
||||
with pytest.raises(TypeError, match="is not serializable"):
|
||||
MsgpackEncoder().encode({"chat_template_kwargs": full})
|
||||
|
||||
engine_kwargs = self._engine_kwargs(request)
|
||||
encoder = MsgpackEncoder()
|
||||
decoder = MsgpackDecoder(dict[str, Any])
|
||||
round_tripped = decoder.decode(
|
||||
encoder.encode({"chat_template_kwargs": engine_kwargs})[0]
|
||||
)
|
||||
assert round_tripped["chat_template_kwargs"] == engine_kwargs
|
||||
|
||||
def test_original_dict_not_mutated(self):
|
||||
"""The caller still passes the full dict to the in-process parser,
|
||||
so the filter must return a copy.
|
||||
"""
|
||||
request = _make_request(documents=[{"id": "d1", "data": {"text": "hi"}}])
|
||||
full = _convert(request).chat_template_kwargs
|
||||
before = set(full)
|
||||
|
||||
self._engine_kwargs(request)
|
||||
assert set(full) == before
|
||||
|
||||
def test_base_implementation_is_a_passthrough(self):
|
||||
"""Only the Cohere subclass filters; plain chat completions must
|
||||
keep forwarding whatever the request asked for.
|
||||
"""
|
||||
kwargs = {
|
||||
"documents": [{"id": "d1"}],
|
||||
POSITION_TO_SOURCE_KEY: {(0, 0): object()},
|
||||
}
|
||||
passed_through = OpenAIServingChat._engine_chat_template_kwargs(
|
||||
_FakeServing(), kwargs
|
||||
)
|
||||
assert passed_through == kwargs
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Tool-message content forwarding
|
||||
# (chat_template_kwargs["_tool_message_v2_content"])
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestToolMessageV2ContentForwarding:
|
||||
"""The cohere renderer needs the original v2 tool-message content
|
||||
(text / document blocks) to reconstruct melody's ``document``-block
|
||||
shape, but the OpenAI content-parts validator on the way in only
|
||||
accepts text / image / audio / etc. blocks. So
|
||||
:meth:`CohereServingChatV2._convert_tool_message` rewrites each
|
||||
content block into a plain ``text`` part on the request that
|
||||
survives validation, and
|
||||
:meth:`CohereServingChatV2._collect_v2_tool_message_content`
|
||||
forwards the *original* v2 content blocks -- as ``model_dump``'d
|
||||
dicts -- to the renderer under
|
||||
:data:`TOOL_MESSAGE_V2_CONTENT_KEY`. These tests pin the
|
||||
forwarding contract (presence, dict-shape, index-alignment); the
|
||||
v2 → melody mapping the renderer applies to these blocks lives in
|
||||
``tests/renderers/test_cohere.py``.
|
||||
"""
|
||||
|
||||
def _request_with_tool_message(self, content: Any) -> CohereChatV2Request:
|
||||
return _make_request(
|
||||
messages=[
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": content},
|
||||
]
|
||||
)
|
||||
|
||||
def test_absent_when_no_tool_messages(self):
|
||||
result = _convert(_make_request())
|
||||
# ``chat_template_kwargs`` may be ``None`` here (no cohere-side
|
||||
# forwarding happened at all) or a dict without the key; both
|
||||
# mean the same thing to the renderer.
|
||||
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
|
||||
|
||||
def test_absent_for_string_tool_content(self):
|
||||
# Bare-string tool content works through the renderer's default
|
||||
# ``_content_blocks`` path, so no forwarded blocks are needed.
|
||||
req = self._request_with_tool_message("plain text")
|
||||
result = _convert(req)
|
||||
assert not (result.chat_template_kwargs or {}).get(TOOL_MESSAGE_V2_CONTENT_KEY)
|
||||
|
||||
def test_forwarded_blocks_are_plain_v2_shape_dicts(self):
|
||||
# The forwarded map must contain the original v2 content
|
||||
# blocks verbatim (as plain dicts, not Pydantic instances) so
|
||||
# the renderer can drive the v2 → melody mapping and the map
|
||||
# stays trivially serializable.
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "K2 is 8611m."},
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
blocks_by_index = _forwarded_v2_content(req)
|
||||
# Index 2 = the tool message (user=0, assistant=1, tool=2).
|
||||
assert list(blocks_by_index.keys()) == [2]
|
||||
assert blocks_by_index[2] == [
|
||||
{"type": "text", "text": "K2 is 8611m."},
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
},
|
||||
]
|
||||
# Guard against a regression where Pydantic instances leak into
|
||||
# the forwarded map -- msgpack can't serialize them, and even
|
||||
# for keys filtered out of the engine-core payload we want the
|
||||
# renderer to see plain data.
|
||||
for block in blocks_by_index[2]:
|
||||
assert isinstance(block, dict)
|
||||
|
||||
def test_forwarded_blocks_preserve_input_order(self):
|
||||
# Melody's tool-result rendering keys result slots by their
|
||||
# position within a tool message's content list, and the
|
||||
# outbound citation source map (built by
|
||||
# ``_walk_citable_positions``) uses that same positional index.
|
||||
# Reordering would drop citations pointing at slot ``1`` onto
|
||||
# the wrong payload.
|
||||
req = self._request_with_tool_message(
|
||||
[
|
||||
{"type": "text", "text": "leading note"},
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"data": {"body": "doc body"}, "id": "d0"},
|
||||
},
|
||||
{"type": "text", "text": '{"parsed": "yes"}'},
|
||||
]
|
||||
)
|
||||
blocks = _forwarded_v2_content(req)[2]
|
||||
assert [b["type"] for b in blocks] == ["text", "document", "text"]
|
||||
|
||||
def test_index_matches_original_request_position(self):
|
||||
# Entries are keyed by *request-message* index so the renderer,
|
||||
# walking the OpenAI-shaped conversation with the same
|
||||
# enumeration, resolves the right slot. If ``_convert_messages``
|
||||
# ever splits or merges messages this invariant breaks, so pin
|
||||
# the current 1-to-1 behavior with a two-tool-message request.
|
||||
req = _make_request(
|
||||
messages=[
|
||||
{"role": "user", "content": "hi"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "c1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "c2",
|
||||
"type": "function",
|
||||
"function": {"name": "g", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
"content": [{"type": "text", "text": "first result"}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c2",
|
||||
"content": [{"type": "text", "text": "second result"}],
|
||||
},
|
||||
]
|
||||
)
|
||||
blocks_by_index = _forwarded_v2_content(req)
|
||||
assert sorted(blocks_by_index.keys()) == [2, 3]
|
||||
assert blocks_by_index[2] == [{"type": "text", "text": "first result"}]
|
||||
assert blocks_by_index[3] == [{"type": "text", "text": "second result"}]
|
||||
|
||||
|
||||
def _forwarded_v2_content(
|
||||
request: CohereChatV2Request,
|
||||
) -> dict[int, list[dict[str, Any]]]:
|
||||
"""Extract the forwarded v2 tool-message content from a converted
|
||||
request, or fail the test if the key isn't present.
|
||||
"""
|
||||
result = _convert(request)
|
||||
kwargs = result.chat_template_kwargs or {}
|
||||
assert TOOL_MESSAGE_V2_CONTENT_KEY in kwargs, (
|
||||
f"expected {TOOL_MESSAGE_V2_CONTENT_KEY} in chat_template_kwargs, "
|
||||
f"got keys {list(kwargs)}"
|
||||
)
|
||||
return kwargs[TOOL_MESSAGE_V2_CONTENT_KEY]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Message-citation forwarding (chat_template_kwargs["_messages_citations"])
|
||||
# ======================================================================
|
||||
|
||||
@@ -655,3 +655,123 @@ class TestMelodySourceResolution:
|
||||
assert out is not None
|
||||
assert out[0].type == "THINKING_CONTENT"
|
||||
assert out[0].sources[0].id == "d0"
|
||||
|
||||
|
||||
class TestParserStreamingEndToEnd:
|
||||
"""End-to-end streaming shape: raw model output text is tokenized
|
||||
with the mock byte-level tokenizer and fed one token at a time
|
||||
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
|
||||
We collect the ordered sequence of ``DeltaMessage``s the parser
|
||||
hands back -- i.e. what the client would receive on the wire --
|
||||
and pin the exact shape.
|
||||
|
||||
Combines the three flavors that reach ``delta``: a thinking block
|
||||
(populates ``reasoning``), a text block (populates ``content``),
|
||||
and a resolved citation (populates ``citations``) referencing
|
||||
``(start, end)`` offsets into the already-streamed content.
|
||||
"""
|
||||
|
||||
_MODEL_OUTPUT = (
|
||||
"<|START_THINKING|>Let me check.<|END_THINKING|>"
|
||||
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _drive_parser(
|
||||
parser: CohereCommand3ReasoningParser, text: str, tok: MockCohereTokenizer
|
||||
) -> list[Any]:
|
||||
"""Feed ``text`` through the parser one token at a time
|
||||
(byte-level) and return the ordered list of non-``None``
|
||||
``DeltaMessage``s.
|
||||
"""
|
||||
deltas: list[Any] = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
for token_str in _token_deltas(tok, text):
|
||||
current_text = previous_text + token_str
|
||||
current_token_ids = previous_token_ids + [0]
|
||||
delta = parser.extract_reasoning_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=token_str,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=[0],
|
||||
)
|
||||
if delta is not None:
|
||||
deltas.append(delta)
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
return deltas
|
||||
|
||||
def test_full_wire_stream_shape(self, tokenizer):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
|
||||
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
|
||||
# installs on a request whose tool_call_index=0 tool result at
|
||||
# index 1 is the "France" document.
|
||||
position_to_source = {
|
||||
(0, 1): CitationSource(
|
||||
type="document",
|
||||
id="doc-paris",
|
||||
document={"id": "doc-paris", "title": "France"},
|
||||
),
|
||||
}
|
||||
|
||||
parser = CohereCommand3ReasoningParser(
|
||||
tokenizer,
|
||||
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
|
||||
)
|
||||
|
||||
deltas = self._drive_parser(parser, self._MODEL_OUTPUT, tokenizer)
|
||||
|
||||
# -- 1. Reasoning and content stream separately and reach
|
||||
# the wire only as their real payload. Framing tokens
|
||||
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
|
||||
# state machine and never produce a delta.
|
||||
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
|
||||
content_stream = [d.content for d in deltas if d.content is not None]
|
||||
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
|
||||
|
||||
assert "".join(reasoning_stream) == "Let me check."
|
||||
assert "".join(content_stream) == "The capital is Paris."
|
||||
|
||||
# -- 2. Exactly one citation reaches the wire, resolved to the
|
||||
# source from ``position_to_source`` (not the raw
|
||||
# ``(bucket, idx)`` coordinates).
|
||||
assert len(citation_deltas) == 1
|
||||
(cite_delta,) = citation_deltas
|
||||
assert cite_delta.citations is not None
|
||||
assert len(cite_delta.citations) == 1
|
||||
cite = cite_delta.citations[0]
|
||||
assert cite.text == "Paris"
|
||||
# Char offsets are into the accumulated ``content``: "The
|
||||
# capital is " is 15 chars, "Paris" is 5.
|
||||
assert cite.start == 15
|
||||
assert cite.end == 20
|
||||
assert len(cite.sources) == 1
|
||||
assert cite.sources[0].id == "doc-paris"
|
||||
assert cite.sources[0].type == "document"
|
||||
|
||||
# -- 3. The citation's ``(start, end)`` offsets point into
|
||||
# bytes that were already emitted as content-deltas earlier
|
||||
# in the stream -- i.e. the citation is anchored to real
|
||||
# already-shipped content.
|
||||
cite_delta_idx = deltas.index(cite_delta)
|
||||
earlier_content = "".join(
|
||||
d.content for d in deltas[:cite_delta_idx] if d.content is not None
|
||||
)
|
||||
assert earlier_content[cite.start : cite.end] == "Paris"
|
||||
|
||||
# Everything after the citation delta is the trailing period.
|
||||
later_content = "".join(
|
||||
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
|
||||
)
|
||||
assert later_content == "."
|
||||
|
||||
# -- 4. Reasoning and content are never mixed on the same
|
||||
# delta: the parser flips modes on the ``<|END_THINKING|>``
|
||||
# boundary and never emits a delta with both fields set.
|
||||
for d in deltas:
|
||||
assert not (d.reasoning is not None and d.content is not None)
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.renderers.cohere import (
|
||||
_normalize_tool_call,
|
||||
_role_to_melody,
|
||||
_tool_to_melody,
|
||||
_v2_tool_content_to_melody_block,
|
||||
)
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
|
||||
@@ -304,6 +305,108 @@ class TestToolToMelody:
|
||||
_tool_to_melody(42) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _v2_tool_content_to_melody_block
|
||||
# ======================================================================
|
||||
|
||||
|
||||
class TestV2ToolContentToMelodyBlock:
|
||||
"""Pins the v2 → melody mapping applied to every tool-message
|
||||
content block. The serving layer forwards the original v2 blocks
|
||||
verbatim under :data:`TOOL_MESSAGE_V2_CONTENT_KEY`; the renderer
|
||||
owns this mapping so cmd3 / cmd4 render each result field as its
|
||||
own entry rather than a single stringified-JSON blob.
|
||||
"""
|
||||
|
||||
def test_document_block_unwraps_data_into_melody_document(self):
|
||||
# ``document.data`` becomes the melody ``document`` payload
|
||||
# directly. Keeping the wrapping ``data`` key would produce a
|
||||
# redundant ``{"data": {...}}`` nesting at render time.
|
||||
# ``document.id`` is dropped here on purpose: outbound
|
||||
# citation binding tracks IDs through the separate
|
||||
# ``POSITION_TO_SOURCE_KEY`` map.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{
|
||||
"type": "document",
|
||||
"document": {
|
||||
"id": "d1",
|
||||
"data": {"result": "The second tallest mountain is K2."},
|
||||
},
|
||||
}
|
||||
)
|
||||
assert result == {
|
||||
"type": "document",
|
||||
"document": {"result": "The second tallest mountain is K2."},
|
||||
}
|
||||
|
||||
def test_document_block_with_missing_data_yields_empty_payload(self):
|
||||
# A document block whose ``data`` isn't a mapping (e.g.
|
||||
# missing entirely) still produces a document block so
|
||||
# per-slot numbering stays aligned with the outbound citation
|
||||
# walk in ``CohereServingChatV2._walk_citable_positions``.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "document", "document": {"id": "d1"}}
|
||||
)
|
||||
assert result == {"type": "document", "document": {}}
|
||||
|
||||
def test_json_parseable_text_becomes_document_dict(self):
|
||||
# A ``text`` block whose payload parses to a JSON object lands
|
||||
# directly under ``document`` (no ``content`` wrapper) so
|
||||
# cmd3/cmd4 render each field individually.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "text", "text": '{"temperature": "20C"}'}
|
||||
)
|
||||
assert result == {"type": "document", "document": {"temperature": "20C"}}
|
||||
|
||||
def test_non_json_text_wrapped_under_content_key(self):
|
||||
# Plain-text tool output gets wrapped under a ``content`` key
|
||||
# so cmd3/cmd4 templates have a stable field to look up.
|
||||
result = _v2_tool_content_to_melody_block(
|
||||
{"type": "text", "text": "K2 is 8611m."}
|
||||
)
|
||||
assert result == {"type": "document", "document": {"content": "K2 is 8611m."}}
|
||||
|
||||
def test_json_text_that_is_not_a_dict_is_treated_as_plain_text(self):
|
||||
# A ``text`` block whose payload parses as a JSON *array* or
|
||||
# *scalar* isn't a valid melody document (which must be a
|
||||
# mapping). Fall back to the ``{"content": <text>}`` wrapper so
|
||||
# the raw text still reaches the prompt.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text", "text": "[1, 2, 3]"})
|
||||
assert result == {"type": "document", "document": {"content": "[1, 2, 3]"}}
|
||||
|
||||
def test_empty_text_still_produces_content_block(self):
|
||||
# Even an empty text block must produce an entry so melody's
|
||||
# per-content-block numbering stays aligned with the
|
||||
# citation-side walk in ``_walk_citable_positions``.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text", "text": ""})
|
||||
assert result == {"type": "document", "document": {"content": ""}}
|
||||
|
||||
def test_missing_text_field_treated_as_empty(self):
|
||||
# ``model_dump(exclude_none=True)`` on a ``TextToolContent``
|
||||
# with an explicitly empty text still emits ``text: ""``, but
|
||||
# guard against a future v2 shape that omits it entirely.
|
||||
result = _v2_tool_content_to_melody_block({"type": "text"})
|
||||
assert result == {"type": "document", "document": {"content": ""}}
|
||||
|
||||
def test_unknown_block_type_returns_none(self):
|
||||
# Callers use ``None`` to know they should skip the slot; a
|
||||
# future v2 addition shouldn't silently render as an empty
|
||||
# document.
|
||||
assert _v2_tool_content_to_melody_block({"type": "audio"}) is None
|
||||
assert _v2_tool_content_to_melody_block({"not": "a block"}) is None
|
||||
|
||||
def test_document_block_with_non_dict_document_returns_none(self):
|
||||
# Malformed ``document`` field -- e.g. someone forwarding a
|
||||
# raw string rather than the ``{id, data}`` wrapper -- is
|
||||
# unusable, so return ``None`` so the caller skips it.
|
||||
assert (
|
||||
_v2_tool_content_to_melody_block(
|
||||
{"type": "document", "document": "not a dict"}
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _conversation_to_melody_messages
|
||||
# ======================================================================
|
||||
@@ -444,6 +547,71 @@ class TestConversationToMelody:
|
||||
out = _conversation_to_melody_messages(conv, {5: [{"anything": 1}]}) # type: ignore[arg-type]
|
||||
assert "citations" not in out[0]
|
||||
|
||||
def test_tool_message_v2_content_replaces_default_content(self):
|
||||
# When ``tool_message_v2_content`` has an entry for a message
|
||||
# index, the renderer maps those v2 blocks through
|
||||
# :func:`_v2_tool_content_to_melody_block` and uses the result
|
||||
# instead of the block list ``_content_blocks`` would derive
|
||||
# from ``msg["content"]``.
|
||||
conv = [
|
||||
{"role": "user", "content": "q"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
# ``_convert_tool_message`` on the serving side flattens
|
||||
# the original ``document`` block to this text part.
|
||||
# The renderer should ignore this flattened content
|
||||
# because the forwarded map carries the structural
|
||||
# truth.
|
||||
"content": '[{"type":"text","text":"{\\"result\\":\\"K2\\"}"}]',
|
||||
},
|
||||
]
|
||||
v2_content = {
|
||||
1: [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "d1", "data": {"result": "K2"}},
|
||||
}
|
||||
]
|
||||
}
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=v2_content,
|
||||
)
|
||||
# User message (no forwarded entry) still goes through the
|
||||
# default ``_content_blocks`` path.
|
||||
assert out[0]["content"] == [{"type": "text", "text": "q"}]
|
||||
# Tool message picks up the mapped melody document block.
|
||||
assert out[1]["content"] == [{"type": "document", "document": {"result": "K2"}}]
|
||||
assert out[1]["tool_call_id"] == "c1"
|
||||
|
||||
def test_tool_message_v2_content_skips_unrecognized_blocks(self):
|
||||
# ``_v2_tool_content_to_melody_block`` returns ``None`` for
|
||||
# shapes it doesn't recognise; the renderer must skip those
|
||||
# rather than let a ``None`` reach melody.
|
||||
conv = [{"role": "tool", "tool_call_id": "c", "content": "ignored"}]
|
||||
v2_content = {
|
||||
0: [
|
||||
{"type": "text", "text": "keep me"},
|
||||
{"type": "mystery", "unexpected": True},
|
||||
]
|
||||
}
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=v2_content,
|
||||
)
|
||||
assert out[0]["content"] == [
|
||||
{"type": "document", "document": {"content": "keep me"}}
|
||||
]
|
||||
|
||||
def test_tool_message_v2_content_none_falls_back_to_default(self):
|
||||
conv = [{"role": "tool", "tool_call_id": "c", "content": "plain text"}]
|
||||
out = _conversation_to_melody_messages(
|
||||
conv, # type: ignore[arg-type]
|
||||
tool_message_v2_content=None,
|
||||
)
|
||||
assert out[0]["content"] == [{"type": "text", "text": "plain text"}]
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# _build_render_config
|
||||
@@ -762,6 +930,40 @@ class TestBuildRenderConfig:
|
||||
) # type: ignore[arg-type]
|
||||
assert "template_jinja" not in cfg
|
||||
|
||||
def test_tool_message_v2_content_key_picked_up_from_kwargs(self):
|
||||
# ``_build_render_config`` must pull
|
||||
# ``TOOL_MESSAGE_V2_CONTENT_KEY`` out of
|
||||
# ``chat_template_kwargs`` and hand it to
|
||||
# ``_conversation_to_melody_messages`` so the renderer's
|
||||
# v2 → melody mapping runs over the original v2 blocks rather
|
||||
# than the flattened string content on the OpenAI-shape tool
|
||||
# message.
|
||||
from vllm.renderers.cohere import TOOL_MESSAGE_V2_CONTENT_KEY
|
||||
|
||||
conv = [
|
||||
{"role": "user", "content": "q"},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": '{"r":"k"}'},
|
||||
]
|
||||
kwargs = {
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY: {
|
||||
1: [
|
||||
{
|
||||
"type": "document",
|
||||
"document": {"id": "d1", "data": {"r": "k"}},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
_, cfg = _build_render_config(conv, kwargs) # type: ignore[arg-type]
|
||||
tool_msg = cfg["messages"][1]
|
||||
assert tool_msg["content"] == [{"type": "document", "document": {"r": "k"}}]
|
||||
# And the key must be *consumed* -- not surfaced as a stray
|
||||
# Jinja variable (``_RENDERER_CONSUMED_KEYS`` covers this but
|
||||
# pin it here so a removal from that set surfaces immediately).
|
||||
assert TOOL_MESSAGE_V2_CONTENT_KEY not in cfg.get(
|
||||
"additional_template_fields", {}
|
||||
)
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# End-to-end async rendering (mirrors ``test_mistral.py``)
|
||||
|
||||
@@ -89,7 +89,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.renderers.cohere import MESSAGES_CITATIONS_KEY, POSITION_TO_SOURCE_KEY
|
||||
from vllm.renderers.cohere import (
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
@@ -97,6 +101,19 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Reserved ``chat_template_kwargs`` entries this handler populates for
|
||||
# the API-server-side parser and renderer only. Stripped before the dict
|
||||
# is forwarded to the engine core -- see
|
||||
# :meth:`CohereServingChatV2._engine_chat_template_kwargs`.
|
||||
_API_SERVER_ONLY_TEMPLATE_KWARGS = frozenset(
|
||||
{
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SSE helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -304,6 +321,28 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
case _:
|
||||
return self._chat_completion_stream_to_v2(generator, request)
|
||||
|
||||
def _engine_chat_template_kwargs(
|
||||
self, chat_template_kwargs: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""Keep this handler's internal entries out of the engine hop.
|
||||
|
||||
Overrides :meth:`OpenAIServingChat._engine_chat_template_kwargs`.
|
||||
The reserved keys in :data:`_API_SERVER_ONLY_TEMPLATE_KWARGS`
|
||||
exist only to hand request-side state to the API-server-side
|
||||
parser and renderer; the engine-core-side reasoning parser (used
|
||||
for structured-output reasoning-end gating) never reads them. They
|
||||
also can't survive the trip: ``POSITION_TO_SOURCE_KEY`` holds
|
||||
:class:`CitationSource` models, and msgpack -- which encodes
|
||||
``reasoning_parser_kwargs`` on the way to the engine core -- has
|
||||
no native encoding for Pydantic models, so forwarding them fails
|
||||
the request with a ``TypeError``.
|
||||
"""
|
||||
return {
|
||||
k: v
|
||||
for k, v in chat_template_kwargs.items()
|
||||
if k not in _API_SERVER_ONLY_TEMPLATE_KWARGS
|
||||
}
|
||||
|
||||
# ==================================================================
|
||||
# Request conversion: Cohere V2 -> ChatCompletionRequest
|
||||
# ==================================================================
|
||||
@@ -438,46 +477,53 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
"content": msg.content,
|
||||
}
|
||||
|
||||
# When the tool result is text-only, flatten to a string for
|
||||
# maximum compatibility with standard chat templates. When it
|
||||
# includes documents, preserve them as structured content parts
|
||||
# so the cohere renderer can surface them as grounding sources
|
||||
# (the :class:`CohereRenderer` understands
|
||||
# ``{type: document, document: {...}}`` blocks). Non-cohere
|
||||
# renderers may not honor document blocks, but that matches
|
||||
# the broader "documents are no-op for OSS models" contract
|
||||
# documented on this endpoint.
|
||||
# Map every content block to a plain ``{"type": "text", "text":
|
||||
# ...}`` part, mirroring the shape :meth:`_convert_user_message`
|
||||
# emits for multi-part user content. vLLM's
|
||||
# ``_parse_chat_message_content`` collapses a text-only tool
|
||||
# part list back into ``"\n".join(...)`` on the way to the
|
||||
# renderer, so simple chat templates that expect string tool
|
||||
# content still work; keeping the list here just preserves the
|
||||
# per-block structure at the OpenAI-shape request level for
|
||||
# symmetry with the user-message conversion.
|
||||
#
|
||||
# Tool message content uses ``ToolMessageV2Content``, which is a
|
||||
# union of ``TextToolContent`` and ``DocumentToolContent`` -
|
||||
# distinct from the user-message text/image union. We discriminate
|
||||
# on the ``type`` literal so we don't have to import each variant.
|
||||
has_documents = any(block.type == "document" for block in msg.content)
|
||||
if not has_documents:
|
||||
text = "\n".join(
|
||||
block.text for block in msg.content if block.type == "text"
|
||||
)
|
||||
return {
|
||||
"role": "tool",
|
||||
"tool_call_id": msg.tool_call_id,
|
||||
"content": text,
|
||||
}
|
||||
|
||||
parts: list[dict[str, Any]] = []
|
||||
# Tool message content uses ``ToolMessageV2Content``, a union of
|
||||
# ``TextToolContent`` and ``DocumentToolContent``. The latter
|
||||
# produces ``{"type": "document", "document": {...}}`` blocks,
|
||||
# which vLLM's shared ``_parse_chat_message_content_parts`` in
|
||||
# ``vllm/entrypoints/chat_utils.py`` rejects because the OpenAI
|
||||
# ``ChatCompletionContentPartParam`` union only knows text /
|
||||
# image / audio / etc. Emitting them anyway triggers a Pydantic
|
||||
# ``ValidatorIterator`` failure at chat-parsing time, so we
|
||||
# JSON-serialize each document into a text part instead.
|
||||
#
|
||||
# The cohere renderer doesn't consume this flattened content:
|
||||
# :meth:`_collect_v2_tool_message_content` forwards the
|
||||
# original v2 content blocks under
|
||||
# :data:`TOOL_MESSAGE_V2_CONTENT_KEY`, and the renderer applies
|
||||
# :func:`vllm.renderers.cohere._v2_tool_content_to_melody_block`
|
||||
# to produce the melody-shape ``document`` blocks cmd3/cmd4
|
||||
# tool-result rendering expects. Outbound citation binding is
|
||||
# unaffected either way: sources resolve through the
|
||||
# ``POSITION_TO_SOURCE_KEY`` map (built from the original v2
|
||||
# messages, not the converted OpenAI messages).
|
||||
content_parts: list[dict[str, Any]] = []
|
||||
for block in msg.content:
|
||||
if block.type == "text":
|
||||
parts.append({"type": "text", "text": block.text})
|
||||
content_parts.append({"type": "text", "text": block.text})
|
||||
elif block.type == "document":
|
||||
parts.append(
|
||||
content_parts.append(
|
||||
{
|
||||
"type": "document",
|
||||
"document": block.document.model_dump(exclude_none=True),
|
||||
"type": "text",
|
||||
"text": json.dumps(
|
||||
block.document.model_dump(exclude_none=True)
|
||||
),
|
||||
}
|
||||
)
|
||||
return {
|
||||
"role": "tool",
|
||||
"tool_call_id": msg.tool_call_id,
|
||||
"content": parts,
|
||||
"content": content_parts,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -660,6 +706,16 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
if position_to_source:
|
||||
kwargs.setdefault(POSITION_TO_SOURCE_KEY, position_to_source)
|
||||
|
||||
# Forward the original v2 tool-message content blocks so the
|
||||
# renderer can apply its v2 → melody mapping over them (the
|
||||
# renderer owns the melody-shape rules, this handler only
|
||||
# ships the raw structural info that OpenAI validation would
|
||||
# otherwise destroy -- see the flatten comment in
|
||||
# :meth:`_convert_tool_message`).
|
||||
v2_tool_content = cls._collect_v2_tool_message_content(request)
|
||||
if v2_tool_content:
|
||||
kwargs.setdefault(TOOL_MESSAGE_V2_CONTENT_KEY, v2_tool_content)
|
||||
|
||||
if kwargs:
|
||||
chat_req.chat_template_kwargs = kwargs
|
||||
|
||||
@@ -1262,6 +1318,49 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _collect_v2_tool_message_content(
|
||||
cls, request: CohereChatV2Request
|
||||
) -> dict[int, list[dict[str, Any]]]:
|
||||
"""Collect the original v2 tool-message content blocks so the
|
||||
renderer can apply its v2 → melody mapping.
|
||||
|
||||
Builds the map that
|
||||
:meth:`_apply_cohere_template_kwargs` stashes under
|
||||
:data:`TOOL_MESSAGE_V2_CONTENT_KEY` for the renderer to
|
||||
consume (see the constant definition in
|
||||
:mod:`vllm.renderers.cohere` for why the map exists). Walks
|
||||
``request.messages`` -- not the OpenAI-shape messages built by
|
||||
:meth:`_convert_messages` -- because that flattening step
|
||||
already threw away the structural distinction between text and
|
||||
document blocks. Each surviving block is
|
||||
:meth:`~pydantic.BaseModel.model_dump`'d to a plain dict so the
|
||||
forwarded map carries no Pydantic instances (msgpack- /
|
||||
general-serialization-safe on principle even though internal
|
||||
underscore-prefixed keys are filtered before crossing the
|
||||
engine-core boundary).
|
||||
|
||||
Only emits an entry for a tool message whose content is a
|
||||
non-empty list of blocks; a bare-string tool message renders
|
||||
fine through the default text-block path in
|
||||
:func:`vllm.renderers.cohere._content_blocks`, so no forwarded
|
||||
blocks are needed there. Returns an empty dict when there is
|
||||
nothing to forward, so the caller can use truthiness to decide
|
||||
whether to stash the key at all.
|
||||
"""
|
||||
blocks_by_index: dict[int, list[dict[str, Any]]] = {}
|
||||
for i, msg in enumerate(request.messages):
|
||||
if not isinstance(msg, ToolChatMessageV2):
|
||||
continue
|
||||
if isinstance(msg.content, str) or not msg.content:
|
||||
continue
|
||||
v2_blocks: list[dict[str, Any]] = []
|
||||
for block in msg.content:
|
||||
v2_blocks.append(block.model_dump(exclude_none=True))
|
||||
if v2_blocks:
|
||||
blocks_by_index[i] = v2_blocks
|
||||
return blocks_by_index
|
||||
|
||||
@staticmethod
|
||||
def _build_usage(response: ChatCompletionResponse) -> CohereUsage | None:
|
||||
if response.usage is None:
|
||||
|
||||
@@ -196,6 +196,24 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
.chat_template_kwargs
|
||||
)
|
||||
|
||||
def _engine_chat_template_kwargs(
|
||||
self, chat_template_kwargs: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""Subclass hook to narrow ``chat_template_kwargs`` for the engine.
|
||||
|
||||
The same dict is used twice: to build the API-server-side parser
|
||||
instance, and to populate
|
||||
``EngineCoreRequest.reasoning_parser_kwargs`` for the engine-core
|
||||
side. The latter crosses ZMQ as msgpack, so a handler that stashes
|
||||
request-scoped state only the API-server-side parser needs (values
|
||||
msgpack can't encode, or payloads not worth shipping) can drop
|
||||
those entries here without affecting the in-process parser.
|
||||
|
||||
Must not mutate the argument -- the caller still needs the full
|
||||
dict. The default forwards it unchanged.
|
||||
"""
|
||||
return chat_template_kwargs
|
||||
|
||||
async def render_chat_request(
|
||||
self,
|
||||
request: ChatCompletionRequest,
|
||||
@@ -360,7 +378,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
session_id=session_id,
|
||||
reasoning_ended=reasoning_ended,
|
||||
reasoning_parser_kwargs={
|
||||
"chat_template_kwargs": chat_template_kwargs,
|
||||
"chat_template_kwargs": self._engine_chat_template_kwargs(
|
||||
chat_template_kwargs
|
||||
),
|
||||
}
|
||||
if parser is not None and parser.reasoning_parser is not None
|
||||
else None,
|
||||
|
||||
@@ -681,27 +681,12 @@ class BaseCohereCommandReasoningParser(ReasoningParser):
|
||||
return request
|
||||
|
||||
|
||||
# melody's streaming filter only buffers a partial ``<co: ...>`` citation
|
||||
# across ``write_decoded`` calls when ``stream_non_grounded_answer`` is
|
||||
# set: otherwise, the moment an opening ``<co`` is seen without a
|
||||
# closing ``</co: ...>`` in the same delta, the filter emits the partial
|
||||
# marker bytes verbatim as plain content. In vLLM's streaming path the
|
||||
# parser is fed one token (1-4 chars) per call, so an unbuffered filter
|
||||
# will leak ``<co: 0>``-style markers into ``delta.content`` and never
|
||||
# emit a ``FilterCitation`` for them. Enabling the flag flips the
|
||||
# partial-match branch in melody's ``parse_citations`` (see
|
||||
# ``src/parsing/citations_filter.rs``) to ``return (None, 0)`` -- i.e.
|
||||
# keep buffering -- which lets a full citation eventually resolve.
|
||||
# Non-streaming (unary) parsing receives the whole output in one call so
|
||||
# the flag is a no-op there and we leave ``unary_opts`` alone.
|
||||
class CohereCommand3ReasoningParser(BaseCohereCommandReasoningParser):
|
||||
def __init__(self, tokenizer: TokenizerLike, *args, **kwargs):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
*args,
|
||||
streaming_opts=(
|
||||
PyFilterOptions().cmd3().no_tools().stream_non_grounded_answer()
|
||||
),
|
||||
streaming_opts=PyFilterOptions().cmd3().no_tools(),
|
||||
unary_opts=PyFilterOptions().cmd3().no_tools(),
|
||||
**kwargs,
|
||||
)
|
||||
@@ -712,9 +697,7 @@ class CohereCommand4ReasoningParser(BaseCohereCommandReasoningParser):
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
*args,
|
||||
streaming_opts=(
|
||||
PyFilterOptions().cmd4().no_tools().stream_non_grounded_answer()
|
||||
),
|
||||
streaming_opts=PyFilterOptions().cmd4().no_tools(),
|
||||
unary_opts=PyFilterOptions().cmd4().no_tools(),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
+131
-6
@@ -157,6 +157,40 @@ MESSAGES_CITATIONS_KEY = "_messages_citations"
|
||||
POSITION_TO_SOURCE_KEY = "_position_to_source"
|
||||
|
||||
|
||||
# Reserved ``chat_template_kwargs`` key carrying the original Cohere v2
|
||||
# tool-message content, keyed by request-message index. Populated by
|
||||
# :meth:`CohereServingChatV2._apply_cohere_template_kwargs` and consumed
|
||||
# by :func:`_conversation_to_melody_messages`, which runs each block
|
||||
# through :func:`_v2_tool_content_to_melody_block` to produce the
|
||||
# melody-shape content list the renderer emits.
|
||||
#
|
||||
# Motivation: OpenAI's ``ChatCompletionContentPartParam`` union (in
|
||||
# ``vllm/entrypoints/chat_utils.py``) only allows ``text`` / ``image_url``
|
||||
# / ``audio`` / ``video`` / ``file`` content parts on tool messages, so
|
||||
# ``{"type": "document", "document": {...}}`` blocks -- valid in Cohere
|
||||
# v2 -- get rejected by vLLM's shared chat validator before they can
|
||||
# reach the renderer.
|
||||
# ``CohereServingChatV2._convert_tool_message`` works around that by
|
||||
# rewriting each content block into a plain ``text`` part (with document
|
||||
# blocks JSON-serialized inline), but that flattening loses the
|
||||
# structural distinction the cohere renderer needs to shape tool-result
|
||||
# payloads for cmd3/cmd4. The serving layer therefore forwards the
|
||||
# original v2 content blocks verbatim (as plain dicts) under this key,
|
||||
# and the renderer owns the v2-shape → melody-shape mapping.
|
||||
#
|
||||
# Value type: ``dict[int, list[dict[str, Any]]]``. Each list entry is a
|
||||
# ``model_dump``'d ``TextToolContent`` / ``DocumentToolContent`` block
|
||||
# (e.g. ``{"type": "text", "text": ...}`` or ``{"type": "document",
|
||||
# "document": {"id": ..., "data": {...}}}``). Index is the position of
|
||||
# the tool message in the request's ``messages`` list, which -- since
|
||||
# ``_convert_messages`` is 1-to-1 -- also matches its position in the
|
||||
# ``ConversationMessage`` list handed to the renderer. Leading
|
||||
# underscore signals "internal, not part of the public
|
||||
# ``chat_template_kwargs`` surface" -- clients should not set this key
|
||||
# directly.
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY = "_tool_message_v2_content"
|
||||
|
||||
|
||||
# Keys this renderer interprets directly from ``chat_template_kwargs`` and
|
||||
# maps onto typed melody render-config fields. Everything *not* in this
|
||||
# set is forwarded verbatim to melody as ``additional_template_fields``
|
||||
@@ -196,6 +230,7 @@ _RENDERER_CONSUMED_KEYS = frozenset(
|
||||
"platform_instruction",
|
||||
MESSAGES_CITATIONS_KEY,
|
||||
POSITION_TO_SOURCE_KEY,
|
||||
TOOL_MESSAGE_V2_CONTENT_KEY,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -377,6 +412,69 @@ def _content_blocks(content: Any) -> list[dict[str, Any]]:
|
||||
return blocks
|
||||
|
||||
|
||||
def _v2_tool_content_to_melody_block(block: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Convert one Cohere v2 tool-message content block into a melody
|
||||
``document`` content block.
|
||||
|
||||
Cohere's chat v2 admits two shapes for tool-message content parts:
|
||||
|
||||
* ``TextToolContent`` -- ``{"type": "text", "text": "..."}``.
|
||||
* ``DocumentToolContent`` -- ``{"type": "document", "document":
|
||||
{"id"?: str, "data": {...}}}``.
|
||||
|
||||
Both are emitted as melody ``document`` blocks, mirroring the cohere
|
||||
api, which routes every tool content part through the same
|
||||
"tool output as document" conversion. The rules:
|
||||
|
||||
* Text that parses to a JSON object becomes that object directly
|
||||
(so structured tool output flows through as first-class fields).
|
||||
* Text that doesn't parse as an object -- empty, non-JSON,
|
||||
or JSON scalar/array -- gets wrapped as ``{"content": <text>}``
|
||||
so templates have a stable field name to look up.
|
||||
* ``document`` blocks unwrap ``document.data`` (a flat map) into
|
||||
the melody payload; ``document.id`` is intentionally dropped
|
||||
here because outbound citation binding tracks it separately via
|
||||
``POSITION_TO_SOURCE_KEY``.
|
||||
|
||||
Only the JSON-object case actually changes what gets rendered. The
|
||||
cmd3/cmd4 templates already wrap a bare ``text`` content block as
|
||||
``{"content": <text>}`` themselves, so routing plain text through
|
||||
here is a no-op on the prompt -- it is the JSON-object unwrap that
|
||||
promotes structured tool output from an escaped string blob into
|
||||
individually addressable (and citable) result fields.
|
||||
|
||||
Note the payload is the *flat* field map: melody's ``Content``
|
||||
carries ``document`` as a plain map, so re-nesting it under a
|
||||
``data`` key would render a spurious ``{"data": {...}}`` wrapper
|
||||
into the tool-result block.
|
||||
|
||||
Returns ``None`` for unrecognised shapes; the caller should skip
|
||||
those blocks (the citation-numbering walk in
|
||||
``CohereServingChatV2._walk_citable_positions`` advances its own
|
||||
cursor over unknown blocks, so slot alignment is preserved).
|
||||
"""
|
||||
block_type = block.get("type")
|
||||
if block_type == "text":
|
||||
text = block.get("text") or ""
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
parsed = None
|
||||
if isinstance(parsed, dict):
|
||||
payload: dict[str, Any] = dict(parsed)
|
||||
else:
|
||||
payload = {"content": text}
|
||||
return {"type": MelodyContentType.DOCUMENT, "document": payload}
|
||||
if block_type == "document":
|
||||
doc = block.get("document")
|
||||
if not isinstance(doc, dict):
|
||||
return None
|
||||
data = doc.get("data")
|
||||
payload = dict(data) if isinstance(data, dict) else {}
|
||||
return {"type": MelodyContentType.DOCUMENT, "document": payload}
|
||||
return None
|
||||
|
||||
|
||||
def _document_to_melody(doc: Any) -> dict[str, Any]:
|
||||
"""Coerce a Cohere v2 document into melody's ``Document`` (dict) shape."""
|
||||
match doc:
|
||||
@@ -421,6 +519,7 @@ def _tool_to_melody(tool: Any) -> dict[str, Any]:
|
||||
def _conversation_to_melody_messages(
|
||||
conversation: list[ConversationMessage],
|
||||
messages_citations: dict[int, list[dict[str, Any]]] | None = None,
|
||||
tool_message_v2_content: dict[int, list[dict[str, Any]]] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Convert vLLM ``ConversationMessage``s into melody's message dict shape.
|
||||
|
||||
@@ -429,17 +528,42 @@ def _conversation_to_melody_messages(
|
||||
``CohereServingChatV2`` under the
|
||||
:data:`MESSAGES_CITATIONS_KEY` chat_template_kwargs entry. Values
|
||||
must already be in melody's ``FilterCitation`` dict shape (see
|
||||
``CohereServingChatV2._sdk_citation_to_melody``). Index is the
|
||||
position in the *input* ``conversation`` list, which currently maps
|
||||
1-to-1 with ``CohereChatV2Request.messages`` for text-only Cohere
|
||||
requests. This mapping breaks silently if ``parse_chat_messages``
|
||||
ever expands a single input message into multiple entries (e.g.
|
||||
``CohereServingChatV2._sdk_citation_to_melody``).
|
||||
|
||||
``tool_message_v2_content`` is an optional index-keyed lookup of
|
||||
the original Cohere v2 tool-message content blocks (as plain
|
||||
dicts), populated by ``CohereServingChatV2`` under the
|
||||
:data:`TOOL_MESSAGE_V2_CONTENT_KEY` entry. When set for a given
|
||||
index, its blocks -- run through
|
||||
:func:`_v2_tool_content_to_melody_block` to apply the v2 → melody
|
||||
mapping -- replace the block list this function would otherwise
|
||||
derive from ``msg["content"]``. This lets the renderer own the
|
||||
v2-shape → melody-shape conversion for the structured ``document``
|
||||
blocks cohere v2 allows on tool messages (which don't survive
|
||||
OpenAI's content-parts validator on the way in, so
|
||||
``_convert_tool_message`` rewrites them into ``text`` parts).
|
||||
|
||||
Both index lookups key by position in the *input* ``conversation``
|
||||
list, which currently maps 1-to-1 with
|
||||
``CohereChatV2Request.messages`` for text-only Cohere requests.
|
||||
This mapping breaks silently if ``parse_chat_messages`` ever
|
||||
expands a single input message into multiple entries (e.g.
|
||||
multi-modal splitting), which is not the case today.
|
||||
"""
|
||||
out: list[dict[str, Any]] = []
|
||||
for i, msg in enumerate(conversation):
|
||||
role = _role_to_melody(msg.get("role", "user"))
|
||||
content_blocks = _content_blocks(msg.get("content"))
|
||||
if (
|
||||
tool_message_v2_content is not None
|
||||
and (v2_blocks := tool_message_v2_content.get(i)) is not None
|
||||
):
|
||||
content_blocks = []
|
||||
for v2_block in v2_blocks:
|
||||
mapped = _v2_tool_content_to_melody_block(v2_block)
|
||||
if mapped is not None:
|
||||
content_blocks.append(mapped)
|
||||
else:
|
||||
content_blocks = _content_blocks(msg.get("content"))
|
||||
|
||||
# Treat reasoning content as a thinking block on assistant turns
|
||||
# so multi-turn reasoning is preserved in the rendered prompt.
|
||||
@@ -513,6 +637,7 @@ def _build_render_config(
|
||||
"messages": _conversation_to_melody_messages(
|
||||
conversation,
|
||||
chat_template_kwargs.get(MESSAGES_CITATIONS_KEY),
|
||||
chat_template_kwargs.get(TOOL_MESSAGE_V2_CONTENT_KEY),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user