Fix Cohere ChatV2 citation and tool handling issues (#52175)
Signed-off-by: Andrew Berneshawi <andrewberneshawi@cohere.com>
This commit is contained in:
@@ -655,3 +655,123 @@ class TestMelodySourceResolution:
|
||||
assert out is not None
|
||||
assert out[0].type == "THINKING_CONTENT"
|
||||
assert out[0].sources[0].id == "d0"
|
||||
|
||||
|
||||
class TestParserStreamingEndToEnd:
|
||||
"""End-to-end streaming shape: raw model output text is tokenized
|
||||
with the mock byte-level tokenizer and fed one token at a time
|
||||
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
|
||||
We collect the ordered sequence of ``DeltaMessage``s the parser
|
||||
hands back -- i.e. what the client would receive on the wire --
|
||||
and pin the exact shape.
|
||||
|
||||
Combines the three flavors that reach ``delta``: a thinking block
|
||||
(populates ``reasoning``), a text block (populates ``content``),
|
||||
and a resolved citation (populates ``citations``) referencing
|
||||
``(start, end)`` offsets into the already-streamed content.
|
||||
"""
|
||||
|
||||
_MODEL_OUTPUT = (
|
||||
"<|START_THINKING|>Let me check.<|END_THINKING|>"
|
||||
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _drive_parser(
|
||||
parser: CohereCommand3ReasoningParser, text: str, tok: MockCohereTokenizer
|
||||
) -> list[Any]:
|
||||
"""Feed ``text`` through the parser one token at a time
|
||||
(byte-level) and return the ordered list of non-``None``
|
||||
``DeltaMessage``s.
|
||||
"""
|
||||
deltas: list[Any] = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
for token_str in _token_deltas(tok, text):
|
||||
current_text = previous_text + token_str
|
||||
current_token_ids = previous_token_ids + [0]
|
||||
delta = parser.extract_reasoning_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=token_str,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=[0],
|
||||
)
|
||||
if delta is not None:
|
||||
deltas.append(delta)
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
return deltas
|
||||
|
||||
def test_full_wire_stream_shape(self, tokenizer):
|
||||
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
|
||||
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
|
||||
|
||||
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
|
||||
# installs on a request whose tool_call_index=0 tool result at
|
||||
# index 1 is the "France" document.
|
||||
position_to_source = {
|
||||
(0, 1): CitationSource(
|
||||
type="document",
|
||||
id="doc-paris",
|
||||
document={"id": "doc-paris", "title": "France"},
|
||||
),
|
||||
}
|
||||
|
||||
parser = CohereCommand3ReasoningParser(
|
||||
tokenizer,
|
||||
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
|
||||
)
|
||||
|
||||
deltas = self._drive_parser(parser, self._MODEL_OUTPUT, tokenizer)
|
||||
|
||||
# -- 1. Reasoning and content stream separately and reach
|
||||
# the wire only as their real payload. Framing tokens
|
||||
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
|
||||
# state machine and never produce a delta.
|
||||
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
|
||||
content_stream = [d.content for d in deltas if d.content is not None]
|
||||
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
|
||||
|
||||
assert "".join(reasoning_stream) == "Let me check."
|
||||
assert "".join(content_stream) == "The capital is Paris."
|
||||
|
||||
# -- 2. Exactly one citation reaches the wire, resolved to the
|
||||
# source from ``position_to_source`` (not the raw
|
||||
# ``(bucket, idx)`` coordinates).
|
||||
assert len(citation_deltas) == 1
|
||||
(cite_delta,) = citation_deltas
|
||||
assert cite_delta.citations is not None
|
||||
assert len(cite_delta.citations) == 1
|
||||
cite = cite_delta.citations[0]
|
||||
assert cite.text == "Paris"
|
||||
# Char offsets are into the accumulated ``content``: "The
|
||||
# capital is " is 15 chars, "Paris" is 5.
|
||||
assert cite.start == 15
|
||||
assert cite.end == 20
|
||||
assert len(cite.sources) == 1
|
||||
assert cite.sources[0].id == "doc-paris"
|
||||
assert cite.sources[0].type == "document"
|
||||
|
||||
# -- 3. The citation's ``(start, end)`` offsets point into
|
||||
# bytes that were already emitted as content-deltas earlier
|
||||
# in the stream -- i.e. the citation is anchored to real
|
||||
# already-shipped content.
|
||||
cite_delta_idx = deltas.index(cite_delta)
|
||||
earlier_content = "".join(
|
||||
d.content for d in deltas[:cite_delta_idx] if d.content is not None
|
||||
)
|
||||
assert earlier_content[cite.start : cite.end] == "Paris"
|
||||
|
||||
# Everything after the citation delta is the trailing period.
|
||||
later_content = "".join(
|
||||
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
|
||||
)
|
||||
assert later_content == "."
|
||||
|
||||
# -- 4. Reasoning and content are never mixed on the same
|
||||
# delta: the parser flips modes on the ``<|END_THINKING|>``
|
||||
# boundary and never emits a delta with both fields set.
|
||||
for d in deltas:
|
||||
assert not (d.reasoning is not None and d.content is not None)
|
||||
|
||||
Reference in New Issue
Block a user