[Frontend] create unified Cohere parser (#56392)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Walter Beller-Morales
2026-09-12 04:04:46 -04:00
committed by GitHub
co-authored by Flora Feng
parent d406f09eb8
commit 756794a9a7
21 changed files with 1919 additions and 2139 deletions
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+18
View File
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from .utils import MockCohereTokenizer
@pytest.fixture(scope="package")
def tokenizer() -> MockCohereTokenizer:
return MockCohereTokenizer()
@pytest.fixture
def request_obj() -> ChatCompletionRequest:
return ChatCompletionRequest(messages=[], model="test-model")
+172
View File
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Citation source resolution and the streamed citation wire shape."""
from __future__ import annotations
from types import SimpleNamespace
from typing import Any
from vllm.parser.cohere_command import (
_melody_citations_to_vllm,
_melody_sources_to_vllm,
)
from .utils import make_parser, stream_parser
class TestMelodySourceResolution:
"""Pin the parser-side source resolution (see
``_melody_sources_to_vllm``). The parser receives melody's numeric
``(tool_call_index, tool_result_indices)`` addressing and resolves
it against a ``position_to_source`` map handed in by
``CohereServingChatV2._apply_cohere_template_kwargs`` -- the same
map that would otherwise live in the serving layer's resolver.
"""
@staticmethod
def _fake_melody_source(bucket: int, indices: list[int]) -> Any:
return SimpleNamespace(tool_call_index=bucket, tool_result_indices=indices)
def test_multi_index_source_fans_out(self):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
position_map: dict[tuple[int, int], CitationSource] = {
(0, 0): CitationSource(type="document", id="d0", document={"id": "d0"}),
(0, 1): CitationSource(type="document", id="d1", document={"id": "d1"}),
}
raw = [self._fake_melody_source(0, [0, 1])]
out = _melody_sources_to_vllm(raw, position_map)
assert [s.id for s in out] == ["d0", "d1"]
# Verify type / payload were plumbed through, not just the id.
assert out[0].type == "document"
assert out[0].document == {"id": "d0"}
def test_unresolvable_position_skipped(self):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
position_map: dict[tuple[int, int], CitationSource] = {
(0, 0): CitationSource(type="document", id="d0"),
}
raw = [self._fake_melody_source(9, [0])]
assert _melody_sources_to_vllm(raw, position_map) == []
def test_missing_position_map_drops_all_sources(self):
# A parser instance without a position map (parser wired
# outside of ``CohereServingChatV2``) can't attribute anything,
# so every source is dropped. Callers downstream will see the
# citation with empty ``sources`` and drop it entirely.
raw = [self._fake_melody_source(0, [0])]
assert _melody_sources_to_vllm(raw, None) == []
def test_citations_pass_through_is_thinking_tag(self):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
position_map: dict[tuple[int, int], CitationSource] = {
(0, 0): CitationSource(type="document", id="d0"),
}
raw = [
SimpleNamespace(
start_index=0,
end_index=5,
text="hello",
is_thinking=True,
sources=[self._fake_melody_source(0, [0])],
)
]
out = _melody_citations_to_vllm(raw, position_map)
assert out is not None
assert out[0].type == "THINKING_CONTENT"
assert out[0].sources[0].id == "d0"
class TestParserStreamingEndToEnd:
"""End-to-end streaming shape: raw model output text is tokenized
with the mock byte-level tokenizer and fed one token at a time
through :meth:`CohereCommand3ReasoningParser.extract_reasoning_streaming`.
We collect the ordered sequence of ``DeltaMessage``s the parser
hands back -- i.e. what the client would receive on the wire --
and pin the exact shape.
Combines the three flavors that reach ``delta``: a thinking block
(populates ``reasoning``), a text block (populates ``content``),
and a resolved citation (populates ``citations``) referencing
``(start, end)`` offsets into the already-streamed content.
"""
_MODEL_OUTPUT = (
"<|START_THINKING|>Let me check.<|END_THINKING|>"
"<|START_RESPONSE|>The capital is <co>Paris</co: 0:[1]>.<|END_RESPONSE|>"
)
def test_full_wire_stream_shape(self, tokenizer, request_obj):
from vllm.entrypoints.cohere.cohere_chat_message import CitationSource
from vllm.renderers.cohere import POSITION_TO_SOURCE_KEY
# Mimic what ``CohereServingChatV2._apply_cohere_template_kwargs``
# installs on a request whose tool_call_index=0 tool result at
# index 1 is the "France" document.
position_to_source = {
(0, 1): CitationSource(
type="document",
id="doc-paris",
document={"id": "doc-paris", "title": "France"},
),
}
parser = make_parser(
tokenizer,
"cohere_command3",
chat_template_kwargs={POSITION_TO_SOURCE_KEY: position_to_source},
)
deltas = stream_parser(parser, request_obj, tokenizer, self._MODEL_OUTPUT)
# -- 1. Reasoning and content stream separately and reach
# the wire only as their real payload. Framing tokens
# (``<|START_THINKING|>`` etc.) are consumed by the parser's
# state machine and never produce a delta.
reasoning_stream = [d.reasoning for d in deltas if d.reasoning is not None]
content_stream = [d.content for d in deltas if d.content is not None]
citation_deltas = [d for d in deltas if getattr(d, "citations", None)]
assert "".join(reasoning_stream) == "Let me check."
assert "".join(content_stream) == "The capital is Paris."
# -- 2. Exactly one citation reaches the wire, resolved to the
# source from ``position_to_source`` (not the raw
# ``(bucket, idx)`` coordinates).
assert len(citation_deltas) == 1
(cite_delta,) = citation_deltas
assert cite_delta.citations is not None
assert len(cite_delta.citations) == 1
cite = cite_delta.citations[0]
assert cite.text == "Paris"
# Char offsets are into the accumulated ``content``: "The
# capital is " is 15 chars, "Paris" is 5.
assert cite.start == 15
assert cite.end == 20
assert len(cite.sources) == 1
assert cite.sources[0].id == "doc-paris"
assert cite.sources[0].type == "document"
# -- 3. The citation's ``(start, end)`` offsets point into
# bytes that were already emitted as content-deltas earlier
# in the stream -- i.e. the citation is anchored to real
# already-shipped content.
cite_delta_idx = deltas.index(cite_delta)
earlier_content = "".join(
d.content for d in deltas[:cite_delta_idx] if d.content is not None
)
assert earlier_content[cite.start : cite.end] == "Paris"
# Everything after the citation delta is the trailing period.
later_content = "".join(
d.content for d in deltas[cite_delta_idx + 1 :] if d.content is not None
)
assert later_content == "."
# -- 4. Reasoning and content are never mixed on the same
# delta: the parser flips modes on the ``<|END_THINKING|>``
# boundary and never emits a delta with both fields set.
for d in deltas:
assert not (d.reasoning is not None and d.content is not None)
+203
View File
@@ -0,0 +1,203 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reasoning extraction, reasoning-end gating and parser selection for the
unified Cohere Command parser."""
from __future__ import annotations
from dataclasses import dataclass
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser import ParserManager
from vllm.reasoning.cohere_command_reasoning_parser import (
CohereCommand3ReasoningParser,
CohereCommand4ReasoningParser,
)
from .utils import make_parser, stream_parser
@dataclass
class ReasoningCase:
parser_name: str
model_output: str
expected_reasoning: str | None
expected_content: str | None
REASONING_CASES = [
pytest.param(
ReasoningCase(
parser_name="cohere_command3",
model_output="""\
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
[
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
]
<|END_ACTION|>""",
expected_reasoning="i will call foo with query1",
expected_content=None,
),
id="cmd3-single_tool_call",
),
pytest.param(
ReasoningCase(
parser_name="cohere_command4",
model_output="""\
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
[
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
]
<|END_ACTION|>""",
expected_reasoning="i will call foo with query1",
expected_content=None,
),
id="cmd4-single_tool_call",
),
pytest.param(
ReasoningCase(
parser_name="cohere_command3",
model_output="""\
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
expected_reasoning="This is a rainbow emoji: 🌈",
expected_content="foo bar",
),
id="cmd3-citations_with_emoji",
),
pytest.param(
ReasoningCase(
parser_name="cohere_command4",
model_output="""\
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
expected_reasoning="This is a rainbow emoji: 🌈",
expected_content="foo bar",
),
id="cmd4-citations_with_emoji",
),
]
@pytest.mark.parametrize("case", REASONING_CASES)
class TestExtractReasoning:
def test_nonstreaming(self, tokenizer, request_obj, case: ReasoningCase):
parser = make_parser(tokenizer, case.parser_name)
reasoning, content, tool_calls = parser.parse(
case.model_output, request_obj, enable_auto_tools=True
)
assert reasoning == case.expected_reasoning
assert content == case.expected_content
# No tools on the request, so any tool call is dropped, not surfaced.
assert not tool_calls
def test_streaming(self, tokenizer, request_obj, case: ReasoningCase):
parser = make_parser(tokenizer, case.parser_name)
deltas = stream_parser(parser, request_obj, tokenizer, case.model_output)
reasoning = "".join(d.reasoning for d in deltas if d.reasoning) or None
content = "".join(d.content for d in deltas if d.content) or None
assert reasoning == case.expected_reasoning
assert content == case.expected_content
assert all(not d.tool_calls for d in deltas)
class TestFramingTokensStripped:
@pytest.mark.parametrize(
("parser_name", "content_tags"),
[
pytest.param(
"cohere_command4", ("<|START_TEXT|>", "<|END_TEXT|>"), id="cmd4"
),
pytest.param(
"cohere_command3", ("<|START_RESPONSE|>", "<|END_RESPONSE|>"), id="cmd3"
),
],
)
@pytest.mark.parametrize("chunk_size", [1, 4], ids=["per_token", "batched"])
@pytest.mark.parametrize("with_tools", [False, True], ids=["no_tools", "tools"])
def test_content_framing_tokens_stripped(
self, tokenizer, parser_name, content_tags, chunk_size, with_tools
):
parser = make_parser(tokenizer, parser_name)
tools = [{"type": "function", "function": {"name": "foo"}}]
request = ChatCompletionRequest(
messages=[], model="test-model", tools=tools if with_tools else None
)
parser.adjust_request(request)
start_tag, end_tag = content_tags
generation = (
f"<|START_THINKING|>Think deeply. The user greets us.<|END_THINKING|>"
f"{start_tag}I'm doing well, thank you{end_tag}"
)
deltas = stream_parser(parser, request, tokenizer, generation, chunk_size)
assert "".join(d.reasoning or "" for d in deltas) == (
"Think deeply. The user greets us."
)
assert not any(d.tool_calls for d in deltas)
assert "".join(d.content or "" for d in deltas) == "I'm doing well, thank you"
class TestIsReasoningEnd:
@pytest.mark.parametrize(
"parser_cls",
[CohereCommand3ReasoningParser, CohereCommand4ReasoningParser],
ids=["cmd3", "cmd4"],
)
def test_is_reasoning_end(self, tokenizer, parser_cls):
parser = parser_cls(tokenizer)
start_id = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
end_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
chatbot_id = tokenizer.convert_tokens_to_ids("<|CHATBOT_TOKEN|>")
content_ids = [99, 100]
# Generation-only tokens have no chatbot marker, so the whole sequence
# is considered.
assert parser.is_reasoning_end([end_id])
assert parser.is_reasoning_end([start_id, *content_ids, end_id])
assert not parser.is_reasoning_end([start_id, *content_ids])
# Full prompt/history tokens are scoped to the latest chatbot marker,
# so stray thinking tokens from the preamble or previous turns are ignored.
assert not parser.is_reasoning_end([start_id, end_id, chatbot_id, *content_ids])
assert parser.is_reasoning_end(
[start_id, end_id, chatbot_id, start_id, *content_ids, end_id]
)
@pytest.mark.parametrize("parser_name", ["cohere_command3", "cohere_command4"])
def test_count_reasoning_tokens(tokenizer, parser_name):
parser = make_parser(tokenizer, parser_name)
start = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
end = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
assert parser.count_reasoning_tokens([99, start, 11, 12, end, 100]) == 2
assert parser.count_reasoning_tokens([end, start, 1, start, 2, end, 3, end]) == 3
assert parser.count_reasoning_tokens([start, 1, 2]) == 2
assert parser.count_reasoning_tokens([1, 2, end]) == 0
class TestParserSelection:
@pytest.mark.parametrize("with_tool_parser", [True, False])
def test_adjust_request_keeps_framing_tokens(self, tokenizer, with_tool_parser):
"""Melody needs the special tokens, with or without a tool parser."""
cls = ParserManager.get_parser(
"cohere_command4" if with_tool_parser else None,
"cohere_command4",
enable_auto_tools=True,
)
request = ChatCompletionRequest(messages=[], model="m")
assert request.skip_special_tokens
cls(tokenizer).adjust_request(request)
assert request.skip_special_tokens is False
def test_non_cohere_selection_does_not_import_melody_parser(self, monkeypatch):
"""``cohere_melody`` is optional; other parsers must not depend on it."""
import sys
monkeypatch.setitem(sys.modules, "vllm.parser.cohere_command", None)
assert ParserManager.get_parser("hermes", "deepseek_r1", True) is not None
+362
View File
@@ -0,0 +1,362 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""``adjust_request``: folding tools and JSON schemas into Cohere structural tags."""
from __future__ import annotations
import json
from collections import UserDict
from types import SimpleNamespace
import pytest
from pydantic import ValidationError
from vllm.entrypoints.generate.base.protocol import (
JsonSchemaResponseFormat,
ResponseFormat,
StructuralTagResponseFormat,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.cohere_command import (
CohereCommandParser,
_has_effective_tools,
_response_format_type,
_schema_dict_from_structured_outputs,
convert_schema_to_structural_tags,
)
from vllm.sampling_params import StructuredOutputsParams
from .utils import MockCohereTokenizer, make_parser
SCHEMA_A = {"type": "object", "properties": {"a": {"type": "string"}}}
SCHEMA_B = {"type": "object", "properties": {"b": {"type": "number"}}}
GET_WEATHER_TOOL = {
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
VALID_STRUCTURAL_TAG = {
"type": "structural_tag",
"format": {
"type": "triggered_tags",
"tags": [
{
"begin": "<tool>",
"content": {"type": "any_text"},
"end": "</tool>",
}
],
"triggers": ["<tool>"],
},
}
def _model_config(arch: str) -> SimpleNamespace:
return SimpleNamespace(
architecture=arch,
architectures=[arch],
hf_text_config=SimpleNamespace(architectures=[arch]),
)
def _make_chat_request(**kwargs) -> ChatCompletionRequest:
data = {"messages": [{"role": "user", "content": "hi"}], "model": "m"}
data.update(kwargs)
return ChatCompletionRequest.model_validate(data)
def _first_json_schema(tag_json: str) -> dict | None:
outer = json.loads(tag_json)
for t in (outer.get("format") or {}).get("tags") or []:
c = t.get("content") or {}
if c.get("type") == "json_schema":
js = c.get("json_schema")
return js if isinstance(js, dict) else None
return None
def _content_types(tag_json: str) -> set[str]:
outer = json.loads(tag_json)
out: set[str] = set()
for t in (outer.get("format") or {}).get("tags") or []:
ty = (t.get("content") or {}).get("type")
if isinstance(ty, str):
out.add(ty)
return out
@pytest.fixture(scope="module")
def parser(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
"""Parser configured with a supported Cohere architecture."""
return make_parser(
tokenizer, "cohere_command4", model_config=_model_config("Cohere2ForCausalLM")
)
@pytest.fixture(scope="module")
def parser_no_model_config(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
"""Parser with no ``model_config`` (cannot resolve architecture)."""
return make_parser(tokenizer, "cohere_command4")
@pytest.fixture(scope="module")
def parser_unsupported_arch(tokenizer: MockCohereTokenizer) -> CohereCommandParser:
"""Parser configured with an architecture that has no structural tag style."""
return make_parser(
tokenizer, "cohere_command4", model_config=_model_config("LlamaForCausalLM")
)
class TestAdjustRequestPassthrough:
def test_structured_outputs_structural_tag_not_modified(self, parser) -> None:
tag = json.dumps(VALID_STRUCTURAL_TAG)
r = _make_chat_request(structured_outputs={"structural_tag": tag})
o = parser.adjust_request(r)
assert o.structured_outputs.structural_tag == tag
def test_response_format_structural_tag_short_circuit(self, parser) -> None:
# ``ChatCompletionRequest`` validates ``response_format`` as a union;
# bare ``{"type": "structural_tag"}`` is invalid (use pydantic model).
rf = StructuralTagResponseFormat(
type="structural_tag",
format=VALID_STRUCTURAL_TAG["format"],
)
r = _make_chat_request(response_format=rf)
o = parser.adjust_request(r)
assert _response_format_type(o.response_format) == "structural_tag"
assert o.structured_outputs is None
class TestAdjustRequestNoOp:
def test_no_schema_no_tools(self, parser) -> None:
o = parser.adjust_request(_make_chat_request())
assert o.structured_outputs is None
assert o.response_format is None
def test_no_model_config(self, parser_no_model_config) -> None:
inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
)
o = parser_no_model_config.adjust_request(r)
assert o.response_format is not None
assert o.structured_outputs is None
class TestAdjustRequestUnsupportedArchitecture:
def test_json_schema_raises(self, parser_unsupported_arch) -> None:
inner = JsonSchemaResponseFormat(name="n", json_schema=SCHEMA_A)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
)
with pytest.raises(ValueError, match="does not support"):
parser_unsupported_arch.adjust_request(r)
class TestAdjustRequestFoldFromResponseFormat:
@pytest.mark.parametrize(
"response_format, expected_schema",
[
pytest.param(
ResponseFormat(
type="json_schema",
json_schema=JsonSchemaResponseFormat(
name="n", json_schema=SCHEMA_A
),
),
SCHEMA_A,
id="json_schema_pydantic",
),
pytest.param(
{
"type": "json_schema",
"json_schema": {"name": "n", "schema": SCHEMA_A},
},
SCHEMA_A,
id="json_schema_dict",
),
pytest.param(
{"type": "json_object"},
{"type": "object"},
id="json_object",
),
],
)
def test_response_format_cleared(
self, parser, response_format, expected_schema
) -> None:
r = _make_chat_request(response_format=response_format)
o = parser.adjust_request(r)
assert o.response_format is None
assert (
_first_json_schema(o.structured_outputs.structural_tag) == expected_schema
)
class TestHasEffectiveTools:
@pytest.mark.parametrize(
"tools, expected",
[
pytest.param(None, False, id="none"),
pytest.param([], False, id="empty_list"),
pytest.param(" ", False, id="blank_str"),
pytest.param(
[{"type": "function", "function": {"name": "f"}}],
True,
id="non_empty_list",
),
pytest.param('{"x": 1}', True, id="non_empty_str"),
],
)
def test_has_effective_tools(self, tools, expected) -> None:
assert _has_effective_tools(tools) is expected
def test_convert_schema_json_only_with_empty_tools_list(self) -> None:
tag = convert_schema_to_structural_tags(
schema=SCHEMA_B,
tools=[],
model_architecture="Cohere2ForCausalLM",
)
assert tag is not None
assert _first_json_schema(tag) == SCHEMA_B
class TestAdjustRequestFoldFromStructuredOutputs:
@pytest.mark.parametrize(
"structured_outputs, expected_schema",
[
pytest.param({"json": SCHEMA_B}, SCHEMA_B, id="json_dict"),
pytest.param({"json": json.dumps(SCHEMA_B)}, SCHEMA_B, id="json_string"),
pytest.param(
{"json_object": True}, {"type": "object"}, id="json_object_flag"
),
pytest.param(
StructuredOutputsParams(json=SCHEMA_B),
SCHEMA_B,
id="structured_outputs_dataclass",
),
pytest.param(
{"json": {"name": "n", "schema": SCHEMA_A}},
SCHEMA_A,
id="openai_wrapper_dict_unwrapped",
),
],
)
def test_structured_outputs_folded(
self, parser, structured_outputs, expected_schema
) -> None:
o = parser.adjust_request(
_make_chat_request(structured_outputs=structured_outputs),
)
assert (
_first_json_schema(o.structured_outputs.structural_tag) == expected_schema
)
def test_responses_request_default_empty_tools(self, parser) -> None:
"""``ResponsesRequest.tools`` defaults to ``[]``, not ``None``."""
r = ResponsesRequest.model_validate(
{
"input": "hi",
"model": "m",
"structured_outputs": {"json": SCHEMA_B},
}
)
assert r.tools == []
o = parser.adjust_request(r)
assert _first_json_schema(o.structured_outputs.structural_tag) == SCHEMA_B
def test_json_userdict_mapping_unwrapped(self) -> None:
inner = {"type": "object", "properties": {"u": {"type": "number"}}}
so = StructuredOutputsParams(json=UserDict(inner))
assert _schema_dict_from_structured_outputs(so) == inner
@pytest.mark.parametrize(
"json_value, match",
[
pytest.param("{not json}", "valid JSON", id="invalid_json_string"),
pytest.param(
json.dumps(["a", "b"]), "JSON object", id="non_object_json_string"
),
pytest.param(" ", "empty", id="empty_json_string"),
],
)
def test_structured_outputs_json_string_raises(
self, parser, json_value, match
) -> None:
with pytest.raises(ValueError, match=match):
parser.adjust_request(
_make_chat_request(structured_outputs={"json": json_value}),
)
@pytest.mark.parametrize(
"construct",
[
pytest.param(
lambda: _make_chat_request(structured_outputs={"json": [1, 2, 3]}),
id="chat_completion_request",
),
pytest.param(
lambda: StructuredOutputsParams(json=[1, 2, 3]), # type: ignore[arg-type]
id="structured_outputs_params",
),
],
)
def test_json_wrong_type_raises(self, construct) -> None:
"""Non-str / non-dict ``json`` fails at Pydantic validation."""
with pytest.raises(ValidationError):
construct()
class TestAdjustRequestPrecedence:
def test_response_format_over_structured_outputs_json(self, parser) -> None:
s_rf = {"type": "object", "properties": {"rf": {"type": "string"}}}
s_so = {"type": "object", "properties": {"so": {"type": "number"}}}
inner = JsonSchemaResponseFormat(name="n", json_schema=s_rf)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
structured_outputs={"json": s_so},
)
o = parser.adjust_request(r)
assert _first_json_schema(o.structured_outputs.structural_tag) == s_rf
class TestAdjustRequestTextPlusStructuredOutputs:
def test_text_response_format_preserved(self, parser) -> None:
sch = {"type": "object", "properties": {"k": {"type": "string"}}}
r = _make_chat_request(
response_format=ResponseFormat(type="text"),
structured_outputs={"json": sch},
)
o = parser.adjust_request(r)
assert o.response_format is not None
assert o.response_format.type == "text"
assert _first_json_schema(o.structured_outputs.structural_tag) == sch
class TestAdjustRequestTools:
def test_tools_only_command_a_grammar(self, parser) -> None:
o = parser.adjust_request(
_make_chat_request(tools=[GET_WEATHER_TOOL], tool_choice="auto"),
)
assert "grammar" in _content_types(o.structured_outputs.structural_tag)
def test_tools_plus_json_schema_both_kinds(self, parser) -> None:
inner = JsonSchemaResponseFormat(
name="n",
json_schema={"type": "object", "properties": {"r": {"type": "string"}}},
)
r = _make_chat_request(
response_format=ResponseFormat(type="json_schema", json_schema=inner),
tools=[GET_WEATHER_TOOL],
tool_choice="auto",
)
o = parser.adjust_request(r)
types = _content_types(o.structured_outputs.structural_tag)
assert "grammar" in types
assert "json_schema" in types
+288
View File
@@ -0,0 +1,288 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tool-call parsing for the unified Cohere Command parser.
The ``cohere_command3`` / ``cohere_command4`` tool parsers are registry shims;
``CohereCommandParser`` (resolved through ``ParserManager``) does the parsing.
"""
from __future__ import annotations
import json
from dataclasses import dataclass, field
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.abstract_parser import Parser
from .utils import MockCohereTokenizer, drive_parser, make_parser, token_deltas
SPECIAL_TOKEN_MARKERS = (
"<|START_THINKING|>",
"<|END_THINKING|>",
"<|START_RESPONSE|>",
"<|END_RESPONSE|>",
"<|START_ACTION|>",
"<|END_ACTION|>",
"<|START_TEXT|>",
"<|END_TEXT|>",
)
DUMMY_TOOLS = [
{"type": "function", "function": {"name": name, "parameters": {"type": "object"}}}
for name in ("foo", "bar")
]
# (registry name, answer framing tag)
VARIANTS = (("cohere_command3", "RESPONSE"), ("cohere_command4", "TEXT"))
@dataclass
class ExpectedToolCall:
id: str
name: str
arguments: dict
@dataclass
class ToolCallCase:
parser_name: str
model_output: str
expected_tool_calls: list[ExpectedToolCall] = field(default_factory=list)
expected_reasoning: str | None = None
expected_content: str | None = None
SINGLE_TOOL_CALL = """\
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
[
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
]
<|END_ACTION|>"""
MULTI_TOOL_CALL = """\
<|START_THINKING|>first I think about foo<|END_THINKING|><|START_ACTION|>
[
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}},
{"tool_call_id": "1", "tool_name": "bar", "parameters": {"x": 42}}
]
<|END_ACTION|>"""
CITATIONS_NO_TOOL_CALL = (
"<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>\n"
"<|START_{tag}|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_{tag}|>"
)
# (case id, model output template, expectations)
_CASE_TABLE = (
(
"single_tool_call",
SINGLE_TOOL_CALL,
dict(
expected_tool_calls=[ExpectedToolCall("0", "foo", {"query": "query1"})],
expected_reasoning="i will call foo with query1",
),
),
(
"citations_no_tool_calls",
CITATIONS_NO_TOOL_CALL,
dict(
expected_reasoning="This is a rainbow emoji: 🌈", expected_content="foo bar"
),
),
(
"multiple_tool_calls",
MULTI_TOOL_CALL,
dict(
expected_tool_calls=[
ExpectedToolCall("0", "foo", {"query": "query1"}),
ExpectedToolCall("1", "bar", {"x": 42}),
],
expected_reasoning="first I think about foo",
),
),
(
"reasoning_only",
"<|START_THINKING|>just think, no response<|END_THINKING|>",
dict(expected_reasoning="just think, no response"),
),
)
TOOL_CALL_CASES = [
pytest.param(
ToolCallCase(name, output.replace("{tag}", tag), **expected),
id=f"{name}-{case_id}",
)
for name, tag in VARIANTS
for case_id, output, expected in _CASE_TABLE
]
PARSER_NAMES = pytest.mark.parametrize("parser_name", [name for name, _ in VARIANTS])
def _tool_parser(tokenizer: MockCohereTokenizer, parser_name: str) -> Parser:
return make_parser(tokenizer, parser_name, DUMMY_TOOLS)
def _tool_request() -> ChatCompletionRequest:
return ChatCompletionRequest(
messages=[], model="test-model", tools=DUMMY_TOOLS, tool_choice="auto"
)
@dataclass
class StreamingResult:
tool_calls: dict[int, dict]
reasoning: str | None
content: str | None
def run_streaming(
tokenizer: MockCohereTokenizer,
parser_name: str,
model_output: str,
chunk_size: int = 1,
) -> StreamingResult:
deltas = drive_parser(
_tool_parser(tokenizer, parser_name),
_tool_request(),
token_deltas(tokenizer, model_output, chunk_size),
)
accumulated: dict[int, dict] = {}
for delta in deltas:
for tc in delta.tool_calls or []:
slot = accumulated.setdefault(
tc.index, {"id": "", "name": "", "arguments": ""}
)
if tc.id:
slot["id"] = tc.id
if tc.function and tc.function.name:
slot["name"] = tc.function.name
if tc.function and tc.function.arguments:
slot["arguments"] += tc.function.arguments
return StreamingResult(
tool_calls=accumulated,
reasoning="".join(d.reasoning for d in deltas if d.reasoning) or None,
content="".join(d.content for d in deltas if d.content) or None,
)
def run_nonstreaming(
tokenizer: MockCohereTokenizer, parser_name: str, model_output: str
) -> tuple[str | None, str | None, list]:
reasoning, content, tool_calls = _tool_parser(tokenizer, parser_name).parse(
model_output, _tool_request(), enable_auto_tools=True
)
return reasoning, content, tool_calls or []
@pytest.mark.parametrize("case", TOOL_CALL_CASES)
class TestExtractToolCalls:
def test_streaming(self, tokenizer: MockCohereTokenizer, case: ToolCallCase):
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
assert len(streamed.tool_calls) == len(case.expected_tool_calls)
for i, expected in enumerate(case.expected_tool_calls):
tc = streamed.tool_calls[i]
assert tc["id"] == expected.id
assert tc["name"] == expected.name
assert json.loads(tc["arguments"]) == expected.arguments
def test_streaming_reasoning(
self, tokenizer: MockCohereTokenizer, case: ToolCallCase
):
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
assert streamed.reasoning == case.expected_reasoning
def test_streaming_content(
self, tokenizer: MockCohereTokenizer, case: ToolCallCase
):
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
assert streamed.content == case.expected_content
def test_nonstreaming(self, tokenizer: MockCohereTokenizer, case: ToolCallCase):
reasoning, content, tool_calls = run_nonstreaming(
tokenizer, case.parser_name, case.model_output
)
assert reasoning == case.expected_reasoning
assert content == case.expected_content
assert len(tool_calls) == len(case.expected_tool_calls)
for actual, expected in zip(tool_calls, case.expected_tool_calls):
assert actual.name == expected.name
assert json.loads(actual.arguments) == expected.arguments
def test_streaming_nonstreaming_agree(
self, tokenizer: MockCohereTokenizer, case: ToolCallCase
):
streamed = run_streaming(tokenizer, case.parser_name, case.model_output)
_, _, tool_calls = run_nonstreaming(
tokenizer, case.parser_name, case.model_output
)
assert len(streamed.tool_calls) == len(tool_calls)
for i, actual in enumerate(tool_calls):
assert streamed.tool_calls[i]["name"] == actual.name
assert json.loads(streamed.tool_calls[i]["arguments"]) == json.loads(
actual.arguments
)
class TestSpeculativeDecodingMultiTokenDelta:
@PARSER_NAMES
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
def test_no_special_token_leak_in_streaming_deltas(
self, tokenizer: MockCohereTokenizer, parser_name: str, chunk_size: int
):
deltas = drive_parser(
_tool_parser(tokenizer, parser_name),
_tool_request(),
token_deltas(tokenizer, SINGLE_TOOL_CALL, chunk_size),
)
for delta in deltas:
fields: list[tuple[str, str | None]] = [
("reasoning", delta.reasoning),
("content", delta.content),
]
for tc in delta.tool_calls or []:
if tc.function:
fields.append(("tool_call.name", tc.function.name))
fields.append(("tool_call.arguments", tc.function.arguments))
for marker in SPECIAL_TOKEN_MARKERS:
for field_name, value in fields:
assert value is None or marker not in value, (
f"special token {marker!r} leaked into {field_name} "
f"with chunk_size={chunk_size} delta={delta!r}"
)
@PARSER_NAMES
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 6])
def test_multi_token_chunks_still_produce_correct_tool_call(
self, tokenizer: MockCohereTokenizer, parser_name: str, chunk_size: int
):
streamed = run_streaming(tokenizer, parser_name, SINGLE_TOOL_CALL, chunk_size)
assert len(streamed.tool_calls) == 1
tc = streamed.tool_calls[0]
assert tc["id"] == "0"
assert tc["name"] == "foo"
assert json.loads(tc["arguments"]) == {"query": "query1"}
class TestStreamingDeltaShape:
@PARSER_NAMES
def test_reasoning_and_tool_calls_are_separate_deltas(
self, tokenizer: MockCohereTokenizer, parser_name: str
):
deltas = drive_parser(
_tool_parser(tokenizer, parser_name),
_tool_request(),
token_deltas(tokenizer, SINGLE_TOOL_CALL),
)
for delta in deltas:
populated = [
delta.content is not None,
delta.reasoning is not None,
bool(delta.tool_calls),
]
assert sum(populated) == 1, (
"A single streaming delta must carry exactly one of "
f"content/reasoning/tool_calls, got {delta!r}"
)
assert any(d.reasoning is not None for d in deltas)
assert any(d.tool_calls for d in deltas)
+131
View File
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the unified Cohere Command parser tests."""
from __future__ import annotations
import itertools
import regex as re
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser import ParserManager
from vllm.parser.abstract_parser import Parser
_SPECIAL_TOKEN_RE = re.compile(r"(<\|[A-Z_]+\|>)")
REPLACEMENT_CHAR = "�"
class MockCohereTokenizer:
"""Byte-level stand-in for the Cohere tokenizer.
Text round-trips through UTF-8 bytes so a multi-byte character can split
across "tokens" (reproducing trailing U+FFFD buffering), while each
``<|...|>`` marker is a single id so token-id reasoning-end gating behaves
as with the real tokenizer.
"""
_SPECIAL_TOKEN_IDS = {
tok: 256 + i
for i, tok in enumerate(
(
"<|START_THINKING|>",
"<|END_THINKING|>",
"<|CHATBOT_TOKEN|>",
"<|START_RESPONSE|>",
"<|END_RESPONSE|>",
"<|START_TEXT|>",
"<|END_TEXT|>",
"<|START_ACTION|>",
"<|END_ACTION|>",
)
)
}
_ID_TO_SPECIAL_TOKEN = {v: k for k, v in _SPECIAL_TOKEN_IDS.items()}
def convert_tokens_to_ids(self, token: str) -> int:
return self._SPECIAL_TOKEN_IDS.get(token, 0)
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
ids: list[int] = []
for part in _SPECIAL_TOKEN_RE.split(text):
if part in self._SPECIAL_TOKEN_IDS:
ids.append(self._SPECIAL_TOKEN_IDS[part])
else:
ids.extend(part.encode("utf-8"))
return ids
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
out: list[str] = []
for special, run in itertools.groupby(
ids, self._ID_TO_SPECIAL_TOKEN.__contains__
):
if not special:
out.append(bytes(run).decode("utf-8", errors="replace"))
elif not skip_special_tokens:
out.extend(self._ID_TO_SPECIAL_TOKEN[i] for i in run)
return "".join(out)
def token_deltas(
tokenizer: MockCohereTokenizer, text: str, chunk_size: int = 1
) -> list[tuple[str, list[int]]]:
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
U+FFFD) into the next step as real streaming does."""
ids = tokenizer.encode(text)
deltas: list[tuple[str, list[int]]] = []
prev = ""
pending: list[int] = []
for start in range(0, len(ids), chunk_size):
end = start + chunk_size
pending.extend(ids[start:end])
current = tokenizer.decode(ids[:end])
if current.endswith(REPLACEMENT_CHAR) and end < len(ids):
continue
deltas.append((current[len(prev) :], pending))
prev, pending = current, []
return deltas
def make_parser(
tokenizer: MockCohereTokenizer,
name: str,
tools: list[dict] | None = None,
chat_template_kwargs: dict | None = None,
model_config=None,
) -> Parser:
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
name (``cohere_command3`` / ``cohere_command4``)."""
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
assert cls is not None
return cls(
tokenizer,
tools,
model_config=model_config,
chat_template_kwargs=chat_template_kwargs or {},
)
def drive_parser(
parser: Parser,
request: ChatCompletionRequest,
deltas: list[tuple[str, list[int]]],
) -> list[DeltaMessage]:
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
return the non-``None`` messages in order."""
out: list[DeltaMessage] = []
for i, (text, ids) in enumerate(deltas):
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
if delta is not None:
out.append(delta)
return out
def stream_parser(parser, request, tokenizer, text: str, chunk_size: int = 1):
"""Stream ``text`` through ``parser`` in ``chunk_size``-token deltas."""
return drive_parser(parser, request, token_deltas(tokenizer, text, chunk_size))