[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-203
View File
@@ -1,203 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reasoning extraction, reasoning-end gating and parser selection for the
unified Cohere Command parser."""
from __future__ import annotations
from dataclasses import dataclass
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser import ParserManager
from vllm.reasoning.cohere_command_reasoning_parser import (
CohereCommand3ReasoningParser,
CohereCommand4ReasoningParser,
)
from .utils import make_parser, stream_parser
@dataclass
class ReasoningCase:
parser_name: str
model_output: str
expected_reasoning: str | None
expected_content: str | None
REASONING_CASES = [
pytest.param(
ReasoningCase(
parser_name="cohere_command3",
model_output="""\
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
[
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
]
<|END_ACTION|>""",
expected_reasoning="i will call foo with query1",
expected_content=None,
),
id="cmd3-single_tool_call",
),
pytest.param(
ReasoningCase(
parser_name="cohere_command4",
model_output="""\
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
[
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
]
<|END_ACTION|>""",
expected_reasoning="i will call foo with query1",
expected_content=None,
),
id="cmd4-single_tool_call",
),
pytest.param(
ReasoningCase(
parser_name="cohere_command3",
model_output="""\
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
expected_reasoning="This is a rainbow emoji: 🌈",
expected_content="foo bar",
),
id="cmd3-citations_with_emoji",
),
pytest.param(
ReasoningCase(
parser_name="cohere_command4",
model_output="""\
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
expected_reasoning="This is a rainbow emoji: 🌈",
expected_content="foo bar",
),
id="cmd4-citations_with_emoji",
),
]
@pytest.mark.parametrize("case", REASONING_CASES)
class TestExtractReasoning:
def test_nonstreaming(self, tokenizer, request_obj, case: ReasoningCase):
parser = make_parser(tokenizer, case.parser_name)
reasoning, content, tool_calls = parser.parse(
case.model_output, request_obj, enable_auto_tools=True
)
assert reasoning == case.expected_reasoning
assert content == case.expected_content
# No tools on the request, so any tool call is dropped, not surfaced.
assert not tool_calls
def test_streaming(self, tokenizer, request_obj, case: ReasoningCase):
parser = make_parser(tokenizer, case.parser_name)
deltas = stream_parser(parser, request_obj, tokenizer, case.model_output)
reasoning = "".join(d.reasoning for d in deltas if d.reasoning) or None
content = "".join(d.content for d in deltas if d.content) or None
assert reasoning == case.expected_reasoning
assert content == case.expected_content
assert all(not d.tool_calls for d in deltas)
class TestFramingTokensStripped:
@pytest.mark.parametrize(
("parser_name", "content_tags"),
[
pytest.param(
"cohere_command4", ("<|START_TEXT|>", "<|END_TEXT|>"), id="cmd4"
),
pytest.param(
"cohere_command3", ("<|START_RESPONSE|>", "<|END_RESPONSE|>"), id="cmd3"
),
],
)
@pytest.mark.parametrize("chunk_size", [1, 4], ids=["per_token", "batched"])
@pytest.mark.parametrize("with_tools", [False, True], ids=["no_tools", "tools"])
def test_content_framing_tokens_stripped(
self, tokenizer, parser_name, content_tags, chunk_size, with_tools
):
parser = make_parser(tokenizer, parser_name)
tools = [{"type": "function", "function": {"name": "foo"}}]
request = ChatCompletionRequest(
messages=[], model="test-model", tools=tools if with_tools else None
)
parser.adjust_request(request)
start_tag, end_tag = content_tags
generation = (
f"<|START_THINKING|>Think deeply. The user greets us.<|END_THINKING|>"
f"{start_tag}I'm doing well, thank you{end_tag}"
)
deltas = stream_parser(parser, request, tokenizer, generation, chunk_size)
assert "".join(d.reasoning or "" for d in deltas) == (
"Think deeply. The user greets us."
)
assert not any(d.tool_calls for d in deltas)
assert "".join(d.content or "" for d in deltas) == "I'm doing well, thank you"
class TestIsReasoningEnd:
@pytest.mark.parametrize(
"parser_cls",
[CohereCommand3ReasoningParser, CohereCommand4ReasoningParser],
ids=["cmd3", "cmd4"],
)
def test_is_reasoning_end(self, tokenizer, parser_cls):
parser = parser_cls(tokenizer)
start_id = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
end_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
chatbot_id = tokenizer.convert_tokens_to_ids("<|CHATBOT_TOKEN|>")
content_ids = [99, 100]
# Generation-only tokens have no chatbot marker, so the whole sequence
# is considered.
assert parser.is_reasoning_end([end_id])
assert parser.is_reasoning_end([start_id, *content_ids, end_id])
assert not parser.is_reasoning_end([start_id, *content_ids])
# Full prompt/history tokens are scoped to the latest chatbot marker,
# so stray thinking tokens from the preamble or previous turns are ignored.
assert not parser.is_reasoning_end([start_id, end_id, chatbot_id, *content_ids])
assert parser.is_reasoning_end(
[start_id, end_id, chatbot_id, start_id, *content_ids, end_id]
)
@pytest.mark.parametrize("parser_name", ["cohere_command3", "cohere_command4"])
def test_count_reasoning_tokens(tokenizer, parser_name):
parser = make_parser(tokenizer, parser_name)
start = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
end = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
assert parser.count_reasoning_tokens([99, start, 11, 12, end, 100]) == 2
assert parser.count_reasoning_tokens([end, start, 1, start, 2, end, 3, end]) == 3
assert parser.count_reasoning_tokens([start, 1, 2]) == 2
assert parser.count_reasoning_tokens([1, 2, end]) == 0
class TestParserSelection:
@pytest.mark.parametrize("with_tool_parser", [True, False])
def test_adjust_request_keeps_framing_tokens(self, tokenizer, with_tool_parser):
"""Melody needs the special tokens, with or without a tool parser."""
cls = ParserManager.get_parser(
"cohere_command4" if with_tool_parser else None,
"cohere_command4",
enable_auto_tools=True,
)
request = ChatCompletionRequest(messages=[], model="m")
assert request.skip_special_tokens
cls(tokenizer).adjust_request(request)
assert request.skip_special_tokens is False
def test_non_cohere_selection_does_not_import_melody_parser(self, monkeypatch):
"""``cohere_melody`` is optional; other parsers must not depend on it."""
import sys
monkeypatch.setitem(sys.modules, "vllm.parser.cohere_command", None)
assert ParserManager.get_parser("hermes", "deepseek_r1", True) is not None
-131
View File
@@ -1,131 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared helpers for the unified Cohere Command parser tests."""
from __future__ import annotations
import itertools
import regex as re
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser import ParserManager
from vllm.parser.abstract_parser import Parser
_SPECIAL_TOKEN_RE = re.compile(r"(<\|[A-Z_]+\|>)")
REPLACEMENT_CHAR = "�"
class MockCohereTokenizer:
"""Byte-level stand-in for the Cohere tokenizer.
Text round-trips through UTF-8 bytes so a multi-byte character can split
across "tokens" (reproducing trailing U+FFFD buffering), while each
``<|...|>`` marker is a single id so token-id reasoning-end gating behaves
as with the real tokenizer.
"""
_SPECIAL_TOKEN_IDS = {
tok: 256 + i
for i, tok in enumerate(
(
"<|START_THINKING|>",
"<|END_THINKING|>",
"<|CHATBOT_TOKEN|>",
"<|START_RESPONSE|>",
"<|END_RESPONSE|>",
"<|START_TEXT|>",
"<|END_TEXT|>",
"<|START_ACTION|>",
"<|END_ACTION|>",
)
)
}
_ID_TO_SPECIAL_TOKEN = {v: k for k, v in _SPECIAL_TOKEN_IDS.items()}
def convert_tokens_to_ids(self, token: str) -> int:
return self._SPECIAL_TOKEN_IDS.get(token, 0)
def get_vocab(self) -> dict[str, int]:
return {}
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
ids: list[int] = []
for part in _SPECIAL_TOKEN_RE.split(text):
if part in self._SPECIAL_TOKEN_IDS:
ids.append(self._SPECIAL_TOKEN_IDS[part])
else:
ids.extend(part.encode("utf-8"))
return ids
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
out: list[str] = []
for special, run in itertools.groupby(
ids, self._ID_TO_SPECIAL_TOKEN.__contains__
):
if not special:
out.append(bytes(run).decode("utf-8", errors="replace"))
elif not skip_special_tokens:
out.extend(self._ID_TO_SPECIAL_TOKEN[i] for i in run)
return "".join(out)
def token_deltas(
tokenizer: MockCohereTokenizer, text: str, chunk_size: int = 1
) -> list[tuple[str, list[int]]]:
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
U+FFFD) into the next step as real streaming does."""
ids = tokenizer.encode(text)
deltas: list[tuple[str, list[int]]] = []
prev = ""
pending: list[int] = []
for start in range(0, len(ids), chunk_size):
end = start + chunk_size
pending.extend(ids[start:end])
current = tokenizer.decode(ids[:end])
if current.endswith(REPLACEMENT_CHAR) and end < len(ids):
continue
deltas.append((current[len(prev) :], pending))
prev, pending = current, []
return deltas
def make_parser(
tokenizer: MockCohereTokenizer,
name: str,
tools: list[dict] | None = None,
chat_template_kwargs: dict | None = None,
model_config=None,
) -> Parser:
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
name (``cohere_command3`` / ``cohere_command4``)."""
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
assert cls is not None
return cls(
tokenizer,
tools,
model_config=model_config,
chat_template_kwargs=chat_template_kwargs or {},
)
def drive_parser(
parser: Parser,
request: ChatCompletionRequest,
deltas: list[tuple[str, list[int]]],
) -> list[DeltaMessage]:
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
return the non-``None`` messages in order."""
out: list[DeltaMessage] = []
for i, (text, ids) in enumerate(deltas):
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
if delta is not None:
out.append(delta)
return out
def stream_parser(parser, request, tokenizer, text: str, chunk_size: int = 1):
"""Stream ``text`` through ``parser`` in ``chunk_size``-token deltas."""
return drive_parser(parser, request, token_deltas(tokenizer, text, chunk_size))
-311
View File
@@ -1,311 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DeepSeek V3.2 parser engine semantics.
V3.2 uses the same DSML parameter format as V4 but wraps tool calls in
``<|DSML|function_calls>`` instead of ``<|DSML|tool_calls>`` and has
no reasoning (``<think>``/``</think>``) support.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_content,
collect_function_name,
collect_tool_arguments,
simulate_tool_streaming,
)
from vllm.parser.deepseek_v4 import (
DSML_INVOKE_END,
DSML_INVOKE_NAME_END,
DSML_INVOKE_PREFIX,
)
from vllm.parser.deepseek_v32 import (
DSML_FUNC_END,
DSML_FUNC_START,
DeepSeekV32Parser,
)
from vllm.parser.engine.parser_engine_config import ParserState
_PARAM_OPEN = '|DSML|parameter name="{name}" string="{is_str}">'
_PARAM_CLOSE = "</|DSML|parameter>"
def _param(name: str, is_str: str, value: str) -> str:
return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
def _invoke(name: str, *params: str) -> str:
body = "\n".join(params)
return (
f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
)
def _func_calls(*invocations: str) -> str:
body = "\n".join(invocations)
return f"{DSML_FUNC_START}\n{body}\n{DSML_FUNC_END}"
def _make_tool(name, properties):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": {
"type": "object",
"properties": properties,
},
},
)
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer({})
@pytest.fixture
def mock_request():
from unittest.mock import MagicMock
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
req = MagicMock(spec=ChatCompletionRequest)
req.tools = []
req.tool_choice = "auto"
return req
# ── Non-streaming extraction ────────────────────────────────────────
class TestNonStreaming:
def test_no_tool_call(self, mock_tokenizer, mock_request):
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls("Hello world", mock_request)
assert not result.tools_called
assert result.content == "Hello world"
def test_single_tool(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("get_weather", _param("city", "true", "SF")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"city": "SF"}
def test_parallel_tools(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("get_weather", _param("city", "true", "SF")),
_invoke("get_weather", _param("city", "true", "NYC")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called
assert len(result.tool_calls) == 2
assert json.loads(result.tool_calls[0].function.arguments) == {"city": "SF"}
assert json.loads(result.tool_calls[1].function.arguments) == {"city": "NYC"}
def test_content_before_tool_call(self, mock_tokenizer, mock_request):
text = "Let me check. " + _func_calls(
_invoke("search", _param("q", "true", "vllm")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called
assert result.content is not None
assert "Let me check" in result.content
def test_non_string_params_json_parsed(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke(
"toggle",
_param("enabled", "false", "true"),
_param("count", "false", "42"),
),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
args = json.loads(result.tool_calls[0].function.arguments)
assert args["enabled"] is True
assert args["count"] == 42
def test_wrapper_unwrapping(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
mock_request.tools = [tool]
text = _func_calls(
_invoke(
"get_weather",
_param("arguments", "false", '{"location":"Beijing"}'),
),
)
parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
result = parser.extract_tool_calls(text, mock_request)
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "Beijing"}
# ── Initial state ────────────────────────────────────────────────────
class TestInitialState:
def test_always_content(self, mock_tokenizer):
parser = DeepSeekV32Parser(mock_tokenizer)
cfg = parser.parser_engine_config
assert cfg.initial_state == ParserState.CONTENT
def test_ignores_thinking_kwargs(self, mock_tokenizer):
parser = DeepSeekV32Parser(
mock_tokenizer,
chat_template_kwargs={"thinking": True, "enable_thinking": True},
)
cfg = parser.parser_engine_config
assert cfg.initial_state == ParserState.CONTENT
# ── Streaming ────────────────────────────────────────────────────────
class TestStreaming:
def test_split_parameter_start_is_buffered(self, mock_tokenizer, mock_request):
chunks = [
DSML_FUNC_START,
f"{DSML_INVOKE_PREFIX}edit{DSML_INVOKE_NAME_END}",
f"<{_PARAM_OPEN.format(name='expr', is_str='true')}a<b><|DSML|parameter",
' name="date" string="true">tomorrow',
_PARAM_CLOSE,
DSML_INVOKE_END,
DSML_FUNC_END,
]
results = simulate_tool_streaming(
DeepSeekV32Parser(mock_tokenizer), mock_request, chunks
)
arguments = collect_tool_arguments(results)
assert "<|DSML|parameter" not in arguments
assert json.loads(arguments) == {"expr": "a<b>", "date": "tomorrow"}
def test_single_tool_streaming(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("get_weather", _param("city", "true", "SF")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
assert collect_function_name(results) == "get_weather"
args_json = collect_tool_arguments(results)
assert json.loads(args_json) == {"city": "SF"}
def test_content_before_tool_streaming(self, mock_tokenizer, mock_request):
text = "Checking... " + _func_calls(
_invoke("fn", _param("k", "true", "v")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
content = collect_content(results)
assert "Checking" in content
def test_parallel_tools_streaming(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("fn_a", _param("x", "true", "1")),
_invoke("fn_b", _param("y", "true", "2")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
names = []
for delta, _ in results:
if delta and delta.tool_calls:
for tc in delta.tool_calls:
if tc.function and tc.function.name:
names.append(tc.function.name)
assert "fn_a" in names
assert "fn_b" in names
def test_no_tool_content_only(self, mock_tokenizer, mock_request):
text = "Just some text, no tools."
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
content = collect_content(results)
assert "Just some text" in content
args = collect_tool_arguments(results)
assert args == ""
def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
mock_request.tools = [tool]
parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
chunks = [
DSML_FUNC_START,
_invoke(
"get_weather",
_param("arguments", "false", '{"location": "NYC"}'),
),
DSML_FUNC_END,
]
results = simulate_tool_streaming(parser, mock_request, chunks)
streamed_args = collect_tool_arguments(results)
final_delta, _ = results[-1]
finish_delta = parser.finish_streaming()
extracted = parser._build_extracted_result(final_delta, finish_delta)
assert extracted.tools_called is True
assert len(extracted.tool_calls) == 1
final_args = extracted.tool_calls[0].function.arguments
assert json.loads(final_args) == {"location": "NYC"}
assert '"arguments"' not in streamed_args
assert final_args.startswith(streamed_args)
def test_missing_invoke_end(self, mock_tokenizer, mock_request):
text = (
f"{DSML_FUNC_START}\n"
f"{DSML_INVOKE_PREFIX}fn{DSML_INVOKE_NAME_END}\n"
f"{_param('k', 'true', 'v')}\n"
f"{DSML_FUNC_END}"
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
assert collect_function_name(results) == "fn"
args = json.loads(collect_tool_arguments(results))
assert args == {"k": "v"}
class TestMissingFunctionCallsWrapper:
"""An invoke without the ``<|DSML|function_calls>`` wrapper is still a
tool call (V3.2 counterpart of #48931)."""
def test_non_streaming(self, mock_tokenizer, mock_request):
text = _invoke("fn", _param("k", "true", "v"))
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert result.tool_calls[0].function.name == "fn"
assert json.loads(result.tool_calls[0].function.arguments) == {"k": "v"}
assert result.content is None
def test_streaming(self, mock_tokenizer, mock_request):
text = "Sure.\n" + _invoke("fn", _param("k", "true", "v"))
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
assert collect_function_name(results) == "fn"
assert json.loads(collect_tool_arguments(results)) == {"k": "v"}
assert "DSML" not in collect_content(results)
File diff suppressed because it is too large Load Diff
-148
View File
@@ -1,148 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
MockTokenizer,
_test_request,
collect_output,
replay_streaming,
)
from vllm.parser.deepseek_v41 import deepseek_v41_config
from vllm.parser.parser_manager import ParserManager
CALLS = (
'\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
'<|DSML| parameter name="city" string="true">杭州</|DSML| parameter>\n'
'<|DSML| parameter name="count" string="false">42</|DSML| parameter>\n'
'</|DSML| invoke>\n<|DSML| invoke name="add">\n'
'<|DSML| parameter name="x" string="false">1.5</|DSML| parameter>\n'
'<|DSML| parameter name="y" string="false">2.25</|DSML| parameter>\n'
"</|DSML| invoke>\n</|DSML| calls>"
)
def tokenizer_for(text, special_calls):
vocab = {"<think>": 50, "</think>": 51}
if special_calls:
vocab |= {"<|DSML| calls>": 52, "</|DSML| calls>": 53}
tokens: list[tuple[int, str]] = []
while text:
special = next((marker for marker in vocab if text.startswith(marker)), None)
piece = special or text[0]
tokens.append((vocab[piece] if special else 100 + len(tokens), piece))
text = text[len(piece) :]
return MockTokenizer(vocab, tokens), tokens
def parser_for(tokenizer, controls):
cls = ParserManager.get_parser(
tool_parser_name="deepseek_v41",
reasoning_parser_name="deepseek_v41",
enable_auto_tools=True,
)
return cls(tokenizer, chat_template_kwargs=controls)
@pytest.mark.parametrize("thinking", [False, True])
@pytest.mark.parametrize("special_calls", [False, True])
@pytest.mark.parametrize("chunk_size", [1, 7, 10000])
def test_registered_adapters_parse_parallel_calls_across_chunks(
thinking,
special_calls,
chunk_size,
):
text = ("Plan.</think>" if thinking else "") + "Checking." + CALLS
tokenizer, tokens = tokenizer_for(text, special_calls)
parser = parser_for(tokenizer, {"thinking": thinking})
output = collect_output(
replay_streaming(
parser,
tokens,
chunk_size=chunk_size,
finished_on_last=True,
tools=DUMMY_TOOLS,
prompt_token_ids=[50 if thinking else 51],
)
)
assert output.reasoning == ("Plan." if thinking else "")
assert output.content.strip() == "Checking."
assert [call["name"] for call in output.tool_calls] == ["get_weather", "add"]
assert [json.loads(call["arguments"]) for call in output.tool_calls] == [
{"city": "杭州", "count": 42},
{"x": 1.5, "y": 2.25},
]
@pytest.mark.parametrize("thinking", [False, True])
def test_registered_adapters_parse_complete_output(thinking):
text = ("Plan.</think>" if thinking else "") + CALLS
tokenizer, tokens = tokenizer_for(text, False)
parser = parser_for(tokenizer, {"thinking": thinking})
reasoning, content, calls = parser.parse(
text,
_test_request(tools=DUMMY_TOOLS),
enable_auto_tools=True,
model_output_token_ids=[tid for tid, _ in tokens],
)
assert (reasoning or "") == ("Plan." if thinking else "")
assert not (content or "").strip()
assert [call.name for call in calls] == ["get_weather", "add"]
assert json.loads(calls[0].arguments) == {"city": "杭州", "count": 42}
@pytest.mark.parametrize(
("controls", "text", "expected_reasoning", "reasoning_tokens"),
[
({"thinking": False}, "12", "", 0),
({"enable_thinking": False}, "12", "", 0),
({"thinking": True, "reasoning_effort": "none"}, "12", "", 0),
({}, "</think>12", "", 0),
({}, "Plan.</think>12", "Plan.", 5),
],
)
def test_reasoning_adapter_controls_and_usage(
controls,
text,
expected_reasoning,
reasoning_tokens,
):
tokenizer, tokens = tokenizer_for(text, False)
parser = parser_for(tokenizer, controls)
output = collect_output(
replay_streaming(
parser,
tokens,
chunk_size=1,
finished_on_last=True,
)
)
assert output.reasoning == expected_reasoning
assert output.content == "12"
assert (
parser.reasoning_parser.count_reasoning_tokens([tid for tid, _ in tokens])
== reasoning_tokens
)
assert parser.is_reasoning_end([50, 100, 51])
assert not parser.is_reasoning_end([51, 100, 50])
def test_python_argument_conversion_and_partial_values():
converter = deepseek_v41_config().arg_converter
raw = (
'<|DSML| parameter name="object" string="false">'
'{"a": [true, null]}</|DSML| parameter>'
'<|DSML| parameter name="bad" string="false">[broken</|DSML| parameter>'
'<|DSML| parameter name="bare">7</|DSML| parameter>'
'<|DSML| parameter name="text" string="true"> a<b'
)
assert json.loads(converter(raw, True)) == {
"object": {"a": [True, None]},
"bad": "[broken",
"bare": 7,
"text": " a<b",
}
File diff suppressed because it is too large Load Diff
-230
View File
@@ -1,230 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reasoning-boundary tests for the GLM-4.7 parser.
GLM can transition straight from reasoning into a tool call without ever
emitting ``</think>``, so ``<tool_call>`` is an implicit reasoning
terminator. These cover that path plus the multi-turn prompts where an
earlier turn's markers must not be read as the current turn's state. The
streaming tests check that long tool arguments stream the same JSON as a
non-streaming parse, without converting the whole argument on every delta.
"""
import dataclasses
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.glm47_moe import (
THINK_END,
THINK_START,
TOOL_CALL_END,
TOOL_CALL_START,
Glm47MoeParser,
_glm47_arg_value_extender,
glm47_moe_config,
)
THINK_S, THINK_E = 1000, 1001
TOOL_S, TOOL_E = 1002, 1003
ASSISTANT, OBSERVATION, USER = 1004, 1005, 1006
TEXT = 42 # stand-in for an ordinary reasoning/content token
VOCAB = {
THINK_START: THINK_S,
THINK_END: THINK_E,
TOOL_CALL_START: TOOL_S,
TOOL_CALL_END: TOOL_E,
"<|assistant|>": ASSISTANT,
"<|observation|>": OBSERVATION,
"<|user|>": USER,
}
@pytest.fixture
def parser():
return Glm47MoeParser(make_mock_tokenizer(VOCAB))
@pytest.fixture
def no_thinking_parser():
return Glm47MoeParser(
make_mock_tokenizer(VOCAB),
chat_template_kwargs={"thinking": False},
)
class TestIsReasoningEnd:
def test_open_reasoning(self, parser):
assert not parser.is_reasoning_end([THINK_S, TEXT])
def test_think_end(self, parser):
assert parser.is_reasoning_end([THINK_S, TEXT, THINK_E, TEXT])
def test_tool_call_without_think_end(self, parser):
"""Reasoning -> tool call with no ``</think>`` still ends reasoning."""
assert parser.is_reasoning_end([THINK_S, TEXT, TOOL_S, TEXT])
def test_previous_turn_tool_call_ignored(self, parser):
"""A finished tool call from an earlier turn says nothing about the
turn currently being generated."""
prompt = [THINK_S, TEXT, TOOL_S, TEXT, TOOL_E, OBSERVATION, TEXT, ASSISTANT]
assert not parser.is_reasoning_end(prompt)
def test_previous_turn_think_end_ignored(self, parser):
prompt = [THINK_S, TEXT, THINK_E, TEXT, USER, TEXT, ASSISTANT]
assert not parser.is_reasoning_end(prompt)
def test_empty_input(self, parser):
assert not parser.is_reasoning_end([])
def test_thinking_disabled(self, no_thinking_parser):
assert no_thinking_parser.is_reasoning_end([THINK_S, TEXT])
class TestExtractContentIds:
def test_think_end_wins_over_later_tool_call(self, parser):
"""Content between ``</think>`` and a tool call must survive."""
ids = [THINK_S, TEXT, THINK_E, 20, 21, TOOL_S, 30, TOOL_E]
assert parser.extract_content_ids(ids) == [20, 21, TOOL_S, 30, TOOL_E]
def test_every_tool_call_after_think_end_kept(self, parser):
ids = [THINK_E, 20, TOOL_S, 30, TOOL_E, 21, TOOL_S, 31, TOOL_E]
assert parser.extract_content_ids(ids) == ids[1:]
def test_falls_back_to_tool_call(self, parser):
"""Without ``</think>``, content starts at the opener itself so the
tool parser still receives a well-formed call."""
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E]
assert parser.extract_content_ids(ids) == [TOOL_S, 30, TOOL_E]
def test_falls_back_to_first_tool_call_of_turn(self, parser):
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E, TOOL_S, 31, TOOL_E]
assert parser.extract_content_ids(ids) == ids[2:]
def test_previous_turn_tool_call_ignored(self, parser):
ids = [TOOL_S, 30, TOOL_E, OBSERVATION, TEXT, ASSISTANT, THINK_S, TEXT]
assert parser.extract_content_ids(ids) == ids
def test_no_markers_returns_input_ids(self, parser):
assert parser.extract_content_ids([20, 21]) == [20, 21]
def test_thinking_disabled(self, no_thinking_parser):
ids = [THINK_S, TEXT, TOOL_S]
assert no_thinking_parser.extract_content_ids(ids) == ids
WRITE_TOOL = ChatCompletionToolsParam(
type="function",
function={
"name": "Write",
"parameters": {
"type": "object",
"properties": {
"file_path": {"type": "string"},
"line_count": {"type": "integer"},
"content": {"type": "string"},
},
},
},
)
CALL_HEAD = (
f"{THINK_END}{TOOL_CALL_START}Write\n"
"<arg_key>file_path</arg_key><arg_value>/repo/app.py</arg_value>\n"
"<arg_key>line_count</arg_key><arg_value>120</arg_value>\n"
"<arg_key>content</arg_key><arg_value>"
)
CALL_TAIL = f"</arg_value>\n{TOOL_CALL_END}"
# Escapes, non-ASCII, and tag-like text that must stay inside the value.
CONTENT_LINE = 'if a < b > c: print("x\\y") # </div> </arg_ <arg_key>\t\u00e9\n'
def _pieces(text: str, size: int) -> list[str]:
return [text[i : i + size] for i in range(0, len(text), size)]
def _stream_arguments(parser, request, chunks: list[str]) -> str:
args = []
for i, chunk in enumerate(chunks):
delta = parser.parse_delta(chunk, [], request, finished=i == len(chunks) - 1)
for tool_call in (delta and delta.tool_calls) or []:
if tool_call.function and tool_call.function.arguments:
args.append(tool_call.function.arguments)
return "".join(args)
class TestStreamedToolArguments:
@pytest.mark.parametrize(
"tools, line_count",
[([], '"120"'), ([WRITE_TOOL], "120")],
ids=["no_schema", "schema"],
)
def test_long_call_streams_non_streaming_arguments(
self, mock_request, tools, line_count
):
mock_request.tools = tools
text = CALL_HEAD + CONTENT_LINE * 400 + CALL_TAIL
streamed = _stream_arguments(
Glm47MoeParser(make_mock_tokenizer(VOCAB)), mock_request, _pieces(text, 7)
)
result = Glm47MoeParser(make_mock_tokenizer(VOCAB)).extract_tool_calls(
text, mock_request
)
assert streamed == result.tool_calls[0].function.arguments
assert f'"line_count": {line_count},' in streamed
def test_conversions_do_not_grow_with_argument_length(self, mock_request):
"""The open value streams without re-running the converter."""
mock_request.tools = [WRITE_TOOL]
def streaming_conversions(lines: int) -> int:
calls = 0
base = glm47_moe_config()
def counting_converter(raw_args: str, partial: bool) -> str:
nonlocal calls
if partial:
calls += 1
return base.arg_converter(raw_args, partial)
parser = Glm47MoeParser(
make_mock_tokenizer(VOCAB),
parser_engine_config=dataclasses.replace(
base, arg_converter=counting_converter
),
)
chunks = [
*_pieces(CALL_HEAD, 4),
*_pieces(CONTENT_LINE * lines, 4),
*_pieces(CALL_TAIL, 4),
]
_stream_arguments(parser, mock_request, chunks)
return calls
assert streaming_conversions(lines=40) == streaming_conversions(lines=400)
def test_value_extender_stops_at_split_value_end(self):
extend = _glm47_arg_value_extender("<arg_key>k</arg_key><arg_value>x")
assert [extend(d) for d in ['a"\n', "</arg_", "value>"]] == [
'a\\"\\n',
"</arg_",
None,
]
@pytest.mark.parametrize(
"raw_args",
[
"<arg_key>k</arg_key><arg_value>x</arg_value>",
"<arg_key>k</arg_key><arg_value>1</arg_value>"
"<arg_key>j</arg_key><arg_value>2</arg_value>"
"<arg_key>k</arg_key><arg_value>3",
],
ids=["closed_value", "repeated_key_not_last"],
)
def test_no_value_extender_unless_open_value_is_last(self, raw_args):
assert _glm47_arg_value_extender(raw_args) is None
-234
View File
@@ -1,234 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Granite parser.
Granite emits tool calls as a JSON array following a marker
(``<|tool_call|>`` for 3.0, ``<tool_call>`` for 3.1) with no closing
terminal; the ``]`` ends the region and surrounding prose is content. The
engine's ``tool_call_body_array`` mode splits the array into one call per
element and ``_granite_arg_converter`` carves each element's ``arguments``.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite
from vllm.parser.granite import GraniteParser, _granite_arg_converter
TOOL_TOKEN = "<|tool_call|>"
TOOL_STRING = "<tool_call>"
_GRANITE_VOCAB = {TOOL_TOKEN: 49154}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_GRANITE_VOCAB)
@pytest.fixture
def parser(mock_tokenizer):
return GraniteParser(mock_tokenizer)
def _tokenize(text: str) -> list[tuple[int, str]]:
"""Tokenize like the real stream: ``<|tool_call|>`` is one special token,
plain text becomes one token per character."""
tokens: list[tuple[int, str]] = []
i = 0
while i < len(text):
if text.startswith(TOOL_TOKEN, i):
tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN))
i += len(TOOL_TOKEN)
else:
tokens.append((ord(text[i]), text[i]))
i += 1
return tokens
def _stream(parser, request, text: str, chunk_size: int):
tokens = _tokenize(text)
results = []
previous_text = ""
previous_token_ids: list[int] = []
for start in range(0, len(tokens), chunk_size):
batch = tokens[start : start + chunk_size]
delta_text = "".join(t for _, t in batch)
delta_token_ids = [tid for tid, _ in batch]
current_text = previous_text + delta_text
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=tuple(previous_token_ids),
current_token_ids=tuple(current_token_ids),
delta_token_ids=tuple(delta_token_ids),
request=request,
)
results.append(delta)
previous_text = current_text
previous_token_ids = current_token_ids
finish = parser.finish_streaming()
if finish is not None:
results.append(finish)
return results
def _collect_content(results) -> str:
return "".join(d.content for d in results if d and d.content)
def _collect_names(results) -> list[str]:
names: dict[int, str] = {}
for d in results:
if not (d and d.tool_calls):
continue
for tc in d.tool_calls:
if tc.function and tc.function.name:
names[tc.index] = names.get(tc.index, "") + tc.function.name
return [names[i] for i in sorted(names)]
def _collect_args(results) -> dict[int, str]:
args: dict[int, str] = {}
for d in results:
if not (d and d.tool_calls):
continue
for tc in d.tool_calls:
if tc.function and tc.function.arguments:
args[tc.index] = args.get(tc.index, "") + tc.function.arguments
return args
class TestArgConverter:
def test_complete_wrapper(self):
raw = '{"name": "get_weather", "arguments": {"city": "SF"}}'
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
def test_arguments_before_name(self):
raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}'
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
def test_partial_before_arguments(self):
assert _granite_arg_converter('{"name": "get_w', True) == ""
def test_partial_inside_arguments(self):
raw = '{"name": "x", "arguments": {"a": 1'
assert _granite_arg_converter(raw, True) == '{"a": 1'
def test_prefix_stability(self):
# Each growing prefix must extend the previous converter output.
full = '{"name": "x", "arguments": {"city": "Tokyo"}}'
prev = ""
for i in range(1, len(full) + 1):
out = _granite_arg_converter(full[:i], True)
assert out.startswith(prev) or prev.startswith(out) or out == ""
if out:
prev = out
def test_missing_arguments_defaults(self):
assert _granite_arg_converter('{"name": "x"}', False) == "{}"
assert _granite_arg_converter('{"name": "x"}', True) == ""
def test_non_object_arguments_rejected(self):
with pytest.raises(ValueError, match="JSON object"):
_granite_arg_converter('{"name": "x", "arguments": [1]}', False)
class TestNonStreaming:
def test_plain_text(self, parser, mock_request):
reasoning, content, tools = parser.parse("Just a reply.", mock_request)
assert reasoning is None
assert content == "Just a reply."
assert tools is None
def test_single_tool_token_marker(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
_, content, tools = parser.parse(text, mock_request)
assert content is None
assert [t.name for t in tools] == ["get_weather"]
assert json.loads(tools[0].arguments) == {"city": "SF"}
def test_single_tool_string_marker(self, parser, mock_request):
# Granite 3.1 uses the plain-text ``<tool_call>`` marker.
text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]'
_, content, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["get_time"]
assert json.loads(tools[0].arguments) == {}
def test_parallel_calls_one_array(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, '
f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]'
)
_, _, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["a", "b"]
assert json.loads(tools[0].arguments) == {"x": 1}
assert json.loads(tools[1].arguments) == {"y": [1, 2]}
def test_surrounding_text_is_content(self, parser, mock_request):
text = (
"Let me check.\n"
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
_, content, tools = parser.parse(text, mock_request)
assert content == "Let me check."
assert [t.name for t in tools] == ["get_weather"]
def test_arguments_before_name(self, parser, mock_request):
text = (
f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]'
)
_, _, tools = parser.parse(text, mock_request)
assert [t.name for t in tools] == ["get_weather"]
assert json.loads(tools[0].arguments) == {"city": "SF"}
def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request):
# A non-array body after the marker (malformed) yields no tool call.
text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}'
_, _, tools = parser.parse(text, mock_request)
assert not tools
class TestStreaming:
@pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11])
def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size):
text = (
f'{TOOL_TOKEN} [{{"name": "get_weather", '
f'"arguments": {{"city": "Tokyo"}}}}, '
f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]'
)
results = _stream(parser, mock_request, text, chunk_size)
assert _collect_names(results) == ["get_weather", "get_time"]
args = _collect_args(results)
assert json.loads(args[0]) == {"city": "Tokyo"}
assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"}
@pytest.mark.parametrize("chunk_size", [1, 3, 7])
def test_surrounding_text_streaming(self, parser, mock_request, chunk_size):
text = (
"Let me check.\n"
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
)
results = _stream(parser, mock_request, text, chunk_size)
assert _collect_names(results) == ["get_weather"]
assert "Let me check." in _collect_content(results)
def test_incomplete_array_at_eos(self, parser, mock_request):
# Truncated mid-element must not raise and must not leak the marker.
text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1'
results = _stream(parser, mock_request, text, 3)
assert TOOL_TOKEN not in _collect_content(results)
class TestReplay:
@pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id)
def test_replay_scenarios(self, scenario):
# ``_build_granite`` replays the sample through GraniteParser at
# chunk_size=1 and asserts reasoning/content/tool_calls (validate=True).
_build_granite(scenario)
File diff suppressed because it is too large Load Diff
-106
View File
@@ -1,106 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.parser.engine.replay_harness import MockTokenizer, _test_request
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
def _tokenizer() -> MockTokenizer:
vocab = {
"<think>": 1,
"</think>": 2,
"<tool_call>": 3,
"</tool_call>": 4,
"<arg_key>": 5,
"</arg_key>": 6,
"<arg_value>": 7,
"</arg_value>": 8,
}
return MockTokenizer(vocab=vocab, tokens=[])
def test_ling3_registered():
assert ReasoningParserManager.get_reasoning_parser("ling3") is not None
assert ToolParserManager.get_tool_parser("ling3") is not None
def test_ling3_defaults_thinking_on():
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
parser = parser_cls(_tokenizer())
assert parser.reasoning_start_str == "<think>"
assert parser.reasoning_end_str == "</think>"
reasoning, content = parser.extract_reasoning(
"<think>reason</think>answer", _test_request()
)
assert reasoning == "reason"
assert content == "answer"
def test_ling3_disable_thinking_keeps_reasoning_as_content():
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
parser = parser_cls(
_tokenizer(),
chat_template_kwargs={"enable_thinking": False},
)
reasoning, content = parser.extract_reasoning(
"<think>reason</think>answer", _test_request()
)
assert reasoning is None
assert content == "<think>reason</think>answer"
@pytest.mark.parametrize("model_output", ["only reasoning", "<think>only reasoning"])
def test_ling3_truncated_reasoning_stays_reasoning(model_output):
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
parser = parser_cls(
_tokenizer(),
chat_template_kwargs={"enable_thinking": True},
)
reasoning, content = parser.extract_reasoning(model_output, _test_request())
assert reasoning == "only reasoning"
assert content is None
def test_ling3_tool_call_without_newline():
parser_cls = ToolParserManager.get_tool_parser("ling3")
parser = parser_cls(_tokenizer())
result = parser.extract_tool_calls(
"<tool_call>get_weather<arg_key>city</arg_key>"
"<arg_value>Beijing</arg_value></tool_call>",
request=_test_request(),
)
assert result.tools_called
assert result.content is None
assert result.tool_calls[0].function.name == "get_weather"
assert json.loads(result.tool_calls[0].function.arguments) == {
"city": "Beijing",
}
def test_ling3_tool_call_without_args():
parser_cls = ToolParserManager.get_tool_parser("ling3")
parser = parser_cls(_tokenizer())
result = parser.extract_tool_calls(
"<tool_call>ping</tool_call>",
request=_test_request(),
)
assert result.tools_called
assert result.content is None
assert result.tool_calls[0].function.name == "ping"
assert json.loads(result.tool_calls[0].function.arguments) == {}
@@ -1,459 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MiniCPM-V family parser."""
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.minicpmv import (
EscapedNewlineNormalizer,
MiniCPMVOutputNormalizer,
recover_newlines,
)
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
_VOCAB = {
"<think>": 10,
"</think>": 11,
"<|im_end|>": 12,
"<tool_call>": 13,
"</tool_call>": 14,
}
_TOOLS = [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
)
]
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_VOCAB)
@pytest.fixture
def request_obj():
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
def _make_parser(tokenizer, *, enable_thinking=None, tools=None):
"""Build the parser the serving stack resolves for MiniCPM-V.
Everything is built through ``ParserManager`` so the tests exercise the
``DelegatingParser`` the server actually drives, not the parser engine on
its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that
XML syntax verbatim and has no tool parser of its own.
"""
parser_cls = ParserManager.get_parser(
tool_parser_name="qwen3_coder" if tools else None,
reasoning_parser_name="minicpmv",
enable_auto_tools=bool(tools),
)
assert parser_cls is not None
kwargs = {}
if enable_thinking is not None:
kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
return parser_cls(tokenizer, tools, **kwargs)
def test_reasoning_parser_is_registered():
parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv")
assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter"
def test_thinking_defaults_to_the_chat_template(mock_tokenizer):
"""Without `enable_thinking`, the grammar must wait for reasoning.
The chat template ends the assistant turn with `<think>` unless the flag is
explicitly false, so the model reasons by default. This is the flag
`serving.py` latches into the structured-output gate, so a non-thinking
default both disables the grammar and leaks the thinking into `content`.
"""
def reasoning_pending(**kwargs):
return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2])
assert reasoning_pending() is True # default matches the template
assert reasoning_pending(enable_thinking=True) is True
assert reasoning_pending(enable_thinking=False) is False
def test_no_minicpmv_tool_parser_is_registered():
"""MiniCPM-V deliberately registers no tool parser.
The family emits the Qwen3-Coder XML syntax verbatim, so
``--tool-call-parser qwen3_coder`` already covers it. Registering a
``minicpmv`` tool parser would duplicate that name for no parsing gain.
"""
with pytest.raises(KeyError):
ToolParserManager.get_tool_parser("minicpmv")
def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer):
"""`--tool-call-parser qwen3_coder` must work through the serving Parser."""
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=_TOOLS,
)
parser_cls = ParserManager.get_parser(
tool_parser_name="qwen3_coder",
reasoning_parser_name="minicpmv",
enable_auto_tools=True,
)
assert parser_cls is not None
parser = parser_cls(mock_tokenizer, request.tools)
_, _, tool_calls = parser.parse(
"<tool_call><function=get_weather>"
"<parameter=city>SF</parameter></function></tool_call>",
request,
enable_auto_tools=True,
)
assert tool_calls is not None
assert tool_calls[0].name == "get_weather"
assert "SF" in tool_calls[0].arguments
class TestNonThinking:
@pytest.fixture
def parser(self, mock_tokenizer):
return _make_parser(mock_tokenizer, enable_thinking=False)
def test_plain_content_is_unchanged(self, parser):
reasoning, content = parser.extract_reasoning("The answer is 42.", None)
assert reasoning is None
assert content == "The answer is 42."
def test_reserved_markers_are_removed(self, parser):
output = (
"<reserved_12>reasoning<reserved_13><|reserved_14|>tool call<|reserved_15|>"
)
reasoning, content = parser.extract_reasoning(output, None)
assert reasoning is None
assert content == "reasoningtool call"
def test_unrelated_reserved_text_is_unchanged(self, parser):
reasoning, content = parser.extract_reasoning(
"<reserved_11>content<|reserved_16|>",
None,
)
assert reasoning is None
assert content == "<reserved_11>content<|reserved_16|>"
def test_dangling_standard_think_end_is_removed(self, parser):
output = "first part </think>\n\nfinal part"
reasoning, content = parser.extract_reasoning(output, None)
assert reasoning is None
assert content == "first part \n\nfinal part"
def test_truncated_standard_reasoning_does_not_leak(self, parser):
reasoning, content = parser.extract_reasoning(
"prefix<think>private reasoning",
None,
)
assert reasoning is None
assert content == "prefix"
def test_streaming_split_standard_markers(self, parser):
reasoning, content = simulate_reasoning_streaming(
parser,
[
"<",
"think>",
"private reasoning",
"</",
"think>",
"final answer",
],
)
assert reasoning == ""
assert content == "final answer"
def test_streaming_split_reserved_markers(self, parser):
reasoning, content = simulate_reasoning_streaming(
parser,
[
"<reser",
"ved_12>",
"reasoning",
"<|reserved",
"_13|>",
"final answer",
],
)
assert reasoning == ""
assert content == "reasoningfinal answer"
def test_tool_calls_are_parsed(self, mock_tokenizer):
"""Thinking is off by default, and tool calls must still be parsed."""
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=_TOOLS,
)
reasoning, content, tool_calls = parser.parse(
"<tool_call><function=get_weather>"
"<parameter=city>SF</parameter></function></tool_call>",
request,
enable_auto_tools=True,
)
assert reasoning is None
assert not content
assert tool_calls is not None
assert tool_calls[0].name == "get_weather"
assert "SF" in tool_calls[0].arguments
def test_streaming_tool_calls_are_parsed(self, mock_tokenizer):
"""Regression: tool calls must stream when thinking is explicitly off.
The reasoning parser used to report ``reasoning_ended=False`` in
non-thinking mode, which kept the delegating parser in its reasoning
phase and streamed the raw ``<tool_call>`` XML to the client as content.
"""
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=_TOOLS,
)
delta = parser.parse_delta(
"<tool_call><function=get_weather>"
"<parameter=city>SF</parameter></function></tool_call>",
[],
request,
prompt_token_ids=[],
finished=True,
)
assert delta is not None
assert delta.tool_calls
assert delta.tool_calls[0].function is not None
assert delta.tool_calls[0].function.name == "get_weather"
assert "SF" in (delta.tool_calls[0].function.arguments or "")
assert not delta.content
class TestThinking:
def test_standard_think_tags_are_parsed(self, mock_tokenizer):
parser = _make_parser(mock_tokenizer, enable_thinking=True)
reasoning, content = parser.extract_reasoning(
"<think>private reasoning</think>final answer",
None,
)
assert reasoning == "private reasoning"
assert content == "final answer"
def test_reserved_markers_are_removed(self, mock_tokenizer):
parser = _make_parser(mock_tokenizer, enable_thinking=True)
reasoning, content = parser.extract_reasoning(
"<think><reserved_12>private reasoning</think><|reserved_13|>final answer",
None,
)
assert reasoning == "private reasoning"
assert content == "final answer"
def test_streaming_im_end_is_removed(self):
tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[])
parser = _make_parser(tokenizer, enable_thinking=True)
reasoning, content = simulate_reasoning_streaming(
parser,
["Answer: 111", "<|im_end|>"],
)
assert reasoning == "Answer: 111"
assert content == ""
class TestNewlineRecovery:
"""The tokenizer writes a newline as the two characters ``\\`` and ``n``."""
def test_recover_newlines(self):
text = (
"first\\nsecond\\rthird\\r\\nfourth "
"`code\\nvalue` "
"```python\\nvalue = '\\\\n'\\n``` "
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
"escaped \\\\n"
)
assert recover_newlines(text) == (
"first\nsecond\nthird\nfourth "
"`code\\nvalue` "
"```python\\nvalue = '\\\\n'\\n``` "
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
"escaped \\\\n"
)
def test_streaming_matches_complete_recovery(self):
text = (
"first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline"
)
expected = recover_newlines(text)
for split in range(len(text) + 1):
normalizer = EscapedNewlineNormalizer()
actual = normalizer.feed(text[:split])
actual += normalizer.feed(text[split:], final=True)
assert actual == expected
normalizer = EscapedNewlineNormalizer()
actual = "".join(normalizer.feed(char) for char in text)
actual += normalizer.feed("", final=True)
assert actual == expected
def test_backslash_runs_stay_consistent_across_splits(self):
# A run of backslashes can be split over chunks. Its parity decides
# whether a following `n` is a newline, so streaming and non-streaming
# must agree for every split point.
for count in range(1, 7):
text = "\\" * count + "n"
expected = recover_newlines(text)
for split in range(len(text) + 1):
normalizer = EscapedNewlineNormalizer()
actual = normalizer.feed(text[:split])
actual += normalizer.feed(text[split:], final=True)
assert actual == expected, (count, split)
def test_streaming_recovers_reasoning_and_content(self):
normalizer = MiniCPMVOutputNormalizer()
first = normalizer.normalize_delta(
DeltaMessage(reasoning="reason\\"),
finished=False,
)
second = normalizer.normalize_delta(
DeltaMessage(reasoning="nnext\\", content="answer\\"),
finished=False,
)
final = normalizer.normalize_delta(
DeltaMessage(content="nend"),
finished=True,
)
assert first == DeltaMessage(reasoning="reason")
assert second == DeltaMessage(reasoning="\nnext\\", content="answer")
assert final == DeltaMessage(content="\nend")
def test_streaming_does_not_modify_tool_arguments(self):
normalizer = MiniCPMVOutputNormalizer()
arguments = '{"value":"first\\\\nsecond"}'
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=0,
function=DeltaFunctionCall(arguments=arguments),
)
]
)
result = normalizer.normalize_delta(delta, finished=True)
assert result is not None
assert result.tool_calls[0].function is not None
assert result.tool_calls[0].function.arguments == arguments
def test_parse_recovers_content(self, mock_tokenizer, request_obj):
parser = _make_parser(mock_tokenizer, enable_thinking=False)
reasoning, content, tool_calls = parser.parse(
"first\\nsecond",
request_obj,
)
assert reasoning is None
assert content == "first\nsecond"
assert not tool_calls
def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj):
parser = _make_parser(mock_tokenizer, enable_thinking=False)
chunk = parser.parse_delta(
"first\\",
[],
request_obj,
finished=False,
)
rest = parser.parse_delta(
"nsecond",
[],
request_obj,
finished=True,
)
# The trailing half of the escape is held back until it is complete.
assert chunk is not None
assert chunk.content == "first"
assert rest is not None
assert rest.content == "\nsecond"
def test_buffered_delta_is_dropped_not_sent_empty(
self, mock_tokenizer, request_obj
):
parser = _make_parser(mock_tokenizer, enable_thinking=False)
parser.parse_delta("hello", [], request_obj, finished=False)
# The lone backtick is held back for the next chunk. Reporting it as an
# all-None DeltaMessage would reach the client as an empty SSE chunk.
assert parser.parse_delta("`", [], request_obj, finished=False) is None
def test_normalizer_drops_empty_delta(self):
normalizer = MiniCPMVOutputNormalizer()
assert (
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
is None
)
def test_normalizer_flushes_buffered_content_when_finished(self):
normalizer = MiniCPMVOutputNormalizer()
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
final = normalizer.normalize_delta(DeltaMessage(), finished=True)
assert final is not None
assert final.content == "`"
-385
View File
@@ -1,385 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_function_name,
collect_tool_arguments,
simulate_tool_streaming,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.minimax_m2 import (
THINK_END,
TOOL_CALL_END,
TOOL_CALL_START,
MinimaxM2Parser,
_minimax_m2_arg_value_extender,
minimax_m2_config,
)
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(
{
THINK_END: 99,
TOOL_CALL_START: 100,
TOOL_CALL_END: 101,
}
)
@pytest.fixture
def parser(mock_tokenizer):
return MinimaxM2Parser(mock_tokenizer)
def make_tools(*names: str):
return [
ChatCompletionToolsParam(
function=FunctionDefinition(
name=name,
parameters={
"type": "object",
"properties": {},
},
),
)
for name in names
]
class TestNonStreaming:
def test_no_tool_calls(self, parser, mock_request):
result = parser.extract_tool_calls(
"</think>This is a regular response without tool calls.",
mock_request,
)
assert result.tools_called is False
assert result.tool_calls == []
assert result.content == "This is a regular response without tool calls."
def test_single_tool_call(self, parser, mock_request):
result = parser.extract_tool_calls(
'<minimax:tool_call><invoke name="get_weather">'
'<parameter name="city">Seattle</parameter>'
"</invoke></minimax:tool_call>",
mock_request,
)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
assert json.loads(result.tool_calls[0].function.arguments) == {
"city": "Seattle",
}
def test_multiple_invokes(self, parser, mock_request):
result = parser.extract_tool_calls(
"<minimax:tool_call>"
'<invoke name="search"><parameter name="q">OpenAI</parameter></invoke>'
'<invoke name="search"><parameter name="q">vLLM</parameter></invoke>'
"</minimax:tool_call>",
mock_request,
)
assert result.tools_called is True
assert [tc.function.name for tc in result.tool_calls] == ["search", "search"]
assert json.loads(result.tool_calls[0].function.arguments) == {"q": "OpenAI"}
assert json.loads(result.tool_calls[1].function.arguments) == {"q": "vLLM"}
def test_schema_type_coercion(self, mock_tokenizer, mock_request):
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="forecast",
parameters={
"type": "object",
"properties": {
"days": {"type": "integer"},
"include_hourly": {"type": "boolean"},
},
},
),
)
]
parser = MinimaxM2Parser(mock_tokenizer, tools=tools)
mock_request.tools = tools
result = parser.extract_tool_calls(
'<minimax:tool_call><invoke name="forecast">'
'<parameter name="days">5</parameter>'
'<parameter name="include_hourly">true</parameter>'
"</invoke></minimax:tool_call>",
mock_request,
)
assert json.loads(result.tool_calls[0].function.arguments) == {
"days": 5,
"include_hourly": True,
}
def test_invalid_tool_name_is_rejected(self, mock_tokenizer, mock_request):
tools = make_tools("search")
parser = MinimaxM2Parser(mock_tokenizer)
mock_request.tools = tools
result = parser.extract_tool_calls(
'<minimax:tool_call><invoke name="img_gen">'
'<parameter name="prompt">a cat</parameter>'
"</invoke></minimax:tool_call>",
mock_request,
)
assert result.tools_called is False
assert result.tool_calls == []
def test_mixed_tool_names_only_return_valid(self, mock_tokenizer, mock_request):
tools = make_tools("search")
parser = MinimaxM2Parser(mock_tokenizer)
mock_request.tools = tools
result = parser.extract_tool_calls(
"<minimax:tool_call>"
'<invoke name="img_gen"><parameter name="prompt">cat</parameter></invoke>'
'<invoke name="search"><parameter name="query">news</parameter></invoke>'
"</minimax:tool_call>",
mock_request,
)
assert result.tools_called is True
assert [tc.function.name for tc in result.tool_calls] == ["search"]
assert json.loads(result.tool_calls[0].function.arguments) == {
"query": "news",
}
class TestStreaming:
def test_streaming_single_tool_call(self, parser, mock_request):
results = simulate_tool_streaming(
parser,
mock_request,
[
"<minimax:tool_call>",
'<invoke name="get_weather">',
'<parameter name="city">Seattle</parameter>',
"</invoke></minimax:tool_call>",
],
)
assert collect_function_name(results) == "get_weather"
assert json.loads(collect_tool_arguments(results)) == {
"city": "Seattle",
}
def test_streaming_multiple_invokes(self, parser, mock_request):
results = simulate_tool_streaming(
parser,
mock_request,
[
"<minimax:tool_call>",
'<invoke name="a"><parameter name="x">1</parameter></invoke>',
'<invoke name="b"><parameter name="y">2</parameter></invoke>',
"</minimax:tool_call>",
],
)
tool_names = [
tc.function.name
for delta, _ in results
if delta and delta.tool_calls
for tc in delta.tool_calls
if tc.function and tc.function.name
]
assert tool_names == ["a", "b"]
def test_streaming_invoke_prefix_split_before_quote(self, parser, mock_request):
results = simulate_tool_streaming(
parser,
mock_request,
[
"<minimax:tool_call>",
"<invoke name=",
'"get_weather">',
'<parameter name="city">Seattle</parameter>',
"</invoke></minimax:tool_call>",
],
)
assert collect_function_name(results) == "get_weather"
assert json.loads(collect_tool_arguments(results)) == {
"city": "Seattle",
}
def test_streaming_invalid_tool_name_is_rejected(
self, mock_tokenizer, mock_request
):
tools = make_tools("search")
parser = MinimaxM2Parser(mock_tokenizer)
mock_request.tools = tools
results = simulate_tool_streaming(
parser,
mock_request,
[
"<minimax:tool_call>",
'<invoke name="img_gen">',
'<parameter name="prompt">cat</parameter>',
"</invoke></minimax:tool_call>",
],
)
assert collect_function_name(results) is None
assert collect_tool_arguments(results) == ""
WRITE_CALL_HEAD = (
'<minimax:tool_call>\n<invoke name="write_file">\n'
'<parameter name="path">/repo/app.py</parameter>\n'
'<parameter name="mode">644</parameter>\n'
'<parameter name="content">'
)
WRITE_CALL_TAIL = "</parameter>\n</invoke>\n</minimax:tool_call>"
# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
# the value, including prefixes of the closing and next-parameter tags.
CONTENT_LINE = 'if a < b > c: print("x\\y") # </p> </param <parameter\té\n\n'
class TestArgValueExtender:
"""The open parameter value streams without re-running the converter."""
@pytest.fixture
def write_tool(self):
return ChatCompletionToolsParam(
function=FunctionDefinition(
name="write_file",
parameters={
"type": "object",
"properties": {
"path": {"type": "string"},
"mode": {"type": "integer"},
"content": {"type": "string"},
},
},
),
)
@pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
def test_long_call_streams_non_streaming_arguments(
self, mock_tokenizer, mock_request, write_tool, use_schema
):
mock_request.tools = [write_tool] if use_schema else []
text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
results = simulate_tool_streaming(
MinimaxM2Parser(mock_tokenizer),
mock_request,
[text[i : i + 7] for i in range(0, len(text), 7)],
)
result = MinimaxM2Parser(mock_tokenizer).extract_tool_calls(text, mock_request)
arguments = collect_tool_arguments(results)
assert arguments == result.tool_calls[0].function.arguments
assert json.loads(arguments)["content"] == CONTENT_LINE * 200
assert ('"mode": 644,' if use_schema else '"mode": "644",') in arguments
def test_conversions_do_not_grow_with_argument_length(
self, mock_tokenizer, mock_request, write_tool
):
mock_request.tools = [write_tool]
base = minimax_m2_config()
converter = base.arg_converter
assert converter is not None
def streaming_conversions(lines: int) -> int:
calls = 0
def counting_converter(raw_args: str, partial: bool) -> str:
nonlocal calls
calls += partial
return converter(raw_args, partial)
parser = ParserEngine(
mock_tokenizer,
parser_engine_config=dataclasses.replace(
base, arg_converter=counting_converter
),
)
text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
simulate_tool_streaming(
parser,
mock_request,
[text[i : i + 4] for i in range(0, len(text), 4)],
)
return calls
assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
def test_extender_stops_at_split_end_tag(self):
extend = _minimax_m2_arg_value_extender('<parameter name="k">')
assert extend is not None
deltas = ['a"\n', "</param", "eter>"]
assert [extend(d) for d in deltas] == ['a\\"\\n', "</param", None]
def test_extender_stops_at_split_next_parameter(self):
extend = _minimax_m2_arg_value_extender("<parameter name='k'>x")
assert extend is not None
assert [extend(d) for d in ["< parameter", " name ="]] == ["< parameter", None]
@pytest.mark.parametrize(
"raw_args",
[
'<parameter name="k">x</parameter>',
'<parameter name="k',
(
'<parameter name="k">1</parameter><parameter name="j">2</parameter>'
'<parameter name="k">3'
),
'<parameter name="a<parameter name=k>x',
'<parameter name="k>x',
],
ids=[
"closed_value",
"open_name",
"repeated_key_not_last",
"earlier_unmatched_opener",
"bare_name_opens_quote",
],
)
def test_no_extender_unless_open_value_is_stable(self, raw_args):
assert _minimax_m2_arg_value_extender(raw_args) is None
class TestReasoning:
def test_extract_reasoning_without_start_token(self, parser, mock_request):
reasoning, content = parser.extract_reasoning(
"This is reasoning</think>This is content",
mock_request,
)
assert reasoning == "This is reasoning"
assert content == "This is content"
def test_extract_reasoning_without_end_token(self, parser, mock_request):
reasoning, content = parser.extract_reasoning(
"This is still reasoning",
mock_request,
)
assert reasoning == "This is still reasoning"
assert content is None
def test_extract_content_ids_without_end_token(self, parser):
assert parser.extract_content_ids([1, 2, 3]) == []
def test_extract_content_ids_after_end_token(self, parser):
assert parser.extract_content_ids([1, 99, 2, 3]) == [2, 3]
-253
View File
@@ -1,253 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_content,
collect_function_name,
collect_tool_arguments,
simulate_tool_streaming,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.parser.minimax_m3 import (
INVOKE_END,
MAX_ELEMENT_DEPTH,
NAMESPACE,
TOOL_CALL_END,
TOOL_CALL_START,
MinimaxM3Parser,
)
NS = NAMESPACE
TOOLS = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="create_order",
parameters={
"type": "object",
"properties": {
"user_id": {"type": "integer"},
"note": {"type": "string"},
"tags": {"type": "array", "items": {"type": "string"}},
"shipping": {
"type": "object",
"properties": {"zip": {"type": "integer"}},
},
"either": {
"anyOf": [
{"type": "array", "items": {"type": "integer"}},
{"type": "string"},
]
},
"free": {"type": "object"},
},
},
),
)
]
@pytest.fixture
def parser():
return MinimaxM3Parser(make_mock_tokenizer({}), tools=TOOLS)
def element(name: str, body: str) -> str:
return f"{NS}<{name}>{body}{NS}</{name}>"
def tool_block(*invokes: tuple[str, str]) -> str:
body = "\n".join(
f'{NS}<invoke name="{name}">{params}{INVOKE_END}' for name, params in invokes
)
return f"{TOOL_CALL_START}\n{body}\n{TOOL_CALL_END}"
def parse_arguments(parser, request, params: str, name: str = "create_order"):
result = parser.extract_tool_calls(tool_block((name, params)), request)
assert [tc.function.name for tc in result.tool_calls] == [name]
return json.loads(result.tool_calls[0].function.arguments)
class TestArguments:
def test_nested_elements_follow_schema(self, parser, mock_request):
params = (
element("tags", element("item", "a") + element("item", "b"))
+ element("shipping", element("zip", "018956"))
+ element("either", element("item", "1") + element("item", "2"))
+ element("free", element("t", "x") + element("t", "y"))
)
assert parse_arguments(parser, mock_request, params) == {
"tags": ["a", "b"],
"shipping": {"zip": 18956},
"either": [1, 2],
"free": {"t": ["x", "y"]},
}
def test_empty_elements_follow_schema(self, parser, mock_request):
# The chat template renders [] and {} as empty elements.
params = (
element("tags", "")
+ element("shipping", "")
+ element("note", "")
+ element("either", "")
)
assert parse_arguments(parser, mock_request, params) == {
"tags": [],
"shipping": {},
"note": "",
"either": "",
}
def test_unknown_tool_keeps_text_and_nests_objects(self, parser, mock_request):
params = element("n", "1") + element(
"list", element("item", "a") + element("item", "b")
)
assert parse_arguments(parser, mock_request, params, name="other") == {
"n": "1",
"list": {"item": ["a", "b"]},
}
def test_mixed_text_uses_reserved_field(self, parser, mock_request):
params = element("free", "hi " + element("k", "v") + " there") + element(
"shipping", "x" + element("$text", "child")
)
assert parse_arguments(parser, mock_request, params) == {
"free": {"k": "v", "$text": "hi there"},
"shipping": {"$text": "child", "$$text": "x"},
}
def test_leaf_text_is_verbatim_for_strings(self, parser, mock_request):
params = element("note", "\n a <b> & c\n") + element("user_id", " 7 ")
assert parse_arguments(parser, mock_request, params) == {
"note": "\n a <b> & c\n",
"user_id": 7,
}
def test_block_marker_named_element_is_argument(self, parser, mock_request):
params = element("free", element("tool_call", "v"))
assert parse_arguments(parser, mock_request, params) == {
"free": {"tool_call": "v"},
}
@pytest.mark.parametrize(
"tail",
[
"junk" + element("note", "dropped"),
f"{NS}<note>a{NS}</other>",
f"{NS}<note>unterminated",
],
)
def test_malformed_element_keeps_preceding_params(self, parser, mock_request, tail):
params = element("user_id", "1") + tail
assert parse_arguments(parser, mock_request, params) == {"user_id": 1}
def test_depth_limit(self, parser, mock_request):
def nested(depth: int) -> str:
body = "v"
for _ in range(depth - 1):
body = element("n", body)
return element("free", body)
accepted = parse_arguments(parser, mock_request, nested(MAX_ELEMENT_DEPTH))
assert "free" in accepted
rejected = parse_arguments(parser, mock_request, nested(MAX_ELEMENT_DEPTH + 1))
assert rejected == {}
class TestToolBlock:
def test_multiple_invokes_and_prefix_content(self, parser, mock_request):
result = parser.extract_tool_calls(
"Checking.\n"
+ tool_block(
("create_order", element("user_id", "1")),
("create_order", element("user_id", "2")),
),
mock_request,
)
assert result.content == "Checking.\n"
assert [json.loads(tc.function.arguments) for tc in result.tool_calls] == [
{"user_id": 1},
{"user_id": 2},
]
def test_text_after_tool_block_is_dropped(self, parser, mock_request):
result = parser.extract_tool_calls(
"pre" + tool_block(("create_order", "")) + "post", mock_request
)
assert result.content == "pre"
assert len(result.tool_calls) == 1
def test_invoke_outside_tool_block_is_content(self, parser, mock_request):
text = f'{NS}<invoke name="create_order">{INVOKE_END}'
result = parser.extract_tool_calls(text, mock_request)
assert not result.tools_called
assert result.content == text
class TestStreaming:
def test_arguments_arrive_with_invoke_end(self, parser, mock_request):
block = tool_block(
("create_order", element("user_id", "42") + element("tags", ""))
)
invoke_end = block.index(INVOKE_END)
results = simulate_tool_streaming(
parser,
mock_request,
["Sure. ", block[:invoke_end], block[invoke_end:]],
)
before_end, at_end = results[1][0], results[2][0]
assert before_end is not None and before_end.tool_calls
assert before_end.tool_calls[0].function.name == "create_order"
assert not before_end.tool_calls[0].function.arguments
assert collect_content(results) == "Sure. "
assert json.loads(collect_tool_arguments([(at_end, "")])) == {
"user_id": 42,
"tags": [],
}
def test_char_chunks_match_complete_parse(self, parser, mock_request):
text = "a" + tool_block(
("create_order", element("shipping", element("zip", "1"))),
("create_order", element("note", "x")),
)
results = simulate_tool_streaming(parser, mock_request, list(text))
assert collect_content(results) == "a"
assert collect_function_name(results) == "create_order"
streamed = [
tc.function.arguments
for delta, _ in results
if delta and delta.tool_calls
for tc in delta.tool_calls
if tc.function and tc.function.arguments
]
assert [json.loads(args) for args in streamed] == [
{"shipping": {"zip": 1}},
{"note": "x"},
]
def test_truncated_invoke_keeps_completed_params(self, parser, mock_request):
block = tool_block(
("create_order", element("user_id", "1") + element("note", "x"))
)
truncated = block[: block.index(f"{NS}</note>")]
results = simulate_tool_streaming(parser, mock_request, [truncated])
results.append((parser.finish_streaming(), truncated))
assert collect_function_name(results) == "create_order"
assert json.loads(collect_tool_arguments(results)) == {"user_id": 1}
def test_adjust_request_keeps_special_tokens_skipped(parser, mock_request):
mock_request.skip_special_tokens = True
assert parser.adjust_request(mock_request).skip_special_tokens is True
-303
View File
@@ -1,303 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Nemotron V3 parser.
Validates that ``NemotronV3Parser`` correctly handles:
- ``<think>``/``</think>`` reasoning with ``<tool_call>`` XML tool calls
(same format as Qwen3)
- Nemotron-specific reasoning/content swap when ``enable_thinking=False``
or ``force_nonempty_content=True``
"""
import json
from unittest.mock import MagicMock
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_function_name,
collect_tool_arguments,
simulate_tool_streaming,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.parser.nemotron_v3 import NemotronV3Parser
_THINK_START_ID = 50
_THINK_END_ID = 51
_TOOL_CALL_ID = 60
_TOOL_CALL_END_ID = 61
_TEXT_ID = 100
_VOCAB = {
"<think>": _THINK_START_ID,
"</think>": _THINK_END_ID,
"<tool_call>": _TOOL_CALL_ID,
"</tool_call>": _TOOL_CALL_END_ID,
}
def _make_request(**chat_template_kwargs):
request = MagicMock(spec=ChatCompletionRequest)
request.tools = []
request.tool_choice = "auto"
request.include_reasoning = True
request.chat_template_kwargs = chat_template_kwargs or None
return request
@pytest.fixture
def parser():
return NemotronV3Parser(make_mock_tokenizer(_VOCAB))
class TestNemotronSwap:
def test_enable_thinking_false_swaps(self, parser):
"""When enable_thinking=False, model output without think tags
should have reasoning swapped to content."""
text = "The answer is 42."
request = _make_request(enable_thinking=False)
reasoning, content = parser.extract_reasoning(text, request)
assert content == "The answer is 42."
assert reasoning is None
def test_force_nonempty_content_swaps(self, parser):
"""force_nonempty_content=True triggers swap when content empty."""
text = "The answer is 42."
request = _make_request(force_nonempty_content=True)
reasoning, content = parser.extract_reasoning(text, request)
assert content == "The answer is 42."
assert reasoning is None
def test_no_swap_when_content_exists(self, parser):
"""With enable_thinking=False but real </think> giving content,
no swap occurs."""
text = "Some reasoning.</think>Actual content here."
request = _make_request(enable_thinking=False)
reasoning, content = parser.extract_reasoning(text, request)
assert reasoning == "Some reasoning."
assert content == "Actual content here."
def test_no_swap_when_enable_thinking_true(self, parser):
"""Normal thinking mode: no swap, even when content is empty."""
text = "Still thinking..."
request = _make_request(enable_thinking=True)
reasoning, content = parser.extract_reasoning(text, request)
assert reasoning == "Still thinking..."
assert content is None
def test_no_swap_with_none_request(self, parser):
"""Graceful handling when request is None."""
text = "Some text."
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "Some text."
assert content is None
def test_no_swap_with_no_kwargs(self, parser):
"""No swap when chat_template_kwargs is absent."""
text = "Some text."
request = _make_request()
reasoning, content = parser.extract_reasoning(text, request)
assert reasoning == "Some text."
assert content is None
def test_swap_with_whitespace_only_content(self, parser):
"""Swap occurs when content is whitespace-only."""
text = "The answer.</think> "
request = _make_request(enable_thinking=False)
reasoning, content = parser.extract_reasoning(text, request)
assert content == "The answer."
assert reasoning == " "
class TestNonStreamingToolCalls:
def test_single_tool_call(self, parser):
text = (
"<tool_call>\n"
"<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
"</function>\n"
"</tool_call>"
)
request = _make_request()
result = parser.extract_tool_calls(text, request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"city": "Tokyo"}
def test_parallel_tool_calls(self, parser):
text = (
"<tool_call>\n"
"<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
"</function>\n"
"</tool_call>"
"<tool_call>\n"
"<function=get_time>\n"
"<parameter=timezone>Asia/Tokyo</parameter>\n"
"</function>\n"
"</tool_call>"
)
request = _make_request()
result = parser.extract_tool_calls(text, request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "get_weather"
assert result.tool_calls[1].function.name == "get_time"
def test_no_tool_calls(self, parser):
request = _make_request()
result = parser.extract_tool_calls("Hello, how can I help?", request)
assert result.tools_called is False
# Parser starts in REASONING state, so plain text is classified
# as reasoning (not content) when there are no tool calls.
assert result.content is None
class TestStreaming:
def test_streaming_tool_calls(self, parser):
request = _make_request()
chunks = [
"<tool_call>\n",
"<function=get_weather>\n",
"<parameter=city>Tokyo",
"</parameter>\n",
"</function>\n",
"</tool_call>",
]
results = simulate_tool_streaming(parser, request, chunks)
name = collect_function_name(results)
assert name == "get_weather"
args_text = collect_tool_arguments(results)
assert args_text
parsed = json.loads(args_text)
assert parsed == {"city": "Tokyo"}
class TestParseDeltaTokenIdFiltering:
"""parse_delta must not trigger tool call parsing when <tool_call>
appears as regular text rather than as a special token ID."""
def test_tool_call_text_in_reasoning_is_not_parsed(self, parser):
"""Literal <tool_call> in model reasoning should be content,
not a tool call."""
request = _make_request()
text = (
"The test uses <tool_call> syntax:\n"
"<tool_call>\n"
"<function=Bash>\n"
"<parameter=command>ls</parameter>\n"
"</function>\n"
"</tool_call>"
)
result = parser.parse_delta(
delta_text=text,
delta_token_ids=[_TEXT_ID] * 6,
request=request,
prompt_token_ids=[],
finished=True,
)
assert result is not None
assert result.reasoning is not None
assert "<tool_call>" in result.reasoning
assert not result.tool_calls
def test_special_token_id_still_triggers_tool_call(self, parser):
"""When the scanner matches a special token ID, the tool call
must still be parsed correctly."""
request = _make_request()
parser.parse_delta(
delta_text="Let me check.",
delta_token_ids=[_TEXT_ID, _TEXT_ID, _TEXT_ID],
request=request,
prompt_token_ids=[],
finished=False,
)
parser.parse_delta(
delta_text="<tool_call>",
delta_token_ids=[_TOOL_CALL_ID],
request=request,
finished=False,
)
parser.parse_delta(
delta_text=(
"\n<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
"</function>\n"
),
delta_token_ids=[_TEXT_ID] * 5,
request=request,
finished=False,
)
parser.parse_delta(
delta_text="</tool_call>",
delta_token_ids=[_TOOL_CALL_END_ID],
request=request,
finished=True,
)
assert any(s.name == "get_weather" for s in parser._tool_slots)
def test_text_discussion_then_real_tool_call(self, parser):
"""Model discusses tool syntax in reasoning, then makes a real
tool call via special tokens."""
request = _make_request()
r1 = parser.parse_delta(
delta_text="Use <tool_call> to invoke tools.",
delta_token_ids=[_TEXT_ID] * 6,
request=request,
prompt_token_ids=[],
finished=False,
)
r2 = parser.parse_delta(
delta_text="</think>",
delta_token_ids=[_THINK_END_ID],
request=request,
finished=False,
)
r3 = parser.parse_delta(
delta_text="<tool_call>",
delta_token_ids=[_TOOL_CALL_ID],
request=request,
finished=False,
)
r4 = parser.parse_delta(
delta_text=("\n<function=test>\n<parameter=x>1</parameter>\n</function>\n"),
delta_token_ids=[_TEXT_ID] * 4,
request=request,
finished=False,
)
r5 = parser.parse_delta(
delta_text="</tool_call>",
delta_token_ids=[_TOOL_CALL_END_ID],
request=request,
finished=True,
)
results = [r1, r2, r3, r4, r5]
reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
assert "<tool_call>" in reasoning
names = [
tc.function.name
for r in results
if r and r.tool_calls
for tc in r.tool_calls
if tc.function and tc.function.name
]
assert "test" in names
File diff suppressed because it is too large Load Diff
-556
View File
@@ -1,556 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""PLaMo3 grammar, truncation, and serving-adapter regressions."""
from __future__ import annotations
import json
import pytest
import regex as re
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
_test_request,
collect_output,
replay_streaming,
replay_with_text_holdback,
)
from vllm.parser.parser_manager import ParserManager
from vllm.parser.plamo3 import (
BEGIN_THINK,
BEGIN_TOOL_ARGUMENTS,
BEGIN_TOOL_NAME,
BEGIN_TOOL_REQUEST,
BEGIN_TOOL_REQUESTS,
END_THINK,
END_TOOL_ARGUMENTS,
END_TOOL_NAME,
END_TOOL_REQUEST,
END_TOOL_REQUESTS,
EOT,
PLAMO_MARKER_TOKENS,
Plamo3Parser,
)
def _tool_header(name="weather"):
return (
BEGIN_TOOL_REQUEST
+ BEGIN_TOOL_NAME
+ name
+ END_TOOL_NAME
+ BEGIN_TOOL_ARGUMENTS
)
def _tool_call(name, arguments):
return _tool_header(name) + arguments + END_TOOL_ARGUMENTS + END_TOOL_REQUEST
@pytest.fixture
def mock_tokenizer():
special = [*sorted(PLAMO_MARKER_TOKENS), EOT, "<|plamo:bos|>"]
atoms = [*special, "<|plamo:constrain|>", "<|plamo:msg|>"]
vocab = {token: 100 + i for i, token in enumerate(atoms)}
inverse = {i: token for token, i in vocab.items()}
pattern = re.compile("|".join(map(re.escape, atoms)) + r"|[\s\S]")
tokenizer = make_mock_tokenizer(vocab, special_tokens=special)
tokenizer.encode.side_effect = lambda text, **kwargs: [
vocab[piece] if piece in vocab else 1000 + ord(piece)
for piece in pattern.findall(text)
]
tokenizer.decode.side_effect = lambda ids, **kwargs: "".join(
inverse[i] if i in inverse else chr(i - 1000) for i in ids
)
return tokenizer
@pytest.fixture(params=["direct", "registered"])
def parser_cls(request):
if request.param == "direct":
return Plamo3Parser
return ParserManager.get_parser(
tool_parser_name="plamo3",
reasoning_parser_name="plamo3",
enable_auto_tools=True,
)
def _stream(parser, tokenizer, output, chunk_size=1, finished_on_last=False):
tokens = [
(i, tokenizer.decode([i]))
for i in tokenizer.encode(output, add_special_tokens=False)
]
deltas = replay_streaming(
parser,
tokens,
chunk_size=chunk_size,
finished_on_last=finished_on_last,
tools=DUMMY_TOOLS,
)
if not finished_on_last:
deltas.append(
parser.parse_delta("", [], _test_request(DUMMY_TOOLS), finished=True)
)
return deltas
@pytest.mark.parametrize("thinking", [False, True])
@pytest.mark.parametrize("with_markers", [False, True])
def test_reasoning_mode(
parser_cls, mock_tokenizer, mock_request, thinking, with_markers
):
output = (
BEGIN_THINK + "Reasoning." + END_THINK + "Answer."
if with_markers
else "Answer."
)
if thinking:
expected = ("Reasoning.", "Answer.") if with_markers else ("Answer.", "")
else:
expected = ("", output)
kwargs = {"chat_template_kwargs": {"enable_thinking": thinking}}
reasoning, content, calls = parser_cls(mock_tokenizer, **kwargs).parse(
output, mock_request
)
assert (reasoning or "", content or "") == expected
assert not calls
result = collect_output(
_stream(parser_cls(mock_tokenizer, **kwargs), mock_tokenizer, output)
)
assert (result.reasoning, result.content) == expected
assert not result.tool_calls
def test_duplicate_think_end_does_not_leak_into_content(
parser_cls, mock_tokenizer, mock_request
):
output = END_THINK + "A" + END_THINK + "B"
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (reasoning, content) == (None, "AB")
assert not calls
def test_content_can_start_a_new_reasoning_block(
parser_cls, mock_tokenizer, mock_request
):
output = END_THINK + "A" + BEGIN_THINK + "R" + END_THINK
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (reasoning, content) == ("R", "A")
assert not calls
@pytest.mark.parametrize("chunk_size", [1, 7, None])
@pytest.mark.parametrize("finished_on_last", [False, True])
@pytest.mark.parametrize(
"closing",
[
"",
END_TOOL_ARGUMENTS + END_TOOL_REQUEST,
END_TOOL_ARGUMENTS + END_TOOL_REQUEST + END_TOOL_REQUESTS + EOT,
],
)
def test_complete_arguments_survive_missing_closers(
parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last, closing
):
output = (
BEGIN_THINK
+ "Reasoning. "
+ END_THINK
+ "Before tools. "
+ BEGIN_TOOL_REQUESTS
+ _tool_header()
+ '{"city":"東京"}'
+ closing
)
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (reasoning, content) == ("Reasoning. ", "Before tools. ")
assert calls and [(c.name, json.loads(c.arguments)) for c in calls] == [
("weather", {"city": "東京"})
]
result = collect_output(
_stream(
parser_cls(mock_tokenizer),
mock_tokenizer,
output,
chunk_size,
finished_on_last,
)
)
assert (result.reasoning, result.content) == (reasoning, content)
assert result.tool_calls == [{"name": "weather", "arguments": '{"city":"東京"}'}]
def test_tool_name_cutoffs_do_not_leak_markers(
parser_cls, mock_tokenizer, mock_request
):
header = BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "weather" + END_TOOL_NAME
header_ids = mock_tokenizer.encode(header, add_special_tokens=False)
name_start = len(
mock_tokenizer.encode(
BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME, add_special_tokens=False
)
)
for end in range(len(header_ids)):
partial = mock_tokenizer.decode(header_ids[:end])
output = END_THINK + "Before tools." + BEGIN_TOOL_REQUESTS + partial
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (reasoning, content) == (None, "Before tools."), end
if end <= name_start:
expected_name = ""
else:
name_text = mock_tokenizer.decode(header_ids[name_start:end])
if "<|plamo:" in name_text:
name_text = name_text[: name_text.rfind("<|plamo:")]
expected_name = name_text[: len("weather")]
assert [call.name for call in calls or []] == (
[expected_name] if expected_name else []
), end
result = collect_output(
_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)
)
assert [call["name"] for call in result.tool_calls] == (
[expected_name] if expected_name else []
), end
assert all("<" not in call["name"] for call in result.tool_calls), end
assert result.content == "Before tools.", end
@pytest.mark.parametrize("chunk_size", [1, 7, None])
@pytest.mark.parametrize("finished_on_last", [False, True])
def test_all_argument_closer_cutoffs_do_not_leak(
parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last
):
head = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header()
closer_ids = mock_tokenizer.encode(END_TOOL_ARGUMENTS, add_special_tokens=False)
for end in range(len(closer_ids)):
partial = mock_tokenizer.decode(closer_ids[:end])
output = head + '{"city":"東京"}' + partial
_, _, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert calls and calls[0].arguments == '{"city":"東京"}', end
result = collect_output(
_stream(
parser_cls(mock_tokenizer),
mock_tokenizer,
output,
chunk_size,
finished_on_last,
)
)
assert result.tool_calls == [
{"name": "weather", "arguments": '{"city":"東京"}'}
], end
@pytest.mark.parametrize(
"arguments",
[
"{}",
'{"name":"Alice","arguments":{"x":1},"parameters":true}',
'{"text":"literal <|plamo:end_',
],
)
def test_arguments_are_not_rewritten(
parser_cls, mock_tokenizer, mock_request, arguments
):
output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header() + arguments
_, _, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert calls and calls[0].arguments == arguments
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert result.tool_calls == [{"name": "weather", "arguments": arguments}]
@pytest.mark.parametrize("truncated_second", [False, True])
def test_multiple_calls_keep_indices_ids_and_arguments_separate(
parser_cls, mock_tokenizer, mock_request, truncated_second
):
second = (
BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "clo"
if truncated_second
else _tool_call("clock", '{"timezone":"Asia/Tokyo"}')
)
output = (
END_THINK
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", '{"city":"東京"}')
+ second
)
expected = [{"name": "weather", "arguments": '{"city":"東京"}'}]
if truncated_second:
expected.append({"name": "clo", "arguments": "{}"})
else:
output += END_TOOL_REQUESTS
expected.append({"name": "clock", "arguments": '{"timezone":"Asia/Tokyo"}'})
stream_expected = expected
_, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (
calls
and [{"name": c.name, "arguments": c.arguments} for c in calls] == expected
)
deltas = _stream(parser_cls(mock_tokenizer), mock_tokenizer, output)
result = collect_output(deltas)
assert result.tool_calls == stream_expected
assert result.content == (content or "") == ""
announced = [
call for delta in deltas if delta for call in delta.tool_calls or [] if call.id
]
assert [call.index for call in announced] == list(range(len(expected)))
assert len({call.id for call in announced}) == len(expected)
def test_arguments_are_available_before_outer_closer(mock_tokenizer, mock_request):
parser = Plamo3Parser(mock_tokenizer)
output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_call("weather", '{"city":"Tokyo"}')
first = parser.parse_delta(output, [], mock_request, finished=False)
assert collect_output([first]).tool_calls == [
{"name": "weather", "arguments": '{"city":"Tokyo"}'}
]
assert (
parser.parse_delta(END_TOOL_REQUESTS[:10], [], mock_request, finished=False)
is None
)
assert (
parser.parse_delta(END_TOOL_REQUESTS[10:], [], mock_request, finished=True)
is None
)
def test_missing_tool_request_start_does_not_break_streaming(
parser_cls, mock_tokenizer, mock_request
):
output = (
BEGIN_TOOL_REQUESTS
+ BEGIN_TOOL_NAME
+ "weather"
+ END_TOOL_NAME
+ BEGIN_TOOL_ARGUMENTS
+ "{}"
+ END_TOOL_ARGUMENTS
+ END_TOOL_REQUESTS
)
delta = parser_cls(mock_tokenizer).parse_delta(
output, [], mock_request, finished=False
)
assert delta is not None
assert collect_output([delta]).tool_calls == [
{"name": "weather", "arguments": "{}"}
]
def test_reusing_parser_drops_pending_names(mock_tokenizer, mock_request):
parser = Plamo3Parser(mock_tokenizer)
parser.parse(
END_THINK
+ BEGIN_TOOL_REQUESTS
+ BEGIN_TOOL_REQUEST
+ BEGIN_TOOL_NAME
+ "partial",
mock_request,
)
output = (
END_THINK
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
)
_, _, calls = parser.parse(output, mock_request)
assert calls and calls[0].name == "weather"
@pytest.mark.parametrize(
"text", ["", "a < b", "literal <", "literal <|plamo:unknown", " text\n"]
)
def test_plain_content_is_preserved(parser_cls, mock_tokenizer, mock_request, text):
output = END_THINK + text + EOT
_, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (content or "") == text
assert not calls
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert (result.content, result.tool_calls) == (text, [])
@pytest.mark.parametrize(
"text, ended",
[
("", False),
(BEGIN_THINK + "reasoning", False),
(END_THINK + "answer", True),
(END_THINK + "answer" + BEGIN_THINK + "new reasoning", False),
(END_THINK + "answer" + EOT + "assistant", False),
],
)
def test_reasoning_end_checks_the_latest_turn(mock_tokenizer, text, ended):
parser = Plamo3Parser(mock_tokenizer)
ids = mock_tokenizer.encode(text)
assert parser.is_reasoning_end(ids) == ended
if ended:
assert mock_tokenizer.decode(parser.extract_content_ids(ids)) == "answer"
@pytest.mark.parametrize("suffix", ["<|plamo:end_", "<|plamo:end_think"])
def test_partial_end_think_is_consistent_between_streaming_and_non_streaming(
parser_cls, mock_tokenizer, mock_request, suffix
):
output = "visible" + suffix
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
assert (reasoning, content) == ("visible", None)
assert not calls
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert (result.reasoning, result.content, result.tool_calls) == ("visible", "", [])
@pytest.mark.parametrize(
"suffix",
[
"<|plamo:begin_",
"<|plamo:begin_tool_name",
"<|plamo:end_tool",
],
)
def test_unfinished_marker_prefixes_are_stripped(mock_tokenizer, suffix):
parser = Plamo3Parser(mock_tokenizer)
assert parser._strip_unfinished_marker("visible" + suffix) == "visible"
@pytest.mark.parametrize("suffix", ["<|plamo:", "<|plamo:b"])
def test_non_token_prefix_is_not_stripped(mock_tokenizer, suffix):
parser = Plamo3Parser(mock_tokenizer)
text = "visible" + suffix
assert parser._strip_unfinished_marker(text) == text
def test_complete_markers_are_not_stripped(mock_tokenizer):
parser = Plamo3Parser(mock_tokenizer)
text = "visible" + BEGIN_TOOL_NAME
assert parser._strip_unfinished_marker(text) == text
def test_short_prefix_is_not_stripped(mock_tokenizer):
parser = Plamo3Parser(mock_tokenizer)
text = "visible<"
assert parser._strip_unfinished_marker(text) == text
def test_tool_block_can_end_reasoning_implicitly(
parser_cls, mock_tokenizer, mock_request
):
output = (
"reasoning"
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
)
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
output, mock_request, enable_auto_tools=True
)
assert (reasoning, content) == ("reasoning", None)
assert calls and calls[0].name == "weather"
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
assert (result.reasoning, result.content) == ("reasoning", "")
assert result.tool_calls == [{"name": "weather", "arguments": "{}"}]
def test_every_cut_point_matches_non_streaming(
parser_cls, mock_tokenizer, mock_request
):
output = (
BEGIN_THINK
+ "Reasoning. "
+ END_THINK
+ "Before tools. "
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", '{"city":"東京"}')
+ END_TOOL_REQUESTS
+ EOT
)
ids = mock_tokenizer.encode(output, add_special_tokens=False)
for k in range(1, len(ids) + 1):
prefix = mock_tokenizer.decode(ids[:k])
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
prefix, mock_request, enable_auto_tools=True
)
result = collect_output(
_stream(parser_cls(mock_tokenizer), mock_tokenizer, prefix)
)
assert (result.reasoning, result.content) == (
(reasoning or ""),
(content or ""),
), k
expected = [{"name": c.name, "arguments": c.arguments} for c in (calls or [])]
assert result.tool_calls == expected, (k, expected, result.tool_calls)
@pytest.mark.parametrize(
"choice",
[
"auto",
"required",
"none",
{"type": "function", "function": {"name": "weather"}},
],
)
def test_tool_choice_uses_the_plamo_format(parser_cls, mock_tokenizer, choice):
request = _test_request(
[
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}},
},
]
)
request = type(request).model_validate(
request.model_dump() | {"tool_choice": choice}
)
output = (
END_THINK
+ "Before tools."
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
)
_, content, calls = parser_cls(mock_tokenizer).parse(
output, request, enable_auto_tools=True
)
assert content == "Before tools."
assert [c.name for c in calls or []] == ([] if choice == "none" else ["weather"])
parser = parser_cls(mock_tokenizer)
delta = parser.parse_delta(output, [], request, finished=True)
result = collect_output([delta])
assert result.content == "Before tools."
assert result.tool_calls == (
[] if choice == "none" else [{"name": "weather", "arguments": "{}"}]
)
@pytest.mark.parametrize("delay", [1, 3])
def test_delayed_text_keeps_split_markers_intact(parser_cls, mock_tokenizer, delay):
output = (
BEGIN_THINK
+ "reasoning"
+ END_THINK
+ "content"
+ BEGIN_TOOL_REQUESTS
+ _tool_call("weather", "{}")
+ END_TOOL_REQUESTS
+ EOT
)
tokens = [(i, mock_tokenizer.decode([i])) for i in mock_tokenizer.encode(output)]
result = collect_output(
replay_with_text_holdback(
parser_cls(mock_tokenizer), tokens, text_delay=delay, tools=DUMMY_TOOLS
)
)
assert (result.reasoning, result.content) == ("reasoning", "content")
assert result.tool_calls == [{"name": "weather", "arguments": "{}"}]
File diff suppressed because it is too large Load Diff
-235
View File
@@ -1,235 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based seed_oss parser.
seed_oss is Qwen3 with four overridden wrapper tokens, so the shared grammar
(arg types, multiline values, parallel calls, streaming mechanics, …) is
already covered by ``test_qwen3.py``/``test_qwen3_reasoning.py``. These tests
cover only what is seed_oss-specific: that the ``seed:`` token overrides are
wired through, the reasoning→tool boundary holds with them, the malformed
header from #46314 no longer drops sibling calls, and the registered adapters
resolve. Seed-specific budget-reflect tags inside reasoning are also covered
here because the old dedicated parser tests exercised them.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_function_name,
collect_tool_arguments,
simulate_reasoning_streaming,
simulate_tool_streaming,
)
from vllm.parser.engine.registered_adapters import (
SeedOssParserReasoningAdapter,
SeedOssParserToolAdapter,
)
from vllm.parser.seed_oss import SeedOssParser
TOOL_CALL_START = "<seed:tool_call>"
TOOL_CALL_END = "</seed:tool_call>"
THINK_START = "<seed:think>"
THINK_END = "</seed:think>"
TURN_START = "<seed:bos>"
TURN_END = "<seed:eos>"
_THINK_START_ID = 50
_THINK_END_ID = 51
_TOOL_CALL_START_ID = 60
_TOOL_CALL_END_ID = 61
_SEED_BOS_ID = 70
_SEED_EOS_ID = 71
_TEXT_ID = 100
_SEED_OSS_VOCAB = {
THINK_START: _THINK_START_ID,
THINK_END: _THINK_END_ID,
TOOL_CALL_START: _TOOL_CALL_START_ID,
TOOL_CALL_END: _TOOL_CALL_END_ID,
TURN_START: _SEED_BOS_ID,
TURN_END: _SEED_EOS_ID,
}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_SEED_OSS_VOCAB)
@pytest.fixture
def tool_parser(mock_tokenizer):
return SeedOssParser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
)
@pytest.fixture
def parser(mock_tokenizer):
return SeedOssParser(mock_tokenizer)
def test_token_overrides_wired(parser):
assert parser.parser_engine_config.name == "seed_oss"
assert parser.reasoning_start_str == THINK_START
assert parser.reasoning_end_str == THINK_END
assert parser.parser_engine_config.turn_boundary_tokens == frozenset(
(TURN_START, TURN_END)
)
class TestSeedTurnBoundaries:
"""The Seed-OSS chat template wraps every message in ``<seed:bos>role`` /
``<seed:eos>`` and replays prior-turn ``reasoning_content`` inside
``<seed:think>...</seed:think>``, so a stale ``</seed:think>`` in history
is the normal multi-turn case. The backward walk must stop at the Seed
boundary tokens; the inherited ChatML tokens are absent from the Seed
vocab and would disable the scoping entirely.
"""
def test_replayed_reasoning_in_history_not_end(self, parser):
assert not parser.is_reasoning_end(
[
_SEED_BOS_ID,
_THINK_START_ID,
_TEXT_ID,
_THINK_END_ID,
_TEXT_ID,
_SEED_EOS_ID,
_SEED_BOS_ID,
]
)
def test_think_end_in_current_turn_is_end(self, parser):
assert parser.is_reasoning_end(
[_SEED_BOS_ID, _TEXT_ID, _SEED_EOS_ID, _SEED_BOS_ID, _THINK_END_ID]
)
def test_boundary_with_thinking_disabled_is_end(self, mock_tokenizer):
parser = SeedOssParser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
)
assert parser.is_reasoning_end([_SEED_BOS_ID, _TEXT_ID])
def test_single_tool_call(tool_parser, mock_request):
text = (
f"{TOOL_CALL_START}\n<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
f"</function>\n{TOOL_CALL_END}"
)
result = tool_parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert result.tool_calls[0].function.name == "get_weather"
assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Tokyo"}
def test_malformed_function_end_does_not_drop_siblings(tool_parser, mock_request):
"""Regression for #46314: a malformed ``</function>`` with no closing ``>``
on the header must not discard the other, well-formed calls."""
text = (
f"{TOOL_CALL_START}\n<function=broken</function>\n{TOOL_CALL_END}"
f"{TOOL_CALL_START}\n<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
f"</function>\n{TOOL_CALL_END}"
)
result = tool_parser.extract_tool_calls(text, mock_request)
weather = next(tc for tc in result.tool_calls if tc.function.name == "get_weather")
assert json.loads(weather.function.arguments) == {"city": "Tokyo"}
def test_basic_streaming(tool_parser, mock_request):
chunks = [
f"{TOOL_CALL_START}\n",
"<function=get_weather>\n",
"<parameter=city>Tokyo",
"</parameter>\n",
"</function>\n",
f"{TOOL_CALL_END}",
]
results = simulate_tool_streaming(tool_parser, mock_request, chunks)
assert collect_function_name(results) == "get_weather"
assert json.loads(collect_tool_arguments(results)) == {"city": "Tokyo"}
def test_reasoning_then_tool_call(parser):
text = (
f"{THINK_START}I need to read the file.{THINK_END}"
f"{TOOL_CALL_START}\n<function=read>\n"
"<parameter=path>/tmp/x</parameter>\n"
f"</function>\n{TOOL_CALL_END}"
)
reasoning, _ = parser.extract_reasoning(text, None)
assert reasoning == "I need to read the file."
assert TOOL_CALL_START not in reasoning
def test_streaming_think_end_and_tool_call_same_delta(parser):
"""``</seed:think>`` and ``<seed:tool_call>`` arriving in one delta must
not leak the terminal tokens into the reasoning text."""
reasoning, content = simulate_reasoning_streaming(
parser,
[
"Let me list the directory.",
f"{THINK_END}{TOOL_CALL_START}",
"<function=read>",
],
[(1,), (_THINK_END_ID, _TOOL_CALL_START_ID), (2,)],
)
assert reasoning == "Let me list the directory."
assert THINK_END not in reasoning
assert TOOL_CALL_START not in reasoning
assert content is not None
def test_end_to_end_through_registered_adapters(mock_tokenizer, mock_request):
reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer)
tool_parser = SeedOssParserToolAdapter(mock_tokenizer)
text = (
f"{THINK_START}Plan the call.{THINK_END}"
f"{TOOL_CALL_START}\n<function=get_weather>\n"
"<parameter=city>Tokyo</parameter>\n"
f"</function>\n{TOOL_CALL_END}"
)
reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request)
assert reasoning == "Plan the call."
tool_result = tool_parser.extract_tool_calls(remaining, mock_request)
assert tool_result.tool_calls[0].function.name == "get_weather"
assert json.loads(tool_result.tool_calls[0].function.arguments) == {"city": "Tokyo"}
def test_budget_reflect_tags_do_not_break_adapter_pipeline(
mock_tokenizer,
mock_request,
):
reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer)
tool_parser = SeedOssParserToolAdapter(mock_tokenizer)
text = (
f"{THINK_START}"
"The user's current thinking budget is 512.</seed:cot_budget_reflect>\n"
"I need the weather.\n"
"<seed:cot_budget_reflect>I have used 131 tokens."
"</seed:cot_budget_reflect>\n"
f"{THINK_END}"
f"{TOOL_CALL_START}\n<function=get_weather>\n"
"<parameter=city>Barcelona</parameter>\n"
f"</function>\n{TOOL_CALL_END}"
)
reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request)
assert reasoning is not None
assert "current thinking budget is 512" in reasoning
assert "<seed:cot_budget_reflect>" in reasoning
assert "</seed:cot_budget_reflect>" in reasoning
tool_result = tool_parser.extract_tool_calls(remaining, mock_request)
assert tool_result.tool_calls[0].function.name == "get_weather"
assert json.loads(tool_result.tool_calls[0].function.arguments) == {
"city": "Barcelona"
}
-122
View File
@@ -1,122 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Step-3.5 parser.
Step-3.5 reuses the Qwen3 grammar, which ``test_qwen3.py`` covers. These tests
cover what differs: thinking is always on, the newline the model emits before
``</think>`` is dropped, prior-turn reasoning does not end the current turn,
and the ``step3p5`` names resolve to the engine adapters.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
from vllm.parser.engine.registered_adapters import (
Step3p5ParserReasoningAdapter,
Step3p5ParserToolAdapter,
)
from vllm.parser.step3p5 import Step3p5Parser
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
_THINK_START_ID = 50
_THINK_END_ID = 51
_IM_START_ID = 70
_IM_END_ID = 71
_TEXT_ID = 100
_STEP3P5_VOCAB = {
"<think>": _THINK_START_ID,
"</think>": _THINK_END_ID,
"<tool_call>": 60,
"</tool_call>": 61,
"<|im_start|>": _IM_START_ID,
"<|im_end|>": _IM_END_ID,
}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_STEP3P5_VOCAB)
@pytest.fixture
def parser(mock_tokenizer):
return Step3p5Parser(mock_tokenizer)
def test_parser_names_resolve_to_engine_adapters():
tool_parser_cls = ToolParserManager.get_tool_parser("step3p5")
assert issubclass(tool_parser_cls, Step3p5ParserToolAdapter)
assert tool_parser_cls.structural_tag_model == "qwen_3_coder"
reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser("step3p5")
assert reasoning_parser_cls is Step3p5ParserReasoningAdapter
def test_enable_thinking_false_is_ignored(mock_tokenizer):
# The chat template always prefills "<think>\n" and has no thinking switch.
parser = Step3p5Parser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
)
assert parser.extract_reasoning("plan</think>answer", None) == ("plan", "answer")
assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID, _THINK_START_ID])
def test_newline_before_think_end_dropped(parser):
text = "Line one.\nLine two.\n</think>\nThe answer."
assert parser.extract_reasoning(text, None) == (
"Line one.\nLine two.",
"\nThe answer.",
)
def test_newline_before_think_end_dropped_streaming(parser):
reasoning, content = simulate_reasoning_streaming(
parser,
["Line one.\n", "Line two.\n", "</think>", "\nThe answer."],
[(_TEXT_ID,), (_TEXT_ID,), (_THINK_END_ID,), (_TEXT_ID,)],
)
assert reasoning == "Line one.\nLine two."
assert content == "\nThe answer."
class TestPriorTurnReasoning:
"""The template replays earlier assistant turns as
``<think>\\n...\\n</think>\\n``, so a ``</think>`` from history must not end
reasoning for the new turn (#34211)."""
_HISTORY = [
_IM_START_ID,
_TEXT_ID,
_THINK_START_ID,
_TEXT_ID,
_THINK_END_ID,
_TEXT_ID,
_IM_END_ID,
]
def test_prior_turn_think_end_not_end(self, parser):
assert not parser.is_reasoning_end([*self._HISTORY, _IM_START_ID, _TEXT_ID])
def test_think_end_in_current_turn_is_end(self, parser):
assert parser.is_reasoning_end(
[*self._HISTORY, _IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID]
)
def test_reasoning_then_tool_call(parser, mock_request):
text = (
"Need the weather.\n</think>\n<tool_call>\n<function=get_weather>\n"
"<parameter=city>\nTokyo\n</parameter>\n</function>\n</tool_call>"
)
reasoning, content, tool_calls = parser.parse(
text, mock_request, enable_auto_tools=True
)
assert reasoning == "Need the weather."
assert content is None
assert tool_calls is not None
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Tokyo"}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-599
View File
@@ -1,599 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from mistral_common.protocol.instruct.messages import (
AssistantMessage,
ThinkChunk,
ToolMessage,
UserMessage,
)
from mistral_common.protocol.instruct.request import InstructRequest
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
from tests.reasoning.utils import run_reasoning_extraction_mistral
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.parser_manager import ParserManager
from vllm.reasoning import ReasoningParser, ReasoningParserManager
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tokenizers.mistral import MistralTokenizer
_PARSER_NAME = "mistral"
_MODEL_V13 = "mistralai/Magistral-Small-2509"
_MODEL_V11 = "mistralai/Magistral-Small-2506"
_SAMPLE_TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a city.",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
]
@pytest.fixture(scope="module")
def mistral_tokenizer() -> MistralTokenizer:
"""v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens."""
return MistralTokenizer.from_pretrained(_MODEL_V13)
@pytest.fixture(scope="module")
def mistral_v11_tokenizer() -> MistralTokenizer:
"""v11 Magistral tokenizer using plain-text `<think>`/`</think>` tags."""
return MistralTokenizer.from_pretrained(_MODEL_V11)
def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]:
"""Encode output string with `[THINK]`/`[/THINK]` markers into token IDs.
`[THINK]` and `[/THINK]` placeholders in `output` are replaced by their
special-token IDs; all surrounding text is encoded with the base tokenizer.
Mirrors the encoding helper from the deleted
`tests/reasoning/test_mistral_reasoning_parser.py`.
"""
think_start = "[THINK]"
think_end = "[/THINK]"
len_start = len(think_start)
len_end = len(think_end)
index_start = output.find(think_start)
index_end = output.find(think_end)
out: list[int] = []
if index_start != -1:
out += tokenizer.tokenizer.encode(output[:index_start], False, False)
out += [tokenizer.instruct.BEGIN_THINK]
if index_end != -1:
middle = output[index_start + len_start : index_end]
suffix = output[index_end + len_end :]
out += tokenizer.tokenizer.encode(middle, False, False)
out += [tokenizer.instruct.END_THINK]
out += tokenizer.tokenizer.encode(suffix, False, False)
else:
out += tokenizer.tokenizer.encode(
output[index_start + len_start :], False, False
)
elif index_end != -1:
out += tokenizer.tokenizer.encode(output[:index_end], False, False)
out += [tokenizer.instruct.END_THINK]
out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False)
else:
out += tokenizer.tokenizer.encode(output, False, False)
return out
_TEST_CASES = [
# v13 special-token encoding: [THINK]…[/THINK] with trailing content.
pytest.param(
False,
"[THINK]r[/THINK]c",
"r",
"c",
id="v13_basic",
),
pytest.param(
True,
"[THINK]r[/THINK]c",
"r",
"c",
id="v13_basic_streaming",
),
# No trailing content after reasoning ends.
pytest.param(
False,
"[THINK]r[/THINK]",
"r",
None,
id="v13_no_trailing_content",
),
pytest.param(
True,
"[THINK]r[/THINK]",
"r",
None,
id="v13_no_trailing_content_streaming",
),
# Multi-line reasoning and multi-line content.
pytest.param(
False,
"[THINK]a\nb[/THINK]c\nd",
"a\nb",
"c\nd",
id="v13_multiline",
),
pytest.param(
True,
"[THINK]a\nb[/THINK]c\nd",
"a\nb",
"c\nd",
id="v13_multiline_streaming",
),
# No reasoning at all: output is pure content.
pytest.param(
False,
"hello world",
None,
"hello world",
id="v13_no_reasoning",
),
pytest.param(
True,
"hello world",
None,
"hello world",
id="v13_no_reasoning_streaming",
),
# [THINK] opened but never closed; reasoning accumulates, content is None.
pytest.param(
False,
"[THINK]r",
"r",
None,
id="v13_think_no_end",
),
pytest.param(
True,
"[THINK]r",
"r",
None,
id="v13_think_no_end_streaming",
),
# Stray [/THINK] with no matching [THINK]: no reasoning, content follows.
pytest.param(
False,
"[/THINK]This is the rest",
None,
"This is the rest",
id="v13_stray_think_end",
),
pytest.param(
True,
"[/THINK]This is the rest",
None,
"This is the rest",
id="v13_stray_think_end_streaming",
),
# Content before [THINK]: leading segment joins with trailing segment.
pytest.param(
False,
"Before\n[THINK]r[/THINK]\nAfter",
"r",
"Before\n\nAfter",
id="v13_leading_content",
),
pytest.param(
True,
"Before\n[THINK]r[/THINK]\nAfter",
"r",
"Before\n\nAfter",
id="v13_leading_content_streaming",
),
# Empty output: no reasoning, no content.
pytest.param(
False,
"",
None,
None,
id="v13_empty",
),
pytest.param(
True,
"",
None,
None,
id="v13_empty_streaming",
),
]
@pytest.mark.parametrize(
"streaming, output, expected_reasoning, expected_content",
_TEST_CASES,
)
def test_mistral_reasoning_v13(
streaming: bool,
output: str,
expected_reasoning: str | None,
expected_content: str | None,
mistral_tokenizer: MistralTokenizer,
) -> None:
output_tokens = _encode_v13(mistral_tokenizer, output)
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
mistral_tokenizer
)
reasoning, content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == expected_reasoning
assert content == expected_content
def test_system_prompt_think_ignored(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""Streaming content extraction must not
include `[THINK]`/`[/THINK]` examples from the system prompt.
"""
output = "[THINK]real[/THINK]answer"
output_tokens = _encode_v13(mistral_tokenizer, output)
for streaming in (False, True):
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
_PARSER_NAME
)(mistral_tokenizer)
reasoning, content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == "real"
assert content == "answer"
def test_reasoning_with_tool_calls(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""Reasoning extraction with a tool call in the generated output."""
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
if tool_calls_token_id is None:
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
# Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1}
output_tokens = (
[mistral_tokenizer.instruct.BEGIN_THINK]
+ mistral_tokenizer.tokenizer.encode("r", False, False)
+ [mistral_tokenizer.instruct.END_THINK]
+ [tool_calls_token_id]
+ mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False)
)
for streaming in (False, True):
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
_PARSER_NAME
)(mistral_tokenizer)
reasoning, _content = run_reasoning_extraction_mistral(
parser, output_tokens, streaming=streaming
)
assert reasoning == "r"
assert "[TOOL_CALLS]" not in (reasoning or "")
@pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"])
def test_reasoning_v11_plain_text_think(
mistral_v11_tokenizer: MistralTokenizer, streaming: bool
) -> None:
"""v11 tokenizers use plain-text `<think>`/`</think>` (no special tokens)."""
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
mistral_v11_tokenizer
)
tokens = mistral_v11_tokenizer.tokenizer.encode("<think>r</think>c", False, False)
reasoning, content = run_reasoning_extraction_mistral(
parser, tokens, streaming=streaming
)
assert reasoning == "r"
assert content == "c"
def test_is_reasoning_end(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""is_reasoning_end returns True iff reasoning has finished."""
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
mistral_tokenizer
)
# Complete reasoning: END_THINK present → True.
complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c")
assert parser.is_reasoning_end(complete_ids)
# Open-only: no END_THINK, BEGIN_THINK present → False.
open_ids = _encode_v13(mistral_tokenizer, "[THINK]r")
assert not parser.is_reasoning_end(open_ids)
# No reasoning markers at all → False.
no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello")
assert not parser.is_reasoning_end(no_reasoning_ids)
# Explicit END_THINK closes reasoning → True.
explicit_end_ids = (
[mistral_tokenizer.instruct.BEGIN_THINK]
+ mistral_tokenizer.tokenizer.encode("r", False, False)
+ [mistral_tokenizer.instruct.END_THINK]
)
assert parser.is_reasoning_end(explicit_end_ids)
# [TOOL_CALLS] acts as implicit reasoning-end marker.
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
if tool_calls_token_id is None:
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
implicit_end_ids = (
[mistral_tokenizer.instruct.BEGIN_THINK]
+ mistral_tokenizer.tokenizer.encode("r", False, False)
+ [tool_calls_token_id]
)
assert parser.is_reasoning_end(implicit_end_ids)
def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids):
"""Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content."""
reasoning = ""
content = ""
previous_tokens: list[str] | None = None
prefix_offset = 0
read_offset = 0
for i, tok in enumerate(gen_ids):
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer=tokenizer,
all_input_ids=gen_ids[: i + 1],
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=True,
spaces_between_special_tokens=True,
)
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
delta_message = parser.parse_delta(
delta_text=delta_text,
delta_token_ids=[tok],
request=request,
prompt_token_ids=prompt_ids,
finished=(i == len(gen_ids) - 1),
)
if delta_message is not None:
reasoning += delta_message.reasoning or ""
content += delta_message.content or ""
return reasoning, content
def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""A closed `[THINK]` block in the prompt must not finalize reasoning.
Regression: reasoning system prompts end the prompt with a closed
`[THINK]...[/THINK]` block. With a parser-injected grammar the generation's
reasoning must still be captured as ``reasoning_content`` rather than
leaking into ``content`` (the prompt-based reasoning-end heuristic must be
skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check,
which the isolated reasoning-parser tests do not cover.
"""
parser_cls = ParserManager.get_parser(
tool_parser_name="mistral",
reasoning_parser_name="mistral",
enable_auto_tools=True,
)
parser = parser_cls(mistral_tokenizer, None)
request = parser.adjust_request(
ChatCompletionRequest(
messages=[],
model="test",
tools=_SAMPLE_TOOLS,
tool_choice="auto",
)
)
assert getattr(request, "_grammar_from_parser", False)
prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]")
gen_ids = _encode_v13(
mistral_tokenizer,
"[THINK]because two plus two is four[/THINK]The answer is 4.",
)
reasoning, content = _stream_parse_delta(
parser, mistral_tokenizer, gen_ids, request, prompt_ids
)
assert "because two plus two is four" in reasoning
assert "because two plus two is four" not in content
assert "The answer is 4." in content
def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]:
"""Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]`
to their special-token IDs; all other text uses the base tokenizer.
"""
vocab = tokenizer.get_vocab()
markers: dict[str, int] = {
"[THINK]": tokenizer.instruct.BEGIN_THINK,
"[/THINK]": tokenizer.instruct.END_THINK,
"[TOOL_CALLS]": vocab["[TOOL_CALLS]"],
"[ARGS]": vocab["[ARGS]"],
}
out: list[int] = []
remaining = text
while remaining:
earliest: int | None = None
earliest_marker: str | None = None
for marker in markers:
idx = remaining.find(marker)
if idx != -1 and (earliest is None or idx < earliest):
earliest = idx
earliest_marker = marker
if earliest_marker is None:
out += tokenizer.tokenizer.encode(remaining, False, False)
break
assert earliest is not None # set together with earliest_marker
if earliest > 0:
out += tokenizer.tokenizer.encode(remaining[:earliest], False, False)
out.append(markers[earliest_marker])
remaining = remaining[earliest + len(earliest_marker) :]
return out
def _stream_turn(
parser,
tokenizer: MistralTokenizer,
gen_ids: list[int],
request,
prompt_ids: list[int],
) -> tuple[str, str, list[tuple[str, str]]]:
"""Stream `gen_ids` through `parse_delta`, returning `(reasoning, content,
tool_calls)`.
`tool_calls` is a list of `(name, arguments)` tuples ordered by
`DeltaToolCall.index`. Arguments are concatenated across deltas.
"""
reasoning = ""
content = ""
tool_call_acc: dict[int, dict[str, str]] = {}
previous_tokens: list[str] | None = None
prefix_offset = 0
read_offset = 0
for i, tok in enumerate(gen_ids):
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
tokenizer=tokenizer,
all_input_ids=gen_ids[: i + 1],
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=True,
spaces_between_special_tokens=True,
)
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
delta_message = parser.parse_delta(
delta_text=delta_text,
delta_token_ids=[tok],
request=request,
prompt_token_ids=prompt_ids,
finished=(i == len(gen_ids) - 1),
)
if delta_message is not None:
reasoning += delta_message.reasoning or ""
content += delta_message.content or ""
for tc in delta_message.tool_calls:
if tc.index not in tool_call_acc:
tool_call_acc[tc.index] = {"name": "", "args": ""}
if tc.function:
if tc.function.name:
tool_call_acc[tc.index]["name"] = tc.function.name
if tc.function.arguments:
tool_call_acc[tc.index]["args"] += tc.function.arguments
tool_calls = [
(tool_call_acc[k]["name"], tool_call_acc[k]["args"])
for k in sorted(tool_call_acc)
]
return reasoning, content, tool_calls
def test_parse_delta_multi_turn_reason_and_tool(
mistral_tokenizer: MistralTokenizer,
) -> None:
"""Two-turn loop: each assistant turn carries reasoning + a tool call.
For each turn, the mistral-common conversation is encoded into `prompt_ids`
before streaming the next generation. Turn 2's prompt therefore contains
a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the
accumulated tool-call/result history that single-turn tests never exercise.
Asserts that across both turns the parser correctly classifies think tokens
as `reasoning_content` (never leaking into `content`), and that the tool
call name/arguments are reconstructed correctly from the streaming deltas.
"""
parser_cls = ParserManager.get_parser(
tool_parser_name="mistral",
reasoning_parser_name="mistral",
enable_auto_tools=True,
)
request = parser_cls(mistral_tokenizer, None).adjust_request(
ChatCompletionRequest(
messages=[],
model="test",
tools=_SAMPLE_TOOLS,
tool_choice="auto",
)
)
assert getattr(request, "_grammar_from_parser", False)
cities = ["Dallas", "Paris"]
conversation: list = [UserMessage(content="What is the weather in Dallas?")]
for i, city in enumerate(cities):
# Fresh parser per turn: streaming state must not bleed across turns.
parser = parser_cls(mistral_tokenizer, None)
prompt_ids = mistral_tokenizer.instruct.encode_instruct(
InstructRequest(messages=conversation)
).tokens
gen_str = (
f"[THINK]I should check {city} weather.[/THINK]"
f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}'
)
gen_ids = _encode_gen(mistral_tokenizer, gen_str)
reasoning, content, tool_calls = _stream_turn(
parser, mistral_tokenizer, gen_ids, request, prompt_ids
)
assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured"
assert f"check {city} weather" not in content, (
f"turn {i}: reasoning leaked into content"
)
assert "[THINK]" not in content, f"turn {i}: [THINK] leaked"
assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked"
assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call"
name, args = tool_calls[0]
assert name == "get_weather", f"turn {i}: wrong tool name {name!r}"
assert city in args, f"turn {i}: city not in args {args!r}"
# Extend conversation with this turn's history for the next turn.
tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001"
conversation.extend(
[
AssistantMessage(
content=[
ThinkChunk(
thinking=f"I should check {city} weather.",
closed=True,
)
],
tool_calls=[
ToolCall(
id=tc_id,
function=FunctionCall(
name="get_weather",
arguments=json.dumps({"city": city}),
),
)
],
),
ToolMessage(
content=f"{city}: 70F sunny",
tool_call_id=tc_id,
),
]
)
if i < len(cities) - 1:
conversation.append(UserMessage(content=f"And in {cities[i + 1]}?"))
File diff suppressed because it is too large Load Diff
-747
View File
@@ -1,747 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from copy import deepcopy
from types import SimpleNamespace
from typing import Any
import pytest
from openai.types.responses import FunctionTool, ToolChoiceFunction
from xgrammar import Grammar
from xgrammar.testing import _is_grammar_accept_string
from vllm.config.reasoning import ReasoningConfig
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.entrypoints.openai.responses.protocol import (
ResponsesRequest,
ResponsesResponse,
)
from vllm.exceptions import VLLMValidationError
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
from vllm.parser.parser_manager import ParserManager
from vllm.parser.response_template import ResponseTemplateParser
from vllm.reasoning import ReasoningParserManager
from vllm.tool_parsers import ToolParserManager
# The `response_template` of the Beam checkpoints.
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
"defaults": {"role": "assistant"},
"start_anchor_pattern": r"<\|start\|>assistant",
"fields": {
"thinking": {
"open": "<|channel|>analysis<|message|>",
"close": "<|end|><|start|>assistant",
"content": "text",
},
"content": {
"open": "<|channel|>final<|message|>",
"close": ["<|return|>", "<|end|>", "<|call|>"],
"content": "text",
},
"tool_calls": {
"open_pattern": (
r" to=(?P<name>[^\s<]+)"
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
),
"close": ["<|end|><|start|>assistant", "<|call|>"],
"repeats": True,
"content": "xml-inline",
"content_args": {
"tag_pattern": (
r'<rfl:parameter name="(?P<key>[^"]+)">'
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
),
"value_parser": {"name": "text", "args": {"strip": False}},
},
"transform": {
"type": "function",
"function": {"name": "{name}", "arguments": "{content}"},
},
},
},
}
VOCAB = {
"<|start|>": 200,
"<|channel|>": 201,
"<|message|>": 202,
"<|end|>": 203,
"<|call|>": 204,
"<|return|>": 205,
"<|constrain|>": 206,
}
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
PROMPT_TOKEN_IDS = [200, 100]
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
FINAL = "<|channel|>final<|message|>"
NEXT_MESSAGE = "<|end|><|start|>assistant"
def _tool(name, properties, *, strict=None):
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": {"type": "object", "properties": properties},
"strict": strict,
},
)
TOOLS = [
_tool("get_weather", {"city": {"type": "string"}}),
_tool("set_count", {"count": {"type": "integer"}}),
]
STRICT_TOOLS = [
_tool("get_weather", {"city": {"type": "string"}}),
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
]
class BeamTokenizer:
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
self.response_template = response_template
self.init_kwargs = {}
self.name_or_path = ""
self.all_special_tokens = list(VOCAB)
self.all_special_ids = list(VOCAB.values())
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
return "".join(
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
for token_id in token_ids
)
def get_vocab(self):
return VOCAB
def encode(self, text, add_special_tokens=False):
del add_special_tokens
return list(text.encode())
def request(**overrides):
values = dict(
tool_choice="auto",
include_reasoning=True,
tools=TOOLS,
parallel_tool_calls=True,
)
values.update(overrides)
return SimpleNamespace(**values)
def parameter(name, value):
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
def call(name, body, *, terminator="<|call|>"):
return (
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
)
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
parser_cls = ParserManager.get_parser(
tool_parser_name="beam",
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
assert parser_cls is not None
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
parser.set_prompt_token_ids(prompt_token_ids)
return parser
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
reasoning, content, calls = make_parser(prompt_token_ids).parse(
text, request(), enable_auto_tools=True
)
return reasoning, content, normalize_calls(calls)
def normalize_calls(calls):
return [(call.name, json.loads(call.arguments)) for call in calls or []]
def test_beam_parsers_are_registered():
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
def test_parser_manager_selects_beam_parser():
parser_cls = ParserManager.get_parser(
tool_parser_name="beam",
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
assert parser_cls is not None
assert issubclass(parser_cls, BeamParser)
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
assert parser_cls.tool_parser_cls is BeamToolParser
assert parser_cls._enable_auto_tools
@pytest.mark.parametrize("other", ["hermes", "hf"])
def test_beam_parser_rejects_other_parsers(other):
with pytest.raises(TypeError, match="cannot be combined"):
ParserManager.get_parser(
tool_parser_name=other,
reasoning_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(),
)
def test_beam_parser_requires_checkpoint_metadata_at_startup():
with pytest.raises(
TypeError, match="--reasoning-parser beam requires `response_template`"
):
ParserManager.get_parser(
reasoning_parser_name="beam",
tokenizer=BeamTokenizer(response_template=None),
)
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
del template["fields"]["tool_calls"]
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
ParserManager.get_parser(
tool_parser_name="beam",
enable_auto_tools=True,
tokenizer=BeamTokenizer(template),
)
def test_reasoning_and_final_answer():
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
def test_prefilled_reasoning_continues_from_prompt():
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
assert parse(text, prompt) == (" more.", "Done.", [])
def test_parallel_calls_are_coerced_to_their_schemas():
text = (
THOUGHT
+ call(
"get_weather",
parameter("city", "Paris"),
terminator="<|end|><|start|>assistant",
)
+ call("set_count", parameter("count", "7"))
)
assert parse(text) == (
"plan",
None,
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
)
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
def test_output_cut_before_the_next_message_start_ends_its_region(end):
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
its reasoning or call and lacks only the next message's start."""
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
assert parse(THOUGHT + weather) == (
"plan",
None,
[("get_weather", {"city": "Paris"})],
)
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
FINAL_IDS = [201, 102, 202]
@pytest.mark.parametrize(
("prompt_token_ids", "token_ids", "count"),
[
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
],
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
)
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
parser = make_parser(prompt_token_ids)
assert parser.count_reasoning_tokens(token_ids) == count
def test_reasoning_tokens_are_counted_as_the_generation_grows():
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
parser = make_parser()
assert [
parser.count_reasoning_tokens(token_ids[:end])
for end in range(1, len(token_ids) + 1)
] == [
make_parser().count_reasoning_tokens(token_ids[:end])
for end in range(1, len(token_ids) + 1)
]
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
def test_call_stop_token_ends_the_answer():
assert parse(FINAL + "done<|call|>") == (None, "done", [])
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
def test_streaming_matches_non_streaming(chunk_size):
text = (
THOUGHT
+ call(
"get_weather",
parameter("city", "Paris"),
terminator="<|end|><|start|>assistant",
)
+ call("set_count", parameter("count", "7"))
)
parser = make_parser()
reasoning, calls = [], []
for start in range(0, len(text), chunk_size):
end = min(start + chunk_size, len(text))
delta = parser.parse_delta(
text[start:end], [], request(), finished=end == len(text)
)
if delta is not None:
reasoning.append(delta.reasoning or "")
calls.extend(delta.tool_calls)
assert "".join(reasoning) == "plan"
assert [
(call.function.name, json.loads(call.function.arguments)) for call in calls
] == parse(text)[2]
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
WEATHER = [("get_weather", {"city": "Paris"})]
def chat_request(tools=STRICT_TOOLS, **body):
return ChatCompletionRequest(
model="beam",
messages=[{"role": "user", "content": "Weather?"}],
tools=tools and [tool.model_dump() for tool in tools],
**body,
)
def named(name):
return {"type": "function", "function": {"name": name}}
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
return req
def accepts(req, text):
grammar = Grammar.from_structural_tag(
req.extract_structured_outputs().structural_tag
)
return _is_grammar_accept_string(grammar, text)
@pytest.mark.parametrize(
("tool_choice", "text", "calls"),
[
("required", THOUGHT + WEATHER_CALL, WEATHER),
("required", WEATHER_CALL, None),
(
"required",
THOUGHT
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
+ call("set_count", parameter("count", "7")),
[*WEATHER, ("set_count", {"count": 7})],
),
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
(
"required",
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
None,
),
("required", THOUGHT + FINAL + "Paris<|return|>", None),
(named("set_count"), THOUGHT + WEATHER_CALL, None),
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
(
named("get_weather"),
THOUGHT
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
+ WEATHER_CALL,
None,
),
],
)
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
"""From the message start, the grammar opens with the reasoning, and
accepts only calls the parser reads back."""
req = adjust(chat_request(tool_choice=tool_choice))
assert req._grammar_from_parser
assert accepts(req, text) is (calls is not None)
if calls is not None:
assert parse(text)[2] == calls
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
req = adjust(
chat_request(tool_choice="required"),
chat_template_kwargs={"enable_thinking": False},
)
assert accepts(req, WEATHER_CALL)
assert accepts(req, THOUGHT + WEATHER_CALL)
def test_parallel_tool_calls_false_allows_a_single_call():
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
assert accepts(req, THOUGHT + WEATHER_CALL)
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
@pytest.mark.parametrize("strict", [False, True])
@pytest.mark.parametrize(
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
)
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
"""A model that skips a parameter's opener must not write whitespace until
`max_tokens`."""
write = _tool(
"write",
{"path": {"type": "string"}, "content": {"type": "string"}},
strict=strict,
)
req = adjust(chat_request([write], tool_choice="required"))
body = parameter("path", "a.py") + separator + parameter("content", "x")
assert accepts(req, THOUGHT + call("write", body)) is accepted
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
def test_forced_calls_in_prefilled_reasoning_start_after_it():
req = adjust(
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
)
assert not req._grammar_from_parser
assert accepts(req, WEATHER_CALL)
def test_forced_calls_cannot_follow_a_prefilled_answer():
with pytest.raises(VLLMValidationError, match="required tool choice"):
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
def test_prefilled_answer_can_only_be_finished():
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
assert accepts(req, "Sunny.<|return|>")
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
def test_forced_call_with_unencodable_tool_name_is_rejected():
tools = [_tool("get weather", {"city": {"type": "string"}})]
with pytest.raises(VLLMValidationError, match="tool name"):
adjust(chat_request(tools, tool_choice="required"))
def test_string_formats_and_patterns_are_held_in_raw_text():
"""Like XGrammar, a strict string keeps its format or pattern, unless a
match could end the value early."""
contact = _tool(
"set_contact",
{
"email": {"type": "string", "format": "email"},
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
"n": {"type": "integer"},
},
strict=True,
)
contact.function.parameters["required"] = ["email", "code", "n"]
req = adjust(chat_request([contact], tool_choice="required"))
def contact_call(email, code, *rest):
body = parameter("email", email) + parameter("code", code) + "".join(rest)
return THOUGHT + call("set_contact", body)
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
assert accepts(req, text)
assert normalize_calls(calls) == [
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
]
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
# Line breaks could end the value.
dotted = _tool(
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
)
with pytest.raises(VLLMValidationError, match="required tool choice"):
adjust(chat_request([dotted], tool_choice="required"))
@pytest.mark.parametrize(
("text", "accepted"),
[
(THOUGHT + WEATHER_CALL, True),
(THOUGHT + FINAL + "Sunny.<|return|>", True),
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
(THOUGHT + call("set_count", parameter("count", "x")), False),
],
)
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
assert accepts(adjust(chat_request()), text) is accepted
@pytest.mark.parametrize(
("text", "accepted"),
[
(THOUGHT + FINAL + "Sunny.<|return|>", True),
(FINAL + "Sunny.<|return|>", False),
(WEATHER_CALL, False),
(THOUGHT + WEATHER_CALL, False),
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
],
)
def test_tool_choice_none_allows_only_an_answer(text, accepted):
req = adjust(chat_request(TOOLS, tool_choice="none"))
assert req._grammar_from_parser
assert accepts(req, text) is accepted
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
req = adjust(chat_request(tools, tool_choice=tool_choice))
count_call = THOUGHT + call("set_count", parameter("count", "3"))
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
assert not accepts(req, FINAL + "Sunny.<|return|>")
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
assert not accepts(req, THOUGHT + "Sunny.")
assert accepts(req, count_call) is (tool_choice == "auto")
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
@pytest.mark.parametrize("strict", [None, True])
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
def test_automatic_choice_lets_calls_name_undeclared_tools(
strict, tool_choice, response_format
):
"""Unless a forced choice limits calls to the declared tools. The parser
passes such calls on, so the harness can tell the model the tool does not
exist."""
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
req = adjust(
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
)
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
parser = make_parser(tools=[bash])
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
assert accepts(req, undeclared) is (tool_choice == "auto")
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
@pytest.mark.parametrize(
("name", "body", "accepted"),
[
("set_count", parameter("count", "1"), True),
("set_count", parameter("count", "x"), False),
("get_weather", parameter("town", "Paris"), False),
# The grammar cannot enforce `minProperties`.
("find", parameter("x", "1") + parameter("x", "y"), True),
("search_files", parameter("x", "1") + parameter("x", "y"), True),
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
],
)
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
name, body, accepted
):
find = _tool("find", {"q": {"type": "string"}})
find.function.parameters["minProperties"] = 1
req = adjust(chat_request([*TOOLS, find]))
assert accepts(req, THOUGHT + call(name, body)) is accepted
def test_value_ends_before_an_opener_on_the_next_line():
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
req = adjust(chat_request([write], tool_choice="required"))
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
text = THOUGHT + call("write", body)
assert accepts(req, text)
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
assert normalize_calls(calls) == [
("write", {"content": "print(1)\n", "path": "a.py"})
]
def test_always_constrained_output_can_be_disabled(monkeypatch):
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
assert adjust(chat_request(TOOLS)).structured_outputs is None
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
dotted = _tool(
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
)
assert adjust(chat_request([dotted])).structured_outputs is None
def test_generic_parser_does_not_derive_the_template_grammar():
req = chat_request(TOOLS, tool_choice="required")
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
with pytest.raises(VLLMValidationError, match="required tool choice"):
parser.adjust_request(req)
@pytest.mark.parametrize("tools", [None, TOOLS])
def test_structured_output_is_framed_as_the_answer(tools):
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
req = adjust(
chat_request(
tools,
response_format={
"type": "json_schema",
"json_schema": {"name": "answer", "schema": schema},
},
)
)
assert req.response_format is None
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
def test_responses_named_tool_choice_follows_the_template_grammar():
req = adjust(
ResponsesRequest(
input="Weather?",
tools=[
FunctionTool(
type="function",
name="get_weather",
parameters={
"type": "object",
"properties": {"city": {"type": "string"}},
},
strict=True,
)
],
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
text={"format": {"type": "json_object"}},
)
)
assert req._grammar_from_parser
assert req.text.format.type == "json_object"
assert accepts(req, THOUGHT + WEATHER_CALL)
assert not accepts(req, FINAL + "{}<|return|>")
@pytest.mark.parametrize("status", ["in_progress", "completed"])
@pytest.mark.parametrize("inside_reasoning", [False, True])
def test_responses_preserves_text_config_with_template_grammar(
status, inside_reasoning
):
text = {
"format": {
"type": "json_schema",
"name": "answer",
"strict": True,
"schema": {
"type": "object",
"properties": {"a": {"type": "integer"}},
"required": ["a"],
"additionalProperties": False,
},
},
"verbosity": "low",
}
req = ResponsesRequest(input="Answer", text=text)
expected_text = req.text.model_dump(by_alias=True)
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
adjust(req, prompt_token_ids=prompt)
params = req.to_sampling_params(default_max_tokens=128)
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
prefix = "" if inside_reasoning else THOUGHT
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
assert response.model_dump(by_alias=True)["text"] == expected_text
@pytest.mark.parametrize("reverse_alternatives", [False, True])
def test_reasoning_budget_initializes_from_checkpoint_pattern(
monkeypatch, reverse_alternatives
):
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
alternatives = [
r"^<\|channel\|>analysis<\|message\|>",
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
]
if reverse_alternatives:
alternatives.reverse()
template["fields"]["thinking"] = {
"open_pattern": "(?:" + "|".join(alternatives) + ")",
"close": "<|end|>",
"content": "text",
}
tokenizer = BeamTokenizer(template)
monkeypatch.setattr(
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
)
config = ReasoningConfig(reasoning_parser="beam")
config.initialize_token_ids(SimpleNamespace())
assert config.enabled
# The marker must work both after the prompt's assistant header and in
# subsequent complete assistant messages, regardless of regex branch order.
assert config.reasoning_start_token_ids == tokenizer.encode(
"<|channel|>analysis<|message|>"
)
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
thinking = template["fields"]["thinking"]
del thinking["open"]
thinking["open_pattern"] = pattern
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
-440
View File
@@ -1,440 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for include_reasoning suppression in the unified Parser interface.
Covers non-streaming (parser.parse() + build_response_output_items),
streaming (parse_delta), and ParsableContext.append_output() paths.
"""
import json
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
class ThinkReasoningParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return "<think>"
@property
def end_token(self) -> str:
return "</think>"
MODEL_OUTPUT_REASONING_AND_CONTENT = (
"<think>let me think about this</think>The answer is 42."
)
MODEL_OUTPUT_REASONING_AND_TOOL = (
"<think>I need to call a tool</think>"
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
MODEL_OUTPUT_CONTENT_ONLY = "The answer is 42."
@pytest.fixture(scope="module")
def tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")
def make_responses_request(**kwargs) -> ResponsesRequest:
defaults = dict(model="test-model", input="test input")
defaults.update(kwargs)
return ResponsesRequest(**defaults)
def make_chat_request(**kwargs) -> ChatCompletionRequest:
defaults = dict(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
)
defaults.update(kwargs)
return ChatCompletionRequest(**defaults)
def make_parser(tokenizer, reasoning=False, tool=False):
class TestParser(DelegatingParser):
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
tool_parser_cls = Hermes2ProToolParser if tool else None
return TestParser(tokenizer)
# ── Non-streaming: parser.parse() + build_response_output_items ──────
def parse_and_build(parser, request, model_output, enable_auto_tools=False):
"""Mirrors the non-streaming path in _make_response_output_items /
ParsableContext.append_output(): parse → suppress reasoning → build items.
"""
from vllm.entrypoints.openai.responses.utils import (
build_response_output_items,
)
reasoning, content, tool_calls = parser.parse(
model_output, request, enable_auto_tools=enable_auto_tools
)
if not request.include_reasoning:
reasoning = None
return build_response_output_items(
reasoning=reasoning,
content=content,
tool_calls=tool_calls,
)
class TestNonStreamingIncludeReasoning:
def test_include_reasoning_true_has_reasoning_item(self, tokenizer):
"""Default: reasoning items appear in output."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=True)
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
types = [o.type for o in outputs]
assert "reasoning" in types
assert "message" in types
def test_include_reasoning_false_no_reasoning_item(self, tokenizer):
"""Reasoning item is suppressed when include_reasoning=False."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=False)
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
types = [o.type for o in outputs]
assert "reasoning" not in types
assert "message" in types
assert outputs[0].content[0].text == "The answer is 42."
def test_include_reasoning_false_content_preserved(self, tokenizer):
"""Content is extracted correctly even when reasoning is suppressed."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=False)
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
message = next(o for o in outputs if o.type == "message")
assert message.content[0].text == "The answer is 42."
def test_include_reasoning_false_tool_calls_preserved(self, tokenizer):
"""Tool calls still work when reasoning is suppressed."""
parser = make_parser(tokenizer, reasoning=True, tool=True)
request = make_responses_request(
include_reasoning=False,
tools=[
{
"type": "function",
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
}
],
)
outputs = parse_and_build(
parser,
request,
MODEL_OUTPUT_REASONING_AND_TOOL,
enable_auto_tools=True,
)
types = [o.type for o in outputs]
assert "reasoning" not in types
assert "function_call" in types
fc = next(o for o in outputs if o.type == "function_call")
assert fc.name == "get_weather"
assert json.loads(fc.arguments) == {"city": "Dallas"}
def test_no_reasoning_parser_include_false_is_noop(self, tokenizer):
"""include_reasoning=False is harmless when no reasoning parser."""
parser = make_parser(tokenizer, reasoning=False)
request = make_responses_request(include_reasoning=False)
outputs = parse_and_build(parser, request, MODEL_OUTPUT_CONTENT_ONLY)
assert len(outputs) == 1
assert outputs[0].type == "message"
assert outputs[0].content[0].text == MODEL_OUTPUT_CONTENT_ONLY
def test_default_include_reasoning_is_true(self, tokenizer):
"""ResponsesRequest defaults to include_reasoning=True."""
request = make_responses_request()
assert request.include_reasoning is True
def test_include_reasoning_false_suppresses_all_reasoning(self, tokenizer):
"""Reasoning is suppressed regardless of request type."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=False)
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
assert all(o.type != "reasoning" for o in outputs)
# ── Streaming: parse_delta ───────────────────────────────────────────
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
token_ids = tokenizer.encode(text, add_special_tokens=False)
results: list[DeltaMessage | None] = []
for i, tid in enumerate(token_ids):
delta_text = tokenizer.decode([tid])
is_last = i == len(token_ids) - 1
result = parser.parse_delta(
delta_text,
[tid],
request,
prompt_token_ids=prompt_token_ids,
finished=is_last,
)
prompt_token_ids = None
results.append(result)
return results
def collect_fields(results):
all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
all_content = "".join(r.content for r in results if r and r.content)
all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
return all_reasoning, all_content, all_tool_calls
class TestParseDeltaIncludeReasoning:
def test_streaming_include_true_emits_reasoning(self, tokenizer):
"""With include_reasoning=True, reasoning deltas are emitted."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=True)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_REASONING_AND_CONTENT,
request,
prompt_token_ids=[],
)
reasoning, content, _ = collect_fields(results)
assert "let me think about this" in reasoning
assert "42" in content
def test_streaming_include_false_suppresses_reasoning(self, tokenizer):
"""With include_reasoning=False, no reasoning deltas are emitted."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=False)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_REASONING_AND_CONTENT,
request,
prompt_token_ids=[],
)
reasoning, content, _ = collect_fields(results)
assert reasoning == ""
assert "42" in content
def test_streaming_include_false_content_still_works(self, tokenizer):
"""Content is correctly extracted in streaming even with suppression."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=False)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_REASONING_AND_CONTENT,
request,
prompt_token_ids=[],
)
_, content, _ = collect_fields(results)
assert "The answer is 42" in content
def test_streaming_include_false_tool_calls_preserved(self, tokenizer):
"""Tool calls stream correctly when reasoning is suppressed."""
parser = make_parser(tokenizer, reasoning=True, tool=True)
request = make_responses_request(
include_reasoning=False,
tools=[
{
"type": "function",
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
}
],
)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_REASONING_AND_TOOL,
request,
prompt_token_ids=[],
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_streaming_no_reasoning_parser_include_false(self, tokenizer):
"""No crash when reasoning parser absent and include_reasoning=False."""
parser = make_parser(tokenizer, reasoning=False)
request = make_responses_request(include_reasoning=False)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_CONTENT_ONLY,
request,
prompt_token_ids=[],
)
reasoning, content, _ = collect_fields(results)
assert reasoning == ""
assert "42" in content
def test_streaming_chat_completion_include_false(self, tokenizer):
"""parse_delta also respects ChatCompletionRequest.include_reasoning."""
parser = make_parser(tokenizer, reasoning=True)
request = make_chat_request(include_reasoning=False)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_REASONING_AND_CONTENT,
request,
prompt_token_ids=[],
)
reasoning, content, _ = collect_fields(results)
assert reasoning == ""
assert "42" in content
def test_streaming_reasoning_only_deltas_become_none(self, tokenizer):
"""Deltas that carry only reasoning become None (not empty)."""
parser = make_parser(tokenizer, reasoning=True)
request = make_responses_request(include_reasoning=False)
results = stream_text(
parser,
tokenizer,
MODEL_OUTPUT_REASONING_AND_CONTENT,
request,
prompt_token_ids=[],
)
for r in results:
if r is not None:
assert r.reasoning is None
# ── ParsableContext.append_output() ───────────────────────────────────
class TestParsableContextIncludeReasoning:
def _make_context(self, tokenizer, request):
from vllm.entrypoints.openai.responses.context import ParsableContext
class TestParser(DelegatingParser):
reasoning_parser_cls = ThinkReasoningParser
tool_parser_cls = None
return ParsableContext(
tokenizer=tokenizer,
parser_cls=TestParser,
response_messages=[],
request=request,
available_tools=None,
chat_template=None,
chat_template_content_format="auto",
)
def test_process_include_false_suppresses_reasoning(self, tokenizer):
"""ParsableContext.process() suppresses reasoning items."""
from vllm.outputs import CompletionOutput, RequestOutput
request = make_responses_request(include_reasoning=False)
ctx = self._make_context(tokenizer, request)
output = RequestOutput(
request_id="test",
prompt=None,
prompt_token_ids=[],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=MODEL_OUTPUT_REASONING_AND_CONTENT,
token_ids=tokenizer.encode(
MODEL_OUTPUT_REASONING_AND_CONTENT,
add_special_tokens=False,
),
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
)
ctx.append_output(output)
types = [getattr(m, "type", None) for m in ctx.response_messages]
assert "reasoning" not in types
assert "message" in types
def test_process_include_true_has_reasoning(self, tokenizer):
"""ParsableContext.process() includes reasoning by default."""
from vllm.outputs import CompletionOutput, RequestOutput
request = make_responses_request(include_reasoning=True)
ctx = self._make_context(tokenizer, request)
output = RequestOutput(
request_id="test",
prompt=None,
prompt_token_ids=[],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text=MODEL_OUTPUT_REASONING_AND_CONTENT,
token_ids=tokenizer.encode(
MODEL_OUTPUT_REASONING_AND_CONTENT,
add_special_tokens=False,
),
cumulative_logprob=None,
logprobs=None,
finish_reason="stop",
)
],
finished=True,
)
ctx.append_output(output)
types = [getattr(m, "type", None) for m in ctx.response_messages]
assert "reasoning" in types
assert "message" in types
-598
View File
@@ -1,598 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from types import SimpleNamespace
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.parser_manager import ParserManager
from vllm.parser.qwen3 import qwen3_config
from vllm.parser.utils import count_history_tool_calls
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
@pytest.fixture(autouse=True)
def enable_hermes_required_named_parsing(monkeypatch):
# With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets
# supports_required_and_named=False and uses structural-tag guided tool
# calling. Force it True to test the non-guided JSON required/named parse path.
monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True)
class ThinkReasoningParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return "<think>"
@property
def end_token(self) -> str:
return "</think>"
MODEL_OUTPUT = (
"<think>let me think about this</think>"
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
PLAIN_TEXT = "The weather in Dallas is sunny and 75°F."
TOOL_CALL_ONLY = (
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
TOOL_ARGUMENTS = '{"city": "Dallas"}'
@pytest.fixture(scope="module")
def tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")
def make_request(**overrides):
base = {
"model": "test-model",
"messages": [{"role": "user", "content": "hi"}],
}
base.update(overrides)
return ChatCompletionRequest.model_validate(base)
TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
},
}
]
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
hf_overrides=None,
)
HISTORY_MESSAGES = [
{"role": "user", "content": "first"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "functions.get_current_weather:0",
"type": "function",
"function": {
"name": "get_current_weather",
"arguments": "{}",
},
},
{
"id": "functions.get_forecast:1",
"type": "function",
"function": {
"name": "get_forecast",
"arguments": "{}",
},
},
],
},
{
"role": "tool",
"tool_call_id": "functions.get_current_weather:0",
"content": "{}",
},
{
"role": "tool",
"tool_call_id": "functions.get_forecast:1",
"content": "{}",
},
{"role": "user", "content": "again"},
]
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
class TestParser(DelegatingParser):
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
tool_parser_cls = Hermes2ProToolParser if tool else None
return TestParser(tokenizer, **kwargs)
@pytest.mark.parametrize(
"reasoning,tool",
[(False, False), (False, True)],
ids=["neither", "tool-only"],
)
def test_parse_plain_text_no_reasoning_parser(tokenizer, reasoning, tool):
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
request = make_request()
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
assert r is None
assert content == PLAIN_TEXT
assert tool_calls is not None
assert len(tool_calls) == 0
@pytest.mark.parametrize(
"reasoning,tool",
[(True, False), (True, True)],
ids=["reasoning-only", "both"],
)
def test_parse_plain_text_with_reasoning_parser(tokenizer, reasoning, tool):
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
request = make_request()
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
assert r == PLAIN_TEXT
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_both_parsers(tokenizer):
parser = make_parser(tokenizer, reasoning=True, tool=True)
request = make_request(tools=TOOLS)
reasoning, content, tool_calls = parser.parse(
MODEL_OUTPUT, request, enable_auto_tools=True
)
assert reasoning is not None
assert "let me think about this" in reasoning
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
assert not content or content.strip() == ""
def test_parse_reasoning_only(tokenizer):
parser = make_parser(tokenizer, reasoning=True, tool=False)
request = make_request()
reasoning, content, tool_calls = parser.parse(MODEL_OUTPUT, request)
assert reasoning is not None
assert "let me think about this" in reasoning
assert content is not None
assert "<tool_call>" in content
assert "get_weather" in content
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_tool_only(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(tools=TOOLS)
reasoning, content, tool_calls = parser.parse(
MODEL_OUTPUT, request, enable_auto_tools=True
)
assert reasoning is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
def test_parse_named_tool_choice(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
reasoning, content, tool_calls = parser.parse(
TOOL_ARGUMENTS, request, enable_auto_tools=True
)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert tool_calls[0].arguments == TOOL_ARGUMENTS
def test_parse_named_tool_choice_with_reasoning(tokenizer):
parser = make_parser(tokenizer, reasoning=True, tool=True)
model_output = f"<think>thinking</think>{TOOL_ARGUMENTS}"
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
reasoning, content, tool_calls = parser.parse(
model_output, request, enable_auto_tools=True
)
assert reasoning is not None
assert "thinking" in reasoning
assert content is None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
assert tool_calls[0].arguments == TOOL_ARGUMENTS
def test_parse_required_tool_choice(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
functions_json = json.dumps(
[
{"name": "get_weather", "parameters": {"city": "Dallas"}},
{"name": "get_time", "parameters": {"timezone": "UTC"}},
]
)
request = make_request(tools=TOOLS, tool_choice="required")
reasoning, content, tool_calls = parser.parse(
functions_json, request, enable_auto_tools=True
)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 2
assert tool_calls[0].name == "get_weather"
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
assert tool_calls[1].name == "get_time"
assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
def test_parse_required_tool_choice_kimi_k2_ids(tokenizer):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
functions_json = json.dumps(
[
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 2}},
]
)
request = make_request(tools=TOOLS, tool_choice="required")
_, content, tool_calls = parser.parse(
functions_json, request, enable_auto_tools=True
)
assert content is None
assert tool_calls is not None
assert [tc.id for tc in tool_calls] == [
"functions.get_current_weather:0",
"functions.get_forecast:1",
]
def test_parse_required_tool_choice_kimi_k2_ids_after_history(tokenizer):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
functions_json = json.dumps(
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
)
request = make_request(
messages=HISTORY_MESSAGES,
tools=TOOLS,
tool_choice="required",
)
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
assert tool_calls is not None
assert tool_calls[0].id == "functions.get_current_weather:2"
def test_count_history_tool_calls_responses_request():
request = ResponsesRequest.model_validate(
{
"model": "test-model",
"input": [
{
"type": "function_call",
"call_id": "call_0",
"name": "get_current_weather",
"arguments": "{}",
},
{
"type": "function_call",
"call_id": "call_1",
"name": "get_forecast",
"arguments": "{}",
},
],
}
)
assert count_history_tool_calls(request) == 2
def test_parse_required_tool_choice_random_ids_deferred(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
functions_json = json.dumps(
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
)
request = make_request(
messages=HISTORY_MESSAGES,
tools=TOOLS,
tool_choice="required",
)
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
assert tool_calls is not None
assert tool_calls[0].id is None
def test_parse_named_tool_choice_kimi_k2_id(tokenizer):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
_, content, tool_calls = parser.parse(
TOOL_ARGUMENTS, request, enable_auto_tools=True
)
assert content is None
assert tool_calls is not None
assert tool_calls[0].id == "functions.get_weather:0"
def test_parse_named_tool_choice_content_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(
tools=TOOLS,
tool_choice={
"type": "function",
"function": {"name": "get_weather"},
},
)
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
assert reasoning is None
assert content is None
assert tool_calls is not None
def test_parse_required_tool_choice_content_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(tools=TOOLS, tool_choice="required")
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
assert reasoning is None
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_auto_tools_no_parser(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=False)
request = make_request()
reasoning, content, tool_calls = parser.parse(
TOOL_CALL_ONLY, request, enable_auto_tools=True
)
assert reasoning is None
assert content == TOOL_CALL_ONLY
assert tool_calls is not None
assert len(tool_calls) == 0
def test_parse_auto_tools_no_calls_returns_none(tokenizer):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = make_request(tools=TOOLS)
reasoning, content, tool_calls = parser.parse(
PLAIN_TEXT, request, enable_auto_tools=True
)
assert reasoning is None
assert content == PLAIN_TEXT
assert tool_calls is None
# --------------------------------------------------------------------------
# Prompt that already closes the reasoning block
# --------------------------------------------------------------------------
CONTINUE_MESSAGES = [
{"role": "user", "content": "hi"},
{"role": "assistant", "content": "Sure,"},
]
REASONED_OUTPUT = "<think>\nlet me think\n</think>\n\n" + PLAIN_TEXT
TOOL_TURN_MESSAGES = [
{"role": "user", "content": "Weather in Paris?"},
{
"role": "assistant",
"content": "<think>\nneed a tool\n</think>\n\n",
"tool_calls": [
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
]
def reasoning_parser(tokenizer, name="qwen3"):
parser_cls = ParserManager.get_parser(
reasoning_parser_name=name, tokenizer=tokenizer
)
return parser_cls(tokenizer, None)
def encode(tokenizer, text):
return tokenizer.encode(text, add_special_tokens=False)
def prompt_case(tokenizer, case):
"""(rendered prompt ids, request, model output) for each prompt shape."""
if case == "continue_final_message":
# The template renders an empty, closed think block before "Sure,".
prompt = tokenizer.apply_chat_template(
CONTINUE_MESSAGES,
tokenize=False,
add_generation_prompt=False,
continue_final_message=True,
)
request = make_request(
messages=CONTINUE_MESSAGES,
add_generation_prompt=False,
continue_final_message=True,
)
return encode(tokenizer, prompt), request, " here is the answer"
if case == "template_closed_thinking":
# The template switches thinking off, but the parser is not told
# through chat_template_kwargs (#53284).
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "hi"}],
tokenize=False,
add_generation_prompt=True,
enable_thinking=False,
)
return encode(tokenizer, prompt), make_request(), PLAIN_TEXT
if case == "earlier_turn_closed_reasoning":
# Only a previous assistant turn closes reasoning; this one is open.
prompt = tokenizer.apply_chat_template(
TOOL_TURN_MESSAGES,
tools=[{"type": "function", "function": {"name": "get_weather"}}],
tokenize=False,
add_generation_prompt=True,
)
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
assert case == "reasoning_open"
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": "hi"}],
tokenize=False,
add_generation_prompt=True,
)
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"])
def test_parse_prompt_closed_reasoning_is_content(tokenizer, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
parser = reasoning_parser(tokenizer)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning is None
assert content == output
assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0
@pytest.mark.parametrize(
("parser_name", "case"),
[
("qwen3", "reasoning_open"),
("qwen3", "earlier_turn_closed_reasoning"),
# BaseThinkingReasoningParser scans the prompt without turn
# boundaries, so it must not decide non-streaming classification.
("deepseek_r1", "earlier_turn_closed_reasoning"),
],
)
def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
parser = reasoning_parser(tokenizer, parser_name)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning.strip() == "let me think"
assert content.strip() == PLAIN_TEXT
assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0
def test_parse_prompt_ignored_without_turn_boundaries(tokenizer):
"""Without turn boundaries, an earlier turn's </think> looks like this
turn's, so the engine must not start the output in content."""
prompt_ids, request, output = prompt_case(
tokenizer, "earlier_turn_closed_reasoning"
)
parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True))
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=encode(tokenizer, output)
)
assert reasoning.strip() == "let me think"
assert content.strip() == PLAIN_TEXT
@pytest.mark.parametrize(
"case",
[
"continue_final_message",
"template_closed_thinking",
"reasoning_open",
"earlier_turn_closed_reasoning",
],
)
def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case):
prompt_ids, request, output = prompt_case(tokenizer, case)
output_ids = encode(tokenizer, output)
parser = reasoning_parser(tokenizer)
parser.set_prompt_token_ids(prompt_ids)
reasoning, content, _ = parser.parse(
output, request, model_output_token_ids=output_ids
)
non_streaming = (reasoning or "", content or "")
non_streaming_count = parser.count_reasoning_tokens(output_ids)
parser = reasoning_parser(tokenizer)
streamed_reasoning = streamed_content = ""
for i, token_id in enumerate(output_ids):
delta = parser.parse_delta(
tokenizer.decode([token_id]),
[token_id],
request,
prompt_token_ids=prompt_ids,
finished=i == len(output_ids) - 1,
)
if delta is not None:
streamed_reasoning += delta.reasoning or ""
streamed_content += delta.content or ""
assert (streamed_reasoning, streamed_content) == non_streaming
assert parser.count_reasoning_tokens(output_ids) == non_streaming_count
File diff suppressed because it is too large Load Diff
-835
View File
@@ -1,835 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from types import SimpleNamespace
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
@pytest.fixture(autouse=True)
def enable_hermes_required_named_parsing(monkeypatch):
# With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets
# supports_required_and_named=False and uses structural-tag guided tool
# calling. Force it True to test the non-guided JSON required/named parse path.
monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True)
class ThinkReasoningParser(BaseThinkingReasoningParser):
@property
def start_token(self) -> str:
return "<think>"
@property
def end_token(self) -> str:
return "</think>"
MODEL_OUTPUT = (
"<think>let me think about this</think>"
'<tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
@pytest.fixture(scope="module")
def tokenizer():
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")
TOOLS = [
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {"type": "object", "properties": {}},
},
}
]
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
hf_overrides=None,
)
HISTORY_MESSAGES = [
{"role": "user", "content": "first"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "functions.get_current_weather:0",
"type": "function",
"function": {
"name": "get_current_weather",
"arguments": "{}",
},
}
],
},
{
"role": "tool",
"tool_call_id": "functions.get_current_weather:0",
"content": "{}",
},
{"role": "user", "content": "again"},
]
@pytest.fixture
def request_obj():
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=TOOLS,
tool_choice="auto",
)
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
class TestParser(DelegatingParser):
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
tool_parser_cls = Hermes2ProToolParser if tool else None
return TestParser(tokenizer, **kwargs)
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
token_ids = tokenizer.encode(text, add_special_tokens=False)
results: list[DeltaMessage | None] = []
for tid in token_ids:
delta_text = tokenizer.decode([tid])
result = parser.parse_delta(
delta_text,
[tid],
request,
prompt_token_ids=prompt_token_ids,
finished=False,
)
prompt_token_ids = None
results.append(result)
return results
def collect_fields(results):
all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
all_content = "".join(r.content for r in results if r and r.content)
all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
return all_reasoning, all_content, all_tool_calls
def test_parse_delta_neither_parser(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=False, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ""
assert len(tool_calls) == 0
assert "<think>" in content
assert "let me think about this" in content
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_tool_parser_only(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ""
assert "<think>" in content
assert "let me think about this" in content
assert "</think>" in content
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_parse_delta_reasoning_parser_only(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
assert "</tool_call>" in content
def test_parse_delta_both_parsers(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=True, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def stream_chunks(parser, tokenizer, chunks, request_obj):
"""Stream pre-split token-ID chunks through the parser."""
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for chunk in chunks:
delta_text = tokenizer.decode(chunk)
result = parser.parse_delta(
delta_text,
chunk,
request_obj,
prompt_token_ids=prompt_token_ids,
finished=False,
)
prompt_token_ids = None
results.append(result)
return results
def _boundary_chunks(tokenizer, parser, end_token_id=None):
"""Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
if end_token_id is None:
end_token_id = parser._reasoning_parser.end_token_id
end_idx = token_ids.index(end_token_id)
return [
token_ids[: end_idx - 1],
token_ids[end_idx - 1 : end_idx + 2],
token_ids[end_idx + 2 :],
]
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
"""Regression: reasoning must not be lost when a multi-token delta
spans the reasoning/tool-call boundary."""
parser = make_parser(tokenizer, reasoning=True, tool=True)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
"""When no tool parser is active, boundary-spanning chunks must still
preserve reasoning and pass post-</think> text as content."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
chunks = _boundary_chunks(tokenizer, parser)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
"""Regression: </think> must not leak into content when streaming
token-by-token with reasoning=True, tool=False."""
parser = make_parser(tokenizer, reasoning=True, tool=False)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert "</think>" not in content
assert "<think>" not in content
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
"""Regression test for vllm-project/vllm#40466.
When enable_thinking=False, the chat template places <think>\\n\\n</think>
in the prompt. The model then generates pure content (no think tokens).
All streaming output must go to delta.content, not delta.reasoning.
"""
parser = make_parser(tokenizer, reasoning=True, tool=False)
end_token_id = parser._reasoning_parser.end_token_id
prompt_token_ids = [1, 2, end_token_id, 3]
content_text = "Hello! How can I assist you today?"
results = stream_text(
parser,
tokenizer,
content_text,
request_obj,
prompt_token_ids=prompt_token_ids,
)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == "", f"Expected no reasoning, got: {reasoning!r}"
assert "Hello" in content
assert "assist" in content
assert len(tool_calls) == 0
def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, request_obj):
"""When finished=True but the final parse_delta produces no
tool-call delta, unstreamed args are not flushed."""
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
_, _, tool_calls = collect_fields(results)
assert len(tool_calls) > 0
streamed = parser._tool_parser.streamed_args_for_tool[0]
assert len(streamed) > 5
parser._tool_parser.streamed_args_for_tool[0] = streamed[:-5]
# Prevent normal extraction from catching the gap — without a
# tool-call delta to merge into, the flush is skipped.
parser._tool_parser.extract_tool_calls_streaming = lambda *a, **kw: None
flush_result = parser.parse_delta("", [], request_obj, finished=True)
assert flush_result is None or flush_result.tool_calls is None
def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, request_obj):
"""When all args have been streamed, finished=True must not
produce extra or duplicate arguments."""
parser = make_parser(tokenizer, reasoning=False, tool=True)
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
_, _, tool_calls = collect_fields(results)
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
flush_result = parser.parse_delta("", [], request_obj, finished=True)
assert flush_result is None or flush_result.tool_calls is None
def test_parse_delta_finished_appends_remaining_args(tokenizer, request_obj):
"""When finished=True and the tool parser has unstreamed args,
parse_delta appends the remaining arguments to the tool-call delta."""
parser = make_parser(tokenizer, reasoning=False, tool=True)
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
remainder = ',"unit":"celsius"}'
prompt_ids: list[int] | None = []
results: list[DeltaMessage | None] = []
for i, tid in enumerate(token_ids):
prev = results[-1] if results else None
prev_had_args = (
prev
and prev.tool_calls
and any(tc.function and tc.function.arguments for tc in prev.tool_calls)
)
if prev_had_args:
parser._tool_parser.get_remaining_unstreamed_args = lambda: remainder
result = parser.parse_delta(
tokenizer.decode([tid]),
[tid],
request_obj,
prompt_token_ids=prompt_ids,
finished=prev_had_args,
)
prompt_ids = None
results.append(result)
if prev_had_args:
break
_, _, tool_calls = collect_fields(results)
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert tool_args.endswith(remainder)
def test_parse_delta_tool_choice_none(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=False, tool=True)
request = request_obj.model_copy(update={"tool_choice": "none"})
results = stream_text(parser, tokenizer, MODEL_OUTPUT, request, prompt_token_ids=[])
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ""
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj):
parser = make_parser(tokenizer, reasoning=True, tool=True)
request = request_obj.model_copy(update={"tool_choice": "none"})
results = stream_text(parser, tokenizer, MODEL_OUTPUT, request, prompt_token_ids=[])
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert len(tool_calls) == 0
assert "<tool_call>" in content
assert "get_weather" in content
def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
request = request_obj.model_copy(update={"tool_choice": "required"})
output = json.dumps(
[
{
"name": "get_current_weather",
"parameters": {"city": "Dallas"},
}
]
)
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for i in range(0, len(output), 3):
chunk = output[i : i + 3]
results.append(
parser.parse_delta(
chunk,
[],
request,
prompt_token_ids=prompt_token_ids,
finished=False,
)
)
prompt_token_ids = None
_, content, tool_calls = collect_fields(results)
assert content == ""
assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls)
assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls)
def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history(
tokenizer, request_obj
):
parser = make_parser(
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
)
request = request_obj.model_copy(
update={"messages": HISTORY_MESSAGES, "tool_choice": "required"}
)
output = json.dumps(
[
{
"name": "get_current_weather",
"parameters": {"city": "Dallas"},
}
]
)
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for i in range(0, len(output), 3):
chunk = output[i : i + 3]
results.append(
parser.parse_delta(
chunk,
[],
request,
prompt_token_ids=prompt_token_ids,
finished=False,
)
)
prompt_token_ids = None
_, _, tool_calls = collect_fields(results)
assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls)
assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls)
# ── Engine-based reasoning + non-engine tool parser (Qwen3 + Hermes) ──
class Qwen3ReasoningHermesToolParser(DelegatingParser):
reasoning_parser_cls = Qwen3ParserReasoningAdapter
tool_parser_cls = Hermes2ProToolParser
def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj):
"""Qwen3 engine reasoning + Hermes tool parser, token-by-token.
Sanity check that the mixed engine/non-engine configuration works
when tokens arrive one at a time (no deferred content)."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
assert parser._reasoning_parser.engine_based_streaming is True
assert parser._tool_parser.engine_based_streaming is False
assert parser._engine_based is False
results = stream_text(
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj):
"""Qwen3 engine reasoning + Hermes tool parser, boundary chunks.
When </think> and <tool_call> are in the same chunk with aligned
text and token IDs, the engine processes both terminals and returns
the <tool_call> text as content."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id)
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "think about this" in reasoning
assert content == ""
assert len(tool_calls) > 0
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
assert "tool_call" not in content
def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj):
"""Qwen3 engine reasoning + Hermes tool parser with engine holdback.
Simulates stream_interval > 1 where a batched delta contains
'</think><'. The '<' is a regular character token — not the
<tool_call> special token — so the engine's incremental lexer
buffers it (it could be the start of a text terminal like
<tool_call>). The buffered '<' is only recoverable via
finish_streaming().
Without the fix, finish_streaming() is never called at the
reasoning->tool transition when _engine_based is False, so the '<'
is lost and the Hermes parser sees 'tool_call>...' instead of
'<tool_call>...'."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
vocab = tokenizer.get_vocab()
think_end_id = vocab["</think>"]
lt_id = vocab["<"]
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
end_idx = token_ids.index(think_end_id)
# Reasoning tokens (aligned text + IDs)
pre_ids = token_ids[:end_idx]
pre_text = tokenizer.decode(pre_ids)
# Batched delta: '</think><' — the engine recognises </think> as
# THINK_END but the trailing '<' is consumed by the engine's lexer
# and held back (it could be the start of <tool_call>). The '<'
# is NOT in delta_message.content; it is only in the engine's
# internal buffer, recoverable via finish_streaming().
holdback_ids = [think_end_id, lt_id]
holdback_text = "</think><"
# Remaining text: 'tool_call>\n{...}\n</tool_call>' — the model
# generated <tool_call> as character tokens (not the special token),
# and the '<' was consumed above. Encode separately to get the
# correct token IDs for this substring.
rest_text = (
'tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</tool_call>'
)
rest_ids = tokenizer.encode(rest_text, add_special_tokens=False)
results: list[DeltaMessage | None] = []
results.append(
parser.parse_delta(
pre_text,
pre_ids,
request_obj,
prompt_token_ids=[],
finished=False,
)
)
results.append(
parser.parse_delta(
holdback_text,
holdback_ids,
request_obj,
finished=False,
)
)
results.append(
parser.parse_delta(
rest_text,
rest_ids,
request_obj,
finished=False,
)
)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert len(tool_calls) > 0, (
"Tool calls lost at engine-reasoning -> tool transition. "
"finish_streaming() not called when _engine_based is False."
)
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": "Dallas"}
assert "tool_call" not in content
# ── Engine-based reasoning WITHOUT a tool parser (Qwen3 only) ──
class Qwen3ReasoningNoToolParser(DelegatingParser):
reasoning_parser_cls = Qwen3ParserReasoningAdapter
tool_parser_cls = None
def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request_obj):
"""Qwen3 engine reasoning with NO tool parser, batched boundary.
The three mixed-parser tests above all pair the engine reasoning
parser with Hermes; none exercise the engine-reasoning-only path
through the hoisted finish_streaming() transition (where
``_engine_based`` is True and there is no tool parser). A single
batched delta carries ``</think>`` plus the following content
(as happens with stream_interval > 1). The post-``</think>`` text
must be emitted as content -- not dropped, not reclassified as
reasoning -- and the ``</think>`` marker must not leak either way."""
parser = Qwen3ReasoningNoToolParser(tokenizer)
assert parser._reasoning_parser.engine_based_streaming is True
assert parser._tool_parser is None
model_output = "<think>let me think about this</think>The answer is 42."
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
token_ids = tokenizer.encode(model_output, add_special_tokens=False)
end_idx = token_ids.index(end_token_id)
chunks = [token_ids[:end_idx], token_ids[end_idx:]]
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert content == "The answer is 42."
assert "</think>" not in content
assert "</think>" not in reasoning
assert len(tool_calls) == 0
@pytest.mark.parametrize(
"text_chunks",
[
# A trailing "<" that the engine holds back (it may start a tag).
["<think>let me think</think>a <", "x> b"],
# Whitespace after "</think>" in the same delta as the marker.
["<think>let me think</think>\n\n", "<b>bold</b> done"],
# The delta that ends reasoning carries only held-back text.
["<think>let me think", "</think><", "b>bold</b> done"],
],
)
def test_engine_reasoning_no_tool_batched_holdback_not_dropped(
tokenizer, request_obj, text_chunks
):
"""Text the engine buffers when reasoning ends inside a batched delta is
returned by finish_streaming(); with no tool parser it must still be
emitted as content, not dropped."""
parser = Qwen3ReasoningNoToolParser(tokenizer)
chunks = [tokenizer.encode(t, add_special_tokens=False) for t in text_chunks]
results = stream_chunks(parser, tokenizer, chunks, request_obj)
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == "let me think"
assert content == "".join(text_chunks).split("</think>", 1)[1]
assert len(tool_calls) == 0
@pytest.mark.parametrize(
("text_chunks", "include_reasoning", "expected_content"),
[
# The delta that ends reasoning is also the last one: the held-back
# "<" must be emitted exactly once (the end-of-stream flush skips the
# reasoning parser once reasoning has ended).
(["<think>let me think</think>a <"], True, "a <"),
# Whitespace-only content after the marker at the end of the stream.
(["<think>let me think</think>\n\n"], True, "\n\n"),
# Hiding reasoning must not hide the flushed content.
(["<think>let me think</think>\n\n", "<b>bold</b>"], False, "\n\n<b>bold</b>"),
],
)
def test_engine_reasoning_no_tool_holdback_finished_and_hidden_reasoning(
tokenizer, text_chunks, include_reasoning, expected_content
):
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
include_reasoning=include_reasoning,
)
parser = Qwen3ReasoningNoToolParser(tokenizer)
chunks = [tokenizer.encode(t, add_special_tokens=False) for t in text_chunks]
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for i, chunk in enumerate(chunks):
results.append(
parser.parse_delta(
tokenizer.decode(chunk),
chunk,
request,
prompt_token_ids=prompt_token_ids,
finished=i == len(chunks) - 1,
)
)
prompt_token_ids = None
reasoning, content, tool_calls = collect_fields(results)
assert reasoning == ("let me think" if include_reasoning else "")
assert content == expected_content
assert len(tool_calls) == 0
def _decode_stream_deltas(tokenizer, groups):
"""Decode token-ID groups into ``(delta_text, group)`` pairs via the real
incremental ``DecodeStream``.
This mirrors how vLLM's detokenizer feeds ``parse_delta`` in
production: byte-level UTF-8 hold-back means a character whose bytes
span multiple tokens is only surfaced once complete (a naive
per-token ``decode`` would instead emit U+FFFD replacement chars)."""
from tokenizers.decoders import DecodeStream
stream = DecodeStream(skip_special_tokens=False)
inner = tokenizer._tokenizer
pairs = []
for group in groups:
text = ""
for token_id in group:
piece = stream.step(inner, token_id)
if piece:
text += piece
pairs.append((text, group))
return pairs
def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj):
"""Multi-token character across the reasoning->tool boundary.
Extends the ASCII '<' hold-back guard with bbrowning's multi-token
*character* concern. Two things must both hold:
1. The batched ``</think><`` delta relies on the hoisted
finish_streaming() to recover the engine-buffered '<'. Without
the fix the Hermes parser never sees ``<tool_call>`` and emits no
tool call at all.
2. The tool-call arguments carry ``東京🧑\u200d🚀``; the astronaut
ZWJ sequence's bytes span multiple Qwen3 tokens, so the rest of
the stream is fed one token at a time through the real
``DecodeStream``. Its UTF-8 hold-back yields the correct
character round-trip (a naive per-token decode would corrupt it),
verifying the boundary stays byte-safe for multi-token
characters."""
parser = Qwen3ReasoningHermesToolParser(tokenizer)
vocab = tokenizer.get_vocab()
think_end_id = vocab["</think>"]
lt_id = vocab["<"]
city = "東京🧑\u200d🚀"
# Faithfulness precondition: the emoji really is a multi-token char.
assert len(tokenizer.encode("🧑\u200d🚀", add_special_tokens=False)) > 1
model_output = (
"<think>let me think about this</think>"
'<tool_call>\n{"name": "get_weather", '
f'"arguments": {{"city": "{city}"}}}}\n</tool_call>'
)
token_ids = tokenizer.encode(model_output, add_special_tokens=False)
end_idx = token_ids.index(think_end_id)
pre_ids = token_ids[:end_idx]
rest_text = (
'tool_call>\n{"name": "get_weather", '
f'"arguments": {{"city": "{city}"}}}}\n</tool_call>'
)
rest_ids = tokenizer.encode(rest_text, add_special_tokens=False)
# Deltas: reasoning, then a batched '</think><' (the engine buffers
# the '<'), then the remaining tokens one at a time so the
# multi-token character is genuinely split across deltas by the
# detokenizer.
groups = [pre_ids, [think_end_id, lt_id]] + [[tid] for tid in rest_ids]
pairs = _decode_stream_deltas(tokenizer, groups)
results: list[DeltaMessage | None] = []
prompt_token_ids: list[int] | None = []
for delta_text, group in pairs:
results.append(
parser.parse_delta(
delta_text,
group,
request_obj,
prompt_token_ids=prompt_token_ids,
finished=False,
)
)
prompt_token_ids = None
reasoning, content, tool_calls = collect_fields(results)
assert "let me think about this" in reasoning
assert len(tool_calls) > 0, (
"Tool call lost at engine-reasoning -> tool transition; the "
"buffered '<' was not recovered by finish_streaming()."
)
assert tool_calls[0].function.name == "get_weather"
tool_args = "".join(
tc.function.arguments for tc in tool_calls if tc.function.arguments
)
assert json.loads(tool_args) == {"city": city}
assert content == ""