[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,203 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reasoning extraction, reasoning-end gating and parser selection for the
|
||||
unified Cohere Command parser."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.reasoning.cohere_command_reasoning_parser import (
|
||||
CohereCommand3ReasoningParser,
|
||||
CohereCommand4ReasoningParser,
|
||||
)
|
||||
|
||||
from .utils import make_parser, stream_parser
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReasoningCase:
|
||||
parser_name: str
|
||||
model_output: str
|
||||
expected_reasoning: str | None
|
||||
expected_content: str | None
|
||||
|
||||
|
||||
REASONING_CASES = [
|
||||
pytest.param(
|
||||
ReasoningCase(
|
||||
parser_name="cohere_command3",
|
||||
model_output="""\
|
||||
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_reasoning="i will call foo with query1",
|
||||
expected_content=None,
|
||||
),
|
||||
id="cmd3-single_tool_call",
|
||||
),
|
||||
pytest.param(
|
||||
ReasoningCase(
|
||||
parser_name="cohere_command4",
|
||||
model_output="""\
|
||||
<|START_THINKING|> i will call foo with query1<|END_THINKING|><|START_ACTION|>
|
||||
[
|
||||
{"tool_call_id": "0", "tool_name": "foo", "parameters": {"query": "query1"}}
|
||||
]
|
||||
<|END_ACTION|>""",
|
||||
expected_reasoning="i will call foo with query1",
|
||||
expected_content=None,
|
||||
),
|
||||
id="cmd4-single_tool_call",
|
||||
),
|
||||
pytest.param(
|
||||
ReasoningCase(
|
||||
parser_name="cohere_command3",
|
||||
model_output="""\
|
||||
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
|
||||
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
|
||||
expected_reasoning="This is a rainbow emoji: 🌈",
|
||||
expected_content="foo bar",
|
||||
),
|
||||
id="cmd3-citations_with_emoji",
|
||||
),
|
||||
pytest.param(
|
||||
ReasoningCase(
|
||||
parser_name="cohere_command4",
|
||||
model_output="""\
|
||||
<|START_THINKING|>This is a rainbow <co>emoji: 🌈</co: 0:[1]><|END_THINKING|>
|
||||
<|START_RESPONSE|>foo <co>bar</co: 0:[1,2],1:[3,4]><|END_RESPONSE|>""",
|
||||
expected_reasoning="This is a rainbow emoji: 🌈",
|
||||
expected_content="foo bar",
|
||||
),
|
||||
id="cmd4-citations_with_emoji",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", REASONING_CASES)
|
||||
class TestExtractReasoning:
|
||||
def test_nonstreaming(self, tokenizer, request_obj, case: ReasoningCase):
|
||||
parser = make_parser(tokenizer, case.parser_name)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
case.model_output, request_obj, enable_auto_tools=True
|
||||
)
|
||||
assert reasoning == case.expected_reasoning
|
||||
assert content == case.expected_content
|
||||
# No tools on the request, so any tool call is dropped, not surfaced.
|
||||
assert not tool_calls
|
||||
|
||||
def test_streaming(self, tokenizer, request_obj, case: ReasoningCase):
|
||||
parser = make_parser(tokenizer, case.parser_name)
|
||||
deltas = stream_parser(parser, request_obj, tokenizer, case.model_output)
|
||||
|
||||
reasoning = "".join(d.reasoning for d in deltas if d.reasoning) or None
|
||||
content = "".join(d.content for d in deltas if d.content) or None
|
||||
assert reasoning == case.expected_reasoning
|
||||
assert content == case.expected_content
|
||||
assert all(not d.tool_calls for d in deltas)
|
||||
|
||||
|
||||
class TestFramingTokensStripped:
|
||||
@pytest.mark.parametrize(
|
||||
("parser_name", "content_tags"),
|
||||
[
|
||||
pytest.param(
|
||||
"cohere_command4", ("<|START_TEXT|>", "<|END_TEXT|>"), id="cmd4"
|
||||
),
|
||||
pytest.param(
|
||||
"cohere_command3", ("<|START_RESPONSE|>", "<|END_RESPONSE|>"), id="cmd3"
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", [1, 4], ids=["per_token", "batched"])
|
||||
@pytest.mark.parametrize("with_tools", [False, True], ids=["no_tools", "tools"])
|
||||
def test_content_framing_tokens_stripped(
|
||||
self, tokenizer, parser_name, content_tags, chunk_size, with_tools
|
||||
):
|
||||
parser = make_parser(tokenizer, parser_name)
|
||||
tools = [{"type": "function", "function": {"name": "foo"}}]
|
||||
request = ChatCompletionRequest(
|
||||
messages=[], model="test-model", tools=tools if with_tools else None
|
||||
)
|
||||
parser.adjust_request(request)
|
||||
|
||||
start_tag, end_tag = content_tags
|
||||
generation = (
|
||||
f"<|START_THINKING|>Think deeply. The user greets us.<|END_THINKING|>"
|
||||
f"{start_tag}I'm doing well, thank you{end_tag}"
|
||||
)
|
||||
deltas = stream_parser(parser, request, tokenizer, generation, chunk_size)
|
||||
|
||||
assert "".join(d.reasoning or "" for d in deltas) == (
|
||||
"Think deeply. The user greets us."
|
||||
)
|
||||
assert not any(d.tool_calls for d in deltas)
|
||||
assert "".join(d.content or "" for d in deltas) == "I'm doing well, thank you"
|
||||
|
||||
|
||||
class TestIsReasoningEnd:
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls",
|
||||
[CohereCommand3ReasoningParser, CohereCommand4ReasoningParser],
|
||||
ids=["cmd3", "cmd4"],
|
||||
)
|
||||
def test_is_reasoning_end(self, tokenizer, parser_cls):
|
||||
parser = parser_cls(tokenizer)
|
||||
start_id = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
|
||||
end_id = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
|
||||
chatbot_id = tokenizer.convert_tokens_to_ids("<|CHATBOT_TOKEN|>")
|
||||
content_ids = [99, 100]
|
||||
|
||||
# Generation-only tokens have no chatbot marker, so the whole sequence
|
||||
# is considered.
|
||||
assert parser.is_reasoning_end([end_id])
|
||||
assert parser.is_reasoning_end([start_id, *content_ids, end_id])
|
||||
assert not parser.is_reasoning_end([start_id, *content_ids])
|
||||
|
||||
# Full prompt/history tokens are scoped to the latest chatbot marker,
|
||||
# so stray thinking tokens from the preamble or previous turns are ignored.
|
||||
assert not parser.is_reasoning_end([start_id, end_id, chatbot_id, *content_ids])
|
||||
assert parser.is_reasoning_end(
|
||||
[start_id, end_id, chatbot_id, start_id, *content_ids, end_id]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("parser_name", ["cohere_command3", "cohere_command4"])
|
||||
def test_count_reasoning_tokens(tokenizer, parser_name):
|
||||
parser = make_parser(tokenizer, parser_name)
|
||||
start = tokenizer.convert_tokens_to_ids("<|START_THINKING|>")
|
||||
end = tokenizer.convert_tokens_to_ids("<|END_THINKING|>")
|
||||
|
||||
assert parser.count_reasoning_tokens([99, start, 11, 12, end, 100]) == 2
|
||||
assert parser.count_reasoning_tokens([end, start, 1, start, 2, end, 3, end]) == 3
|
||||
assert parser.count_reasoning_tokens([start, 1, 2]) == 2
|
||||
assert parser.count_reasoning_tokens([1, 2, end]) == 0
|
||||
|
||||
|
||||
class TestParserSelection:
|
||||
@pytest.mark.parametrize("with_tool_parser", [True, False])
|
||||
def test_adjust_request_keeps_framing_tokens(self, tokenizer, with_tool_parser):
|
||||
"""Melody needs the special tokens, with or without a tool parser."""
|
||||
cls = ParserManager.get_parser(
|
||||
"cohere_command4" if with_tool_parser else None,
|
||||
"cohere_command4",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
request = ChatCompletionRequest(messages=[], model="m")
|
||||
assert request.skip_special_tokens
|
||||
cls(tokenizer).adjust_request(request)
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
def test_non_cohere_selection_does_not_import_melody_parser(self, monkeypatch):
|
||||
"""``cohere_melody`` is optional; other parsers must not depend on it."""
|
||||
import sys
|
||||
|
||||
monkeypatch.setitem(sys.modules, "vllm.parser.cohere_command", None)
|
||||
assert ParserManager.get_parser("hermes", "deepseek_r1", True) is not None
|
||||
@@ -1,131 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the unified Cohere Command parser tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser import ParserManager
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
|
||||
_SPECIAL_TOKEN_RE = re.compile(r"(<\|[A-Z_]+\|>)")
|
||||
REPLACEMENT_CHAR = "�"
|
||||
|
||||
|
||||
class MockCohereTokenizer:
|
||||
"""Byte-level stand-in for the Cohere tokenizer.
|
||||
|
||||
Text round-trips through UTF-8 bytes so a multi-byte character can split
|
||||
across "tokens" (reproducing trailing U+FFFD buffering), while each
|
||||
``<|...|>`` marker is a single id so token-id reasoning-end gating behaves
|
||||
as with the real tokenizer.
|
||||
"""
|
||||
|
||||
_SPECIAL_TOKEN_IDS = {
|
||||
tok: 256 + i
|
||||
for i, tok in enumerate(
|
||||
(
|
||||
"<|START_THINKING|>",
|
||||
"<|END_THINKING|>",
|
||||
"<|CHATBOT_TOKEN|>",
|
||||
"<|START_RESPONSE|>",
|
||||
"<|END_RESPONSE|>",
|
||||
"<|START_TEXT|>",
|
||||
"<|END_TEXT|>",
|
||||
"<|START_ACTION|>",
|
||||
"<|END_ACTION|>",
|
||||
)
|
||||
)
|
||||
}
|
||||
_ID_TO_SPECIAL_TOKEN = {v: k for k, v in _SPECIAL_TOKEN_IDS.items()}
|
||||
|
||||
def convert_tokens_to_ids(self, token: str) -> int:
|
||||
return self._SPECIAL_TOKEN_IDS.get(token, 0)
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {}
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
ids: list[int] = []
|
||||
for part in _SPECIAL_TOKEN_RE.split(text):
|
||||
if part in self._SPECIAL_TOKEN_IDS:
|
||||
ids.append(self._SPECIAL_TOKEN_IDS[part])
|
||||
else:
|
||||
ids.extend(part.encode("utf-8"))
|
||||
return ids
|
||||
|
||||
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
out: list[str] = []
|
||||
for special, run in itertools.groupby(
|
||||
ids, self._ID_TO_SPECIAL_TOKEN.__contains__
|
||||
):
|
||||
if not special:
|
||||
out.append(bytes(run).decode("utf-8", errors="replace"))
|
||||
elif not skip_special_tokens:
|
||||
out.extend(self._ID_TO_SPECIAL_TOKEN[i] for i in run)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def token_deltas(
|
||||
tokenizer: MockCohereTokenizer, text: str, chunk_size: int = 1
|
||||
) -> list[tuple[str, list[int]]]:
|
||||
"""Split ``text`` into ``(delta_text, delta_token_ids)`` steps of
|
||||
``chunk_size`` tokens, buffering incomplete multi-byte sequences (trailing
|
||||
U+FFFD) into the next step as real streaming does."""
|
||||
ids = tokenizer.encode(text)
|
||||
deltas: list[tuple[str, list[int]]] = []
|
||||
prev = ""
|
||||
pending: list[int] = []
|
||||
for start in range(0, len(ids), chunk_size):
|
||||
end = start + chunk_size
|
||||
pending.extend(ids[start:end])
|
||||
current = tokenizer.decode(ids[:end])
|
||||
if current.endswith(REPLACEMENT_CHAR) and end < len(ids):
|
||||
continue
|
||||
deltas.append((current[len(prev) :], pending))
|
||||
prev, pending = current, []
|
||||
return deltas
|
||||
|
||||
|
||||
def make_parser(
|
||||
tokenizer: MockCohereTokenizer,
|
||||
name: str,
|
||||
tools: list[dict] | None = None,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
model_config=None,
|
||||
) -> Parser:
|
||||
"""Resolve the unified Cohere parser through ``ParserManager`` by registry
|
||||
name (``cohere_command3`` / ``cohere_command4``)."""
|
||||
cls = ParserManager.get_parser(name, name, enable_auto_tools=True)
|
||||
assert cls is not None
|
||||
return cls(
|
||||
tokenizer,
|
||||
tools,
|
||||
model_config=model_config,
|
||||
chat_template_kwargs=chat_template_kwargs or {},
|
||||
)
|
||||
|
||||
|
||||
def drive_parser(
|
||||
parser: Parser,
|
||||
request: ChatCompletionRequest,
|
||||
deltas: list[tuple[str, list[int]]],
|
||||
) -> list[DeltaMessage]:
|
||||
"""Feed ``deltas`` through ``parse_delta`` (flushing on the last one) and
|
||||
return the non-``None`` messages in order."""
|
||||
out: list[DeltaMessage] = []
|
||||
for i, (text, ids) in enumerate(deltas):
|
||||
delta = parser.parse_delta(text, ids, request, finished=i == len(deltas) - 1)
|
||||
if delta is not None:
|
||||
out.append(delta)
|
||||
return out
|
||||
|
||||
|
||||
def stream_parser(parser, request, tokenizer, text: str, chunk_size: int = 1):
|
||||
"""Stream ``text`` through ``parser`` in ``chunk_size``-token deltas."""
|
||||
return drive_parser(parser, request, token_deltas(tokenizer, text, chunk_size))
|
||||
@@ -1,311 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for DeepSeek V3.2 parser engine semantics.
|
||||
|
||||
V3.2 uses the same DSML parameter format as V4 but wraps tool calls in
|
||||
``<|DSML|function_calls>`` instead of ``<|DSML|tool_calls>`` and has
|
||||
no reasoning (``<think>``/``</think>``) support.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import (
|
||||
collect_content,
|
||||
collect_function_name,
|
||||
collect_tool_arguments,
|
||||
simulate_tool_streaming,
|
||||
)
|
||||
from vllm.parser.deepseek_v4 import (
|
||||
DSML_INVOKE_END,
|
||||
DSML_INVOKE_NAME_END,
|
||||
DSML_INVOKE_PREFIX,
|
||||
)
|
||||
from vllm.parser.deepseek_v32 import (
|
||||
DSML_FUNC_END,
|
||||
DSML_FUNC_START,
|
||||
DeepSeekV32Parser,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import ParserState
|
||||
|
||||
_PARAM_OPEN = '|DSML|parameter name="{name}" string="{is_str}">'
|
||||
_PARAM_CLOSE = "</|DSML|parameter>"
|
||||
|
||||
|
||||
def _param(name: str, is_str: str, value: str) -> str:
|
||||
return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
|
||||
|
||||
|
||||
def _invoke(name: str, *params: str) -> str:
|
||||
body = "\n".join(params)
|
||||
return (
|
||||
f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
|
||||
)
|
||||
|
||||
|
||||
def _func_calls(*invocations: str) -> str:
|
||||
body = "\n".join(invocations)
|
||||
return f"{DSML_FUNC_START}\n{body}\n{DSML_FUNC_END}"
|
||||
|
||||
|
||||
def _make_tool(name, properties):
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer({})
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request():
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
|
||||
req = MagicMock(spec=ChatCompletionRequest)
|
||||
req.tools = []
|
||||
req.tool_choice = "auto"
|
||||
return req
|
||||
|
||||
|
||||
# ── Non-streaming extraction ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_no_tool_call(self, mock_tokenizer, mock_request):
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
result = parser.extract_tool_calls("Hello world", mock_request)
|
||||
assert not result.tools_called
|
||||
assert result.content == "Hello world"
|
||||
|
||||
def test_single_tool(self, mock_tokenizer, mock_request):
|
||||
text = _func_calls(
|
||||
_invoke("get_weather", _param("city", "true", "SF")),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"city": "SF"}
|
||||
|
||||
def test_parallel_tools(self, mock_tokenizer, mock_request):
|
||||
text = _func_calls(
|
||||
_invoke("get_weather", _param("city", "true", "SF")),
|
||||
_invoke("get_weather", _param("city", "true", "NYC")),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 2
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {"city": "SF"}
|
||||
assert json.loads(result.tool_calls[1].function.arguments) == {"city": "NYC"}
|
||||
|
||||
def test_content_before_tool_call(self, mock_tokenizer, mock_request):
|
||||
text = "Let me check. " + _func_calls(
|
||||
_invoke("search", _param("q", "true", "vllm")),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
assert result.tools_called
|
||||
assert result.content is not None
|
||||
assert "Let me check" in result.content
|
||||
|
||||
def test_non_string_params_json_parsed(self, mock_tokenizer, mock_request):
|
||||
text = _func_calls(
|
||||
_invoke(
|
||||
"toggle",
|
||||
_param("enabled", "false", "true"),
|
||||
_param("count", "false", "42"),
|
||||
),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["enabled"] is True
|
||||
assert args["count"] == 42
|
||||
|
||||
def test_wrapper_unwrapping(self, mock_tokenizer, mock_request):
|
||||
tool = _make_tool("get_weather", {"location": {"type": "string"}})
|
||||
mock_request.tools = [tool]
|
||||
text = _func_calls(
|
||||
_invoke(
|
||||
"get_weather",
|
||||
_param("arguments", "false", '{"location":"Beijing"}'),
|
||||
),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "Beijing"}
|
||||
|
||||
|
||||
# ── Initial state ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestInitialState:
|
||||
def test_always_content(self, mock_tokenizer):
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
cfg = parser.parser_engine_config
|
||||
assert cfg.initial_state == ParserState.CONTENT
|
||||
|
||||
def test_ignores_thinking_kwargs(self, mock_tokenizer):
|
||||
parser = DeepSeekV32Parser(
|
||||
mock_tokenizer,
|
||||
chat_template_kwargs={"thinking": True, "enable_thinking": True},
|
||||
)
|
||||
cfg = parser.parser_engine_config
|
||||
assert cfg.initial_state == ParserState.CONTENT
|
||||
|
||||
|
||||
# ── Streaming ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_split_parameter_start_is_buffered(self, mock_tokenizer, mock_request):
|
||||
chunks = [
|
||||
DSML_FUNC_START,
|
||||
f"{DSML_INVOKE_PREFIX}edit{DSML_INVOKE_NAME_END}",
|
||||
f"<{_PARAM_OPEN.format(name='expr', is_str='true')}a<b><|DSML|parameter",
|
||||
' name="date" string="true">tomorrow',
|
||||
_PARAM_CLOSE,
|
||||
DSML_INVOKE_END,
|
||||
DSML_FUNC_END,
|
||||
]
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
DeepSeekV32Parser(mock_tokenizer), mock_request, chunks
|
||||
)
|
||||
|
||||
arguments = collect_tool_arguments(results)
|
||||
assert "<|DSML|parameter" not in arguments
|
||||
assert json.loads(arguments) == {"expr": "a<b>", "date": "tomorrow"}
|
||||
|
||||
def test_single_tool_streaming(self, mock_tokenizer, mock_request):
|
||||
text = _func_calls(
|
||||
_invoke("get_weather", _param("city", "true", "SF")),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
assert collect_function_name(results) == "get_weather"
|
||||
args_json = collect_tool_arguments(results)
|
||||
assert json.loads(args_json) == {"city": "SF"}
|
||||
|
||||
def test_content_before_tool_streaming(self, mock_tokenizer, mock_request):
|
||||
text = "Checking... " + _func_calls(
|
||||
_invoke("fn", _param("k", "true", "v")),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
content = collect_content(results)
|
||||
assert "Checking" in content
|
||||
|
||||
def test_parallel_tools_streaming(self, mock_tokenizer, mock_request):
|
||||
text = _func_calls(
|
||||
_invoke("fn_a", _param("x", "true", "1")),
|
||||
_invoke("fn_b", _param("y", "true", "2")),
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
|
||||
names = []
|
||||
for delta, _ in results:
|
||||
if delta and delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
if tc.function and tc.function.name:
|
||||
names.append(tc.function.name)
|
||||
assert "fn_a" in names
|
||||
assert "fn_b" in names
|
||||
|
||||
def test_no_tool_content_only(self, mock_tokenizer, mock_request):
|
||||
text = "Just some text, no tools."
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
content = collect_content(results)
|
||||
assert "Just some text" in content
|
||||
args = collect_tool_arguments(results)
|
||||
assert args == ""
|
||||
|
||||
def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
|
||||
tool = _make_tool("get_weather", {"location": {"type": "string"}})
|
||||
mock_request.tools = [tool]
|
||||
parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
|
||||
|
||||
chunks = [
|
||||
DSML_FUNC_START,
|
||||
_invoke(
|
||||
"get_weather",
|
||||
_param("arguments", "false", '{"location": "NYC"}'),
|
||||
),
|
||||
DSML_FUNC_END,
|
||||
]
|
||||
|
||||
results = simulate_tool_streaming(parser, mock_request, chunks)
|
||||
streamed_args = collect_tool_arguments(results)
|
||||
|
||||
final_delta, _ = results[-1]
|
||||
finish_delta = parser.finish_streaming()
|
||||
extracted = parser._build_extracted_result(final_delta, finish_delta)
|
||||
|
||||
assert extracted.tools_called is True
|
||||
assert len(extracted.tool_calls) == 1
|
||||
final_args = extracted.tool_calls[0].function.arguments
|
||||
assert json.loads(final_args) == {"location": "NYC"}
|
||||
assert '"arguments"' not in streamed_args
|
||||
assert final_args.startswith(streamed_args)
|
||||
|
||||
def test_missing_invoke_end(self, mock_tokenizer, mock_request):
|
||||
text = (
|
||||
f"{DSML_FUNC_START}\n"
|
||||
f"{DSML_INVOKE_PREFIX}fn{DSML_INVOKE_NAME_END}\n"
|
||||
f"{_param('k', 'true', 'v')}\n"
|
||||
f"{DSML_FUNC_END}"
|
||||
)
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
assert collect_function_name(results) == "fn"
|
||||
args = json.loads(collect_tool_arguments(results))
|
||||
assert args == {"k": "v"}
|
||||
|
||||
|
||||
class TestMissingFunctionCallsWrapper:
|
||||
"""An invoke without the ``<|DSML|function_calls>`` wrapper is still a
|
||||
tool call (V3.2 counterpart of #48931)."""
|
||||
|
||||
def test_non_streaming(self, mock_tokenizer, mock_request):
|
||||
text = _invoke("fn", _param("k", "true", "v"))
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "fn"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {"k": "v"}
|
||||
assert result.content is None
|
||||
|
||||
def test_streaming(self, mock_tokenizer, mock_request):
|
||||
text = "Sure.\n" + _invoke("fn", _param("k", "true", "v"))
|
||||
parser = DeepSeekV32Parser(mock_tokenizer)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
|
||||
assert collect_function_name(results) == "fn"
|
||||
assert json.loads(collect_tool_arguments(results)) == {"k": "v"}
|
||||
assert "DSML" not in collect_content(results)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,148 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
DUMMY_TOOLS,
|
||||
MockTokenizer,
|
||||
_test_request,
|
||||
collect_output,
|
||||
replay_streaming,
|
||||
)
|
||||
from vllm.parser.deepseek_v41 import deepseek_v41_config
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
CALLS = (
|
||||
'\n\n<|DSML| calls>\n<|DSML| invoke name="get_weather">\n'
|
||||
'<|DSML| parameter name="city" string="true">杭州</|DSML| parameter>\n'
|
||||
'<|DSML| parameter name="count" string="false">42</|DSML| parameter>\n'
|
||||
'</|DSML| invoke>\n<|DSML| invoke name="add">\n'
|
||||
'<|DSML| parameter name="x" string="false">1.5</|DSML| parameter>\n'
|
||||
'<|DSML| parameter name="y" string="false">2.25</|DSML| parameter>\n'
|
||||
"</|DSML| invoke>\n</|DSML| calls>"
|
||||
)
|
||||
|
||||
|
||||
def tokenizer_for(text, special_calls):
|
||||
vocab = {"<think>": 50, "</think>": 51}
|
||||
if special_calls:
|
||||
vocab |= {"<|DSML| calls>": 52, "</|DSML| calls>": 53}
|
||||
tokens: list[tuple[int, str]] = []
|
||||
while text:
|
||||
special = next((marker for marker in vocab if text.startswith(marker)), None)
|
||||
piece = special or text[0]
|
||||
tokens.append((vocab[piece] if special else 100 + len(tokens), piece))
|
||||
text = text[len(piece) :]
|
||||
return MockTokenizer(vocab, tokens), tokens
|
||||
|
||||
|
||||
def parser_for(tokenizer, controls):
|
||||
cls = ParserManager.get_parser(
|
||||
tool_parser_name="deepseek_v41",
|
||||
reasoning_parser_name="deepseek_v41",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
return cls(tokenizer, chat_template_kwargs=controls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", [False, True])
|
||||
@pytest.mark.parametrize("special_calls", [False, True])
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, 10000])
|
||||
def test_registered_adapters_parse_parallel_calls_across_chunks(
|
||||
thinking,
|
||||
special_calls,
|
||||
chunk_size,
|
||||
):
|
||||
text = ("Plan.</think>" if thinking else "") + "Checking." + CALLS
|
||||
tokenizer, tokens = tokenizer_for(text, special_calls)
|
||||
parser = parser_for(tokenizer, {"thinking": thinking})
|
||||
output = collect_output(
|
||||
replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=True,
|
||||
tools=DUMMY_TOOLS,
|
||||
prompt_token_ids=[50 if thinking else 51],
|
||||
)
|
||||
)
|
||||
assert output.reasoning == ("Plan." if thinking else "")
|
||||
assert output.content.strip() == "Checking."
|
||||
assert [call["name"] for call in output.tool_calls] == ["get_weather", "add"]
|
||||
assert [json.loads(call["arguments"]) for call in output.tool_calls] == [
|
||||
{"city": "杭州", "count": 42},
|
||||
{"x": 1.5, "y": 2.25},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", [False, True])
|
||||
def test_registered_adapters_parse_complete_output(thinking):
|
||||
text = ("Plan.</think>" if thinking else "") + CALLS
|
||||
tokenizer, tokens = tokenizer_for(text, False)
|
||||
parser = parser_for(tokenizer, {"thinking": thinking})
|
||||
reasoning, content, calls = parser.parse(
|
||||
text,
|
||||
_test_request(tools=DUMMY_TOOLS),
|
||||
enable_auto_tools=True,
|
||||
model_output_token_ids=[tid for tid, _ in tokens],
|
||||
)
|
||||
assert (reasoning or "") == ("Plan." if thinking else "")
|
||||
assert not (content or "").strip()
|
||||
assert [call.name for call in calls] == ["get_weather", "add"]
|
||||
assert json.loads(calls[0].arguments) == {"city": "杭州", "count": 42}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("controls", "text", "expected_reasoning", "reasoning_tokens"),
|
||||
[
|
||||
({"thinking": False}, "12", "", 0),
|
||||
({"enable_thinking": False}, "12", "", 0),
|
||||
({"thinking": True, "reasoning_effort": "none"}, "12", "", 0),
|
||||
({}, "</think>12", "", 0),
|
||||
({}, "Plan.</think>12", "Plan.", 5),
|
||||
],
|
||||
)
|
||||
def test_reasoning_adapter_controls_and_usage(
|
||||
controls,
|
||||
text,
|
||||
expected_reasoning,
|
||||
reasoning_tokens,
|
||||
):
|
||||
tokenizer, tokens = tokenizer_for(text, False)
|
||||
parser = parser_for(tokenizer, controls)
|
||||
output = collect_output(
|
||||
replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
chunk_size=1,
|
||||
finished_on_last=True,
|
||||
)
|
||||
)
|
||||
assert output.reasoning == expected_reasoning
|
||||
assert output.content == "12"
|
||||
assert (
|
||||
parser.reasoning_parser.count_reasoning_tokens([tid for tid, _ in tokens])
|
||||
== reasoning_tokens
|
||||
)
|
||||
assert parser.is_reasoning_end([50, 100, 51])
|
||||
assert not parser.is_reasoning_end([51, 100, 50])
|
||||
|
||||
|
||||
def test_python_argument_conversion_and_partial_values():
|
||||
converter = deepseek_v41_config().arg_converter
|
||||
raw = (
|
||||
'<|DSML| parameter name="object" string="false">'
|
||||
'{"a": [true, null]}</|DSML| parameter>'
|
||||
'<|DSML| parameter name="bad" string="false">[broken</|DSML| parameter>'
|
||||
'<|DSML| parameter name="bare">7</|DSML| parameter>'
|
||||
'<|DSML| parameter name="text" string="true"> a<b'
|
||||
)
|
||||
assert json.loads(converter(raw, True)) == {
|
||||
"object": {"a": [True, None]},
|
||||
"bad": "[broken",
|
||||
"bare": 7,
|
||||
"text": " a<b",
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,230 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reasoning-boundary tests for the GLM-4.7 parser.
|
||||
|
||||
GLM can transition straight from reasoning into a tool call without ever
|
||||
emitting ``</think>``, so ``<tool_call>`` is an implicit reasoning
|
||||
terminator. These cover that path plus the multi-turn prompts where an
|
||||
earlier turn's markers must not be read as the current turn's state. The
|
||||
streaming tests check that long tool arguments stream the same JSON as a
|
||||
non-streaming parse, without converting the whole argument on every delta.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.glm47_moe import (
|
||||
THINK_END,
|
||||
THINK_START,
|
||||
TOOL_CALL_END,
|
||||
TOOL_CALL_START,
|
||||
Glm47MoeParser,
|
||||
_glm47_arg_value_extender,
|
||||
glm47_moe_config,
|
||||
)
|
||||
|
||||
THINK_S, THINK_E = 1000, 1001
|
||||
TOOL_S, TOOL_E = 1002, 1003
|
||||
ASSISTANT, OBSERVATION, USER = 1004, 1005, 1006
|
||||
TEXT = 42 # stand-in for an ordinary reasoning/content token
|
||||
|
||||
VOCAB = {
|
||||
THINK_START: THINK_S,
|
||||
THINK_END: THINK_E,
|
||||
TOOL_CALL_START: TOOL_S,
|
||||
TOOL_CALL_END: TOOL_E,
|
||||
"<|assistant|>": ASSISTANT,
|
||||
"<|observation|>": OBSERVATION,
|
||||
"<|user|>": USER,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser():
|
||||
return Glm47MoeParser(make_mock_tokenizer(VOCAB))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_thinking_parser():
|
||||
return Glm47MoeParser(
|
||||
make_mock_tokenizer(VOCAB),
|
||||
chat_template_kwargs={"thinking": False},
|
||||
)
|
||||
|
||||
|
||||
class TestIsReasoningEnd:
|
||||
def test_open_reasoning(self, parser):
|
||||
assert not parser.is_reasoning_end([THINK_S, TEXT])
|
||||
|
||||
def test_think_end(self, parser):
|
||||
assert parser.is_reasoning_end([THINK_S, TEXT, THINK_E, TEXT])
|
||||
|
||||
def test_tool_call_without_think_end(self, parser):
|
||||
"""Reasoning -> tool call with no ``</think>`` still ends reasoning."""
|
||||
assert parser.is_reasoning_end([THINK_S, TEXT, TOOL_S, TEXT])
|
||||
|
||||
def test_previous_turn_tool_call_ignored(self, parser):
|
||||
"""A finished tool call from an earlier turn says nothing about the
|
||||
turn currently being generated."""
|
||||
prompt = [THINK_S, TEXT, TOOL_S, TEXT, TOOL_E, OBSERVATION, TEXT, ASSISTANT]
|
||||
assert not parser.is_reasoning_end(prompt)
|
||||
|
||||
def test_previous_turn_think_end_ignored(self, parser):
|
||||
prompt = [THINK_S, TEXT, THINK_E, TEXT, USER, TEXT, ASSISTANT]
|
||||
assert not parser.is_reasoning_end(prompt)
|
||||
|
||||
def test_empty_input(self, parser):
|
||||
assert not parser.is_reasoning_end([])
|
||||
|
||||
def test_thinking_disabled(self, no_thinking_parser):
|
||||
assert no_thinking_parser.is_reasoning_end([THINK_S, TEXT])
|
||||
|
||||
|
||||
class TestExtractContentIds:
|
||||
def test_think_end_wins_over_later_tool_call(self, parser):
|
||||
"""Content between ``</think>`` and a tool call must survive."""
|
||||
ids = [THINK_S, TEXT, THINK_E, 20, 21, TOOL_S, 30, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == [20, 21, TOOL_S, 30, TOOL_E]
|
||||
|
||||
def test_every_tool_call_after_think_end_kept(self, parser):
|
||||
ids = [THINK_E, 20, TOOL_S, 30, TOOL_E, 21, TOOL_S, 31, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == ids[1:]
|
||||
|
||||
def test_falls_back_to_tool_call(self, parser):
|
||||
"""Without ``</think>``, content starts at the opener itself so the
|
||||
tool parser still receives a well-formed call."""
|
||||
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == [TOOL_S, 30, TOOL_E]
|
||||
|
||||
def test_falls_back_to_first_tool_call_of_turn(self, parser):
|
||||
ids = [THINK_S, TEXT, TOOL_S, 30, TOOL_E, TOOL_S, 31, TOOL_E]
|
||||
assert parser.extract_content_ids(ids) == ids[2:]
|
||||
|
||||
def test_previous_turn_tool_call_ignored(self, parser):
|
||||
ids = [TOOL_S, 30, TOOL_E, OBSERVATION, TEXT, ASSISTANT, THINK_S, TEXT]
|
||||
assert parser.extract_content_ids(ids) == ids
|
||||
|
||||
def test_no_markers_returns_input_ids(self, parser):
|
||||
assert parser.extract_content_ids([20, 21]) == [20, 21]
|
||||
|
||||
def test_thinking_disabled(self, no_thinking_parser):
|
||||
ids = [THINK_S, TEXT, TOOL_S]
|
||||
assert no_thinking_parser.extract_content_ids(ids) == ids
|
||||
|
||||
|
||||
WRITE_TOOL = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "Write",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"file_path": {"type": "string"},
|
||||
"line_count": {"type": "integer"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
CALL_HEAD = (
|
||||
f"{THINK_END}{TOOL_CALL_START}Write\n"
|
||||
"<arg_key>file_path</arg_key><arg_value>/repo/app.py</arg_value>\n"
|
||||
"<arg_key>line_count</arg_key><arg_value>120</arg_value>\n"
|
||||
"<arg_key>content</arg_key><arg_value>"
|
||||
)
|
||||
CALL_TAIL = f"</arg_value>\n{TOOL_CALL_END}"
|
||||
# Escapes, non-ASCII, and tag-like text that must stay inside the value.
|
||||
CONTENT_LINE = 'if a < b > c: print("x\\y") # </div> </arg_ <arg_key>\t\u00e9\n'
|
||||
|
||||
|
||||
def _pieces(text: str, size: int) -> list[str]:
|
||||
return [text[i : i + size] for i in range(0, len(text), size)]
|
||||
|
||||
|
||||
def _stream_arguments(parser, request, chunks: list[str]) -> str:
|
||||
args = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
delta = parser.parse_delta(chunk, [], request, finished=i == len(chunks) - 1)
|
||||
for tool_call in (delta and delta.tool_calls) or []:
|
||||
if tool_call.function and tool_call.function.arguments:
|
||||
args.append(tool_call.function.arguments)
|
||||
return "".join(args)
|
||||
|
||||
|
||||
class TestStreamedToolArguments:
|
||||
@pytest.mark.parametrize(
|
||||
"tools, line_count",
|
||||
[([], '"120"'), ([WRITE_TOOL], "120")],
|
||||
ids=["no_schema", "schema"],
|
||||
)
|
||||
def test_long_call_streams_non_streaming_arguments(
|
||||
self, mock_request, tools, line_count
|
||||
):
|
||||
mock_request.tools = tools
|
||||
text = CALL_HEAD + CONTENT_LINE * 400 + CALL_TAIL
|
||||
|
||||
streamed = _stream_arguments(
|
||||
Glm47MoeParser(make_mock_tokenizer(VOCAB)), mock_request, _pieces(text, 7)
|
||||
)
|
||||
result = Glm47MoeParser(make_mock_tokenizer(VOCAB)).extract_tool_calls(
|
||||
text, mock_request
|
||||
)
|
||||
|
||||
assert streamed == result.tool_calls[0].function.arguments
|
||||
assert f'"line_count": {line_count},' in streamed
|
||||
|
||||
def test_conversions_do_not_grow_with_argument_length(self, mock_request):
|
||||
"""The open value streams without re-running the converter."""
|
||||
mock_request.tools = [WRITE_TOOL]
|
||||
|
||||
def streaming_conversions(lines: int) -> int:
|
||||
calls = 0
|
||||
base = glm47_moe_config()
|
||||
|
||||
def counting_converter(raw_args: str, partial: bool) -> str:
|
||||
nonlocal calls
|
||||
if partial:
|
||||
calls += 1
|
||||
return base.arg_converter(raw_args, partial)
|
||||
|
||||
parser = Glm47MoeParser(
|
||||
make_mock_tokenizer(VOCAB),
|
||||
parser_engine_config=dataclasses.replace(
|
||||
base, arg_converter=counting_converter
|
||||
),
|
||||
)
|
||||
chunks = [
|
||||
*_pieces(CALL_HEAD, 4),
|
||||
*_pieces(CONTENT_LINE * lines, 4),
|
||||
*_pieces(CALL_TAIL, 4),
|
||||
]
|
||||
_stream_arguments(parser, mock_request, chunks)
|
||||
return calls
|
||||
|
||||
assert streaming_conversions(lines=40) == streaming_conversions(lines=400)
|
||||
|
||||
def test_value_extender_stops_at_split_value_end(self):
|
||||
extend = _glm47_arg_value_extender("<arg_key>k</arg_key><arg_value>x")
|
||||
assert [extend(d) for d in ['a"\n', "</arg_", "value>"]] == [
|
||||
'a\\"\\n',
|
||||
"</arg_",
|
||||
None,
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_args",
|
||||
[
|
||||
"<arg_key>k</arg_key><arg_value>x</arg_value>",
|
||||
"<arg_key>k</arg_key><arg_value>1</arg_value>"
|
||||
"<arg_key>j</arg_key><arg_value>2</arg_value>"
|
||||
"<arg_key>k</arg_key><arg_value>3",
|
||||
],
|
||||
ids=["closed_value", "repeated_key_not_last"],
|
||||
)
|
||||
def test_no_value_extender_unless_open_value_is_last(self, raw_args):
|
||||
assert _glm47_arg_value_extender(raw_args) is None
|
||||
@@ -1,234 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the engine-based Granite parser.
|
||||
|
||||
Granite emits tool calls as a JSON array following a marker
|
||||
(``<|tool_call|>`` for 3.0, ``<tool_call>`` for 3.1) with no closing
|
||||
terminal; the ``]`` ends the region and surrounding prose is content. The
|
||||
engine's ``tool_call_body_array`` mode splits the array into one call per
|
||||
element and ``_granite_arg_converter`` carves each element's ``arguments``.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.trace_builder import _GRANITE_SCENARIOS, _build_granite
|
||||
from vllm.parser.granite import GraniteParser, _granite_arg_converter
|
||||
|
||||
TOOL_TOKEN = "<|tool_call|>"
|
||||
TOOL_STRING = "<tool_call>"
|
||||
|
||||
_GRANITE_VOCAB = {TOOL_TOKEN: 49154}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_GRANITE_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return GraniteParser(mock_tokenizer)
|
||||
|
||||
|
||||
def _tokenize(text: str) -> list[tuple[int, str]]:
|
||||
"""Tokenize like the real stream: ``<|tool_call|>`` is one special token,
|
||||
plain text becomes one token per character."""
|
||||
tokens: list[tuple[int, str]] = []
|
||||
i = 0
|
||||
while i < len(text):
|
||||
if text.startswith(TOOL_TOKEN, i):
|
||||
tokens.append((_GRANITE_VOCAB[TOOL_TOKEN], TOOL_TOKEN))
|
||||
i += len(TOOL_TOKEN)
|
||||
else:
|
||||
tokens.append((ord(text[i]), text[i]))
|
||||
i += 1
|
||||
return tokens
|
||||
|
||||
|
||||
def _stream(parser, request, text: str, chunk_size: int):
|
||||
tokens = _tokenize(text)
|
||||
results = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
for start in range(0, len(tokens), chunk_size):
|
||||
batch = tokens[start : start + chunk_size]
|
||||
delta_text = "".join(t for _, t in batch)
|
||||
delta_token_ids = [tid for tid, _ in batch]
|
||||
current_text = previous_text + delta_text
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=tuple(previous_token_ids),
|
||||
current_token_ids=tuple(current_token_ids),
|
||||
delta_token_ids=tuple(delta_token_ids),
|
||||
request=request,
|
||||
)
|
||||
results.append(delta)
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
finish = parser.finish_streaming()
|
||||
if finish is not None:
|
||||
results.append(finish)
|
||||
return results
|
||||
|
||||
|
||||
def _collect_content(results) -> str:
|
||||
return "".join(d.content for d in results if d and d.content)
|
||||
|
||||
|
||||
def _collect_names(results) -> list[str]:
|
||||
names: dict[int, str] = {}
|
||||
for d in results:
|
||||
if not (d and d.tool_calls):
|
||||
continue
|
||||
for tc in d.tool_calls:
|
||||
if tc.function and tc.function.name:
|
||||
names[tc.index] = names.get(tc.index, "") + tc.function.name
|
||||
return [names[i] for i in sorted(names)]
|
||||
|
||||
|
||||
def _collect_args(results) -> dict[int, str]:
|
||||
args: dict[int, str] = {}
|
||||
for d in results:
|
||||
if not (d and d.tool_calls):
|
||||
continue
|
||||
for tc in d.tool_calls:
|
||||
if tc.function and tc.function.arguments:
|
||||
args[tc.index] = args.get(tc.index, "") + tc.function.arguments
|
||||
return args
|
||||
|
||||
|
||||
class TestArgConverter:
|
||||
def test_complete_wrapper(self):
|
||||
raw = '{"name": "get_weather", "arguments": {"city": "SF"}}'
|
||||
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
|
||||
|
||||
def test_arguments_before_name(self):
|
||||
raw = '{"arguments": {"city": "SF"}, "name": "get_weather"}'
|
||||
assert _granite_arg_converter(raw, False) == '{"city": "SF"}'
|
||||
|
||||
def test_partial_before_arguments(self):
|
||||
assert _granite_arg_converter('{"name": "get_w', True) == ""
|
||||
|
||||
def test_partial_inside_arguments(self):
|
||||
raw = '{"name": "x", "arguments": {"a": 1'
|
||||
assert _granite_arg_converter(raw, True) == '{"a": 1'
|
||||
|
||||
def test_prefix_stability(self):
|
||||
# Each growing prefix must extend the previous converter output.
|
||||
full = '{"name": "x", "arguments": {"city": "Tokyo"}}'
|
||||
prev = ""
|
||||
for i in range(1, len(full) + 1):
|
||||
out = _granite_arg_converter(full[:i], True)
|
||||
assert out.startswith(prev) or prev.startswith(out) or out == ""
|
||||
if out:
|
||||
prev = out
|
||||
|
||||
def test_missing_arguments_defaults(self):
|
||||
assert _granite_arg_converter('{"name": "x"}', False) == "{}"
|
||||
assert _granite_arg_converter('{"name": "x"}', True) == ""
|
||||
|
||||
def test_non_object_arguments_rejected(self):
|
||||
with pytest.raises(ValueError, match="JSON object"):
|
||||
_granite_arg_converter('{"name": "x", "arguments": [1]}', False)
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_plain_text(self, parser, mock_request):
|
||||
reasoning, content, tools = parser.parse("Just a reply.", mock_request)
|
||||
assert reasoning is None
|
||||
assert content == "Just a reply."
|
||||
assert tools is None
|
||||
|
||||
def test_single_tool_token_marker(self, parser, mock_request):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
|
||||
)
|
||||
_, content, tools = parser.parse(text, mock_request)
|
||||
assert content is None
|
||||
assert [t.name for t in tools] == ["get_weather"]
|
||||
assert json.loads(tools[0].arguments) == {"city": "SF"}
|
||||
|
||||
def test_single_tool_string_marker(self, parser, mock_request):
|
||||
# Granite 3.1 uses the plain-text ``<tool_call>`` marker.
|
||||
text = f'{TOOL_STRING} [{{"name": "get_time", "arguments": {{}}}}]'
|
||||
_, content, tools = parser.parse(text, mock_request)
|
||||
assert [t.name for t in tools] == ["get_time"]
|
||||
assert json.loads(tools[0].arguments) == {}
|
||||
|
||||
def test_parallel_calls_one_array(self, parser, mock_request):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"name": "a", "arguments": {{"x": 1}}}}, '
|
||||
f'{{"name": "b", "arguments": {{"y": [1, 2]}}}}]'
|
||||
)
|
||||
_, _, tools = parser.parse(text, mock_request)
|
||||
assert [t.name for t in tools] == ["a", "b"]
|
||||
assert json.loads(tools[0].arguments) == {"x": 1}
|
||||
assert json.loads(tools[1].arguments) == {"y": [1, 2]}
|
||||
|
||||
def test_surrounding_text_is_content(self, parser, mock_request):
|
||||
text = (
|
||||
"Let me check.\n"
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
|
||||
)
|
||||
_, content, tools = parser.parse(text, mock_request)
|
||||
assert content == "Let me check."
|
||||
assert [t.name for t in tools] == ["get_weather"]
|
||||
|
||||
def test_arguments_before_name(self, parser, mock_request):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"arguments": {{"city": "SF"}}, "name": "get_weather"}}]'
|
||||
)
|
||||
_, _, tools = parser.parse(text, mock_request)
|
||||
assert [t.name for t in tools] == ["get_weather"]
|
||||
assert json.loads(tools[0].arguments) == {"city": "SF"}
|
||||
|
||||
def test_marker_without_array_is_not_a_tool_call(self, parser, mock_request):
|
||||
# A non-array body after the marker (malformed) yields no tool call.
|
||||
text = f'{TOOL_TOKEN} {{"name": "func", "arguments": {{}}}}'
|
||||
_, _, tools = parser.parse(text, mock_request)
|
||||
assert not tools
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
@pytest.mark.parametrize("chunk_size", [1, 2, 3, 5, 11])
|
||||
def test_parallel_calls_chunk_invariance(self, parser, mock_request, chunk_size):
|
||||
text = (
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", '
|
||||
f'"arguments": {{"city": "Tokyo"}}}}, '
|
||||
f'{{"name": "get_time", "arguments": {{"timezone": "Asia/Tokyo"}}}}]'
|
||||
)
|
||||
results = _stream(parser, mock_request, text, chunk_size)
|
||||
assert _collect_names(results) == ["get_weather", "get_time"]
|
||||
args = _collect_args(results)
|
||||
assert json.loads(args[0]) == {"city": "Tokyo"}
|
||||
assert json.loads(args[1]) == {"timezone": "Asia/Tokyo"}
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 3, 7])
|
||||
def test_surrounding_text_streaming(self, parser, mock_request, chunk_size):
|
||||
text = (
|
||||
"Let me check.\n"
|
||||
f'{TOOL_TOKEN} [{{"name": "get_weather", "arguments": {{"city": "SF"}}}}]'
|
||||
)
|
||||
results = _stream(parser, mock_request, text, chunk_size)
|
||||
assert _collect_names(results) == ["get_weather"]
|
||||
assert "Let me check." in _collect_content(results)
|
||||
|
||||
def test_incomplete_array_at_eos(self, parser, mock_request):
|
||||
# Truncated mid-element must not raise and must not leak the marker.
|
||||
text = f'{TOOL_TOKEN} [{{"name": "func", "arguments": {{"a": 1'
|
||||
results = _stream(parser, mock_request, text, 3)
|
||||
assert TOOL_TOKEN not in _collect_content(results)
|
||||
|
||||
|
||||
class TestReplay:
|
||||
@pytest.mark.parametrize("scenario", _GRANITE_SCENARIOS, ids=lambda s: s.id)
|
||||
def test_replay_scenarios(self, scenario):
|
||||
# ``_build_granite`` replays the sample through GraniteParser at
|
||||
# chunk_size=1 and asserts reasoning/content/tool_calls (validate=True).
|
||||
_build_granite(scenario)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,106 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.replay_harness import MockTokenizer, _test_request
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
|
||||
def _tokenizer() -> MockTokenizer:
|
||||
vocab = {
|
||||
"<think>": 1,
|
||||
"</think>": 2,
|
||||
"<tool_call>": 3,
|
||||
"</tool_call>": 4,
|
||||
"<arg_key>": 5,
|
||||
"</arg_key>": 6,
|
||||
"<arg_value>": 7,
|
||||
"</arg_value>": 8,
|
||||
}
|
||||
return MockTokenizer(vocab=vocab, tokens=[])
|
||||
|
||||
|
||||
def test_ling3_registered():
|
||||
assert ReasoningParserManager.get_reasoning_parser("ling3") is not None
|
||||
assert ToolParserManager.get_tool_parser("ling3") is not None
|
||||
|
||||
|
||||
def test_ling3_defaults_thinking_on():
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
|
||||
parser = parser_cls(_tokenizer())
|
||||
|
||||
assert parser.reasoning_start_str == "<think>"
|
||||
assert parser.reasoning_end_str == "</think>"
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think>reason</think>answer", _test_request()
|
||||
)
|
||||
|
||||
assert reasoning == "reason"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_ling3_disable_thinking_keeps_reasoning_as_content():
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
|
||||
parser = parser_cls(
|
||||
_tokenizer(),
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think>reason</think>answer", _test_request()
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "<think>reason</think>answer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_output", ["only reasoning", "<think>only reasoning"])
|
||||
def test_ling3_truncated_reasoning_stays_reasoning(model_output):
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("ling3")
|
||||
parser = parser_cls(
|
||||
_tokenizer(),
|
||||
chat_template_kwargs={"enable_thinking": True},
|
||||
)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(model_output, _test_request())
|
||||
|
||||
assert reasoning == "only reasoning"
|
||||
assert content is None
|
||||
|
||||
|
||||
def test_ling3_tool_call_without_newline():
|
||||
parser_cls = ToolParserManager.get_tool_parser("ling3")
|
||||
parser = parser_cls(_tokenizer())
|
||||
|
||||
result = parser.extract_tool_calls(
|
||||
"<tool_call>get_weather<arg_key>city</arg_key>"
|
||||
"<arg_value>Beijing</arg_value></tool_call>",
|
||||
request=_test_request(),
|
||||
)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content is None
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"city": "Beijing",
|
||||
}
|
||||
|
||||
|
||||
def test_ling3_tool_call_without_args():
|
||||
parser_cls = ToolParserManager.get_tool_parser("ling3")
|
||||
parser = parser_cls(_tokenizer())
|
||||
|
||||
result = parser.extract_tool_calls(
|
||||
"<tool_call>ping</tool_call>",
|
||||
request=_test_request(),
|
||||
)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content is None
|
||||
assert result.tool_calls[0].function.name == "ping"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {}
|
||||
@@ -1,459 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MiniCPM-V family parser."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.minicpmv import (
|
||||
EscapedNewlineNormalizer,
|
||||
MiniCPMVOutputNormalizer,
|
||||
recover_newlines,
|
||||
)
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
_VOCAB = {
|
||||
"<think>": 10,
|
||||
"</think>": 11,
|
||||
"<|im_end|>": 12,
|
||||
"<tool_call>": 13,
|
||||
"</tool_call>": 14,
|
||||
}
|
||||
|
||||
_TOOLS = [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj():
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
|
||||
def _make_parser(tokenizer, *, enable_thinking=None, tools=None):
|
||||
"""Build the parser the serving stack resolves for MiniCPM-V.
|
||||
|
||||
Everything is built through ``ParserManager`` so the tests exercise the
|
||||
``DelegatingParser`` the server actually drives, not the parser engine on
|
||||
its own. Tool calling goes through ``qwen3_coder``: MiniCPM-V emits that
|
||||
XML syntax verbatim and has no tool parser of its own.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder" if tools else None,
|
||||
reasoning_parser_name="minicpmv",
|
||||
enable_auto_tools=bool(tools),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
kwargs = {}
|
||||
if enable_thinking is not None:
|
||||
kwargs["chat_template_kwargs"] = {"enable_thinking": enable_thinking}
|
||||
return parser_cls(tokenizer, tools, **kwargs)
|
||||
|
||||
|
||||
def test_reasoning_parser_is_registered():
|
||||
parser_cls = ReasoningParserManager.get_reasoning_parser("minicpmv")
|
||||
|
||||
assert parser_cls.__name__ == "MiniCPMVParserReasoningAdapter"
|
||||
|
||||
|
||||
def test_thinking_defaults_to_the_chat_template(mock_tokenizer):
|
||||
"""Without `enable_thinking`, the grammar must wait for reasoning.
|
||||
|
||||
The chat template ends the assistant turn with `<think>` unless the flag is
|
||||
explicitly false, so the model reasons by default. This is the flag
|
||||
`serving.py` latches into the structured-output gate, so a non-thinking
|
||||
default both disables the grammar and leaks the thinking into `content`.
|
||||
"""
|
||||
|
||||
def reasoning_pending(**kwargs):
|
||||
return not _make_parser(mock_tokenizer, **kwargs).is_reasoning_end([1, 2])
|
||||
|
||||
assert reasoning_pending() is True # default matches the template
|
||||
assert reasoning_pending(enable_thinking=True) is True
|
||||
assert reasoning_pending(enable_thinking=False) is False
|
||||
|
||||
|
||||
def test_no_minicpmv_tool_parser_is_registered():
|
||||
"""MiniCPM-V deliberately registers no tool parser.
|
||||
|
||||
The family emits the Qwen3-Coder XML syntax verbatim, so
|
||||
``--tool-call-parser qwen3_coder`` already covers it. Registering a
|
||||
``minicpmv`` tool parser would duplicate that name for no parsing gain.
|
||||
"""
|
||||
with pytest.raises(KeyError):
|
||||
ToolParserManager.get_tool_parser("minicpmv")
|
||||
|
||||
|
||||
def test_composes_with_qwen3_coder_tool_parser(mock_tokenizer):
|
||||
"""`--tool-call-parser qwen3_coder` must work through the serving Parser."""
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="qwen3_coder",
|
||||
reasoning_parser_name="minicpmv",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(mock_tokenizer, request.tools)
|
||||
|
||||
_, _, tool_calls = parser.parse(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert "SF" in tool_calls[0].arguments
|
||||
|
||||
|
||||
class TestNonThinking:
|
||||
@pytest.fixture
|
||||
def parser(self, mock_tokenizer):
|
||||
return _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
def test_plain_content_is_unchanged(self, parser):
|
||||
reasoning, content = parser.extract_reasoning("The answer is 42.", None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "The answer is 42."
|
||||
|
||||
def test_reserved_markers_are_removed(self, parser):
|
||||
output = (
|
||||
"<reserved_12>reasoning<reserved_13><|reserved_14|>tool call<|reserved_15|>"
|
||||
)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(output, None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "reasoningtool call"
|
||||
|
||||
def test_unrelated_reserved_text_is_unchanged(self, parser):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<reserved_11>content<|reserved_16|>",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "<reserved_11>content<|reserved_16|>"
|
||||
|
||||
def test_dangling_standard_think_end_is_removed(self, parser):
|
||||
output = "first part </think>\n\nfinal part"
|
||||
|
||||
reasoning, content = parser.extract_reasoning(output, None)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "first part \n\nfinal part"
|
||||
|
||||
def test_truncated_standard_reasoning_does_not_leak(self, parser):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"prefix<think>private reasoning",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "prefix"
|
||||
|
||||
def test_streaming_split_standard_markers(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"<",
|
||||
"think>",
|
||||
"private reasoning",
|
||||
"</",
|
||||
"think>",
|
||||
"final answer",
|
||||
],
|
||||
)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content == "final answer"
|
||||
|
||||
def test_streaming_split_reserved_markers(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"<reser",
|
||||
"ved_12>",
|
||||
"reasoning",
|
||||
"<|reserved",
|
||||
"_13|>",
|
||||
"final answer",
|
||||
],
|
||||
)
|
||||
|
||||
assert reasoning == ""
|
||||
assert content == "reasoningfinal answer"
|
||||
|
||||
def test_tool_calls_are_parsed(self, mock_tokenizer):
|
||||
"""Thinking is off by default, and tool calls must still be parsed."""
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert not content
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert "SF" in tool_calls[0].arguments
|
||||
|
||||
def test_streaming_tool_calls_are_parsed(self, mock_tokenizer):
|
||||
"""Regression: tool calls must stream when thinking is explicitly off.
|
||||
|
||||
The reasoning parser used to report ``reasoning_ended=False`` in
|
||||
non-thinking mode, which kept the delegating parser in its reasoning
|
||||
phase and streamed the raw ``<tool_call>`` XML to the client as content.
|
||||
"""
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False, tools=_TOOLS)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=_TOOLS,
|
||||
)
|
||||
|
||||
delta = parser.parse_delta(
|
||||
"<tool_call><function=get_weather>"
|
||||
"<parameter=city>SF</parameter></function></tool_call>",
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert delta is not None
|
||||
assert delta.tool_calls
|
||||
assert delta.tool_calls[0].function is not None
|
||||
assert delta.tool_calls[0].function.name == "get_weather"
|
||||
assert "SF" in (delta.tool_calls[0].function.arguments or "")
|
||||
assert not delta.content
|
||||
|
||||
|
||||
class TestThinking:
|
||||
def test_standard_think_tags_are_parsed(self, mock_tokenizer):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think>private reasoning</think>final answer",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning == "private reasoning"
|
||||
assert content == "final answer"
|
||||
|
||||
def test_reserved_markers_are_removed(self, mock_tokenizer):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"<think><reserved_12>private reasoning</think><|reserved_13|>final answer",
|
||||
None,
|
||||
)
|
||||
|
||||
assert reasoning == "private reasoning"
|
||||
assert content == "final answer"
|
||||
|
||||
def test_streaming_im_end_is_removed(self):
|
||||
tokenizer = make_mock_tokenizer(_VOCAB, special_tokens=[])
|
||||
parser = _make_parser(tokenizer, enable_thinking=True)
|
||||
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["Answer: 111", "<|im_end|>"],
|
||||
)
|
||||
|
||||
assert reasoning == "Answer: 111"
|
||||
assert content == ""
|
||||
|
||||
|
||||
class TestNewlineRecovery:
|
||||
"""The tokenizer writes a newline as the two characters ``\\`` and ``n``."""
|
||||
|
||||
def test_recover_newlines(self):
|
||||
text = (
|
||||
"first\\nsecond\\rthird\\r\\nfourth "
|
||||
"`code\\nvalue` "
|
||||
"```python\\nvalue = '\\\\n'\\n``` "
|
||||
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
|
||||
"escaped \\\\n"
|
||||
)
|
||||
|
||||
assert recover_newlines(text) == (
|
||||
"first\nsecond\nthird\nfourth "
|
||||
"`code\\nvalue` "
|
||||
"```python\\nvalue = '\\\\n'\\n``` "
|
||||
"$x\\ny$ $$x\\ny$$ \\(x\\ny\\) \\[x\\ny\\] "
|
||||
"escaped \\\\n"
|
||||
)
|
||||
|
||||
def test_streaming_matches_complete_recovery(self):
|
||||
text = (
|
||||
"first\\nsecond `code\\nvalue` $x\\ny$ ```text\\nvalue\\n``` last\\r\\nline"
|
||||
)
|
||||
expected = recover_newlines(text)
|
||||
|
||||
for split in range(len(text) + 1):
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = normalizer.feed(text[:split])
|
||||
actual += normalizer.feed(text[split:], final=True)
|
||||
assert actual == expected
|
||||
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = "".join(normalizer.feed(char) for char in text)
|
||||
actual += normalizer.feed("", final=True)
|
||||
assert actual == expected
|
||||
|
||||
def test_backslash_runs_stay_consistent_across_splits(self):
|
||||
# A run of backslashes can be split over chunks. Its parity decides
|
||||
# whether a following `n` is a newline, so streaming and non-streaming
|
||||
# must agree for every split point.
|
||||
for count in range(1, 7):
|
||||
text = "\\" * count + "n"
|
||||
expected = recover_newlines(text)
|
||||
|
||||
for split in range(len(text) + 1):
|
||||
normalizer = EscapedNewlineNormalizer()
|
||||
actual = normalizer.feed(text[:split])
|
||||
actual += normalizer.feed(text[split:], final=True)
|
||||
assert actual == expected, (count, split)
|
||||
|
||||
def test_streaming_recovers_reasoning_and_content(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
first = normalizer.normalize_delta(
|
||||
DeltaMessage(reasoning="reason\\"),
|
||||
finished=False,
|
||||
)
|
||||
second = normalizer.normalize_delta(
|
||||
DeltaMessage(reasoning="nnext\\", content="answer\\"),
|
||||
finished=False,
|
||||
)
|
||||
final = normalizer.normalize_delta(
|
||||
DeltaMessage(content="nend"),
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert first == DeltaMessage(reasoning="reason")
|
||||
assert second == DeltaMessage(reasoning="\nnext\\", content="answer")
|
||||
assert final == DeltaMessage(content="\nend")
|
||||
|
||||
def test_streaming_does_not_modify_tool_arguments(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
arguments = '{"value":"first\\\\nsecond"}'
|
||||
delta = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
index=0,
|
||||
function=DeltaFunctionCall(arguments=arguments),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
result = normalizer.normalize_delta(delta, finished=True)
|
||||
|
||||
assert result is not None
|
||||
assert result.tool_calls[0].function is not None
|
||||
assert result.tool_calls[0].function.arguments == arguments
|
||||
|
||||
def test_parse_recovers_content(self, mock_tokenizer, request_obj):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
"first\\nsecond",
|
||||
request_obj,
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == "first\nsecond"
|
||||
assert not tool_calls
|
||||
|
||||
def test_parse_delta_recovers_content(self, mock_tokenizer, request_obj):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
|
||||
chunk = parser.parse_delta(
|
||||
"first\\",
|
||||
[],
|
||||
request_obj,
|
||||
finished=False,
|
||||
)
|
||||
rest = parser.parse_delta(
|
||||
"nsecond",
|
||||
[],
|
||||
request_obj,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
# The trailing half of the escape is held back until it is complete.
|
||||
assert chunk is not None
|
||||
assert chunk.content == "first"
|
||||
assert rest is not None
|
||||
assert rest.content == "\nsecond"
|
||||
|
||||
def test_buffered_delta_is_dropped_not_sent_empty(
|
||||
self, mock_tokenizer, request_obj
|
||||
):
|
||||
parser = _make_parser(mock_tokenizer, enable_thinking=False)
|
||||
parser.parse_delta("hello", [], request_obj, finished=False)
|
||||
|
||||
# The lone backtick is held back for the next chunk. Reporting it as an
|
||||
# all-None DeltaMessage would reach the client as an empty SSE chunk.
|
||||
assert parser.parse_delta("`", [], request_obj, finished=False) is None
|
||||
|
||||
def test_normalizer_drops_empty_delta(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
|
||||
assert (
|
||||
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
|
||||
is None
|
||||
)
|
||||
|
||||
def test_normalizer_flushes_buffered_content_when_finished(self):
|
||||
normalizer = MiniCPMVOutputNormalizer()
|
||||
normalizer.normalize_delta(DeltaMessage(content="`"), finished=False)
|
||||
|
||||
final = normalizer.normalize_delta(DeltaMessage(), finished=True)
|
||||
|
||||
assert final is not None
|
||||
assert final.content == "`"
|
||||
@@ -1,385 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import (
|
||||
collect_function_name,
|
||||
collect_tool_arguments,
|
||||
simulate_tool_streaming,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.minimax_m2 import (
|
||||
THINK_END,
|
||||
TOOL_CALL_END,
|
||||
TOOL_CALL_START,
|
||||
MinimaxM2Parser,
|
||||
_minimax_m2_arg_value_extender,
|
||||
minimax_m2_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(
|
||||
{
|
||||
THINK_END: 99,
|
||||
TOOL_CALL_START: 100,
|
||||
TOOL_CALL_END: 101,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return MinimaxM2Parser(mock_tokenizer)
|
||||
|
||||
|
||||
def make_tools(*names: str):
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name=name,
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
},
|
||||
),
|
||||
)
|
||||
for name in names
|
||||
]
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_no_tool_calls(self, parser, mock_request):
|
||||
result = parser.extract_tool_calls(
|
||||
"</think>This is a regular response without tool calls.",
|
||||
mock_request,
|
||||
)
|
||||
assert result.tools_called is False
|
||||
assert result.tool_calls == []
|
||||
assert result.content == "This is a regular response without tool calls."
|
||||
|
||||
def test_single_tool_call(self, parser, mock_request):
|
||||
result = parser.extract_tool_calls(
|
||||
'<minimax:tool_call><invoke name="get_weather">'
|
||||
'<parameter name="city">Seattle</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"city": "Seattle",
|
||||
}
|
||||
|
||||
def test_multiple_invokes(self, parser, mock_request):
|
||||
result = parser.extract_tool_calls(
|
||||
"<minimax:tool_call>"
|
||||
'<invoke name="search"><parameter name="q">OpenAI</parameter></invoke>'
|
||||
'<invoke name="search"><parameter name="q">vLLM</parameter></invoke>'
|
||||
"</minimax:tool_call>",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert [tc.function.name for tc in result.tool_calls] == ["search", "search"]
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {"q": "OpenAI"}
|
||||
assert json.loads(result.tool_calls[1].function.arguments) == {"q": "vLLM"}
|
||||
|
||||
def test_schema_type_coercion(self, mock_tokenizer, mock_request):
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="forecast",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"days": {"type": "integer"},
|
||||
"include_hourly": {"type": "boolean"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2Parser(mock_tokenizer, tools=tools)
|
||||
mock_request.tools = tools
|
||||
|
||||
result = parser.extract_tool_calls(
|
||||
'<minimax:tool_call><invoke name="forecast">'
|
||||
'<parameter name="days">5</parameter>'
|
||||
'<parameter name="include_hourly">true</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"days": 5,
|
||||
"include_hourly": True,
|
||||
}
|
||||
|
||||
def test_invalid_tool_name_is_rejected(self, mock_tokenizer, mock_request):
|
||||
tools = make_tools("search")
|
||||
parser = MinimaxM2Parser(mock_tokenizer)
|
||||
mock_request.tools = tools
|
||||
|
||||
result = parser.extract_tool_calls(
|
||||
'<minimax:tool_call><invoke name="img_gen">'
|
||||
'<parameter name="prompt">a cat</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert result.tool_calls == []
|
||||
|
||||
def test_mixed_tool_names_only_return_valid(self, mock_tokenizer, mock_request):
|
||||
tools = make_tools("search")
|
||||
parser = MinimaxM2Parser(mock_tokenizer)
|
||||
mock_request.tools = tools
|
||||
|
||||
result = parser.extract_tool_calls(
|
||||
"<minimax:tool_call>"
|
||||
'<invoke name="img_gen"><parameter name="prompt">cat</parameter></invoke>'
|
||||
'<invoke name="search"><parameter name="query">news</parameter></invoke>'
|
||||
"</minimax:tool_call>",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert [tc.function.name for tc in result.tool_calls] == ["search"]
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"query": "news",
|
||||
}
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_streaming_single_tool_call(self, parser, mock_request):
|
||||
results = simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
'<parameter name="city">Seattle</parameter>',
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
|
||||
assert collect_function_name(results) == "get_weather"
|
||||
assert json.loads(collect_tool_arguments(results)) == {
|
||||
"city": "Seattle",
|
||||
}
|
||||
|
||||
def test_streaming_multiple_invokes(self, parser, mock_request):
|
||||
results = simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="a"><parameter name="x">1</parameter></invoke>',
|
||||
'<invoke name="b"><parameter name="y">2</parameter></invoke>',
|
||||
"</minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
|
||||
tool_names = [
|
||||
tc.function.name
|
||||
for delta, _ in results
|
||||
if delta and delta.tool_calls
|
||||
for tc in delta.tool_calls
|
||||
if tc.function and tc.function.name
|
||||
]
|
||||
assert tool_names == ["a", "b"]
|
||||
|
||||
def test_streaming_invoke_prefix_split_before_quote(self, parser, mock_request):
|
||||
results = simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
"<invoke name=",
|
||||
'"get_weather">',
|
||||
'<parameter name="city">Seattle</parameter>',
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
|
||||
assert collect_function_name(results) == "get_weather"
|
||||
assert json.loads(collect_tool_arguments(results)) == {
|
||||
"city": "Seattle",
|
||||
}
|
||||
|
||||
def test_streaming_invalid_tool_name_is_rejected(
|
||||
self, mock_tokenizer, mock_request
|
||||
):
|
||||
tools = make_tools("search")
|
||||
parser = MinimaxM2Parser(mock_tokenizer)
|
||||
mock_request.tools = tools
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="img_gen">',
|
||||
'<parameter name="prompt">cat</parameter>',
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
|
||||
assert collect_function_name(results) is None
|
||||
assert collect_tool_arguments(results) == ""
|
||||
|
||||
|
||||
WRITE_CALL_HEAD = (
|
||||
'<minimax:tool_call>\n<invoke name="write_file">\n'
|
||||
'<parameter name="path">/repo/app.py</parameter>\n'
|
||||
'<parameter name="mode">644</parameter>\n'
|
||||
'<parameter name="content">'
|
||||
)
|
||||
WRITE_CALL_TAIL = "</parameter>\n</invoke>\n</minimax:tool_call>"
|
||||
# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
|
||||
# the value, including prefixes of the closing and next-parameter tags.
|
||||
CONTENT_LINE = 'if a < b > c: print("x\\y") # </p> </param <parameter\té\n\n'
|
||||
|
||||
|
||||
class TestArgValueExtender:
|
||||
"""The open parameter value streams without re-running the converter."""
|
||||
|
||||
@pytest.fixture
|
||||
def write_tool(self):
|
||||
return ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="write_file",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string"},
|
||||
"mode": {"type": "integer"},
|
||||
"content": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
|
||||
def test_long_call_streams_non_streaming_arguments(
|
||||
self, mock_tokenizer, mock_request, write_tool, use_schema
|
||||
):
|
||||
mock_request.tools = [write_tool] if use_schema else []
|
||||
text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
|
||||
|
||||
results = simulate_tool_streaming(
|
||||
MinimaxM2Parser(mock_tokenizer),
|
||||
mock_request,
|
||||
[text[i : i + 7] for i in range(0, len(text), 7)],
|
||||
)
|
||||
result = MinimaxM2Parser(mock_tokenizer).extract_tool_calls(text, mock_request)
|
||||
|
||||
arguments = collect_tool_arguments(results)
|
||||
assert arguments == result.tool_calls[0].function.arguments
|
||||
assert json.loads(arguments)["content"] == CONTENT_LINE * 200
|
||||
assert ('"mode": 644,' if use_schema else '"mode": "644",') in arguments
|
||||
|
||||
def test_conversions_do_not_grow_with_argument_length(
|
||||
self, mock_tokenizer, mock_request, write_tool
|
||||
):
|
||||
mock_request.tools = [write_tool]
|
||||
base = minimax_m2_config()
|
||||
converter = base.arg_converter
|
||||
assert converter is not None
|
||||
|
||||
def streaming_conversions(lines: int) -> int:
|
||||
calls = 0
|
||||
|
||||
def counting_converter(raw_args: str, partial: bool) -> str:
|
||||
nonlocal calls
|
||||
calls += partial
|
||||
return converter(raw_args, partial)
|
||||
|
||||
parser = ParserEngine(
|
||||
mock_tokenizer,
|
||||
parser_engine_config=dataclasses.replace(
|
||||
base, arg_converter=counting_converter
|
||||
),
|
||||
)
|
||||
text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
|
||||
simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
[text[i : i + 4] for i in range(0, len(text), 4)],
|
||||
)
|
||||
return calls
|
||||
|
||||
assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
|
||||
|
||||
def test_extender_stops_at_split_end_tag(self):
|
||||
extend = _minimax_m2_arg_value_extender('<parameter name="k">')
|
||||
assert extend is not None
|
||||
deltas = ['a"\n', "</param", "eter>"]
|
||||
assert [extend(d) for d in deltas] == ['a\\"\\n', "</param", None]
|
||||
|
||||
def test_extender_stops_at_split_next_parameter(self):
|
||||
extend = _minimax_m2_arg_value_extender("<parameter name='k'>x")
|
||||
assert extend is not None
|
||||
assert [extend(d) for d in ["< parameter", " name ="]] == ["< parameter", None]
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_args",
|
||||
[
|
||||
'<parameter name="k">x</parameter>',
|
||||
'<parameter name="k',
|
||||
(
|
||||
'<parameter name="k">1</parameter><parameter name="j">2</parameter>'
|
||||
'<parameter name="k">3'
|
||||
),
|
||||
'<parameter name="a<parameter name=k>x',
|
||||
'<parameter name="k>x',
|
||||
],
|
||||
ids=[
|
||||
"closed_value",
|
||||
"open_name",
|
||||
"repeated_key_not_last",
|
||||
"earlier_unmatched_opener",
|
||||
"bare_name_opens_quote",
|
||||
],
|
||||
)
|
||||
def test_no_extender_unless_open_value_is_stable(self, raw_args):
|
||||
assert _minimax_m2_arg_value_extender(raw_args) is None
|
||||
|
||||
|
||||
class TestReasoning:
|
||||
def test_extract_reasoning_without_start_token(self, parser, mock_request):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"This is reasoning</think>This is content",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert reasoning == "This is reasoning"
|
||||
assert content == "This is content"
|
||||
|
||||
def test_extract_reasoning_without_end_token(self, parser, mock_request):
|
||||
reasoning, content = parser.extract_reasoning(
|
||||
"This is still reasoning",
|
||||
mock_request,
|
||||
)
|
||||
|
||||
assert reasoning == "This is still reasoning"
|
||||
assert content is None
|
||||
|
||||
def test_extract_content_ids_without_end_token(self, parser):
|
||||
assert parser.extract_content_ids([1, 2, 3]) == []
|
||||
|
||||
def test_extract_content_ids_after_end_token(self, parser):
|
||||
assert parser.extract_content_ids([1, 99, 2, 3]) == [2, 3]
|
||||
@@ -1,253 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import (
|
||||
collect_content,
|
||||
collect_function_name,
|
||||
collect_tool_arguments,
|
||||
simulate_tool_streaming,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.parser.minimax_m3 import (
|
||||
INVOKE_END,
|
||||
MAX_ELEMENT_DEPTH,
|
||||
NAMESPACE,
|
||||
TOOL_CALL_END,
|
||||
TOOL_CALL_START,
|
||||
MinimaxM3Parser,
|
||||
)
|
||||
|
||||
NS = NAMESPACE
|
||||
|
||||
TOOLS = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="create_order",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "integer"},
|
||||
"note": {"type": "string"},
|
||||
"tags": {"type": "array", "items": {"type": "string"}},
|
||||
"shipping": {
|
||||
"type": "object",
|
||||
"properties": {"zip": {"type": "integer"}},
|
||||
},
|
||||
"either": {
|
||||
"anyOf": [
|
||||
{"type": "array", "items": {"type": "integer"}},
|
||||
{"type": "string"},
|
||||
]
|
||||
},
|
||||
"free": {"type": "object"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser():
|
||||
return MinimaxM3Parser(make_mock_tokenizer({}), tools=TOOLS)
|
||||
|
||||
|
||||
def element(name: str, body: str) -> str:
|
||||
return f"{NS}<{name}>{body}{NS}</{name}>"
|
||||
|
||||
|
||||
def tool_block(*invokes: tuple[str, str]) -> str:
|
||||
body = "\n".join(
|
||||
f'{NS}<invoke name="{name}">{params}{INVOKE_END}' for name, params in invokes
|
||||
)
|
||||
return f"{TOOL_CALL_START}\n{body}\n{TOOL_CALL_END}"
|
||||
|
||||
|
||||
def parse_arguments(parser, request, params: str, name: str = "create_order"):
|
||||
result = parser.extract_tool_calls(tool_block((name, params)), request)
|
||||
assert [tc.function.name for tc in result.tool_calls] == [name]
|
||||
return json.loads(result.tool_calls[0].function.arguments)
|
||||
|
||||
|
||||
class TestArguments:
|
||||
def test_nested_elements_follow_schema(self, parser, mock_request):
|
||||
params = (
|
||||
element("tags", element("item", "a") + element("item", "b"))
|
||||
+ element("shipping", element("zip", "018956"))
|
||||
+ element("either", element("item", "1") + element("item", "2"))
|
||||
+ element("free", element("t", "x") + element("t", "y"))
|
||||
)
|
||||
assert parse_arguments(parser, mock_request, params) == {
|
||||
"tags": ["a", "b"],
|
||||
"shipping": {"zip": 18956},
|
||||
"either": [1, 2],
|
||||
"free": {"t": ["x", "y"]},
|
||||
}
|
||||
|
||||
def test_empty_elements_follow_schema(self, parser, mock_request):
|
||||
# The chat template renders [] and {} as empty elements.
|
||||
params = (
|
||||
element("tags", "")
|
||||
+ element("shipping", "")
|
||||
+ element("note", "")
|
||||
+ element("either", "")
|
||||
)
|
||||
assert parse_arguments(parser, mock_request, params) == {
|
||||
"tags": [],
|
||||
"shipping": {},
|
||||
"note": "",
|
||||
"either": "",
|
||||
}
|
||||
|
||||
def test_unknown_tool_keeps_text_and_nests_objects(self, parser, mock_request):
|
||||
params = element("n", "1") + element(
|
||||
"list", element("item", "a") + element("item", "b")
|
||||
)
|
||||
assert parse_arguments(parser, mock_request, params, name="other") == {
|
||||
"n": "1",
|
||||
"list": {"item": ["a", "b"]},
|
||||
}
|
||||
|
||||
def test_mixed_text_uses_reserved_field(self, parser, mock_request):
|
||||
params = element("free", "hi " + element("k", "v") + " there") + element(
|
||||
"shipping", "x" + element("$text", "child")
|
||||
)
|
||||
assert parse_arguments(parser, mock_request, params) == {
|
||||
"free": {"k": "v", "$text": "hi there"},
|
||||
"shipping": {"$text": "child", "$$text": "x"},
|
||||
}
|
||||
|
||||
def test_leaf_text_is_verbatim_for_strings(self, parser, mock_request):
|
||||
params = element("note", "\n a <b> & c\n") + element("user_id", " 7 ")
|
||||
assert parse_arguments(parser, mock_request, params) == {
|
||||
"note": "\n a <b> & c\n",
|
||||
"user_id": 7,
|
||||
}
|
||||
|
||||
def test_block_marker_named_element_is_argument(self, parser, mock_request):
|
||||
params = element("free", element("tool_call", "v"))
|
||||
assert parse_arguments(parser, mock_request, params) == {
|
||||
"free": {"tool_call": "v"},
|
||||
}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tail",
|
||||
[
|
||||
"junk" + element("note", "dropped"),
|
||||
f"{NS}<note>a{NS}</other>",
|
||||
f"{NS}<note>unterminated",
|
||||
],
|
||||
)
|
||||
def test_malformed_element_keeps_preceding_params(self, parser, mock_request, tail):
|
||||
params = element("user_id", "1") + tail
|
||||
assert parse_arguments(parser, mock_request, params) == {"user_id": 1}
|
||||
|
||||
def test_depth_limit(self, parser, mock_request):
|
||||
def nested(depth: int) -> str:
|
||||
body = "v"
|
||||
for _ in range(depth - 1):
|
||||
body = element("n", body)
|
||||
return element("free", body)
|
||||
|
||||
accepted = parse_arguments(parser, mock_request, nested(MAX_ELEMENT_DEPTH))
|
||||
assert "free" in accepted
|
||||
rejected = parse_arguments(parser, mock_request, nested(MAX_ELEMENT_DEPTH + 1))
|
||||
assert rejected == {}
|
||||
|
||||
|
||||
class TestToolBlock:
|
||||
def test_multiple_invokes_and_prefix_content(self, parser, mock_request):
|
||||
result = parser.extract_tool_calls(
|
||||
"Checking.\n"
|
||||
+ tool_block(
|
||||
("create_order", element("user_id", "1")),
|
||||
("create_order", element("user_id", "2")),
|
||||
),
|
||||
mock_request,
|
||||
)
|
||||
assert result.content == "Checking.\n"
|
||||
assert [json.loads(tc.function.arguments) for tc in result.tool_calls] == [
|
||||
{"user_id": 1},
|
||||
{"user_id": 2},
|
||||
]
|
||||
|
||||
def test_text_after_tool_block_is_dropped(self, parser, mock_request):
|
||||
result = parser.extract_tool_calls(
|
||||
"pre" + tool_block(("create_order", "")) + "post", mock_request
|
||||
)
|
||||
assert result.content == "pre"
|
||||
assert len(result.tool_calls) == 1
|
||||
|
||||
def test_invoke_outside_tool_block_is_content(self, parser, mock_request):
|
||||
text = f'{NS}<invoke name="create_order">{INVOKE_END}'
|
||||
result = parser.extract_tool_calls(text, mock_request)
|
||||
assert not result.tools_called
|
||||
assert result.content == text
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_arguments_arrive_with_invoke_end(self, parser, mock_request):
|
||||
block = tool_block(
|
||||
("create_order", element("user_id", "42") + element("tags", ""))
|
||||
)
|
||||
invoke_end = block.index(INVOKE_END)
|
||||
results = simulate_tool_streaming(
|
||||
parser,
|
||||
mock_request,
|
||||
["Sure. ", block[:invoke_end], block[invoke_end:]],
|
||||
)
|
||||
|
||||
before_end, at_end = results[1][0], results[2][0]
|
||||
assert before_end is not None and before_end.tool_calls
|
||||
assert before_end.tool_calls[0].function.name == "create_order"
|
||||
assert not before_end.tool_calls[0].function.arguments
|
||||
assert collect_content(results) == "Sure. "
|
||||
assert json.loads(collect_tool_arguments([(at_end, "")])) == {
|
||||
"user_id": 42,
|
||||
"tags": [],
|
||||
}
|
||||
|
||||
def test_char_chunks_match_complete_parse(self, parser, mock_request):
|
||||
text = "a" + tool_block(
|
||||
("create_order", element("shipping", element("zip", "1"))),
|
||||
("create_order", element("note", "x")),
|
||||
)
|
||||
results = simulate_tool_streaming(parser, mock_request, list(text))
|
||||
|
||||
assert collect_content(results) == "a"
|
||||
assert collect_function_name(results) == "create_order"
|
||||
streamed = [
|
||||
tc.function.arguments
|
||||
for delta, _ in results
|
||||
if delta and delta.tool_calls
|
||||
for tc in delta.tool_calls
|
||||
if tc.function and tc.function.arguments
|
||||
]
|
||||
assert [json.loads(args) for args in streamed] == [
|
||||
{"shipping": {"zip": 1}},
|
||||
{"note": "x"},
|
||||
]
|
||||
|
||||
def test_truncated_invoke_keeps_completed_params(self, parser, mock_request):
|
||||
block = tool_block(
|
||||
("create_order", element("user_id", "1") + element("note", "x"))
|
||||
)
|
||||
truncated = block[: block.index(f"{NS}</note>")]
|
||||
results = simulate_tool_streaming(parser, mock_request, [truncated])
|
||||
results.append((parser.finish_streaming(), truncated))
|
||||
|
||||
assert collect_function_name(results) == "create_order"
|
||||
assert json.loads(collect_tool_arguments(results)) == {"user_id": 1}
|
||||
|
||||
|
||||
def test_adjust_request_keeps_special_tokens_skipped(parser, mock_request):
|
||||
mock_request.skip_special_tokens = True
|
||||
assert parser.adjust_request(mock_request).skip_special_tokens is True
|
||||
@@ -1,303 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the engine-based Nemotron V3 parser.
|
||||
|
||||
Validates that ``NemotronV3Parser`` correctly handles:
|
||||
- ``<think>``/``</think>`` reasoning with ``<tool_call>`` XML tool calls
|
||||
(same format as Qwen3)
|
||||
- Nemotron-specific reasoning/content swap when ``enable_thinking=False``
|
||||
or ``force_nonempty_content=True``
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import (
|
||||
collect_function_name,
|
||||
collect_tool_arguments,
|
||||
simulate_tool_streaming,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.parser.nemotron_v3 import NemotronV3Parser
|
||||
|
||||
_THINK_START_ID = 50
|
||||
_THINK_END_ID = 51
|
||||
_TOOL_CALL_ID = 60
|
||||
_TOOL_CALL_END_ID = 61
|
||||
_TEXT_ID = 100
|
||||
|
||||
_VOCAB = {
|
||||
"<think>": _THINK_START_ID,
|
||||
"</think>": _THINK_END_ID,
|
||||
"<tool_call>": _TOOL_CALL_ID,
|
||||
"</tool_call>": _TOOL_CALL_END_ID,
|
||||
}
|
||||
|
||||
|
||||
def _make_request(**chat_template_kwargs):
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = []
|
||||
request.tool_choice = "auto"
|
||||
request.include_reasoning = True
|
||||
request.chat_template_kwargs = chat_template_kwargs or None
|
||||
return request
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser():
|
||||
return NemotronV3Parser(make_mock_tokenizer(_VOCAB))
|
||||
|
||||
|
||||
class TestNemotronSwap:
|
||||
def test_enable_thinking_false_swaps(self, parser):
|
||||
"""When enable_thinking=False, model output without think tags
|
||||
should have reasoning swapped to content."""
|
||||
text = "The answer is 42."
|
||||
request = _make_request(enable_thinking=False)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
assert content == "The answer is 42."
|
||||
assert reasoning is None
|
||||
|
||||
def test_force_nonempty_content_swaps(self, parser):
|
||||
"""force_nonempty_content=True triggers swap when content empty."""
|
||||
text = "The answer is 42."
|
||||
request = _make_request(force_nonempty_content=True)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
assert content == "The answer is 42."
|
||||
assert reasoning is None
|
||||
|
||||
def test_no_swap_when_content_exists(self, parser):
|
||||
"""With enable_thinking=False but real </think> giving content,
|
||||
no swap occurs."""
|
||||
text = "Some reasoning.</think>Actual content here."
|
||||
request = _make_request(enable_thinking=False)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
assert reasoning == "Some reasoning."
|
||||
assert content == "Actual content here."
|
||||
|
||||
def test_no_swap_when_enable_thinking_true(self, parser):
|
||||
"""Normal thinking mode: no swap, even when content is empty."""
|
||||
text = "Still thinking..."
|
||||
request = _make_request(enable_thinking=True)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
assert reasoning == "Still thinking..."
|
||||
assert content is None
|
||||
|
||||
def test_no_swap_with_none_request(self, parser):
|
||||
"""Graceful handling when request is None."""
|
||||
text = "Some text."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "Some text."
|
||||
assert content is None
|
||||
|
||||
def test_no_swap_with_no_kwargs(self, parser):
|
||||
"""No swap when chat_template_kwargs is absent."""
|
||||
text = "Some text."
|
||||
request = _make_request()
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
assert reasoning == "Some text."
|
||||
assert content is None
|
||||
|
||||
def test_swap_with_whitespace_only_content(self, parser):
|
||||
"""Swap occurs when content is whitespace-only."""
|
||||
text = "The answer.</think> "
|
||||
request = _make_request(enable_thinking=False)
|
||||
reasoning, content = parser.extract_reasoning(text, request)
|
||||
assert content == "The answer."
|
||||
assert reasoning == " "
|
||||
|
||||
|
||||
class TestNonStreamingToolCalls:
|
||||
def test_single_tool_call(self, parser):
|
||||
text = (
|
||||
"<tool_call>\n"
|
||||
"<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
request = _make_request()
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"city": "Tokyo"}
|
||||
|
||||
def test_parallel_tool_calls(self, parser):
|
||||
text = (
|
||||
"<tool_call>\n"
|
||||
"<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>"
|
||||
"<tool_call>\n"
|
||||
"<function=get_time>\n"
|
||||
"<parameter=timezone>Asia/Tokyo</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
request = _make_request()
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert result.tool_calls[1].function.name == "get_time"
|
||||
|
||||
def test_no_tool_calls(self, parser):
|
||||
request = _make_request()
|
||||
result = parser.extract_tool_calls("Hello, how can I help?", request)
|
||||
assert result.tools_called is False
|
||||
# Parser starts in REASONING state, so plain text is classified
|
||||
# as reasoning (not content) when there are no tool calls.
|
||||
assert result.content is None
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_streaming_tool_calls(self, parser):
|
||||
request = _make_request()
|
||||
chunks = [
|
||||
"<tool_call>\n",
|
||||
"<function=get_weather>\n",
|
||||
"<parameter=city>Tokyo",
|
||||
"</parameter>\n",
|
||||
"</function>\n",
|
||||
"</tool_call>",
|
||||
]
|
||||
results = simulate_tool_streaming(parser, request, chunks)
|
||||
name = collect_function_name(results)
|
||||
assert name == "get_weather"
|
||||
args_text = collect_tool_arguments(results)
|
||||
assert args_text
|
||||
parsed = json.loads(args_text)
|
||||
assert parsed == {"city": "Tokyo"}
|
||||
|
||||
|
||||
class TestParseDeltaTokenIdFiltering:
|
||||
"""parse_delta must not trigger tool call parsing when <tool_call>
|
||||
appears as regular text rather than as a special token ID."""
|
||||
|
||||
def test_tool_call_text_in_reasoning_is_not_parsed(self, parser):
|
||||
"""Literal <tool_call> in model reasoning should be content,
|
||||
not a tool call."""
|
||||
request = _make_request()
|
||||
|
||||
text = (
|
||||
"The test uses <tool_call> syntax:\n"
|
||||
"<tool_call>\n"
|
||||
"<function=Bash>\n"
|
||||
"<parameter=command>ls</parameter>\n"
|
||||
"</function>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
result = parser.parse_delta(
|
||||
delta_text=text,
|
||||
delta_token_ids=[_TEXT_ID] * 6,
|
||||
request=request,
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.reasoning is not None
|
||||
assert "<tool_call>" in result.reasoning
|
||||
assert not result.tool_calls
|
||||
|
||||
def test_special_token_id_still_triggers_tool_call(self, parser):
|
||||
"""When the scanner matches a special token ID, the tool call
|
||||
must still be parsed correctly."""
|
||||
request = _make_request()
|
||||
|
||||
parser.parse_delta(
|
||||
delta_text="Let me check.",
|
||||
delta_token_ids=[_TEXT_ID, _TEXT_ID, _TEXT_ID],
|
||||
request=request,
|
||||
prompt_token_ids=[],
|
||||
finished=False,
|
||||
)
|
||||
|
||||
parser.parse_delta(
|
||||
delta_text="<tool_call>",
|
||||
delta_token_ids=[_TOOL_CALL_ID],
|
||||
request=request,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
parser.parse_delta(
|
||||
delta_text=(
|
||||
"\n<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
"</function>\n"
|
||||
),
|
||||
delta_token_ids=[_TEXT_ID] * 5,
|
||||
request=request,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
parser.parse_delta(
|
||||
delta_text="</tool_call>",
|
||||
delta_token_ids=[_TOOL_CALL_END_ID],
|
||||
request=request,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
assert any(s.name == "get_weather" for s in parser._tool_slots)
|
||||
|
||||
def test_text_discussion_then_real_tool_call(self, parser):
|
||||
"""Model discusses tool syntax in reasoning, then makes a real
|
||||
tool call via special tokens."""
|
||||
request = _make_request()
|
||||
|
||||
r1 = parser.parse_delta(
|
||||
delta_text="Use <tool_call> to invoke tools.",
|
||||
delta_token_ids=[_TEXT_ID] * 6,
|
||||
request=request,
|
||||
prompt_token_ids=[],
|
||||
finished=False,
|
||||
)
|
||||
|
||||
r2 = parser.parse_delta(
|
||||
delta_text="</think>",
|
||||
delta_token_ids=[_THINK_END_ID],
|
||||
request=request,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
r3 = parser.parse_delta(
|
||||
delta_text="<tool_call>",
|
||||
delta_token_ids=[_TOOL_CALL_ID],
|
||||
request=request,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
r4 = parser.parse_delta(
|
||||
delta_text=("\n<function=test>\n<parameter=x>1</parameter>\n</function>\n"),
|
||||
delta_token_ids=[_TEXT_ID] * 4,
|
||||
request=request,
|
||||
finished=False,
|
||||
)
|
||||
|
||||
r5 = parser.parse_delta(
|
||||
delta_text="</tool_call>",
|
||||
delta_token_ids=[_TOOL_CALL_END_ID],
|
||||
request=request,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
results = [r1, r2, r3, r4, r5]
|
||||
reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
|
||||
assert "<tool_call>" in reasoning
|
||||
|
||||
names = [
|
||||
tc.function.name
|
||||
for r in results
|
||||
if r and r.tool_calls
|
||||
for tc in r.tool_calls
|
||||
if tc.function and tc.function.name
|
||||
]
|
||||
assert "test" in names
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,556 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""PLaMo3 grammar, truncation, and serving-adapter regressions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.replay_harness import (
|
||||
DUMMY_TOOLS,
|
||||
_test_request,
|
||||
collect_output,
|
||||
replay_streaming,
|
||||
replay_with_text_holdback,
|
||||
)
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.plamo3 import (
|
||||
BEGIN_THINK,
|
||||
BEGIN_TOOL_ARGUMENTS,
|
||||
BEGIN_TOOL_NAME,
|
||||
BEGIN_TOOL_REQUEST,
|
||||
BEGIN_TOOL_REQUESTS,
|
||||
END_THINK,
|
||||
END_TOOL_ARGUMENTS,
|
||||
END_TOOL_NAME,
|
||||
END_TOOL_REQUEST,
|
||||
END_TOOL_REQUESTS,
|
||||
EOT,
|
||||
PLAMO_MARKER_TOKENS,
|
||||
Plamo3Parser,
|
||||
)
|
||||
|
||||
|
||||
def _tool_header(name="weather"):
|
||||
return (
|
||||
BEGIN_TOOL_REQUEST
|
||||
+ BEGIN_TOOL_NAME
|
||||
+ name
|
||||
+ END_TOOL_NAME
|
||||
+ BEGIN_TOOL_ARGUMENTS
|
||||
)
|
||||
|
||||
|
||||
def _tool_call(name, arguments):
|
||||
return _tool_header(name) + arguments + END_TOOL_ARGUMENTS + END_TOOL_REQUEST
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
special = [*sorted(PLAMO_MARKER_TOKENS), EOT, "<|plamo:bos|>"]
|
||||
atoms = [*special, "<|plamo:constrain|>", "<|plamo:msg|>"]
|
||||
vocab = {token: 100 + i for i, token in enumerate(atoms)}
|
||||
inverse = {i: token for token, i in vocab.items()}
|
||||
pattern = re.compile("|".join(map(re.escape, atoms)) + r"|[\s\S]")
|
||||
tokenizer = make_mock_tokenizer(vocab, special_tokens=special)
|
||||
tokenizer.encode.side_effect = lambda text, **kwargs: [
|
||||
vocab[piece] if piece in vocab else 1000 + ord(piece)
|
||||
for piece in pattern.findall(text)
|
||||
]
|
||||
tokenizer.decode.side_effect = lambda ids, **kwargs: "".join(
|
||||
inverse[i] if i in inverse else chr(i - 1000) for i in ids
|
||||
)
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture(params=["direct", "registered"])
|
||||
def parser_cls(request):
|
||||
if request.param == "direct":
|
||||
return Plamo3Parser
|
||||
return ParserManager.get_parser(
|
||||
tool_parser_name="plamo3",
|
||||
reasoning_parser_name="plamo3",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
|
||||
def _stream(parser, tokenizer, output, chunk_size=1, finished_on_last=False):
|
||||
tokens = [
|
||||
(i, tokenizer.decode([i]))
|
||||
for i in tokenizer.encode(output, add_special_tokens=False)
|
||||
]
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
tokens,
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=finished_on_last,
|
||||
tools=DUMMY_TOOLS,
|
||||
)
|
||||
if not finished_on_last:
|
||||
deltas.append(
|
||||
parser.parse_delta("", [], _test_request(DUMMY_TOOLS), finished=True)
|
||||
)
|
||||
return deltas
|
||||
|
||||
|
||||
@pytest.mark.parametrize("thinking", [False, True])
|
||||
@pytest.mark.parametrize("with_markers", [False, True])
|
||||
def test_reasoning_mode(
|
||||
parser_cls, mock_tokenizer, mock_request, thinking, with_markers
|
||||
):
|
||||
output = (
|
||||
BEGIN_THINK + "Reasoning." + END_THINK + "Answer."
|
||||
if with_markers
|
||||
else "Answer."
|
||||
)
|
||||
if thinking:
|
||||
expected = ("Reasoning.", "Answer.") if with_markers else ("Answer.", "")
|
||||
else:
|
||||
expected = ("", output)
|
||||
kwargs = {"chat_template_kwargs": {"enable_thinking": thinking}}
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer, **kwargs).parse(
|
||||
output, mock_request
|
||||
)
|
||||
assert (reasoning or "", content or "") == expected
|
||||
assert not calls
|
||||
result = collect_output(
|
||||
_stream(parser_cls(mock_tokenizer, **kwargs), mock_tokenizer, output)
|
||||
)
|
||||
assert (result.reasoning, result.content) == expected
|
||||
assert not result.tool_calls
|
||||
|
||||
|
||||
def test_duplicate_think_end_does_not_leak_into_content(
|
||||
parser_cls, mock_tokenizer, mock_request
|
||||
):
|
||||
output = END_THINK + "A" + END_THINK + "B"
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
|
||||
assert (reasoning, content) == (None, "AB")
|
||||
assert not calls
|
||||
|
||||
|
||||
def test_content_can_start_a_new_reasoning_block(
|
||||
parser_cls, mock_tokenizer, mock_request
|
||||
):
|
||||
output = END_THINK + "A" + BEGIN_THINK + "R" + END_THINK
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
|
||||
assert (reasoning, content) == ("R", "A")
|
||||
assert not calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, None])
|
||||
@pytest.mark.parametrize("finished_on_last", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"closing",
|
||||
[
|
||||
"",
|
||||
END_TOOL_ARGUMENTS + END_TOOL_REQUEST,
|
||||
END_TOOL_ARGUMENTS + END_TOOL_REQUEST + END_TOOL_REQUESTS + EOT,
|
||||
],
|
||||
)
|
||||
def test_complete_arguments_survive_missing_closers(
|
||||
parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last, closing
|
||||
):
|
||||
output = (
|
||||
BEGIN_THINK
|
||||
+ "Reasoning. "
|
||||
+ END_THINK
|
||||
+ "Before tools. "
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_header()
|
||||
+ '{"city":"東京"}'
|
||||
+ closing
|
||||
)
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert (reasoning, content) == ("Reasoning. ", "Before tools. ")
|
||||
assert calls and [(c.name, json.loads(c.arguments)) for c in calls] == [
|
||||
("weather", {"city": "東京"})
|
||||
]
|
||||
result = collect_output(
|
||||
_stream(
|
||||
parser_cls(mock_tokenizer),
|
||||
mock_tokenizer,
|
||||
output,
|
||||
chunk_size,
|
||||
finished_on_last,
|
||||
)
|
||||
)
|
||||
assert (result.reasoning, result.content) == (reasoning, content)
|
||||
assert result.tool_calls == [{"name": "weather", "arguments": '{"city":"東京"}'}]
|
||||
|
||||
|
||||
def test_tool_name_cutoffs_do_not_leak_markers(
|
||||
parser_cls, mock_tokenizer, mock_request
|
||||
):
|
||||
header = BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "weather" + END_TOOL_NAME
|
||||
header_ids = mock_tokenizer.encode(header, add_special_tokens=False)
|
||||
name_start = len(
|
||||
mock_tokenizer.encode(
|
||||
BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME, add_special_tokens=False
|
||||
)
|
||||
)
|
||||
for end in range(len(header_ids)):
|
||||
partial = mock_tokenizer.decode(header_ids[:end])
|
||||
output = END_THINK + "Before tools." + BEGIN_TOOL_REQUESTS + partial
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert (reasoning, content) == (None, "Before tools."), end
|
||||
if end <= name_start:
|
||||
expected_name = ""
|
||||
else:
|
||||
name_text = mock_tokenizer.decode(header_ids[name_start:end])
|
||||
if "<|plamo:" in name_text:
|
||||
name_text = name_text[: name_text.rfind("<|plamo:")]
|
||||
expected_name = name_text[: len("weather")]
|
||||
assert [call.name for call in calls or []] == (
|
||||
[expected_name] if expected_name else []
|
||||
), end
|
||||
result = collect_output(
|
||||
_stream(parser_cls(mock_tokenizer), mock_tokenizer, output)
|
||||
)
|
||||
assert [call["name"] for call in result.tool_calls] == (
|
||||
[expected_name] if expected_name else []
|
||||
), end
|
||||
assert all("<" not in call["name"] for call in result.tool_calls), end
|
||||
assert result.content == "Before tools.", end
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, None])
|
||||
@pytest.mark.parametrize("finished_on_last", [False, True])
|
||||
def test_all_argument_closer_cutoffs_do_not_leak(
|
||||
parser_cls, mock_tokenizer, mock_request, chunk_size, finished_on_last
|
||||
):
|
||||
head = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header()
|
||||
closer_ids = mock_tokenizer.encode(END_TOOL_ARGUMENTS, add_special_tokens=False)
|
||||
for end in range(len(closer_ids)):
|
||||
partial = mock_tokenizer.decode(closer_ids[:end])
|
||||
output = head + '{"city":"東京"}' + partial
|
||||
_, _, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert calls and calls[0].arguments == '{"city":"東京"}', end
|
||||
result = collect_output(
|
||||
_stream(
|
||||
parser_cls(mock_tokenizer),
|
||||
mock_tokenizer,
|
||||
output,
|
||||
chunk_size,
|
||||
finished_on_last,
|
||||
)
|
||||
)
|
||||
assert result.tool_calls == [
|
||||
{"name": "weather", "arguments": '{"city":"東京"}'}
|
||||
], end
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
[
|
||||
"{}",
|
||||
'{"name":"Alice","arguments":{"x":1},"parameters":true}',
|
||||
'{"text":"literal <|plamo:end_',
|
||||
],
|
||||
)
|
||||
def test_arguments_are_not_rewritten(
|
||||
parser_cls, mock_tokenizer, mock_request, arguments
|
||||
):
|
||||
output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_header() + arguments
|
||||
_, _, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert calls and calls[0].arguments == arguments
|
||||
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
|
||||
assert result.tool_calls == [{"name": "weather", "arguments": arguments}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("truncated_second", [False, True])
|
||||
def test_multiple_calls_keep_indices_ids_and_arguments_separate(
|
||||
parser_cls, mock_tokenizer, mock_request, truncated_second
|
||||
):
|
||||
second = (
|
||||
BEGIN_TOOL_REQUEST + BEGIN_TOOL_NAME + "clo"
|
||||
if truncated_second
|
||||
else _tool_call("clock", '{"timezone":"Asia/Tokyo"}')
|
||||
)
|
||||
output = (
|
||||
END_THINK
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_call("weather", '{"city":"東京"}')
|
||||
+ second
|
||||
)
|
||||
expected = [{"name": "weather", "arguments": '{"city":"東京"}'}]
|
||||
if truncated_second:
|
||||
expected.append({"name": "clo", "arguments": "{}"})
|
||||
else:
|
||||
output += END_TOOL_REQUESTS
|
||||
expected.append({"name": "clock", "arguments": '{"timezone":"Asia/Tokyo"}'})
|
||||
stream_expected = expected
|
||||
_, content, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert (
|
||||
calls
|
||||
and [{"name": c.name, "arguments": c.arguments} for c in calls] == expected
|
||||
)
|
||||
deltas = _stream(parser_cls(mock_tokenizer), mock_tokenizer, output)
|
||||
result = collect_output(deltas)
|
||||
assert result.tool_calls == stream_expected
|
||||
assert result.content == (content or "") == ""
|
||||
announced = [
|
||||
call for delta in deltas if delta for call in delta.tool_calls or [] if call.id
|
||||
]
|
||||
assert [call.index for call in announced] == list(range(len(expected)))
|
||||
assert len({call.id for call in announced}) == len(expected)
|
||||
|
||||
|
||||
def test_arguments_are_available_before_outer_closer(mock_tokenizer, mock_request):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
output = END_THINK + BEGIN_TOOL_REQUESTS + _tool_call("weather", '{"city":"Tokyo"}')
|
||||
first = parser.parse_delta(output, [], mock_request, finished=False)
|
||||
assert collect_output([first]).tool_calls == [
|
||||
{"name": "weather", "arguments": '{"city":"Tokyo"}'}
|
||||
]
|
||||
assert (
|
||||
parser.parse_delta(END_TOOL_REQUESTS[:10], [], mock_request, finished=False)
|
||||
is None
|
||||
)
|
||||
assert (
|
||||
parser.parse_delta(END_TOOL_REQUESTS[10:], [], mock_request, finished=True)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_missing_tool_request_start_does_not_break_streaming(
|
||||
parser_cls, mock_tokenizer, mock_request
|
||||
):
|
||||
output = (
|
||||
BEGIN_TOOL_REQUESTS
|
||||
+ BEGIN_TOOL_NAME
|
||||
+ "weather"
|
||||
+ END_TOOL_NAME
|
||||
+ BEGIN_TOOL_ARGUMENTS
|
||||
+ "{}"
|
||||
+ END_TOOL_ARGUMENTS
|
||||
+ END_TOOL_REQUESTS
|
||||
)
|
||||
delta = parser_cls(mock_tokenizer).parse_delta(
|
||||
output, [], mock_request, finished=False
|
||||
)
|
||||
assert delta is not None
|
||||
assert collect_output([delta]).tool_calls == [
|
||||
{"name": "weather", "arguments": "{}"}
|
||||
]
|
||||
|
||||
|
||||
def test_reusing_parser_drops_pending_names(mock_tokenizer, mock_request):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
parser.parse(
|
||||
END_THINK
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ BEGIN_TOOL_REQUEST
|
||||
+ BEGIN_TOOL_NAME
|
||||
+ "partial",
|
||||
mock_request,
|
||||
)
|
||||
output = (
|
||||
END_THINK
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_call("weather", "{}")
|
||||
+ END_TOOL_REQUESTS
|
||||
)
|
||||
_, _, calls = parser.parse(output, mock_request)
|
||||
assert calls and calls[0].name == "weather"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text", ["", "a < b", "literal <", "literal <|plamo:unknown", " text\n"]
|
||||
)
|
||||
def test_plain_content_is_preserved(parser_cls, mock_tokenizer, mock_request, text):
|
||||
output = END_THINK + text + EOT
|
||||
_, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
|
||||
assert (content or "") == text
|
||||
assert not calls
|
||||
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
|
||||
assert (result.content, result.tool_calls) == (text, [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text, ended",
|
||||
[
|
||||
("", False),
|
||||
(BEGIN_THINK + "reasoning", False),
|
||||
(END_THINK + "answer", True),
|
||||
(END_THINK + "answer" + BEGIN_THINK + "new reasoning", False),
|
||||
(END_THINK + "answer" + EOT + "assistant", False),
|
||||
],
|
||||
)
|
||||
def test_reasoning_end_checks_the_latest_turn(mock_tokenizer, text, ended):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
ids = mock_tokenizer.encode(text)
|
||||
assert parser.is_reasoning_end(ids) == ended
|
||||
if ended:
|
||||
assert mock_tokenizer.decode(parser.extract_content_ids(ids)) == "answer"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", ["<|plamo:end_", "<|plamo:end_think"])
|
||||
def test_partial_end_think_is_consistent_between_streaming_and_non_streaming(
|
||||
parser_cls, mock_tokenizer, mock_request, suffix
|
||||
):
|
||||
output = "visible" + suffix
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(output, mock_request)
|
||||
assert (reasoning, content) == ("visible", None)
|
||||
assert not calls
|
||||
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
|
||||
assert (result.reasoning, result.content, result.tool_calls) == ("visible", "", [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"suffix",
|
||||
[
|
||||
"<|plamo:begin_",
|
||||
"<|plamo:begin_tool_name",
|
||||
"<|plamo:end_tool",
|
||||
],
|
||||
)
|
||||
def test_unfinished_marker_prefixes_are_stripped(mock_tokenizer, suffix):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
assert parser._strip_unfinished_marker("visible" + suffix) == "visible"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("suffix", ["<|plamo:", "<|plamo:b"])
|
||||
def test_non_token_prefix_is_not_stripped(mock_tokenizer, suffix):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
text = "visible" + suffix
|
||||
assert parser._strip_unfinished_marker(text) == text
|
||||
|
||||
|
||||
def test_complete_markers_are_not_stripped(mock_tokenizer):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
text = "visible" + BEGIN_TOOL_NAME
|
||||
assert parser._strip_unfinished_marker(text) == text
|
||||
|
||||
|
||||
def test_short_prefix_is_not_stripped(mock_tokenizer):
|
||||
parser = Plamo3Parser(mock_tokenizer)
|
||||
text = "visible<"
|
||||
assert parser._strip_unfinished_marker(text) == text
|
||||
|
||||
|
||||
def test_tool_block_can_end_reasoning_implicitly(
|
||||
parser_cls, mock_tokenizer, mock_request
|
||||
):
|
||||
output = (
|
||||
"reasoning"
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_call("weather", "{}")
|
||||
+ END_TOOL_REQUESTS
|
||||
)
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert (reasoning, content) == ("reasoning", None)
|
||||
assert calls and calls[0].name == "weather"
|
||||
result = collect_output(_stream(parser_cls(mock_tokenizer), mock_tokenizer, output))
|
||||
assert (result.reasoning, result.content) == ("reasoning", "")
|
||||
assert result.tool_calls == [{"name": "weather", "arguments": "{}"}]
|
||||
|
||||
|
||||
def test_every_cut_point_matches_non_streaming(
|
||||
parser_cls, mock_tokenizer, mock_request
|
||||
):
|
||||
output = (
|
||||
BEGIN_THINK
|
||||
+ "Reasoning. "
|
||||
+ END_THINK
|
||||
+ "Before tools. "
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_call("weather", '{"city":"東京"}')
|
||||
+ END_TOOL_REQUESTS
|
||||
+ EOT
|
||||
)
|
||||
ids = mock_tokenizer.encode(output, add_special_tokens=False)
|
||||
for k in range(1, len(ids) + 1):
|
||||
prefix = mock_tokenizer.decode(ids[:k])
|
||||
reasoning, content, calls = parser_cls(mock_tokenizer).parse(
|
||||
prefix, mock_request, enable_auto_tools=True
|
||||
)
|
||||
result = collect_output(
|
||||
_stream(parser_cls(mock_tokenizer), mock_tokenizer, prefix)
|
||||
)
|
||||
assert (result.reasoning, result.content) == (
|
||||
(reasoning or ""),
|
||||
(content or ""),
|
||||
), k
|
||||
expected = [{"name": c.name, "arguments": c.arguments} for c in (calls or [])]
|
||||
assert result.tool_calls == expected, (k, expected, result.tool_calls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"choice",
|
||||
[
|
||||
"auto",
|
||||
"required",
|
||||
"none",
|
||||
{"type": "function", "function": {"name": "weather"}},
|
||||
],
|
||||
)
|
||||
def test_tool_choice_uses_the_plamo_format(parser_cls, mock_tokenizer, choice):
|
||||
request = _test_request(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "weather", "parameters": {"type": "object"}},
|
||||
},
|
||||
]
|
||||
)
|
||||
request = type(request).model_validate(
|
||||
request.model_dump() | {"tool_choice": choice}
|
||||
)
|
||||
output = (
|
||||
END_THINK
|
||||
+ "Before tools."
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_call("weather", "{}")
|
||||
+ END_TOOL_REQUESTS
|
||||
)
|
||||
_, content, calls = parser_cls(mock_tokenizer).parse(
|
||||
output, request, enable_auto_tools=True
|
||||
)
|
||||
assert content == "Before tools."
|
||||
assert [c.name for c in calls or []] == ([] if choice == "none" else ["weather"])
|
||||
parser = parser_cls(mock_tokenizer)
|
||||
delta = parser.parse_delta(output, [], request, finished=True)
|
||||
result = collect_output([delta])
|
||||
assert result.content == "Before tools."
|
||||
assert result.tool_calls == (
|
||||
[] if choice == "none" else [{"name": "weather", "arguments": "{}"}]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delay", [1, 3])
|
||||
def test_delayed_text_keeps_split_markers_intact(parser_cls, mock_tokenizer, delay):
|
||||
output = (
|
||||
BEGIN_THINK
|
||||
+ "reasoning"
|
||||
+ END_THINK
|
||||
+ "content"
|
||||
+ BEGIN_TOOL_REQUESTS
|
||||
+ _tool_call("weather", "{}")
|
||||
+ END_TOOL_REQUESTS
|
||||
+ EOT
|
||||
)
|
||||
tokens = [(i, mock_tokenizer.decode([i])) for i in mock_tokenizer.encode(output)]
|
||||
result = collect_output(
|
||||
replay_with_text_holdback(
|
||||
parser_cls(mock_tokenizer), tokens, text_delay=delay, tools=DUMMY_TOOLS
|
||||
)
|
||||
)
|
||||
assert (result.reasoning, result.content) == ("reasoning", "content")
|
||||
assert result.tool_calls == [{"name": "weather", "arguments": "{}"}]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,235 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the engine-based seed_oss parser.
|
||||
|
||||
seed_oss is Qwen3 with four overridden wrapper tokens, so the shared grammar
|
||||
(arg types, multiline values, parallel calls, streaming mechanics, …) is
|
||||
already covered by ``test_qwen3.py``/``test_qwen3_reasoning.py``. These tests
|
||||
cover only what is seed_oss-specific: that the ``seed:`` token overrides are
|
||||
wired through, the reasoning→tool boundary holds with them, the malformed
|
||||
header from #46314 no longer drops sibling calls, and the registered adapters
|
||||
resolve. Seed-specific budget-reflect tags inside reasoning are also covered
|
||||
here because the old dedicated parser tests exercised them.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import (
|
||||
collect_function_name,
|
||||
collect_tool_arguments,
|
||||
simulate_reasoning_streaming,
|
||||
simulate_tool_streaming,
|
||||
)
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
SeedOssParserReasoningAdapter,
|
||||
SeedOssParserToolAdapter,
|
||||
)
|
||||
from vllm.parser.seed_oss import SeedOssParser
|
||||
|
||||
TOOL_CALL_START = "<seed:tool_call>"
|
||||
TOOL_CALL_END = "</seed:tool_call>"
|
||||
THINK_START = "<seed:think>"
|
||||
THINK_END = "</seed:think>"
|
||||
TURN_START = "<seed:bos>"
|
||||
TURN_END = "<seed:eos>"
|
||||
|
||||
_THINK_START_ID = 50
|
||||
_THINK_END_ID = 51
|
||||
_TOOL_CALL_START_ID = 60
|
||||
_TOOL_CALL_END_ID = 61
|
||||
_SEED_BOS_ID = 70
|
||||
_SEED_EOS_ID = 71
|
||||
_TEXT_ID = 100
|
||||
|
||||
_SEED_OSS_VOCAB = {
|
||||
THINK_START: _THINK_START_ID,
|
||||
THINK_END: _THINK_END_ID,
|
||||
TOOL_CALL_START: _TOOL_CALL_START_ID,
|
||||
TOOL_CALL_END: _TOOL_CALL_END_ID,
|
||||
TURN_START: _SEED_BOS_ID,
|
||||
TURN_END: _SEED_EOS_ID,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_SEED_OSS_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tool_parser(mock_tokenizer):
|
||||
return SeedOssParser(
|
||||
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return SeedOssParser(mock_tokenizer)
|
||||
|
||||
|
||||
def test_token_overrides_wired(parser):
|
||||
assert parser.parser_engine_config.name == "seed_oss"
|
||||
assert parser.reasoning_start_str == THINK_START
|
||||
assert parser.reasoning_end_str == THINK_END
|
||||
assert parser.parser_engine_config.turn_boundary_tokens == frozenset(
|
||||
(TURN_START, TURN_END)
|
||||
)
|
||||
|
||||
|
||||
class TestSeedTurnBoundaries:
|
||||
"""The Seed-OSS chat template wraps every message in ``<seed:bos>role`` /
|
||||
``<seed:eos>`` and replays prior-turn ``reasoning_content`` inside
|
||||
``<seed:think>...</seed:think>``, so a stale ``</seed:think>`` in history
|
||||
is the normal multi-turn case. The backward walk must stop at the Seed
|
||||
boundary tokens; the inherited ChatML tokens are absent from the Seed
|
||||
vocab and would disable the scoping entirely.
|
||||
"""
|
||||
|
||||
def test_replayed_reasoning_in_history_not_end(self, parser):
|
||||
assert not parser.is_reasoning_end(
|
||||
[
|
||||
_SEED_BOS_ID,
|
||||
_THINK_START_ID,
|
||||
_TEXT_ID,
|
||||
_THINK_END_ID,
|
||||
_TEXT_ID,
|
||||
_SEED_EOS_ID,
|
||||
_SEED_BOS_ID,
|
||||
]
|
||||
)
|
||||
|
||||
def test_think_end_in_current_turn_is_end(self, parser):
|
||||
assert parser.is_reasoning_end(
|
||||
[_SEED_BOS_ID, _TEXT_ID, _SEED_EOS_ID, _SEED_BOS_ID, _THINK_END_ID]
|
||||
)
|
||||
|
||||
def test_boundary_with_thinking_disabled_is_end(self, mock_tokenizer):
|
||||
parser = SeedOssParser(
|
||||
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
assert parser.is_reasoning_end([_SEED_BOS_ID, _TEXT_ID])
|
||||
|
||||
|
||||
def test_single_tool_call(tool_parser, mock_request):
|
||||
text = (
|
||||
f"{TOOL_CALL_START}\n<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
f"</function>\n{TOOL_CALL_END}"
|
||||
)
|
||||
result = tool_parser.extract_tool_calls(text, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Tokyo"}
|
||||
|
||||
|
||||
def test_malformed_function_end_does_not_drop_siblings(tool_parser, mock_request):
|
||||
"""Regression for #46314: a malformed ``</function>`` with no closing ``>``
|
||||
on the header must not discard the other, well-formed calls."""
|
||||
text = (
|
||||
f"{TOOL_CALL_START}\n<function=broken</function>\n{TOOL_CALL_END}"
|
||||
f"{TOOL_CALL_START}\n<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
f"</function>\n{TOOL_CALL_END}"
|
||||
)
|
||||
result = tool_parser.extract_tool_calls(text, mock_request)
|
||||
|
||||
weather = next(tc for tc in result.tool_calls if tc.function.name == "get_weather")
|
||||
assert json.loads(weather.function.arguments) == {"city": "Tokyo"}
|
||||
|
||||
|
||||
def test_basic_streaming(tool_parser, mock_request):
|
||||
chunks = [
|
||||
f"{TOOL_CALL_START}\n",
|
||||
"<function=get_weather>\n",
|
||||
"<parameter=city>Tokyo",
|
||||
"</parameter>\n",
|
||||
"</function>\n",
|
||||
f"{TOOL_CALL_END}",
|
||||
]
|
||||
results = simulate_tool_streaming(tool_parser, mock_request, chunks)
|
||||
|
||||
assert collect_function_name(results) == "get_weather"
|
||||
assert json.loads(collect_tool_arguments(results)) == {"city": "Tokyo"}
|
||||
|
||||
|
||||
def test_reasoning_then_tool_call(parser):
|
||||
text = (
|
||||
f"{THINK_START}I need to read the file.{THINK_END}"
|
||||
f"{TOOL_CALL_START}\n<function=read>\n"
|
||||
"<parameter=path>/tmp/x</parameter>\n"
|
||||
f"</function>\n{TOOL_CALL_END}"
|
||||
)
|
||||
reasoning, _ = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "I need to read the file."
|
||||
assert TOOL_CALL_START not in reasoning
|
||||
|
||||
|
||||
def test_streaming_think_end_and_tool_call_same_delta(parser):
|
||||
"""``</seed:think>`` and ``<seed:tool_call>`` arriving in one delta must
|
||||
not leak the terminal tokens into the reasoning text."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"Let me list the directory.",
|
||||
f"{THINK_END}{TOOL_CALL_START}",
|
||||
"<function=read>",
|
||||
],
|
||||
[(1,), (_THINK_END_ID, _TOOL_CALL_START_ID), (2,)],
|
||||
)
|
||||
assert reasoning == "Let me list the directory."
|
||||
assert THINK_END not in reasoning
|
||||
assert TOOL_CALL_START not in reasoning
|
||||
assert content is not None
|
||||
|
||||
|
||||
def test_end_to_end_through_registered_adapters(mock_tokenizer, mock_request):
|
||||
reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer)
|
||||
tool_parser = SeedOssParserToolAdapter(mock_tokenizer)
|
||||
text = (
|
||||
f"{THINK_START}Plan the call.{THINK_END}"
|
||||
f"{TOOL_CALL_START}\n<function=get_weather>\n"
|
||||
"<parameter=city>Tokyo</parameter>\n"
|
||||
f"</function>\n{TOOL_CALL_END}"
|
||||
)
|
||||
reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request)
|
||||
assert reasoning == "Plan the call."
|
||||
|
||||
tool_result = tool_parser.extract_tool_calls(remaining, mock_request)
|
||||
assert tool_result.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(tool_result.tool_calls[0].function.arguments) == {"city": "Tokyo"}
|
||||
|
||||
|
||||
def test_budget_reflect_tags_do_not_break_adapter_pipeline(
|
||||
mock_tokenizer,
|
||||
mock_request,
|
||||
):
|
||||
reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer)
|
||||
tool_parser = SeedOssParserToolAdapter(mock_tokenizer)
|
||||
text = (
|
||||
f"{THINK_START}"
|
||||
"The user's current thinking budget is 512.</seed:cot_budget_reflect>\n"
|
||||
"I need the weather.\n"
|
||||
"<seed:cot_budget_reflect>I have used 131 tokens."
|
||||
"</seed:cot_budget_reflect>\n"
|
||||
f"{THINK_END}"
|
||||
f"{TOOL_CALL_START}\n<function=get_weather>\n"
|
||||
"<parameter=city>Barcelona</parameter>\n"
|
||||
f"</function>\n{TOOL_CALL_END}"
|
||||
)
|
||||
|
||||
reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request)
|
||||
assert reasoning is not None
|
||||
assert "current thinking budget is 512" in reasoning
|
||||
assert "<seed:cot_budget_reflect>" in reasoning
|
||||
assert "</seed:cot_budget_reflect>" in reasoning
|
||||
|
||||
tool_result = tool_parser.extract_tool_calls(remaining, mock_request)
|
||||
assert tool_result.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(tool_result.tool_calls[0].function.arguments) == {
|
||||
"city": "Barcelona"
|
||||
}
|
||||
@@ -1,122 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the engine-based Step-3.5 parser.
|
||||
|
||||
Step-3.5 reuses the Qwen3 grammar, which ``test_qwen3.py`` covers. These tests
|
||||
cover what differs: thinking is always on, the newline the model emits before
|
||||
``</think>`` is dropped, prior-turn reasoning does not end the current turn,
|
||||
and the ``step3p5`` names resolve to the engine adapters.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Step3p5ParserReasoningAdapter,
|
||||
Step3p5ParserToolAdapter,
|
||||
)
|
||||
from vllm.parser.step3p5 import Step3p5Parser
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
_THINK_START_ID = 50
|
||||
_THINK_END_ID = 51
|
||||
_IM_START_ID = 70
|
||||
_IM_END_ID = 71
|
||||
_TEXT_ID = 100
|
||||
|
||||
_STEP3P5_VOCAB = {
|
||||
"<think>": _THINK_START_ID,
|
||||
"</think>": _THINK_END_ID,
|
||||
"<tool_call>": 60,
|
||||
"</tool_call>": 61,
|
||||
"<|im_start|>": _IM_START_ID,
|
||||
"<|im_end|>": _IM_END_ID,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_STEP3P5_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return Step3p5Parser(mock_tokenizer)
|
||||
|
||||
|
||||
def test_parser_names_resolve_to_engine_adapters():
|
||||
tool_parser_cls = ToolParserManager.get_tool_parser("step3p5")
|
||||
assert issubclass(tool_parser_cls, Step3p5ParserToolAdapter)
|
||||
assert tool_parser_cls.structural_tag_model == "qwen_3_coder"
|
||||
reasoning_parser_cls = ReasoningParserManager.get_reasoning_parser("step3p5")
|
||||
assert reasoning_parser_cls is Step3p5ParserReasoningAdapter
|
||||
|
||||
|
||||
def test_enable_thinking_false_is_ignored(mock_tokenizer):
|
||||
# The chat template always prefills "<think>\n" and has no thinking switch.
|
||||
parser = Step3p5Parser(
|
||||
mock_tokenizer, chat_template_kwargs={"enable_thinking": False}
|
||||
)
|
||||
assert parser.extract_reasoning("plan</think>answer", None) == ("plan", "answer")
|
||||
assert not parser.is_reasoning_end([_IM_START_ID, _TEXT_ID, _THINK_START_ID])
|
||||
|
||||
|
||||
def test_newline_before_think_end_dropped(parser):
|
||||
text = "Line one.\nLine two.\n</think>\nThe answer."
|
||||
assert parser.extract_reasoning(text, None) == (
|
||||
"Line one.\nLine two.",
|
||||
"\nThe answer.",
|
||||
)
|
||||
|
||||
|
||||
def test_newline_before_think_end_dropped_streaming(parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["Line one.\n", "Line two.\n", "</think>", "\nThe answer."],
|
||||
[(_TEXT_ID,), (_TEXT_ID,), (_THINK_END_ID,), (_TEXT_ID,)],
|
||||
)
|
||||
assert reasoning == "Line one.\nLine two."
|
||||
assert content == "\nThe answer."
|
||||
|
||||
|
||||
class TestPriorTurnReasoning:
|
||||
"""The template replays earlier assistant turns as
|
||||
``<think>\\n...\\n</think>\\n``, so a ``</think>`` from history must not end
|
||||
reasoning for the new turn (#34211)."""
|
||||
|
||||
_HISTORY = [
|
||||
_IM_START_ID,
|
||||
_TEXT_ID,
|
||||
_THINK_START_ID,
|
||||
_TEXT_ID,
|
||||
_THINK_END_ID,
|
||||
_TEXT_ID,
|
||||
_IM_END_ID,
|
||||
]
|
||||
|
||||
def test_prior_turn_think_end_not_end(self, parser):
|
||||
assert not parser.is_reasoning_end([*self._HISTORY, _IM_START_ID, _TEXT_ID])
|
||||
|
||||
def test_think_end_in_current_turn_is_end(self, parser):
|
||||
assert parser.is_reasoning_end(
|
||||
[*self._HISTORY, _IM_START_ID, _THINK_START_ID, _TEXT_ID, _THINK_END_ID]
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_then_tool_call(parser, mock_request):
|
||||
text = (
|
||||
"Need the weather.\n</think>\n<tool_call>\n<function=get_weather>\n"
|
||||
"<parameter=city>\nTokyo\n</parameter>\n</function>\n</tool_call>"
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
text, mock_request, enable_auto_tools=True
|
||||
)
|
||||
assert reasoning == "Need the weather."
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Tokyo"}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,599 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from mistral_common.protocol.instruct.messages import (
|
||||
AssistantMessage,
|
||||
ThinkChunk,
|
||||
ToolMessage,
|
||||
UserMessage,
|
||||
)
|
||||
from mistral_common.protocol.instruct.request import InstructRequest
|
||||
from mistral_common.protocol.instruct.tool_calls import FunctionCall, ToolCall
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction_mistral
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
_PARSER_NAME = "mistral"
|
||||
_MODEL_V13 = "mistralai/Magistral-Small-2509"
|
||||
_MODEL_V11 = "mistralai/Magistral-Small-2506"
|
||||
|
||||
_SAMPLE_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the current weather in a city.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mistral_tokenizer() -> MistralTokenizer:
|
||||
"""v13 Magistral tokenizer with `[THINK]`/`[/THINK]` special tokens."""
|
||||
return MistralTokenizer.from_pretrained(_MODEL_V13)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def mistral_v11_tokenizer() -> MistralTokenizer:
|
||||
"""v11 Magistral tokenizer using plain-text `<think>`/`</think>` tags."""
|
||||
return MistralTokenizer.from_pretrained(_MODEL_V11)
|
||||
|
||||
|
||||
def _encode_v13(tokenizer: MistralTokenizer, output: str) -> list[int]:
|
||||
"""Encode output string with `[THINK]`/`[/THINK]` markers into token IDs.
|
||||
|
||||
`[THINK]` and `[/THINK]` placeholders in `output` are replaced by their
|
||||
special-token IDs; all surrounding text is encoded with the base tokenizer.
|
||||
Mirrors the encoding helper from the deleted
|
||||
`tests/reasoning/test_mistral_reasoning_parser.py`.
|
||||
"""
|
||||
think_start = "[THINK]"
|
||||
think_end = "[/THINK]"
|
||||
len_start = len(think_start)
|
||||
len_end = len(think_end)
|
||||
|
||||
index_start = output.find(think_start)
|
||||
index_end = output.find(think_end)
|
||||
|
||||
out: list[int] = []
|
||||
if index_start != -1:
|
||||
out += tokenizer.tokenizer.encode(output[:index_start], False, False)
|
||||
out += [tokenizer.instruct.BEGIN_THINK]
|
||||
|
||||
if index_end != -1:
|
||||
middle = output[index_start + len_start : index_end]
|
||||
suffix = output[index_end + len_end :]
|
||||
out += tokenizer.tokenizer.encode(middle, False, False)
|
||||
out += [tokenizer.instruct.END_THINK]
|
||||
out += tokenizer.tokenizer.encode(suffix, False, False)
|
||||
else:
|
||||
out += tokenizer.tokenizer.encode(
|
||||
output[index_start + len_start :], False, False
|
||||
)
|
||||
elif index_end != -1:
|
||||
out += tokenizer.tokenizer.encode(output[:index_end], False, False)
|
||||
out += [tokenizer.instruct.END_THINK]
|
||||
out += tokenizer.tokenizer.encode(output[index_end + len_end :], False, False)
|
||||
else:
|
||||
out += tokenizer.tokenizer.encode(output, False, False)
|
||||
return out
|
||||
|
||||
|
||||
_TEST_CASES = [
|
||||
# v13 special-token encoding: [THINK]…[/THINK] with trailing content.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]r[/THINK]c",
|
||||
"r",
|
||||
"c",
|
||||
id="v13_basic",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]r[/THINK]c",
|
||||
"r",
|
||||
"c",
|
||||
id="v13_basic_streaming",
|
||||
),
|
||||
# No trailing content after reasoning ends.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]r[/THINK]",
|
||||
"r",
|
||||
None,
|
||||
id="v13_no_trailing_content",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]r[/THINK]",
|
||||
"r",
|
||||
None,
|
||||
id="v13_no_trailing_content_streaming",
|
||||
),
|
||||
# Multi-line reasoning and multi-line content.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]a\nb[/THINK]c\nd",
|
||||
"a\nb",
|
||||
"c\nd",
|
||||
id="v13_multiline",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]a\nb[/THINK]c\nd",
|
||||
"a\nb",
|
||||
"c\nd",
|
||||
id="v13_multiline_streaming",
|
||||
),
|
||||
# No reasoning at all: output is pure content.
|
||||
pytest.param(
|
||||
False,
|
||||
"hello world",
|
||||
None,
|
||||
"hello world",
|
||||
id="v13_no_reasoning",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"hello world",
|
||||
None,
|
||||
"hello world",
|
||||
id="v13_no_reasoning_streaming",
|
||||
),
|
||||
# [THINK] opened but never closed; reasoning accumulates, content is None.
|
||||
pytest.param(
|
||||
False,
|
||||
"[THINK]r",
|
||||
"r",
|
||||
None,
|
||||
id="v13_think_no_end",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[THINK]r",
|
||||
"r",
|
||||
None,
|
||||
id="v13_think_no_end_streaming",
|
||||
),
|
||||
# Stray [/THINK] with no matching [THINK]: no reasoning, content follows.
|
||||
pytest.param(
|
||||
False,
|
||||
"[/THINK]This is the rest",
|
||||
None,
|
||||
"This is the rest",
|
||||
id="v13_stray_think_end",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[/THINK]This is the rest",
|
||||
None,
|
||||
"This is the rest",
|
||||
id="v13_stray_think_end_streaming",
|
||||
),
|
||||
# Content before [THINK]: leading segment joins with trailing segment.
|
||||
pytest.param(
|
||||
False,
|
||||
"Before\n[THINK]r[/THINK]\nAfter",
|
||||
"r",
|
||||
"Before\n\nAfter",
|
||||
id="v13_leading_content",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"Before\n[THINK]r[/THINK]\nAfter",
|
||||
"r",
|
||||
"Before\n\nAfter",
|
||||
id="v13_leading_content_streaming",
|
||||
),
|
||||
# Empty output: no reasoning, no content.
|
||||
pytest.param(
|
||||
False,
|
||||
"",
|
||||
None,
|
||||
None,
|
||||
id="v13_empty",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"",
|
||||
None,
|
||||
None,
|
||||
id="v13_empty_streaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"streaming, output, expected_reasoning, expected_content",
|
||||
_TEST_CASES,
|
||||
)
|
||||
def test_mistral_reasoning_v13(
|
||||
streaming: bool,
|
||||
output: str,
|
||||
expected_reasoning: str | None,
|
||||
expected_content: str | None,
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
output_tokens = _encode_v13(mistral_tokenizer, output)
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
|
||||
mistral_tokenizer
|
||||
)
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == expected_reasoning
|
||||
assert content == expected_content
|
||||
|
||||
|
||||
def test_system_prompt_think_ignored(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""Streaming content extraction must not
|
||||
include `[THINK]`/`[/THINK]` examples from the system prompt.
|
||||
"""
|
||||
output = "[THINK]real[/THINK]answer"
|
||||
output_tokens = _encode_v13(mistral_tokenizer, output)
|
||||
for streaming in (False, True):
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
|
||||
_PARSER_NAME
|
||||
)(mistral_tokenizer)
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == "real"
|
||||
assert content == "answer"
|
||||
|
||||
|
||||
def test_reasoning_with_tool_calls(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""Reasoning extraction with a tool call in the generated output."""
|
||||
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
|
||||
if tool_calls_token_id is None:
|
||||
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
|
||||
|
||||
# Token sequence representing: [THINK]r[/THINK][TOOL_CALLS]f{"a":1}
|
||||
output_tokens = (
|
||||
[mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
+ mistral_tokenizer.tokenizer.encode("r", False, False)
|
||||
+ [mistral_tokenizer.instruct.END_THINK]
|
||||
+ [tool_calls_token_id]
|
||||
+ mistral_tokenizer.tokenizer.encode('f{"a":1}', False, False)
|
||||
)
|
||||
for streaming in (False, True):
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(
|
||||
_PARSER_NAME
|
||||
)(mistral_tokenizer)
|
||||
reasoning, _content = run_reasoning_extraction_mistral(
|
||||
parser, output_tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == "r"
|
||||
assert "[TOOL_CALLS]" not in (reasoning or "")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True], ids=["nonstream", "stream"])
|
||||
def test_reasoning_v11_plain_text_think(
|
||||
mistral_v11_tokenizer: MistralTokenizer, streaming: bool
|
||||
) -> None:
|
||||
"""v11 tokenizers use plain-text `<think>`/`</think>` (no special tokens)."""
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
|
||||
mistral_v11_tokenizer
|
||||
)
|
||||
|
||||
tokens = mistral_v11_tokenizer.tokenizer.encode("<think>r</think>c", False, False)
|
||||
reasoning, content = run_reasoning_extraction_mistral(
|
||||
parser, tokens, streaming=streaming
|
||||
)
|
||||
assert reasoning == "r"
|
||||
assert content == "c"
|
||||
|
||||
|
||||
def test_is_reasoning_end(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""is_reasoning_end returns True iff reasoning has finished."""
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(_PARSER_NAME)(
|
||||
mistral_tokenizer
|
||||
)
|
||||
|
||||
# Complete reasoning: END_THINK present → True.
|
||||
complete_ids = _encode_v13(mistral_tokenizer, "[THINK]r[/THINK]c")
|
||||
assert parser.is_reasoning_end(complete_ids)
|
||||
|
||||
# Open-only: no END_THINK, BEGIN_THINK present → False.
|
||||
open_ids = _encode_v13(mistral_tokenizer, "[THINK]r")
|
||||
assert not parser.is_reasoning_end(open_ids)
|
||||
|
||||
# No reasoning markers at all → False.
|
||||
no_reasoning_ids = _encode_v13(mistral_tokenizer, "hello")
|
||||
assert not parser.is_reasoning_end(no_reasoning_ids)
|
||||
|
||||
# Explicit END_THINK closes reasoning → True.
|
||||
explicit_end_ids = (
|
||||
[mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
+ mistral_tokenizer.tokenizer.encode("r", False, False)
|
||||
+ [mistral_tokenizer.instruct.END_THINK]
|
||||
)
|
||||
assert parser.is_reasoning_end(explicit_end_ids)
|
||||
|
||||
# [TOOL_CALLS] acts as implicit reasoning-end marker.
|
||||
tool_calls_token_id = mistral_tokenizer.get_vocab().get("[TOOL_CALLS]")
|
||||
if tool_calls_token_id is None:
|
||||
pytest.skip("Tokenizer does not have [TOOL_CALLS] in vocab")
|
||||
|
||||
implicit_end_ids = (
|
||||
[mistral_tokenizer.instruct.BEGIN_THINK]
|
||||
+ mistral_tokenizer.tokenizer.encode("r", False, False)
|
||||
+ [tool_calls_token_id]
|
||||
)
|
||||
assert parser.is_reasoning_end(implicit_end_ids)
|
||||
|
||||
|
||||
def _stream_parse_delta(parser, tokenizer, gen_ids, request, prompt_ids):
|
||||
"""Stream `gen_ids` through `parse_delta` and reconstruct reasoning/content."""
|
||||
reasoning = ""
|
||||
content = ""
|
||||
previous_tokens: list[str] | None = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, tok in enumerate(gen_ids):
|
||||
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
tokenizer=tokenizer,
|
||||
all_input_ids=gen_ids[: i + 1],
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=True,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=[tok],
|
||||
request=request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=(i == len(gen_ids) - 1),
|
||||
)
|
||||
if delta_message is not None:
|
||||
reasoning += delta_message.reasoning or ""
|
||||
content += delta_message.content or ""
|
||||
return reasoning, content
|
||||
|
||||
|
||||
def test_parse_delta_keeps_reasoning_open_after_closed_think_prompt(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""A closed `[THINK]` block in the prompt must not finalize reasoning.
|
||||
|
||||
Regression: reasoning system prompts end the prompt with a closed
|
||||
`[THINK]...[/THINK]` block. With a parser-injected grammar the generation's
|
||||
reasoning must still be captured as ``reasoning_content`` rather than
|
||||
leaking into ``content`` (the prompt-based reasoning-end heuristic must be
|
||||
skipped). This exercises ``DelegatingParser.parse_delta``'s prompt check,
|
||||
which the isolated reasoning-parser tests do not cover.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="mistral",
|
||||
reasoning_parser_name="mistral",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
parser = parser_cls(mistral_tokenizer, None)
|
||||
request = parser.adjust_request(
|
||||
ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="test",
|
||||
tools=_SAMPLE_TOOLS,
|
||||
tool_choice="auto",
|
||||
)
|
||||
)
|
||||
assert getattr(request, "_grammar_from_parser", False)
|
||||
|
||||
prompt_ids = _encode_v13(mistral_tokenizer, "[THINK]system reasoning[/THINK]")
|
||||
gen_ids = _encode_v13(
|
||||
mistral_tokenizer,
|
||||
"[THINK]because two plus two is four[/THINK]The answer is 4.",
|
||||
)
|
||||
|
||||
reasoning, content = _stream_parse_delta(
|
||||
parser, mistral_tokenizer, gen_ids, request, prompt_ids
|
||||
)
|
||||
|
||||
assert "because two plus two is four" in reasoning
|
||||
assert "because two plus two is four" not in content
|
||||
assert "The answer is 4." in content
|
||||
|
||||
|
||||
def _encode_gen(tokenizer: MistralTokenizer, text: str) -> list[int]:
|
||||
"""Encode `text` mapping `[THINK]`, `[/THINK]`, `[TOOL_CALLS]`, `[ARGS]`
|
||||
to their special-token IDs; all other text uses the base tokenizer.
|
||||
"""
|
||||
vocab = tokenizer.get_vocab()
|
||||
markers: dict[str, int] = {
|
||||
"[THINK]": tokenizer.instruct.BEGIN_THINK,
|
||||
"[/THINK]": tokenizer.instruct.END_THINK,
|
||||
"[TOOL_CALLS]": vocab["[TOOL_CALLS]"],
|
||||
"[ARGS]": vocab["[ARGS]"],
|
||||
}
|
||||
out: list[int] = []
|
||||
remaining = text
|
||||
while remaining:
|
||||
earliest: int | None = None
|
||||
earliest_marker: str | None = None
|
||||
for marker in markers:
|
||||
idx = remaining.find(marker)
|
||||
if idx != -1 and (earliest is None or idx < earliest):
|
||||
earliest = idx
|
||||
earliest_marker = marker
|
||||
if earliest_marker is None:
|
||||
out += tokenizer.tokenizer.encode(remaining, False, False)
|
||||
break
|
||||
assert earliest is not None # set together with earliest_marker
|
||||
if earliest > 0:
|
||||
out += tokenizer.tokenizer.encode(remaining[:earliest], False, False)
|
||||
out.append(markers[earliest_marker])
|
||||
remaining = remaining[earliest + len(earliest_marker) :]
|
||||
return out
|
||||
|
||||
|
||||
def _stream_turn(
|
||||
parser,
|
||||
tokenizer: MistralTokenizer,
|
||||
gen_ids: list[int],
|
||||
request,
|
||||
prompt_ids: list[int],
|
||||
) -> tuple[str, str, list[tuple[str, str]]]:
|
||||
"""Stream `gen_ids` through `parse_delta`, returning `(reasoning, content,
|
||||
tool_calls)`.
|
||||
|
||||
`tool_calls` is a list of `(name, arguments)` tuples ordered by
|
||||
`DeltaToolCall.index`. Arguments are concatenated across deltas.
|
||||
"""
|
||||
reasoning = ""
|
||||
content = ""
|
||||
tool_call_acc: dict[int, dict[str, str]] = {}
|
||||
previous_tokens: list[str] | None = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, tok in enumerate(gen_ids):
|
||||
new_tokens, delta_text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
tokenizer=tokenizer,
|
||||
all_input_ids=gen_ids[: i + 1],
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=True,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
delta_message = parser.parse_delta(
|
||||
delta_text=delta_text,
|
||||
delta_token_ids=[tok],
|
||||
request=request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=(i == len(gen_ids) - 1),
|
||||
)
|
||||
if delta_message is not None:
|
||||
reasoning += delta_message.reasoning or ""
|
||||
content += delta_message.content or ""
|
||||
for tc in delta_message.tool_calls:
|
||||
if tc.index not in tool_call_acc:
|
||||
tool_call_acc[tc.index] = {"name": "", "args": ""}
|
||||
if tc.function:
|
||||
if tc.function.name:
|
||||
tool_call_acc[tc.index]["name"] = tc.function.name
|
||||
if tc.function.arguments:
|
||||
tool_call_acc[tc.index]["args"] += tc.function.arguments
|
||||
tool_calls = [
|
||||
(tool_call_acc[k]["name"], tool_call_acc[k]["args"])
|
||||
for k in sorted(tool_call_acc)
|
||||
]
|
||||
return reasoning, content, tool_calls
|
||||
|
||||
|
||||
def test_parse_delta_multi_turn_reason_and_tool(
|
||||
mistral_tokenizer: MistralTokenizer,
|
||||
) -> None:
|
||||
"""Two-turn loop: each assistant turn carries reasoning + a tool call.
|
||||
|
||||
For each turn, the mistral-common conversation is encoded into `prompt_ids`
|
||||
before streaming the next generation. Turn 2's prompt therefore contains
|
||||
a `[THINK]`/`[/THINK]` block and `[TOOL_CALLS]` from turn 1 — the
|
||||
accumulated tool-call/result history that single-turn tests never exercise.
|
||||
|
||||
Asserts that across both turns the parser correctly classifies think tokens
|
||||
as `reasoning_content` (never leaking into `content`), and that the tool
|
||||
call name/arguments are reconstructed correctly from the streaming deltas.
|
||||
"""
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="mistral",
|
||||
reasoning_parser_name="mistral",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
request = parser_cls(mistral_tokenizer, None).adjust_request(
|
||||
ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="test",
|
||||
tools=_SAMPLE_TOOLS,
|
||||
tool_choice="auto",
|
||||
)
|
||||
)
|
||||
assert getattr(request, "_grammar_from_parser", False)
|
||||
|
||||
cities = ["Dallas", "Paris"]
|
||||
conversation: list = [UserMessage(content="What is the weather in Dallas?")]
|
||||
|
||||
for i, city in enumerate(cities):
|
||||
# Fresh parser per turn: streaming state must not bleed across turns.
|
||||
parser = parser_cls(mistral_tokenizer, None)
|
||||
|
||||
prompt_ids = mistral_tokenizer.instruct.encode_instruct(
|
||||
InstructRequest(messages=conversation)
|
||||
).tokens
|
||||
|
||||
gen_str = (
|
||||
f"[THINK]I should check {city} weather.[/THINK]"
|
||||
f'[TOOL_CALLS]get_weather[ARGS]{{"city": "{city}"}}'
|
||||
)
|
||||
gen_ids = _encode_gen(mistral_tokenizer, gen_str)
|
||||
|
||||
reasoning, content, tool_calls = _stream_turn(
|
||||
parser, mistral_tokenizer, gen_ids, request, prompt_ids
|
||||
)
|
||||
|
||||
assert f"check {city} weather" in reasoning, f"turn {i}: reasoning not captured"
|
||||
assert f"check {city} weather" not in content, (
|
||||
f"turn {i}: reasoning leaked into content"
|
||||
)
|
||||
assert "[THINK]" not in content, f"turn {i}: [THINK] leaked"
|
||||
assert "[TOOL_CALLS]" not in content, f"turn {i}: [TOOL_CALLS] leaked"
|
||||
assert len(tool_calls) == 1, f"turn {i}: expected 1 tool call"
|
||||
name, args = tool_calls[0]
|
||||
assert name == "get_weather", f"turn {i}: wrong tool name {name!r}"
|
||||
assert city in args, f"turn {i}: city not in args {args!r}"
|
||||
|
||||
# Extend conversation with this turn's history for the next turn.
|
||||
tc_id = f"tc{i:07d}" # 9-char alphanumeric id: "tc0000000", "tc0000001"
|
||||
conversation.extend(
|
||||
[
|
||||
AssistantMessage(
|
||||
content=[
|
||||
ThinkChunk(
|
||||
thinking=f"I should check {city} weather.",
|
||||
closed=True,
|
||||
)
|
||||
],
|
||||
tool_calls=[
|
||||
ToolCall(
|
||||
id=tc_id,
|
||||
function=FunctionCall(
|
||||
name="get_weather",
|
||||
arguments=json.dumps({"city": city}),
|
||||
),
|
||||
)
|
||||
],
|
||||
),
|
||||
ToolMessage(
|
||||
content=f"{city}: 70F sunny",
|
||||
tool_call_id=tc_id,
|
||||
),
|
||||
]
|
||||
)
|
||||
if i < len(cities) - 1:
|
||||
conversation.append(UserMessage(content=f"And in {cities[i + 1]}?"))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,747 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import FunctionTool, ToolChoiceFunction
|
||||
from xgrammar import Grammar
|
||||
from xgrammar.testing import _is_grammar_accept_string
|
||||
|
||||
from vllm.config.reasoning import ReasoningConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import (
|
||||
ResponsesRequest,
|
||||
ResponsesResponse,
|
||||
)
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.parser.beam import BeamParser, BeamReasoningParser, BeamToolParser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.response_template import ResponseTemplateParser
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
# The `response_template` of the Beam checkpoints.
|
||||
BEAM_RESPONSE_TEMPLATE: dict[str, Any] = {
|
||||
"defaults": {"role": "assistant"},
|
||||
"start_anchor_pattern": r"<\|start\|>assistant",
|
||||
"fields": {
|
||||
"thinking": {
|
||||
"open": "<|channel|>analysis<|message|>",
|
||||
"close": "<|end|><|start|>assistant",
|
||||
"content": "text",
|
||||
},
|
||||
"content": {
|
||||
"open": "<|channel|>final<|message|>",
|
||||
"close": ["<|return|>", "<|end|>", "<|call|>"],
|
||||
"content": "text",
|
||||
},
|
||||
"tool_calls": {
|
||||
"open_pattern": (
|
||||
r" to=(?P<name>[^\s<]+)"
|
||||
r"<\|channel\|>analysis <\|constrain\|>xml<\|message\|>"
|
||||
),
|
||||
"close": ["<|end|><|start|>assistant", "<|call|>"],
|
||||
"repeats": True,
|
||||
"content": "xml-inline",
|
||||
"content_args": {
|
||||
"tag_pattern": (
|
||||
r'<rfl:parameter name="(?P<key>[^"]+)">'
|
||||
r'\n(?P<value>.*?)\n(?:</rfl:parameter>|(?=<rfl:parameter name="))'
|
||||
),
|
||||
"value_parser": {"name": "text", "args": {"strip": False}},
|
||||
},
|
||||
"transform": {
|
||||
"type": "function",
|
||||
"function": {"name": "{name}", "arguments": "{content}"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
VOCAB = {
|
||||
"<|start|>": 200,
|
||||
"<|channel|>": 201,
|
||||
"<|message|>": 202,
|
||||
"<|end|>": 203,
|
||||
"<|call|>": 204,
|
||||
"<|return|>": 205,
|
||||
"<|constrain|>": 206,
|
||||
}
|
||||
TEXT_TOKENS = {100: "assistant", 101: "analysis", 102: "final", 103: "plan"}
|
||||
PROMPT_TOKEN_IDS = [200, 100]
|
||||
THOUGHT = "<|channel|>analysis<|message|>plan<|end|><|start|>assistant"
|
||||
FINAL = "<|channel|>final<|message|>"
|
||||
NEXT_MESSAGE = "<|end|><|start|>assistant"
|
||||
|
||||
|
||||
def _tool(name, properties, *, strict=None):
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"parameters": {"type": "object", "properties": properties},
|
||||
"strict": strict,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}),
|
||||
]
|
||||
STRICT_TOOLS = [
|
||||
_tool("get_weather", {"city": {"type": "string"}}),
|
||||
_tool("set_count", {"count": {"type": "integer"}}, strict=True),
|
||||
]
|
||||
|
||||
|
||||
class BeamTokenizer:
|
||||
def __init__(self, response_template=BEAM_RESPONSE_TEMPLATE):
|
||||
self.response_template = response_template
|
||||
self.init_kwargs = {}
|
||||
self.name_or_path = ""
|
||||
self.all_special_tokens = list(VOCAB)
|
||||
self.all_special_ids = list(VOCAB.values())
|
||||
|
||||
def decode(self, token_ids, skip_special_tokens=False, **_kwargs):
|
||||
inverse_vocab = {token_id: token for token, token_id in VOCAB.items()}
|
||||
return "".join(
|
||||
TEXT_TOKENS.get(token_id, inverse_vocab.get(token_id, ""))
|
||||
for token_id in token_ids
|
||||
)
|
||||
|
||||
def get_vocab(self):
|
||||
return VOCAB
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
del add_special_tokens
|
||||
return list(text.encode())
|
||||
|
||||
|
||||
def request(**overrides):
|
||||
values = dict(
|
||||
tool_choice="auto",
|
||||
include_reasoning=True,
|
||||
tools=TOOLS,
|
||||
parallel_tool_calls=True,
|
||||
)
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def parameter(name, value):
|
||||
return f'<rfl:parameter name="{name}">\n{value}\n</rfl:parameter>'
|
||||
|
||||
|
||||
def call(name, body, *, terminator="<|call|>"):
|
||||
return (
|
||||
f" to={name}<|channel|>analysis <|constrain|>xml<|message|>{body}{terminator}"
|
||||
)
|
||||
|
||||
|
||||
def make_parser(prompt_token_ids=PROMPT_TOKEN_IDS, tools=TOOLS, **kwargs):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(BeamTokenizer(), tools, **kwargs)
|
||||
parser.set_prompt_token_ids(prompt_token_ids)
|
||||
return parser
|
||||
|
||||
|
||||
def parse(text, prompt_token_ids=PROMPT_TOKEN_IDS):
|
||||
reasoning, content, calls = make_parser(prompt_token_ids).parse(
|
||||
text, request(), enable_auto_tools=True
|
||||
)
|
||||
return reasoning, content, normalize_calls(calls)
|
||||
|
||||
|
||||
def normalize_calls(calls):
|
||||
return [(call.name, json.loads(call.arguments)) for call in calls or []]
|
||||
|
||||
|
||||
def test_beam_parsers_are_registered():
|
||||
assert ToolParserManager.get_tool_parser("beam") is BeamToolParser
|
||||
assert ReasoningParserManager.get_reasoning_parser("beam") is BeamReasoningParser
|
||||
|
||||
|
||||
def test_parser_manager_selects_beam_parser():
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
assert parser_cls is not None
|
||||
assert issubclass(parser_cls, BeamParser)
|
||||
assert parser_cls.reasoning_parser_cls is BeamReasoningParser
|
||||
assert parser_cls.tool_parser_cls is BeamToolParser
|
||||
assert parser_cls._enable_auto_tools
|
||||
|
||||
|
||||
@pytest.mark.parametrize("other", ["hermes", "hf"])
|
||||
def test_beam_parser_rejects_other_parsers(other):
|
||||
with pytest.raises(TypeError, match="cannot be combined"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name=other,
|
||||
reasoning_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(),
|
||||
)
|
||||
|
||||
|
||||
def test_beam_parser_requires_checkpoint_metadata_at_startup():
|
||||
with pytest.raises(
|
||||
TypeError, match="--reasoning-parser beam requires `response_template`"
|
||||
):
|
||||
ParserManager.get_parser(
|
||||
reasoning_parser_name="beam",
|
||||
tokenizer=BeamTokenizer(response_template=None),
|
||||
)
|
||||
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
del template["fields"]["tool_calls"]
|
||||
with pytest.raises(TypeError, match="--tool-call-parser beam requires"):
|
||||
ParserManager.get_parser(
|
||||
tool_parser_name="beam",
|
||||
enable_auto_tools=True,
|
||||
tokenizer=BeamTokenizer(template),
|
||||
)
|
||||
|
||||
|
||||
def test_reasoning_and_final_answer():
|
||||
assert parse(THOUGHT + FINAL + "Sunny.<|return|>") == ("plan", "Sunny.", [])
|
||||
|
||||
|
||||
def test_prefilled_reasoning_continues_from_prompt():
|
||||
prompt = PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
text = " more.<|end|><|start|>assistant" + FINAL + "Done.<|return|>"
|
||||
|
||||
assert parse(text, prompt) == (" more.", "Done.", [])
|
||||
|
||||
|
||||
def test_parallel_calls_are_coerced_to_their_schemas():
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
|
||||
assert parse(text) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"}), ("set_count", {"count": 7})],
|
||||
)
|
||||
|
||||
|
||||
def test_call_stop_token_removed_from_the_text_still_ends_the_call():
|
||||
text = THOUGHT + call("get_weather", parameter("city", "Paris"), terminator="")
|
||||
|
||||
assert parse(text) == ("plan", None, [("get_weather", {"city": "Paris"})])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("end", ["<|end|>", "<|end|><|start|>"])
|
||||
def test_output_cut_before_the_next_message_start_ends_its_region(end):
|
||||
"""Output that stops after `<|end|>`, such as at `max_tokens`, has ended
|
||||
its reasoning or call and lacks only the next message's start."""
|
||||
weather = call("get_weather", parameter("city", "Paris"), terminator=end)
|
||||
|
||||
assert parse("<|channel|>analysis<|message|>plan" + end) == ("plan", None, [])
|
||||
assert parse(THOUGHT + weather) == (
|
||||
"plan",
|
||||
None,
|
||||
[("get_weather", {"city": "Paris"})],
|
||||
)
|
||||
|
||||
|
||||
THOUGHT_IDS = [201, 101, 202, 103, 203, 200, 100]
|
||||
FINAL_IDS = [201, 102, 202]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("prompt_token_ids", "token_ids", "count"),
|
||||
[
|
||||
(PROMPT_TOKEN_IDS, [*THOUGHT_IDS, *FINAL_IDS, 205], 1),
|
||||
(PROMPT_TOKEN_IDS + [201, 101, 202], [103, 103, 203, 200, 100, *FINAL_IDS], 2),
|
||||
(PROMPT_TOKEN_IDS, [201, 101, 202, 103, 103, 103], 3),
|
||||
],
|
||||
ids=["reasoning_then_answer", "prefilled_reasoning", "cut_off"],
|
||||
)
|
||||
def test_reasoning_tokens_are_counted(prompt_token_ids, token_ids, count):
|
||||
parser = make_parser(prompt_token_ids)
|
||||
|
||||
assert parser.count_reasoning_tokens(token_ids) == count
|
||||
|
||||
|
||||
def test_reasoning_tokens_are_counted_as_the_generation_grows():
|
||||
token_ids = [*THOUGHT_IDS, *FINAL_IDS, 205]
|
||||
parser = make_parser()
|
||||
|
||||
assert [
|
||||
parser.count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
] == [
|
||||
make_parser().count_reasoning_tokens(token_ids[:end])
|
||||
for end in range(1, len(token_ids) + 1)
|
||||
]
|
||||
assert parser.count_reasoning_tokens(FINAL_IDS) == 0
|
||||
|
||||
|
||||
def test_call_stop_token_ends_the_answer():
|
||||
assert parse(FINAL + "done<|call|>") == (None, "done", [])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 7, 10_000])
|
||||
def test_streaming_matches_non_streaming(chunk_size):
|
||||
text = (
|
||||
THOUGHT
|
||||
+ call(
|
||||
"get_weather",
|
||||
parameter("city", "Paris"),
|
||||
terminator="<|end|><|start|>assistant",
|
||||
)
|
||||
+ call("set_count", parameter("count", "7"))
|
||||
)
|
||||
parser = make_parser()
|
||||
reasoning, calls = [], []
|
||||
for start in range(0, len(text), chunk_size):
|
||||
end = min(start + chunk_size, len(text))
|
||||
delta = parser.parse_delta(
|
||||
text[start:end], [], request(), finished=end == len(text)
|
||||
)
|
||||
if delta is not None:
|
||||
reasoning.append(delta.reasoning or "")
|
||||
calls.extend(delta.tool_calls)
|
||||
|
||||
assert "".join(reasoning) == "plan"
|
||||
assert [
|
||||
(call.function.name, json.loads(call.function.arguments)) for call in calls
|
||||
] == parse(text)[2]
|
||||
|
||||
|
||||
WEATHER_CALL = call("get_weather", parameter("city", "Paris"))
|
||||
WEATHER = [("get_weather", {"city": "Paris"})]
|
||||
|
||||
|
||||
def chat_request(tools=STRICT_TOOLS, **body):
|
||||
return ChatCompletionRequest(
|
||||
model="beam",
|
||||
messages=[{"role": "user", "content": "Weather?"}],
|
||||
tools=tools and [tool.model_dump() for tool in tools],
|
||||
**body,
|
||||
)
|
||||
|
||||
|
||||
def named(name):
|
||||
return {"type": "function", "function": {"name": name}}
|
||||
|
||||
|
||||
def adjust(req, prompt_token_ids=PROMPT_TOKEN_IDS, **kwargs):
|
||||
make_parser(prompt_token_ids, req.tools, **kwargs).adjust_request(req)
|
||||
return req
|
||||
|
||||
|
||||
def accepts(req, text):
|
||||
grammar = Grammar.from_structural_tag(
|
||||
req.extract_structured_outputs().structural_tag
|
||||
)
|
||||
return _is_grammar_accept_string(grammar, text)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_choice", "text", "calls"),
|
||||
[
|
||||
("required", THOUGHT + WEATHER_CALL, WEATHER),
|
||||
("required", WEATHER_CALL, None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ call("set_count", parameter("count", "7")),
|
||||
[*WEATHER, ("set_count", {"count": 7})],
|
||||
),
|
||||
("required", THOUGHT + call("set_count", parameter("count", "x")), None),
|
||||
(
|
||||
"required",
|
||||
THOUGHT + call("get_weather", parameter("city", "a<|call|>b")),
|
||||
None,
|
||||
),
|
||||
("required", THOUGHT + FINAL + "Paris<|return|>", None),
|
||||
(named("set_count"), THOUGHT + WEATHER_CALL, None),
|
||||
(named("get_weather"), THOUGHT + WEATHER_CALL, WEATHER),
|
||||
(
|
||||
named("get_weather"),
|
||||
THOUGHT
|
||||
+ call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
+ WEATHER_CALL,
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_forced_tool_calls_follow_the_template_grammar(tool_choice, text, calls):
|
||||
"""From the message start, the grammar opens with the reasoning, and
|
||||
accepts only calls the parser reads back."""
|
||||
req = adjust(chat_request(tool_choice=tool_choice))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is (calls is not None)
|
||||
if calls is not None:
|
||||
assert parse(text)[2] == calls
|
||||
|
||||
|
||||
def test_reasoning_may_be_skipped_when_thinking_is_disabled():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"),
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_parallel_tool_calls_false_allows_a_single_call():
|
||||
req = adjust(chat_request(tool_choice="required", parallel_tool_calls=False))
|
||||
calls = call("get_weather", parameter("city", "Paris"), terminator=NEXT_MESSAGE)
|
||||
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, THOUGHT + calls + WEATHER_CALL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strict", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("separator", "accepted"), [("", True), ("\n", True), ("\n\n", False), (" ", False)]
|
||||
)
|
||||
def test_parameters_are_separated_by_at_most_a_newline(strict, separator, accepted):
|
||||
"""A model that skips a parameter's opener must not write whitespace until
|
||||
`max_tokens`."""
|
||||
write = _tool(
|
||||
"write",
|
||||
{"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
strict=strict,
|
||||
)
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = parameter("path", "a.py") + separator + parameter("content", "x")
|
||||
|
||||
assert accepts(req, THOUGHT + call("write", body)) is accepted
|
||||
assert accepts(req, THOUGHT + call("write", separator + body)) is accepted
|
||||
|
||||
|
||||
def test_forced_calls_in_prefilled_reasoning_start_after_it():
|
||||
req = adjust(
|
||||
chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 101, 202, 103]
|
||||
)
|
||||
|
||||
assert not req._grammar_from_parser
|
||||
assert accepts(req, WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_calls_cannot_follow_a_prefilled_answer():
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request(tool_choice="required"), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
|
||||
def test_prefilled_answer_can_only_be_finished():
|
||||
req = adjust(chat_request(TOOLS), PROMPT_TOKEN_IDS + [201, 102, 202])
|
||||
|
||||
assert accepts(req, "Sunny.<|return|>")
|
||||
assert not accepts(req, "Sunny." + NEXT_MESSAGE + WEATHER_CALL)
|
||||
|
||||
|
||||
def test_forced_call_with_unencodable_tool_name_is_rejected():
|
||||
tools = [_tool("get weather", {"city": {"type": "string"}})]
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="tool name"):
|
||||
adjust(chat_request(tools, tool_choice="required"))
|
||||
|
||||
|
||||
def test_string_formats_and_patterns_are_held_in_raw_text():
|
||||
"""Like XGrammar, a strict string keeps its format or pattern, unless a
|
||||
match could end the value early."""
|
||||
contact = _tool(
|
||||
"set_contact",
|
||||
{
|
||||
"email": {"type": "string", "format": "email"},
|
||||
"code": {"type": "string", "pattern": "^[A-Z]{3}$"},
|
||||
"n": {"type": "integer"},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
contact.function.parameters["required"] = ["email", "code", "n"]
|
||||
req = adjust(chat_request([contact], tool_choice="required"))
|
||||
|
||||
def contact_call(email, code, *rest):
|
||||
body = parameter("email", email) + parameter("code", code) + "".join(rest)
|
||||
return THOUGHT + call("set_contact", body)
|
||||
|
||||
text = contact_call("a.b@x.co", "ABC", parameter("n", "1"))
|
||||
calls = make_parser(tools=[contact]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert accepts(req, text)
|
||||
assert normalize_calls(calls) == [
|
||||
("set_contact", {"email": "a.b@x.co", "code": "ABC", "n": 1})
|
||||
]
|
||||
assert not accepts(req, contact_call("ab", "ABC", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABCD", parameter("n", "1")))
|
||||
assert not accepts(req, contact_call("a.b@x.co", "ABC"))
|
||||
# Line breaks could end the value.
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
adjust(chat_request([dotted], tool_choice="required"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + WEATHER_CALL, True),
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(THOUGHT + FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
(THOUGHT + call("set_count", parameter("count", "x")), False),
|
||||
],
|
||||
)
|
||||
def test_strict_tools_constrain_automatic_tool_choice(text, accepted):
|
||||
assert accepts(adjust(chat_request()), text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "accepted"),
|
||||
[
|
||||
(THOUGHT + FINAL + "Sunny.<|return|>", True),
|
||||
(FINAL + "Sunny.<|return|>", False),
|
||||
(WEATHER_CALL, False),
|
||||
(THOUGHT + WEATHER_CALL, False),
|
||||
(FINAL + "Sunny." + NEXT_MESSAGE + WEATHER_CALL, False),
|
||||
],
|
||||
)
|
||||
def test_tool_choice_none_allows_only_an_answer(text, accepted):
|
||||
req = adjust(chat_request(TOOLS, tool_choice="none"))
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert accepts(req, text) is accepted
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("tools", "tool_choice"), [(TOOLS, "auto"), (None, None)])
|
||||
def test_always_constrained_output_follows_the_template_framing(tools, tool_choice):
|
||||
req = adjust(chat_request(tools, tool_choice=tool_choice))
|
||||
count_call = THOUGHT + call("set_count", parameter("count", "3"))
|
||||
|
||||
assert accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT.replace("plan", "") + FINAL + "Sunny.<|return|>")
|
||||
assert not accepts(req, THOUGHT + "Sunny.")
|
||||
assert accepts(req, count_call) is (tool_choice == "auto")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", [None, {"type": "json_object"}])
|
||||
@pytest.mark.parametrize("strict", [None, True])
|
||||
@pytest.mark.parametrize("tool_choice", ["auto", "required"])
|
||||
def test_automatic_choice_lets_calls_name_undeclared_tools(
|
||||
strict, tool_choice, response_format
|
||||
):
|
||||
"""Unless a forced choice limits calls to the declared tools. The parser
|
||||
passes such calls on, so the harness can tell the model the tool does not
|
||||
exist."""
|
||||
bash = _tool("bash", {"command": {"type": "string"}}, strict=strict)
|
||||
req = adjust(
|
||||
chat_request([bash], tool_choice=tool_choice, response_format=response_format)
|
||||
)
|
||||
undeclared = THOUGHT + call("read_file", parameter("path", "a.py"))
|
||||
parser = make_parser(tools=[bash])
|
||||
|
||||
assert accepts(req, THOUGHT + call("bash", parameter("command", "ls")))
|
||||
assert accepts(req, undeclared) is (tool_choice == "auto")
|
||||
calls = parser.parse(undeclared, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [("read_file", {"path": "a.py"})]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("name", "body", "accepted"),
|
||||
[
|
||||
("set_count", parameter("count", "1"), True),
|
||||
("set_count", parameter("count", "x"), False),
|
||||
("get_weather", parameter("town", "Paris"), False),
|
||||
# The grammar cannot enforce `minProperties`.
|
||||
("find", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("search_files", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_coun", parameter("x", "1") + parameter("x", "y"), True),
|
||||
("set_counts", parameter("x", "1") + parameter("x", "y"), True),
|
||||
],
|
||||
)
|
||||
def test_automatic_calls_enforce_declared_arguments_beside_any_other_name(
|
||||
name, body, accepted
|
||||
):
|
||||
find = _tool("find", {"q": {"type": "string"}})
|
||||
find.function.parameters["minProperties"] = 1
|
||||
req = adjust(chat_request([*TOOLS, find]))
|
||||
|
||||
assert accepts(req, THOUGHT + call(name, body)) is accepted
|
||||
|
||||
|
||||
def test_value_ends_before_an_opener_on_the_next_line():
|
||||
write = _tool("write", {"path": {"type": "string"}, "content": {"type": "string"}})
|
||||
req = adjust(chat_request([write], tool_choice="required"))
|
||||
body = '<rfl:parameter name="content">\nprint(1)\n\n' + parameter("path", "a.py")
|
||||
text = THOUGHT + call("write", body)
|
||||
|
||||
assert accepts(req, text)
|
||||
calls = make_parser(tools=[write]).parse(text, req, enable_auto_tools=True)[2]
|
||||
assert normalize_calls(calls) == [
|
||||
("write", {"content": "print(1)\n", "path": "a.py"})
|
||||
]
|
||||
|
||||
|
||||
def test_always_constrained_output_can_be_disabled(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_BEAM_ALWAYS_CONSTRAIN_OUTPUT", "0")
|
||||
|
||||
assert adjust(chat_request(TOOLS)).structured_outputs is None
|
||||
|
||||
|
||||
def test_strict_tools_the_template_cannot_hold_leave_automatic_choice_unconstrained():
|
||||
dotted = _tool(
|
||||
"set_code", {"code": {"type": "string", "pattern": "^a.b$"}}, strict=True
|
||||
)
|
||||
|
||||
assert adjust(chat_request([dotted])).structured_outputs is None
|
||||
|
||||
|
||||
def test_generic_parser_does_not_derive_the_template_grammar():
|
||||
req = chat_request(TOOLS, tool_choice="required")
|
||||
parser = ResponseTemplateParser(BeamTokenizer(), req.tools, enable_auto_tools=True)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="required tool choice"):
|
||||
parser.adjust_request(req)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tools", [None, TOOLS])
|
||||
def test_structured_output_is_framed_as_the_answer(tools):
|
||||
schema = {"type": "object", "properties": {"a": {"type": "integer"}}}
|
||||
req = adjust(
|
||||
chat_request(
|
||||
tools,
|
||||
response_format={
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": "answer", "schema": schema},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
assert req.response_format is None
|
||||
assert accepts(req, THOUGHT + FINAL + '{"a": 1}<|return|>')
|
||||
assert not accepts(req, THOUGHT + FINAL + "Sunny.<|return|>")
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL) is (tools is not None)
|
||||
|
||||
|
||||
def test_responses_named_tool_choice_follows_the_template_grammar():
|
||||
req = adjust(
|
||||
ResponsesRequest(
|
||||
input="Weather?",
|
||||
tools=[
|
||||
FunctionTool(
|
||||
type="function",
|
||||
name="get_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
],
|
||||
tool_choice=ToolChoiceFunction(type="function", name="get_weather"),
|
||||
text={"format": {"type": "json_object"}},
|
||||
)
|
||||
)
|
||||
|
||||
assert req._grammar_from_parser
|
||||
assert req.text.format.type == "json_object"
|
||||
assert accepts(req, THOUGHT + WEATHER_CALL)
|
||||
assert not accepts(req, FINAL + "{}<|return|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["in_progress", "completed"])
|
||||
@pytest.mark.parametrize("inside_reasoning", [False, True])
|
||||
def test_responses_preserves_text_config_with_template_grammar(
|
||||
status, inside_reasoning
|
||||
):
|
||||
text = {
|
||||
"format": {
|
||||
"type": "json_schema",
|
||||
"name": "answer",
|
||||
"strict": True,
|
||||
"schema": {
|
||||
"type": "object",
|
||||
"properties": {"a": {"type": "integer"}},
|
||||
"required": ["a"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
"verbosity": "low",
|
||||
}
|
||||
req = ResponsesRequest(input="Answer", text=text)
|
||||
expected_text = req.text.model_dump(by_alias=True)
|
||||
prompt = PROMPT_TOKEN_IDS + ([201, 101, 202] if inside_reasoning else [])
|
||||
adjust(req, prompt_token_ids=prompt)
|
||||
params = req.to_sampling_params(default_max_tokens=128)
|
||||
grammar = Grammar.from_structural_tag(params.structured_outputs.structural_tag)
|
||||
prefix = "" if inside_reasoning else THOUGHT
|
||||
assert _is_grammar_accept_string(grammar, prefix + FINAL + '{"a": 1}<|return|>')
|
||||
assert not _is_grammar_accept_string(grammar, prefix + FINAL + "Sunny.<|return|>")
|
||||
response = ResponsesResponse.from_request(req, params, "beam", 0, [], status)
|
||||
assert response.model_dump(by_alias=True)["text"] == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("reverse_alternatives", [False, True])
|
||||
def test_reasoning_budget_initializes_from_checkpoint_pattern(
|
||||
monkeypatch, reverse_alternatives
|
||||
):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
alternatives = [
|
||||
r"^<\|channel\|>analysis<\|message\|>",
|
||||
r"<\|start\|>assistant<\|channel\|>analysis<\|message\|>",
|
||||
]
|
||||
if reverse_alternatives:
|
||||
alternatives.reverse()
|
||||
template["fields"]["thinking"] = {
|
||||
"open_pattern": "(?:" + "|".join(alternatives) + ")",
|
||||
"close": "<|end|>",
|
||||
"content": "text",
|
||||
}
|
||||
tokenizer = BeamTokenizer(template)
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.reasoning.cached_tokenizer_from_config", lambda **_: tokenizer
|
||||
)
|
||||
config = ReasoningConfig(reasoning_parser="beam")
|
||||
config.initialize_token_ids(SimpleNamespace())
|
||||
|
||||
assert config.enabled
|
||||
# The marker must work both after the prompt's assistant header and in
|
||||
# subsequent complete assistant messages, regardless of regex branch order.
|
||||
assert config.reasoning_start_token_ids == tokenizer.encode(
|
||||
"<|channel|>analysis<|message|>"
|
||||
)
|
||||
assert config.reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
assert config.natural_reasoning_end_token_ids == tokenizer.encode("<|end|>")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pattern", [r"analysis.*", r"(?:analysis|thinking)"])
|
||||
def test_reasoning_budget_does_not_guess_a_marker_for_other_patterns(pattern):
|
||||
template = deepcopy(BEAM_RESPONSE_TEMPLATE)
|
||||
thinking = template["fields"]["thinking"]
|
||||
del thinking["open"]
|
||||
thinking["open_pattern"] = pattern
|
||||
|
||||
assert BeamReasoningParser(BeamTokenizer(template)).reasoning_start_str is None
|
||||
@@ -1,440 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for include_reasoning suppression in the unified Parser interface.
|
||||
|
||||
Covers non-streaming (parser.parse() + build_response_output_items),
|
||||
streaming (parse_delta), and ParsableContext.append_output() paths.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
|
||||
|
||||
class ThinkReasoningParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<think>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "</think>"
|
||||
|
||||
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT = (
|
||||
"<think>let me think about this</think>The answer is 42."
|
||||
)
|
||||
|
||||
MODEL_OUTPUT_REASONING_AND_TOOL = (
|
||||
"<think>I need to call a tool</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
MODEL_OUTPUT_CONTENT_ONLY = "The answer is 42."
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer("Qwen/Qwen3-32B")
|
||||
|
||||
|
||||
def make_responses_request(**kwargs) -> ResponsesRequest:
|
||||
defaults = dict(model="test-model", input="test input")
|
||||
defaults.update(kwargs)
|
||||
return ResponsesRequest(**defaults)
|
||||
|
||||
|
||||
def make_chat_request(**kwargs) -> ChatCompletionRequest:
|
||||
defaults = dict(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
defaults.update(kwargs)
|
||||
return ChatCompletionRequest(**defaults)
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False):
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
|
||||
return TestParser(tokenizer)
|
||||
|
||||
|
||||
# ── Non-streaming: parser.parse() + build_response_output_items ──────
|
||||
|
||||
|
||||
def parse_and_build(parser, request, model_output, enable_auto_tools=False):
|
||||
"""Mirrors the non-streaming path in _make_response_output_items /
|
||||
ParsableContext.append_output(): parse → suppress reasoning → build items.
|
||||
"""
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_response_output_items,
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
model_output, request, enable_auto_tools=enable_auto_tools
|
||||
)
|
||||
if not request.include_reasoning:
|
||||
reasoning = None
|
||||
return build_response_output_items(
|
||||
reasoning=reasoning,
|
||||
content=content,
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
|
||||
|
||||
class TestNonStreamingIncludeReasoning:
|
||||
def test_include_reasoning_true_has_reasoning_item(self, tokenizer):
|
||||
"""Default: reasoning items appear in output."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=True)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
types = [o.type for o in outputs]
|
||||
assert "reasoning" in types
|
||||
assert "message" in types
|
||||
|
||||
def test_include_reasoning_false_no_reasoning_item(self, tokenizer):
|
||||
"""Reasoning item is suppressed when include_reasoning=False."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
types = [o.type for o in outputs]
|
||||
assert "reasoning" not in types
|
||||
assert "message" in types
|
||||
assert outputs[0].content[0].text == "The answer is 42."
|
||||
|
||||
def test_include_reasoning_false_content_preserved(self, tokenizer):
|
||||
"""Content is extracted correctly even when reasoning is suppressed."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
message = next(o for o in outputs if o.type == "message")
|
||||
assert message.content[0].text == "The answer is 42."
|
||||
|
||||
def test_include_reasoning_false_tool_calls_preserved(self, tokenizer):
|
||||
"""Tool calls still work when reasoning is suppressed."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = make_responses_request(
|
||||
include_reasoning=False,
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
outputs = parse_and_build(
|
||||
parser,
|
||||
request,
|
||||
MODEL_OUTPUT_REASONING_AND_TOOL,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
types = [o.type for o in outputs]
|
||||
assert "reasoning" not in types
|
||||
assert "function_call" in types
|
||||
fc = next(o for o in outputs if o.type == "function_call")
|
||||
assert fc.name == "get_weather"
|
||||
assert json.loads(fc.arguments) == {"city": "Dallas"}
|
||||
|
||||
def test_no_reasoning_parser_include_false_is_noop(self, tokenizer):
|
||||
"""include_reasoning=False is harmless when no reasoning parser."""
|
||||
parser = make_parser(tokenizer, reasoning=False)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_CONTENT_ONLY)
|
||||
|
||||
assert len(outputs) == 1
|
||||
assert outputs[0].type == "message"
|
||||
assert outputs[0].content[0].text == MODEL_OUTPUT_CONTENT_ONLY
|
||||
|
||||
def test_default_include_reasoning_is_true(self, tokenizer):
|
||||
"""ResponsesRequest defaults to include_reasoning=True."""
|
||||
request = make_responses_request()
|
||||
assert request.include_reasoning is True
|
||||
|
||||
def test_include_reasoning_false_suppresses_all_reasoning(self, tokenizer):
|
||||
"""Reasoning is suppressed regardless of request type."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
outputs = parse_and_build(parser, request, MODEL_OUTPUT_REASONING_AND_CONTENT)
|
||||
|
||||
assert all(o.type != "reasoning" for o in outputs)
|
||||
|
||||
|
||||
# ── Streaming: parse_delta ───────────────────────────────────────────
|
||||
|
||||
|
||||
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
results: list[DeltaMessage | None] = []
|
||||
for i, tid in enumerate(token_ids):
|
||||
delta_text = tokenizer.decode([tid])
|
||||
is_last = i == len(token_ids) - 1
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
[tid],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=is_last,
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def collect_fields(results):
|
||||
all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
|
||||
all_content = "".join(r.content for r in results if r and r.content)
|
||||
all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
|
||||
return all_reasoning, all_content, all_tool_calls
|
||||
|
||||
|
||||
class TestParseDeltaIncludeReasoning:
|
||||
def test_streaming_include_true_emits_reasoning(self, tokenizer):
|
||||
"""With include_reasoning=True, reasoning deltas are emitted."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=True)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_include_false_suppresses_reasoning(self, tokenizer):
|
||||
"""With include_reasoning=False, no reasoning deltas are emitted."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_include_false_content_still_works(self, tokenizer):
|
||||
"""Content is correctly extracted in streaming even with suppression."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
_, content, _ = collect_fields(results)
|
||||
|
||||
assert "The answer is 42" in content
|
||||
|
||||
def test_streaming_include_false_tool_calls_preserved(self, tokenizer):
|
||||
"""Tool calls stream correctly when reasoning is suppressed."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = make_responses_request(
|
||||
include_reasoning=False,
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_TOOL,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
def test_streaming_no_reasoning_parser_include_false(self, tokenizer):
|
||||
"""No crash when reasoning parser absent and include_reasoning=False."""
|
||||
parser = make_parser(tokenizer, reasoning=False)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_CONTENT_ONLY,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_chat_completion_include_false(self, tokenizer):
|
||||
"""parse_delta also respects ChatCompletionRequest.include_reasoning."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_chat_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
reasoning, content, _ = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "42" in content
|
||||
|
||||
def test_streaming_reasoning_only_deltas_become_none(self, tokenizer):
|
||||
"""Deltas that carry only reasoning become None (not empty)."""
|
||||
parser = make_parser(tokenizer, reasoning=True)
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
request,
|
||||
prompt_token_ids=[],
|
||||
)
|
||||
|
||||
for r in results:
|
||||
if r is not None:
|
||||
assert r.reasoning is None
|
||||
|
||||
|
||||
# ── ParsableContext.append_output() ───────────────────────────────────
|
||||
|
||||
|
||||
class TestParsableContextIncludeReasoning:
|
||||
def _make_context(self, tokenizer, request):
|
||||
from vllm.entrypoints.openai.responses.context import ParsableContext
|
||||
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser
|
||||
tool_parser_cls = None
|
||||
|
||||
return ParsableContext(
|
||||
tokenizer=tokenizer,
|
||||
parser_cls=TestParser,
|
||||
response_messages=[],
|
||||
request=request,
|
||||
available_tools=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
def test_process_include_false_suppresses_reasoning(self, tokenizer):
|
||||
"""ParsableContext.process() suppresses reasoning items."""
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
request = make_responses_request(include_reasoning=False)
|
||||
ctx = self._make_context(tokenizer, request)
|
||||
|
||||
output = RequestOutput(
|
||||
request_id="test",
|
||||
prompt=None,
|
||||
prompt_token_ids=[],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
token_ids=tokenizer.encode(
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
ctx.append_output(output)
|
||||
|
||||
types = [getattr(m, "type", None) for m in ctx.response_messages]
|
||||
assert "reasoning" not in types
|
||||
assert "message" in types
|
||||
|
||||
def test_process_include_true_has_reasoning(self, tokenizer):
|
||||
"""ParsableContext.process() includes reasoning by default."""
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
|
||||
request = make_responses_request(include_reasoning=True)
|
||||
ctx = self._make_context(tokenizer, request)
|
||||
|
||||
output = RequestOutput(
|
||||
request_id="test",
|
||||
prompt=None,
|
||||
prompt_token_ids=[],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text=MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
token_ids=tokenizer.encode(
|
||||
MODEL_OUTPUT_REASONING_AND_CONTENT,
|
||||
add_special_tokens=False,
|
||||
),
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
finish_reason="stop",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
ctx.append_output(output)
|
||||
|
||||
types = [getattr(m, "type", None) for m in ctx.response_messages]
|
||||
assert "reasoning" in types
|
||||
assert "message" in types
|
||||
@@ -1,598 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.parser.qwen3 import qwen3_config
|
||||
from vllm.parser.utils import count_history_tool_calls
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def enable_hermes_required_named_parsing(monkeypatch):
|
||||
# With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets
|
||||
# supports_required_and_named=False and uses structural-tag guided tool
|
||||
# calling. Force it True to test the non-guided JSON required/named parse path.
|
||||
monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True)
|
||||
|
||||
|
||||
class ThinkReasoningParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<think>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "</think>"
|
||||
|
||||
|
||||
MODEL_OUTPUT = (
|
||||
"<think>let me think about this</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
PLAIN_TEXT = "The weather in Dallas is sunny and 75°F."
|
||||
|
||||
TOOL_CALL_ONLY = (
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
TOOL_ARGUMENTS = '{"city": "Dallas"}'
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer("Qwen/Qwen3-32B")
|
||||
|
||||
|
||||
def make_request(**overrides):
|
||||
base = {
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
}
|
||||
base.update(overrides)
|
||||
return ChatCompletionRequest.model_validate(base)
|
||||
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
|
||||
hf_overrides=None,
|
||||
)
|
||||
|
||||
HISTORY_MESSAGES = [
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "functions.get_current_weather:0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "functions.get_forecast:1",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_forecast",
|
||||
"arguments": "{}",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "functions.get_current_weather:0",
|
||||
"content": "{}",
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "functions.get_forecast:1",
|
||||
"content": "{}",
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
|
||||
return TestParser(tokenizer, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reasoning,tool",
|
||||
[(False, False), (False, True)],
|
||||
ids=["neither", "tool-only"],
|
||||
)
|
||||
def test_parse_plain_text_no_reasoning_parser(tokenizer, reasoning, tool):
|
||||
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
|
||||
request = make_request()
|
||||
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
|
||||
|
||||
assert r is None
|
||||
assert content == PLAIN_TEXT
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reasoning,tool",
|
||||
[(True, False), (True, True)],
|
||||
ids=["reasoning-only", "both"],
|
||||
)
|
||||
def test_parse_plain_text_with_reasoning_parser(tokenizer, reasoning, tool):
|
||||
parser = make_parser(tokenizer, reasoning=reasoning, tool=tool)
|
||||
request = make_request()
|
||||
r, content, tool_calls = parser.parse(PLAIN_TEXT, request)
|
||||
|
||||
assert r == PLAIN_TEXT
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_both_parsers(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = make_request(tools=TOOLS)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
MODEL_OUTPUT, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is not None
|
||||
assert "let me think about this" in reasoning
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
|
||||
assert not content or content.strip() == ""
|
||||
|
||||
|
||||
def test_parse_reasoning_only(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
request = make_request()
|
||||
reasoning, content, tool_calls = parser.parse(MODEL_OUTPUT, request)
|
||||
|
||||
assert reasoning is not None
|
||||
assert "let me think about this" in reasoning
|
||||
assert content is not None
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_tool_only(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(tools=TOOLS)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
MODEL_OUTPUT, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def test_parse_named_tool_choice(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
TOOL_ARGUMENTS, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert tool_calls[0].arguments == TOOL_ARGUMENTS
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_with_reasoning(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
model_output = f"<think>thinking</think>{TOOL_ARGUMENTS}"
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
model_output, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is not None
|
||||
assert "thinking" in reasoning
|
||||
assert content is None
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert tool_calls[0].arguments == TOOL_ARGUMENTS
|
||||
|
||||
|
||||
def test_parse_required_tool_choice(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
functions_json = json.dumps(
|
||||
[
|
||||
{"name": "get_weather", "parameters": {"city": "Dallas"}},
|
||||
{"name": "get_time", "parameters": {"timezone": "UTC"}},
|
||||
]
|
||||
)
|
||||
request = make_request(tools=TOOLS, tool_choice="required")
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
functions_json, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 2
|
||||
assert tool_calls[0].name == "get_weather"
|
||||
assert json.loads(tool_calls[0].arguments) == {"city": "Dallas"}
|
||||
assert tool_calls[1].name == "get_time"
|
||||
assert json.loads(tool_calls[1].arguments) == {"timezone": "UTC"}
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_kimi_k2_ids(tokenizer):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
functions_json = json.dumps(
|
||||
[
|
||||
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
|
||||
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 2}},
|
||||
]
|
||||
)
|
||||
request = make_request(tools=TOOLS, tool_choice="required")
|
||||
_, content, tool_calls = parser.parse(
|
||||
functions_json, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert [tc.id for tc in tool_calls] == [
|
||||
"functions.get_current_weather:0",
|
||||
"functions.get_forecast:1",
|
||||
]
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_kimi_k2_ids_after_history(tokenizer):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
functions_json = json.dumps(
|
||||
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
|
||||
)
|
||||
request = make_request(
|
||||
messages=HISTORY_MESSAGES,
|
||||
tools=TOOLS,
|
||||
tool_choice="required",
|
||||
)
|
||||
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].id == "functions.get_current_weather:2"
|
||||
|
||||
|
||||
def test_count_history_tool_calls_responses_request():
|
||||
request = ResponsesRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": [
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_0",
|
||||
"name": "get_current_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_1",
|
||||
"name": "get_forecast",
|
||||
"arguments": "{}",
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
assert count_history_tool_calls(request) == 2
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_random_ids_deferred(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
functions_json = json.dumps(
|
||||
[{"name": "get_current_weather", "parameters": {"city": "Dallas"}}]
|
||||
)
|
||||
request = make_request(
|
||||
messages=HISTORY_MESSAGES,
|
||||
tools=TOOLS,
|
||||
tool_choice="required",
|
||||
)
|
||||
_, _, tool_calls = parser.parse(functions_json, request, enable_auto_tools=True)
|
||||
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].id is None
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_kimi_k2_id(tokenizer):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
_, content, tool_calls = parser.parse(
|
||||
TOOL_ARGUMENTS, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert tool_calls[0].id == "functions.get_weather:0"
|
||||
|
||||
|
||||
def test_parse_named_tool_choice_content_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(
|
||||
tools=TOOLS,
|
||||
tool_choice={
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather"},
|
||||
},
|
||||
)
|
||||
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
|
||||
|
||||
def test_parse_required_tool_choice_content_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(tools=TOOLS, tool_choice="required")
|
||||
reasoning, content, tool_calls = parser.parse("", request, enable_auto_tools=True)
|
||||
assert reasoning is None
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_auto_tools_no_parser(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=False)
|
||||
request = make_request()
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
TOOL_CALL_ONLY, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == TOOL_CALL_ONLY
|
||||
assert tool_calls is not None
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_auto_tools_no_calls_returns_none(tokenizer):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = make_request(tools=TOOLS)
|
||||
reasoning, content, tool_calls = parser.parse(
|
||||
PLAIN_TEXT, request, enable_auto_tools=True
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == PLAIN_TEXT
|
||||
assert tool_calls is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Prompt that already closes the reasoning block
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
CONTINUE_MESSAGES = [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "Sure,"},
|
||||
]
|
||||
REASONED_OUTPUT = "<think>\nlet me think\n</think>\n\n" + PLAIN_TEXT
|
||||
TOOL_TURN_MESSAGES = [
|
||||
{"role": "user", "content": "Weather in Paris?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "<think>\nneed a tool\n</think>\n\n",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
|
||||
]
|
||||
|
||||
|
||||
def reasoning_parser(tokenizer, name="qwen3"):
|
||||
parser_cls = ParserManager.get_parser(
|
||||
reasoning_parser_name=name, tokenizer=tokenizer
|
||||
)
|
||||
return parser_cls(tokenizer, None)
|
||||
|
||||
|
||||
def encode(tokenizer, text):
|
||||
return tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
|
||||
def prompt_case(tokenizer, case):
|
||||
"""(rendered prompt ids, request, model output) for each prompt shape."""
|
||||
if case == "continue_final_message":
|
||||
# The template renders an empty, closed think block before "Sure,".
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
CONTINUE_MESSAGES,
|
||||
tokenize=False,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
request = make_request(
|
||||
messages=CONTINUE_MESSAGES,
|
||||
add_generation_prompt=False,
|
||||
continue_final_message=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), request, " here is the answer"
|
||||
if case == "template_closed_thinking":
|
||||
# The template switches thinking off, but the parser is not told
|
||||
# through chat_template_kwargs (#53284).
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), PLAIN_TEXT
|
||||
if case == "earlier_turn_closed_reasoning":
|
||||
# Only a previous assistant turn closes reasoning; this one is open.
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
TOOL_TURN_MESSAGES,
|
||||
tools=[{"type": "function", "function": {"name": "get_weather"}}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
|
||||
assert case == "reasoning_open"
|
||||
prompt = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "hi"}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
return encode(tokenizer, prompt), make_request(), REASONED_OUTPUT
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["continue_final_message", "template_closed_thinking"])
|
||||
def test_parse_prompt_closed_reasoning_is_content(tokenizer, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
parser = reasoning_parser(tokenizer)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning is None
|
||||
assert content == output
|
||||
assert parser.count_reasoning_tokens(encode(tokenizer, output)) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("parser_name", "case"),
|
||||
[
|
||||
("qwen3", "reasoning_open"),
|
||||
("qwen3", "earlier_turn_closed_reasoning"),
|
||||
# BaseThinkingReasoningParser scans the prompt without turn
|
||||
# boundaries, so it must not decide non-streaming classification.
|
||||
("deepseek_r1", "earlier_turn_closed_reasoning"),
|
||||
],
|
||||
)
|
||||
def test_parse_prompt_open_reasoning_still_extracted(tokenizer, parser_name, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
parser = reasoning_parser(tokenizer, parser_name)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning.strip() == "let me think"
|
||||
assert content.strip() == PLAIN_TEXT
|
||||
assert parser.count_reasoning_tokens(encode(tokenizer, output)) > 0
|
||||
|
||||
|
||||
def test_parse_prompt_ignored_without_turn_boundaries(tokenizer):
|
||||
"""Without turn boundaries, an earlier turn's </think> looks like this
|
||||
turn's, so the engine must not start the output in content."""
|
||||
prompt_ids, request, output = prompt_case(
|
||||
tokenizer, "earlier_turn_closed_reasoning"
|
||||
)
|
||||
parser = ParserEngine(tokenizer, parser_engine_config=qwen3_config(thinking=True))
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=encode(tokenizer, output)
|
||||
)
|
||||
|
||||
assert reasoning.strip() == "let me think"
|
||||
assert content.strip() == PLAIN_TEXT
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"continue_final_message",
|
||||
"template_closed_thinking",
|
||||
"reasoning_open",
|
||||
"earlier_turn_closed_reasoning",
|
||||
],
|
||||
)
|
||||
def test_prompt_reasoning_streaming_matches_non_streaming(tokenizer, case):
|
||||
prompt_ids, request, output = prompt_case(tokenizer, case)
|
||||
output_ids = encode(tokenizer, output)
|
||||
|
||||
parser = reasoning_parser(tokenizer)
|
||||
parser.set_prompt_token_ids(prompt_ids)
|
||||
reasoning, content, _ = parser.parse(
|
||||
output, request, model_output_token_ids=output_ids
|
||||
)
|
||||
non_streaming = (reasoning or "", content or "")
|
||||
non_streaming_count = parser.count_reasoning_tokens(output_ids)
|
||||
|
||||
parser = reasoning_parser(tokenizer)
|
||||
streamed_reasoning = streamed_content = ""
|
||||
for i, token_id in enumerate(output_ids):
|
||||
delta = parser.parse_delta(
|
||||
tokenizer.decode([token_id]),
|
||||
[token_id],
|
||||
request,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=i == len(output_ids) - 1,
|
||||
)
|
||||
if delta is not None:
|
||||
streamed_reasoning += delta.reasoning or ""
|
||||
streamed_content += delta.content or ""
|
||||
|
||||
assert (streamed_reasoning, streamed_content) == non_streaming
|
||||
assert parser.count_reasoning_tokens(output_ids) == non_streaming_count
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,835 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.parser.abstract_parser import DelegatingParser
|
||||
from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter
|
||||
from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def enable_hermes_required_named_parsing(monkeypatch):
|
||||
# With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets
|
||||
# supports_required_and_named=False and uses structural-tag guided tool
|
||||
# calling. Force it True to test the non-guided JSON required/named parse path.
|
||||
monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True)
|
||||
|
||||
|
||||
class ThinkReasoningParser(BaseThinkingReasoningParser):
|
||||
@property
|
||||
def start_token(self) -> str:
|
||||
return "<think>"
|
||||
|
||||
@property
|
||||
def end_token(self) -> str:
|
||||
return "</think>"
|
||||
|
||||
|
||||
MODEL_OUTPUT = (
|
||||
"<think>let me think about this</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer("Qwen/Qwen3-32B")
|
||||
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
KIMI_K2_MODEL_CONFIG = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(model_type="kimi_k2"),
|
||||
hf_overrides=None,
|
||||
)
|
||||
|
||||
HISTORY_MESSAGES = [
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "functions.get_current_weather:0",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "functions.get_current_weather:0",
|
||||
"content": "{}",
|
||||
},
|
||||
{"role": "user", "content": "again"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def request_obj():
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=TOOLS,
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
|
||||
def make_parser(tokenizer, reasoning=False, tool=False, **kwargs):
|
||||
class TestParser(DelegatingParser):
|
||||
reasoning_parser_cls = ThinkReasoningParser if reasoning else None
|
||||
tool_parser_cls = Hermes2ProToolParser if tool else None
|
||||
|
||||
return TestParser(tokenizer, **kwargs)
|
||||
|
||||
|
||||
def stream_text(parser, tokenizer, text, request, prompt_token_ids=None):
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
results: list[DeltaMessage | None] = []
|
||||
for tid in token_ids:
|
||||
delta_text = tokenizer.decode([tid])
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
[tid],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def collect_fields(results):
|
||||
all_reasoning = "".join(r.reasoning for r in results if r and r.reasoning)
|
||||
all_content = "".join(r.content for r in results if r and r.content)
|
||||
all_tool_calls = [tc for r in results if r and r.tool_calls for tc in r.tool_calls]
|
||||
return all_reasoning, all_content, all_tool_calls
|
||||
|
||||
|
||||
def test_parse_delta_neither_parser(tokenizer, request_obj):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=False)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert len(tool_calls) == 0
|
||||
assert "<think>" in content
|
||||
assert "let me think about this" in content
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
|
||||
|
||||
def test_parse_delta_tool_parser_only(tokenizer, request_obj):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert "<think>" in content
|
||||
assert "let me think about this" in content
|
||||
assert "</think>" in content
|
||||
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_parser_only(tokenizer, request_obj):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert len(tool_calls) == 0
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
assert "</tool_call>" in content
|
||||
|
||||
|
||||
def test_parse_delta_both_parsers(tokenizer, request_obj):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert content == ""
|
||||
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def stream_chunks(parser, tokenizer, chunks, request_obj):
|
||||
"""Stream pre-split token-ID chunks through the parser."""
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for chunk in chunks:
|
||||
delta_text = tokenizer.decode(chunk)
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
chunk,
|
||||
request_obj,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
prompt_token_ids = None
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def _boundary_chunks(tokenizer, parser, end_token_id=None):
|
||||
"""Split MODEL_OUTPUT into 3 chunks that straddle the </think> boundary."""
|
||||
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
|
||||
if end_token_id is None:
|
||||
end_token_id = parser._reasoning_parser.end_token_id
|
||||
end_idx = token_ids.index(end_token_id)
|
||||
return [
|
||||
token_ids[: end_idx - 1],
|
||||
token_ids[end_idx - 1 : end_idx + 2],
|
||||
token_ids[end_idx + 2 :],
|
||||
]
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_not_dropped_on_boundary(tokenizer, request_obj):
|
||||
"""Regression: reasoning must not be lost when a multi-token delta
|
||||
spans the reasoning/tool-call boundary."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
chunks = _boundary_chunks(tokenizer, parser)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "think about this" in reasoning
|
||||
assert content == ""
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_boundary_no_tool_parser(tokenizer, request_obj):
|
||||
"""When no tool parser is active, boundary-spanning chunks must still
|
||||
preserve reasoning and pass post-</think> text as content."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
chunks = _boundary_chunks(tokenizer, parser)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "think about this" in reasoning
|
||||
assert len(tool_calls) == 0
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_only_no_think_leak(tokenizer, request_obj):
|
||||
"""Regression: </think> must not leak into content when streaming
|
||||
token-by-token with reasoning=True, tool=False."""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert "</think>" not in content
|
||||
assert "<think>" not in content
|
||||
|
||||
|
||||
def test_parse_delta_reasoning_only_thinking_disabled(tokenizer, request_obj):
|
||||
"""Regression test for vllm-project/vllm#40466.
|
||||
|
||||
When enable_thinking=False, the chat template places <think>\\n\\n</think>
|
||||
in the prompt. The model then generates pure content (no think tokens).
|
||||
All streaming output must go to delta.content, not delta.reasoning.
|
||||
"""
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=False)
|
||||
|
||||
end_token_id = parser._reasoning_parser.end_token_id
|
||||
prompt_token_ids = [1, 2, end_token_id, 3]
|
||||
|
||||
content_text = "Hello! How can I assist you today?"
|
||||
results = stream_text(
|
||||
parser,
|
||||
tokenizer,
|
||||
content_text,
|
||||
request_obj,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == "", f"Expected no reasoning, got: {reasoning!r}"
|
||||
assert "Hello" in content
|
||||
assert "assist" in content
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def test_parse_delta_finished_no_flush_without_tool_call_delta(tokenizer, request_obj):
|
||||
"""When finished=True but the final parse_delta produces no
|
||||
tool-call delta, unstreamed args are not flushed."""
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
_, _, tool_calls = collect_fields(results)
|
||||
assert len(tool_calls) > 0
|
||||
|
||||
streamed = parser._tool_parser.streamed_args_for_tool[0]
|
||||
assert len(streamed) > 5
|
||||
parser._tool_parser.streamed_args_for_tool[0] = streamed[:-5]
|
||||
|
||||
# Prevent normal extraction from catching the gap — without a
|
||||
# tool-call delta to merge into, the flush is skipped.
|
||||
parser._tool_parser.extract_tool_calls_streaming = lambda *a, **kw: None
|
||||
|
||||
flush_result = parser.parse_delta("", [], request_obj, finished=True)
|
||||
assert flush_result is None or flush_result.tool_calls is None
|
||||
|
||||
|
||||
def test_parse_delta_finished_no_extra_args_when_fully_streamed(tokenizer, request_obj):
|
||||
"""When all args have been streamed, finished=True must not
|
||||
produce extra or duplicate arguments."""
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
_, _, tool_calls = collect_fields(results)
|
||||
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
flush_result = parser.parse_delta("", [], request_obj, finished=True)
|
||||
assert flush_result is None or flush_result.tool_calls is None
|
||||
|
||||
|
||||
def test_parse_delta_finished_appends_remaining_args(tokenizer, request_obj):
|
||||
"""When finished=True and the tool parser has unstreamed args,
|
||||
parse_delta appends the remaining arguments to the tool-call delta."""
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
|
||||
|
||||
remainder = ',"unit":"celsius"}'
|
||||
prompt_ids: list[int] | None = []
|
||||
results: list[DeltaMessage | None] = []
|
||||
for i, tid in enumerate(token_ids):
|
||||
prev = results[-1] if results else None
|
||||
prev_had_args = (
|
||||
prev
|
||||
and prev.tool_calls
|
||||
and any(tc.function and tc.function.arguments for tc in prev.tool_calls)
|
||||
)
|
||||
|
||||
if prev_had_args:
|
||||
parser._tool_parser.get_remaining_unstreamed_args = lambda: remainder
|
||||
|
||||
result = parser.parse_delta(
|
||||
tokenizer.decode([tid]),
|
||||
[tid],
|
||||
request_obj,
|
||||
prompt_token_ids=prompt_ids,
|
||||
finished=prev_had_args,
|
||||
)
|
||||
prompt_ids = None
|
||||
results.append(result)
|
||||
|
||||
if prev_had_args:
|
||||
break
|
||||
|
||||
_, _, tool_calls = collect_fields(results)
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert tool_args.endswith(remainder)
|
||||
|
||||
|
||||
def test_parse_delta_tool_choice_none(tokenizer, request_obj):
|
||||
parser = make_parser(tokenizer, reasoning=False, tool=True)
|
||||
request = request_obj.model_copy(update={"tool_choice": "none"})
|
||||
results = stream_text(parser, tokenizer, MODEL_OUTPUT, request, prompt_token_ids=[])
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == ""
|
||||
assert len(tool_calls) == 0
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
|
||||
|
||||
def test_parse_delta_tool_choice_none_with_reasoning(tokenizer, request_obj):
|
||||
parser = make_parser(tokenizer, reasoning=True, tool=True)
|
||||
request = request_obj.model_copy(update={"tool_choice": "none"})
|
||||
results = stream_text(parser, tokenizer, MODEL_OUTPUT, request, prompt_token_ids=[])
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert len(tool_calls) == 0
|
||||
assert "<tool_call>" in content
|
||||
assert "get_weather" in content
|
||||
|
||||
|
||||
def test_parse_delta_required_tool_choice_kimi_k2_ids(tokenizer, request_obj):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
request = request_obj.model_copy(update={"tool_choice": "required"})
|
||||
output = json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"parameters": {"city": "Dallas"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for i in range(0, len(output), 3):
|
||||
chunk = output[i : i + 3]
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
chunk,
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
prompt_token_ids = None
|
||||
|
||||
_, content, tool_calls = collect_fields(results)
|
||||
assert content == ""
|
||||
assert any(tc.id == "functions.get_current_weather:0" for tc in tool_calls)
|
||||
assert all(tc.id in (None, "functions.get_current_weather:0") for tc in tool_calls)
|
||||
|
||||
|
||||
def test_parse_delta_required_tool_choice_kimi_k2_ids_after_history(
|
||||
tokenizer, request_obj
|
||||
):
|
||||
parser = make_parser(
|
||||
tokenizer, reasoning=False, tool=True, model_config=KIMI_K2_MODEL_CONFIG
|
||||
)
|
||||
request = request_obj.model_copy(
|
||||
update={"messages": HISTORY_MESSAGES, "tool_choice": "required"}
|
||||
)
|
||||
output = json.dumps(
|
||||
[
|
||||
{
|
||||
"name": "get_current_weather",
|
||||
"parameters": {"city": "Dallas"},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for i in range(0, len(output), 3):
|
||||
chunk = output[i : i + 3]
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
chunk,
|
||||
[],
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
prompt_token_ids = None
|
||||
|
||||
_, _, tool_calls = collect_fields(results)
|
||||
assert any(tc.id == "functions.get_current_weather:1" for tc in tool_calls)
|
||||
assert all(tc.id in (None, "functions.get_current_weather:1") for tc in tool_calls)
|
||||
|
||||
|
||||
# ── Engine-based reasoning + non-engine tool parser (Qwen3 + Hermes) ──
|
||||
|
||||
|
||||
class Qwen3ReasoningHermesToolParser(DelegatingParser):
|
||||
reasoning_parser_cls = Qwen3ParserReasoningAdapter
|
||||
tool_parser_cls = Hermes2ProToolParser
|
||||
|
||||
|
||||
def test_engine_reasoning_hermes_tool_token_by_token(tokenizer, request_obj):
|
||||
"""Qwen3 engine reasoning + Hermes tool parser, token-by-token.
|
||||
|
||||
Sanity check that the mixed engine/non-engine configuration works
|
||||
when tokens arrive one at a time (no deferred content)."""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
|
||||
assert parser._reasoning_parser.engine_based_streaming is True
|
||||
assert parser._tool_parser.engine_based_streaming is False
|
||||
assert parser._engine_based is False
|
||||
|
||||
results = stream_text(
|
||||
parser, tokenizer, MODEL_OUTPUT, request_obj, prompt_token_ids=[]
|
||||
)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert content == ""
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
|
||||
|
||||
def test_engine_reasoning_hermes_tool_boundary(tokenizer, request_obj):
|
||||
"""Qwen3 engine reasoning + Hermes tool parser, boundary chunks.
|
||||
|
||||
When </think> and <tool_call> are in the same chunk with aligned
|
||||
text and token IDs, the engine processes both terminals and returns
|
||||
the <tool_call> text as content."""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
|
||||
chunks = _boundary_chunks(tokenizer, parser, end_token_id=end_token_id)
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "think about this" in reasoning
|
||||
assert content == ""
|
||||
assert len(tool_calls) > 0
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
assert "tool_call" not in content
|
||||
|
||||
|
||||
def test_engine_reasoning_hermes_tool_text_holdback(tokenizer, request_obj):
|
||||
"""Qwen3 engine reasoning + Hermes tool parser with engine holdback.
|
||||
|
||||
Simulates stream_interval > 1 where a batched delta contains
|
||||
'</think><'. The '<' is a regular character token — not the
|
||||
<tool_call> special token — so the engine's incremental lexer
|
||||
buffers it (it could be the start of a text terminal like
|
||||
<tool_call>). The buffered '<' is only recoverable via
|
||||
finish_streaming().
|
||||
|
||||
Without the fix, finish_streaming() is never called at the
|
||||
reasoning->tool transition when _engine_based is False, so the '<'
|
||||
is lost and the Hermes parser sees 'tool_call>...' instead of
|
||||
'<tool_call>...'."""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
vocab = tokenizer.get_vocab()
|
||||
think_end_id = vocab["</think>"]
|
||||
lt_id = vocab["<"]
|
||||
|
||||
token_ids = tokenizer.encode(MODEL_OUTPUT, add_special_tokens=False)
|
||||
end_idx = token_ids.index(think_end_id)
|
||||
|
||||
# Reasoning tokens (aligned text + IDs)
|
||||
pre_ids = token_ids[:end_idx]
|
||||
pre_text = tokenizer.decode(pre_ids)
|
||||
|
||||
# Batched delta: '</think><' — the engine recognises </think> as
|
||||
# THINK_END but the trailing '<' is consumed by the engine's lexer
|
||||
# and held back (it could be the start of <tool_call>). The '<'
|
||||
# is NOT in delta_message.content; it is only in the engine's
|
||||
# internal buffer, recoverable via finish_streaming().
|
||||
holdback_ids = [think_end_id, lt_id]
|
||||
holdback_text = "</think><"
|
||||
|
||||
# Remaining text: 'tool_call>\n{...}\n</tool_call>' — the model
|
||||
# generated <tool_call> as character tokens (not the special token),
|
||||
# and the '<' was consumed above. Encode separately to get the
|
||||
# correct token IDs for this substring.
|
||||
rest_text = (
|
||||
'tool_call>\n{"name": "get_weather", '
|
||||
'"arguments": {"city": "Dallas"}}\n</tool_call>'
|
||||
)
|
||||
rest_ids = tokenizer.encode(rest_text, add_special_tokens=False)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
pre_text,
|
||||
pre_ids,
|
||||
request_obj,
|
||||
prompt_token_ids=[],
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
holdback_text,
|
||||
holdback_ids,
|
||||
request_obj,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
rest_text,
|
||||
rest_ids,
|
||||
request_obj,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert len(tool_calls) > 0, (
|
||||
"Tool calls lost at engine-reasoning -> tool transition. "
|
||||
"finish_streaming() not called when _engine_based is False."
|
||||
)
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": "Dallas"}
|
||||
assert "tool_call" not in content
|
||||
|
||||
|
||||
# ── Engine-based reasoning WITHOUT a tool parser (Qwen3 only) ──
|
||||
|
||||
|
||||
class Qwen3ReasoningNoToolParser(DelegatingParser):
|
||||
reasoning_parser_cls = Qwen3ParserReasoningAdapter
|
||||
tool_parser_cls = None
|
||||
|
||||
|
||||
def test_engine_reasoning_no_tool_batched_content_passthrough(tokenizer, request_obj):
|
||||
"""Qwen3 engine reasoning with NO tool parser, batched boundary.
|
||||
|
||||
The three mixed-parser tests above all pair the engine reasoning
|
||||
parser with Hermes; none exercise the engine-reasoning-only path
|
||||
through the hoisted finish_streaming() transition (where
|
||||
``_engine_based`` is True and there is no tool parser). A single
|
||||
batched delta carries ``</think>`` plus the following content
|
||||
(as happens with stream_interval > 1). The post-``</think>`` text
|
||||
must be emitted as content -- not dropped, not reclassified as
|
||||
reasoning -- and the ``</think>`` marker must not leak either way."""
|
||||
parser = Qwen3ReasoningNoToolParser(tokenizer)
|
||||
assert parser._reasoning_parser.engine_based_streaming is True
|
||||
assert parser._tool_parser is None
|
||||
|
||||
model_output = "<think>let me think about this</think>The answer is 42."
|
||||
end_token_id = parser._reasoning_parser._parser_engine._reasoning_end_token_id
|
||||
token_ids = tokenizer.encode(model_output, add_special_tokens=False)
|
||||
end_idx = token_ids.index(end_token_id)
|
||||
chunks = [token_ids[:end_idx], token_ids[end_idx:]]
|
||||
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert content == "The answer is 42."
|
||||
assert "</think>" not in content
|
||||
assert "</think>" not in reasoning
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text_chunks",
|
||||
[
|
||||
# A trailing "<" that the engine holds back (it may start a tag).
|
||||
["<think>let me think</think>a <", "x> b"],
|
||||
# Whitespace after "</think>" in the same delta as the marker.
|
||||
["<think>let me think</think>\n\n", "<b>bold</b> done"],
|
||||
# The delta that ends reasoning carries only held-back text.
|
||||
["<think>let me think", "</think><", "b>bold</b> done"],
|
||||
],
|
||||
)
|
||||
def test_engine_reasoning_no_tool_batched_holdback_not_dropped(
|
||||
tokenizer, request_obj, text_chunks
|
||||
):
|
||||
"""Text the engine buffers when reasoning ends inside a batched delta is
|
||||
returned by finish_streaming(); with no tool parser it must still be
|
||||
emitted as content, not dropped."""
|
||||
parser = Qwen3ReasoningNoToolParser(tokenizer)
|
||||
chunks = [tokenizer.encode(t, add_special_tokens=False) for t in text_chunks]
|
||||
|
||||
results = stream_chunks(parser, tokenizer, chunks, request_obj)
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == "let me think"
|
||||
assert content == "".join(text_chunks).split("</think>", 1)[1]
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text_chunks", "include_reasoning", "expected_content"),
|
||||
[
|
||||
# The delta that ends reasoning is also the last one: the held-back
|
||||
# "<" must be emitted exactly once (the end-of-stream flush skips the
|
||||
# reasoning parser once reasoning has ended).
|
||||
(["<think>let me think</think>a <"], True, "a <"),
|
||||
# Whitespace-only content after the marker at the end of the stream.
|
||||
(["<think>let me think</think>\n\n"], True, "\n\n"),
|
||||
# Hiding reasoning must not hide the flushed content.
|
||||
(["<think>let me think</think>\n\n", "<b>bold</b>"], False, "\n\n<b>bold</b>"),
|
||||
],
|
||||
)
|
||||
def test_engine_reasoning_no_tool_holdback_finished_and_hidden_reasoning(
|
||||
tokenizer, text_chunks, include_reasoning, expected_content
|
||||
):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
parser = Qwen3ReasoningNoToolParser(tokenizer)
|
||||
chunks = [tokenizer.encode(t, add_special_tokens=False) for t in text_chunks]
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
tokenizer.decode(chunk),
|
||||
chunk,
|
||||
request,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=i == len(chunks) - 1,
|
||||
)
|
||||
)
|
||||
prompt_token_ids = None
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert reasoning == ("let me think" if include_reasoning else "")
|
||||
assert content == expected_content
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
def _decode_stream_deltas(tokenizer, groups):
|
||||
"""Decode token-ID groups into ``(delta_text, group)`` pairs via the real
|
||||
incremental ``DecodeStream``.
|
||||
|
||||
This mirrors how vLLM's detokenizer feeds ``parse_delta`` in
|
||||
production: byte-level UTF-8 hold-back means a character whose bytes
|
||||
span multiple tokens is only surfaced once complete (a naive
|
||||
per-token ``decode`` would instead emit U+FFFD replacement chars)."""
|
||||
from tokenizers.decoders import DecodeStream
|
||||
|
||||
stream = DecodeStream(skip_special_tokens=False)
|
||||
inner = tokenizer._tokenizer
|
||||
pairs = []
|
||||
for group in groups:
|
||||
text = ""
|
||||
for token_id in group:
|
||||
piece = stream.step(inner, token_id)
|
||||
if piece:
|
||||
text += piece
|
||||
pairs.append((text, group))
|
||||
return pairs
|
||||
|
||||
|
||||
def test_engine_reasoning_hermes_tool_multibyte_holdback(tokenizer, request_obj):
|
||||
"""Multi-token character across the reasoning->tool boundary.
|
||||
|
||||
Extends the ASCII '<' hold-back guard with bbrowning's multi-token
|
||||
*character* concern. Two things must both hold:
|
||||
|
||||
1. The batched ``</think><`` delta relies on the hoisted
|
||||
finish_streaming() to recover the engine-buffered '<'. Without
|
||||
the fix the Hermes parser never sees ``<tool_call>`` and emits no
|
||||
tool call at all.
|
||||
2. The tool-call arguments carry ``東京🧑\u200d🚀``; the astronaut
|
||||
ZWJ sequence's bytes span multiple Qwen3 tokens, so the rest of
|
||||
the stream is fed one token at a time through the real
|
||||
``DecodeStream``. Its UTF-8 hold-back yields the correct
|
||||
character round-trip (a naive per-token decode would corrupt it),
|
||||
verifying the boundary stays byte-safe for multi-token
|
||||
characters."""
|
||||
parser = Qwen3ReasoningHermesToolParser(tokenizer)
|
||||
vocab = tokenizer.get_vocab()
|
||||
think_end_id = vocab["</think>"]
|
||||
lt_id = vocab["<"]
|
||||
|
||||
city = "東京🧑\u200d🚀"
|
||||
# Faithfulness precondition: the emoji really is a multi-token char.
|
||||
assert len(tokenizer.encode("🧑\u200d🚀", add_special_tokens=False)) > 1
|
||||
|
||||
model_output = (
|
||||
"<think>let me think about this</think>"
|
||||
'<tool_call>\n{"name": "get_weather", '
|
||||
f'"arguments": {{"city": "{city}"}}}}\n</tool_call>'
|
||||
)
|
||||
token_ids = tokenizer.encode(model_output, add_special_tokens=False)
|
||||
end_idx = token_ids.index(think_end_id)
|
||||
pre_ids = token_ids[:end_idx]
|
||||
|
||||
rest_text = (
|
||||
'tool_call>\n{"name": "get_weather", '
|
||||
f'"arguments": {{"city": "{city}"}}}}\n</tool_call>'
|
||||
)
|
||||
rest_ids = tokenizer.encode(rest_text, add_special_tokens=False)
|
||||
|
||||
# Deltas: reasoning, then a batched '</think><' (the engine buffers
|
||||
# the '<'), then the remaining tokens one at a time so the
|
||||
# multi-token character is genuinely split across deltas by the
|
||||
# detokenizer.
|
||||
groups = [pre_ids, [think_end_id, lt_id]] + [[tid] for tid in rest_ids]
|
||||
pairs = _decode_stream_deltas(tokenizer, groups)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
prompt_token_ids: list[int] | None = []
|
||||
for delta_text, group in pairs:
|
||||
results.append(
|
||||
parser.parse_delta(
|
||||
delta_text,
|
||||
group,
|
||||
request_obj,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
finished=False,
|
||||
)
|
||||
)
|
||||
prompt_token_ids = None
|
||||
|
||||
reasoning, content, tool_calls = collect_fields(results)
|
||||
|
||||
assert "let me think about this" in reasoning
|
||||
assert len(tool_calls) > 0, (
|
||||
"Tool call lost at engine-reasoning -> tool transition; the "
|
||||
"buffered '<' was not recovered by finish_streaming()."
|
||||
)
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
tool_args = "".join(
|
||||
tc.function.arguments for tc in tool_calls if tc.function.arguments
|
||||
)
|
||||
assert json.loads(tool_args) == {"city": city}
|
||||
assert content == ""
|
||||
Reference in New Issue
Block a user