[Frontend] [Parser] Port DeepSeek V4 to streaming parser engine framework (#45877)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
This commit is contained in:
Ben Browning
2026-07-03 20:55:23 -04:00
committed by GitHub
parent d6d39c111e
commit fb5291b35b
22 changed files with 1900 additions and 671 deletions
+268
View File
@@ -0,0 +1,268 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DeepSeek V3.2 parser engine semantics.
V3.2 uses the same DSML parameter format as V4 but wraps tool calls in
``<|DSML|function_calls>`` instead of ``<|DSML|tool_calls>`` and has
no reasoning (``<think>``/``</think>``) support.
"""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import (
collect_content,
collect_function_name,
collect_tool_arguments,
simulate_tool_streaming,
)
from vllm.parser.deepseek_v4 import (
DSML_INVOKE_END,
DSML_INVOKE_NAME_END,
DSML_INVOKE_PREFIX,
)
from vllm.parser.deepseek_v32 import (
DSML_FUNC_END,
DSML_FUNC_START,
DeepSeekV32Parser,
)
from vllm.parser.engine.parser_engine_config import ParserState
_PARAM_OPEN = '|DSML|parameter name="{name}" string="{is_str}">'
_PARAM_CLOSE = "</|DSML|parameter>"
def _param(name: str, is_str: str, value: str) -> str:
return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
def _invoke(name: str, *params: str) -> str:
body = "\n".join(params)
return (
f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
)
def _func_calls(*invocations: str) -> str:
body = "\n".join(invocations)
return f"{DSML_FUNC_START}\n{body}\n{DSML_FUNC_END}"
def _make_tool(name, properties):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": {
"type": "object",
"properties": properties,
},
},
)
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer({})
@pytest.fixture
def mock_request():
from unittest.mock import MagicMock
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
req = MagicMock(spec=ChatCompletionRequest)
req.tools = []
req.tool_choice = "auto"
return req
# ── Non-streaming extraction ────────────────────────────────────────
class TestNonStreaming:
def test_no_tool_call(self, mock_tokenizer, mock_request):
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls("Hello world", mock_request)
assert not result.tools_called
assert result.content == "Hello world"
def test_single_tool(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("get_weather", _param("city", "true", "SF")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"city": "SF"}
def test_parallel_tools(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("get_weather", _param("city", "true", "SF")),
_invoke("get_weather", _param("city", "true", "NYC")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called
assert len(result.tool_calls) == 2
assert json.loads(result.tool_calls[0].function.arguments) == {"city": "SF"}
assert json.loads(result.tool_calls[1].function.arguments) == {"city": "NYC"}
def test_content_before_tool_call(self, mock_tokenizer, mock_request):
text = "Let me check. " + _func_calls(
_invoke("search", _param("q", "true", "vllm")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called
assert result.content is not None
assert "Let me check" in result.content
def test_non_string_params_json_parsed(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke(
"toggle",
_param("enabled", "false", "true"),
_param("count", "false", "42"),
),
)
parser = DeepSeekV32Parser(mock_tokenizer)
result = parser.extract_tool_calls(text, mock_request)
args = json.loads(result.tool_calls[0].function.arguments)
assert args["enabled"] is True
assert args["count"] == 42
def test_wrapper_unwrapping(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
mock_request.tools = [tool]
text = _func_calls(
_invoke(
"get_weather",
_param("arguments", "false", '{"location":"Beijing"}'),
),
)
parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
result = parser.extract_tool_calls(text, mock_request)
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "Beijing"}
# ── Initial state ────────────────────────────────────────────────────
class TestInitialState:
def test_always_content(self, mock_tokenizer):
parser = DeepSeekV32Parser(mock_tokenizer)
cfg = parser.parser_engine_config
assert cfg.initial_state == ParserState.CONTENT
def test_ignores_thinking_kwargs(self, mock_tokenizer):
parser = DeepSeekV32Parser(
mock_tokenizer,
chat_template_kwargs={"thinking": True, "enable_thinking": True},
)
cfg = parser.parser_engine_config
assert cfg.initial_state == ParserState.CONTENT
# ── Streaming ────────────────────────────────────────────────────────
class TestStreaming:
def test_single_tool_streaming(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("get_weather", _param("city", "true", "SF")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
assert collect_function_name(results) == "get_weather"
args_json = collect_tool_arguments(results)
assert json.loads(args_json) == {"city": "SF"}
def test_content_before_tool_streaming(self, mock_tokenizer, mock_request):
text = "Checking... " + _func_calls(
_invoke("fn", _param("k", "true", "v")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
content = collect_content(results)
assert "Checking" in content
def test_parallel_tools_streaming(self, mock_tokenizer, mock_request):
text = _func_calls(
_invoke("fn_a", _param("x", "true", "1")),
_invoke("fn_b", _param("y", "true", "2")),
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
names = []
for delta, _ in results:
if delta and delta.tool_calls:
for tc in delta.tool_calls:
if tc.function and tc.function.name:
names.append(tc.function.name)
assert "fn_a" in names
assert "fn_b" in names
def test_no_tool_content_only(self, mock_tokenizer, mock_request):
text = "Just some text, no tools."
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
content = collect_content(results)
assert "Just some text" in content
args = collect_tool_arguments(results)
assert args == ""
def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
mock_request.tools = [tool]
parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool])
chunks = [
DSML_FUNC_START,
_invoke(
"get_weather",
_param("arguments", "false", '{"location": "NYC"}'),
),
DSML_FUNC_END,
]
results = simulate_tool_streaming(parser, mock_request, chunks)
streamed_args = collect_tool_arguments(results)
final_delta, _ = results[-1]
finish_delta = parser.finish_streaming()
extracted = parser._build_extracted_result(final_delta, finish_delta)
assert extracted.tools_called is True
assert len(extracted.tool_calls) == 1
final_args = extracted.tool_calls[0].function.arguments
assert json.loads(final_args) == {"location": "NYC"}
assert '"arguments"' not in streamed_args
assert final_args.startswith(streamed_args)
def test_missing_invoke_end(self, mock_tokenizer, mock_request):
text = (
f"{DSML_FUNC_START}\n"
f"{DSML_INVOKE_PREFIX}fn{DSML_INVOKE_NAME_END}\n"
f"{_param('k', 'true', 'v')}\n"
f"{DSML_FUNC_END}"
)
parser = DeepSeekV32Parser(mock_tokenizer)
results = simulate_tool_streaming(parser, mock_request, list(text))
assert collect_function_name(results) == "fn"
args = json.loads(collect_tool_arguments(results))
assert args == {"k": "v"}
+922
View File
@@ -0,0 +1,922 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DeepSeek V4-specific parser engine semantics."""
import json
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
MockTokenizer,
_test_request,
collect_output,
replay_streaming,
)
from tests.parser.engine.streaming_helpers import (
collect_content,
collect_function_name,
collect_tool_arguments,
simulate_reasoning_streaming,
simulate_tool_streaming,
)
from vllm.parser.abstract_parser import DelegatingParser
from vllm.parser.deepseek_v4 import (
DSML_INVOKE_END,
DSML_INVOKE_NAME_END,
DSML_INVOKE_PREFIX,
DSML_THINK_END,
DSML_THINK_START,
DSML_TOOL_END,
DSML_TOOL_START,
DeepSeekV4Parser,
_dsml_arg_converter,
_unwrap_wrapper_args,
deepseek_v4_config,
)
from vllm.parser.engine.registered_adapters import (
DeepSeekV4ParserReasoningAdapter,
DeepSeekV4ParserToolAdapter,
)
_THINK_START_ID = 50
_THINK_END_ID = 51
_PARAM_OPEN = '|DSML|parameter name="{name}" string="{is_str}">'
_PARAM_CLOSE = "</|DSML|parameter>"
def _param(name: str, is_str: str, value: str) -> str:
return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}"
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(
{
DSML_THINK_START: _THINK_START_ID,
DSML_THINK_END: _THINK_END_ID,
}
)
# ── Arg converter unit tests ─────────────────────────────────────────
class TestArgConverter:
def _raw(self, *params: tuple[str, str, str]) -> str:
lines = [_param(n, s, v) for n, s, v in params]
return "\n" + "\n".join(lines) + "\n"
def test_string_param(self):
raw = self._raw(("city", "true", "杭州"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result == {"city": "杭州"}
def test_string_with_spaces_and_quotes(self):
raw = self._raw(("msg", "true", 'He said "hello world"'))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["msg"] == 'He said "hello world"'
def test_integer_param(self):
raw = self._raw(("count", "false", "42"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["count"] == 42
assert isinstance(result["count"], int)
def test_float_param(self):
raw = self._raw(("ratio", "false", "3.14"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert abs(result["ratio"] - 3.14) < 1e-9
def test_bool_param(self):
raw = self._raw(("flag", "false", "true"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["flag"] is True
def test_array_param(self):
raw = self._raw(("items", "false", '["a", "b", "c"]'))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["items"] == ["a", "b", "c"]
def test_object_param(self):
raw = self._raw(("opts", "false", '{"key": "val"}'))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["opts"] == {"key": "val"}
def test_mixed_types(self):
raw = self._raw(
("location", "true", "Tokyo"),
("limit", "false", "10"),
("active", "false", "false"),
)
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result == {"location": "Tokyo", "limit": 10, "active": False}
def test_empty_args(self):
result = json.loads(_dsml_arg_converter("", partial=False))
assert result == {}
def test_invalid_json_fallback(self):
raw = self._raw(("data", "false", "[broken"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["data"] == "[broken"
def test_chinese_chars_preserved_in_json(self):
raw = self._raw(("query", "true", "你好世界"))
raw_json = _dsml_arg_converter(raw, partial=False)
assert "你好世界" in raw_json
result = json.loads(raw_json)
assert result["query"] == "你好世界"
def test_partial_complete_plus_in_progress(self):
raw = self._raw(("city", "true", "Tokyo"))
raw += f"<{_PARAM_OPEN.format(name='unit', is_str='true')}celsi"
result = json.loads(_dsml_arg_converter(raw, partial=True))
assert result["city"] == "Tokyo"
assert result["unit"] == "celsi"
def test_partial_no_in_progress(self):
raw = self._raw(("city", "true", "Tokyo"))
result = json.loads(_dsml_arg_converter(raw, partial=True))
assert result == {"city": "Tokyo"}
def test_partial_value_with_angle_bracket(self):
raw = f"<{_PARAM_OPEN.format(name='code', is_str='true')}a<b"
result = json.loads(_dsml_arg_converter(raw, partial=True))
assert result == {"code": "a<b"}
def test_partial_value_with_angle_bracket_and_complete_param(self):
raw = self._raw(("city", "true", "Tokyo"))
raw += f"<{_PARAM_OPEN.format(name='expr', is_str='true')}x<5"
result = json.loads(_dsml_arg_converter(raw, partial=True))
assert result["city"] == "Tokyo"
assert result["expr"] == "x<5"
def test_null_string_false(self):
raw = self._raw(("val", "false", "null"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["val"] is None
def test_string_true_not_json_parsed(self):
raw = self._raw(("n", "true", "42"))
result = json.loads(_dsml_arg_converter(raw, partial=False))
assert result["n"] == "42"
assert isinstance(result["n"], str)
# ── Bare </think> absorption and duplicate <think> absorption ─────────
class TestThinkTagAbsorption:
def test_bare_think_end_not_leaked(self, mock_tokenizer):
parser = DeepSeekV4Parser(mock_tokenizer)
chunks = ["</think>", "Here is the direct answer."]
reasoning, content = simulate_reasoning_streaming(parser, chunks)
assert reasoning == ""
assert "</think>" not in content
assert "Here is the direct answer" in content
def test_duplicate_think_start_absorbed(self, mock_tokenizer):
parser = DeepSeekV4Parser(
mock_tokenizer, chat_template_kwargs={"thinking": True}
)
chunks = [
"<think>\n",
"Some reasoning.\n",
"</think>\n",
"Answer.",
]
reasoning, content = simulate_reasoning_streaming(parser, chunks)
assert "Some reasoning" in reasoning
assert "Answer" in content
# ── Missing </|DSML|invoke> before </|DSML|tool_calls> ────────────
class TestMissingInvokeEnd:
def test_non_streaming(self, mock_tokenizer, mock_request):
parser = DeepSeekV4Parser(mock_tokenizer)
text = (
f"{DSML_TOOL_START}"
f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
f"{_param('location', 'true', 'NYC')}\n"
f"{DSML_TOOL_END}"
)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "NYC"}
def test_streaming_with_trailing_content(self, mock_tokenizer, mock_request):
parser = DeepSeekV4Parser(mock_tokenizer)
chunks = [
DSML_TOOL_START,
f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n"
f"{_param('location', 'true', 'NYC')}\n",
DSML_TOOL_END,
"Done.",
]
results = simulate_tool_streaming(parser, mock_request, chunks)
assert collect_function_name(results) == "get_weather"
args = json.loads(collect_tool_arguments(results))
assert args == {"location": "NYC"}
assert "Done." in collect_content(results)
# ── Thinking mode initial state ──────────────────────────────────────
class TestThinkingModeConfig:
def test_thinking_true_starts_in_reasoning(self):
cfg = deepseek_v4_config(thinking=True)
assert cfg.initial_state.name == "REASONING"
def test_thinking_false_starts_in_content(self):
cfg = deepseek_v4_config(thinking=False)
assert cfg.initial_state.name == "CONTENT"
def test_enable_thinking_kwarg(self, mock_tokenizer):
p = DeepSeekV4Parser(
mock_tokenizer, chat_template_kwargs={"enable_thinking": True}
)
assert p.parser_engine_config.initial_state.name == "REASONING"
def test_no_thinking_kwarg_defaults_to_content(self, mock_tokenizer):
p = DeepSeekV4Parser(mock_tokenizer)
assert p.parser_engine_config.initial_state.name == "CONTENT"
def test_thinking_mode_reasoning_without_tags(self, mock_tokenizer):
parser = DeepSeekV4Parser(
mock_tokenizer, chat_template_kwargs={"thinking": True}
)
chunks = [
"\n\nLet me consider ",
"this carefully.\n",
"</think>\n",
"Here is the result.",
]
reasoning, content = simulate_reasoning_streaming(parser, chunks)
assert "Let me consider" in reasoning
assert "Here is the result" in content
def test_thinking_mode_all_reasoning_no_end_tag(self, mock_tokenizer):
parser = DeepSeekV4Parser(
mock_tokenizer, chat_template_kwargs={"thinking": True}
)
chunks = ["I'll review ", "the PR."]
reasoning, content = simulate_reasoning_streaming(parser, chunks)
assert "review" in reasoning
assert "the PR" in reasoning
assert content == ""
def test_reasoning_effort_none_overrides_enable_thinking(self, mock_tokenizer):
p = DeepSeekV4Parser(
mock_tokenizer,
chat_template_kwargs={
"enable_thinking": True,
"reasoning_effort": "none",
},
)
assert p.parser_engine_config.initial_state.name == "CONTENT"
# ── Implicit reasoning end (missing </think> before tool calls) ─────
class TestImplicitReasoningEnd:
"""Tool call markers end reasoning implicitly when </think> is missing.
DeepSeek V4 models occasionally omit </think> before emitting tool calls.
The (REASONING, TOOL_START) transition handles this gracefully.
"""
@pytest.fixture
def thinking_parser(self, mock_tokenizer):
return DeepSeekV4Parser(mock_tokenizer, chat_template_kwargs={"thinking": True})
def _reasoning_then_tool(self, reasoning_text: str) -> str:
return reasoning_text + _tool_calls(
_invoke("get_weather", ("location", "true", "NYC")),
)
def test_non_streaming_extract_reasoning_implicit_end(self, thinking_parser):
text = self._reasoning_then_tool("Let me look up the weather.\n\n")
reasoning, content = thinking_parser.extract_reasoning(text, None)
assert reasoning == "Let me look up the weather."
assert DSML_TOOL_START not in reasoning
assert DSML_INVOKE_PREFIX not in reasoning
assert content is None
def test_non_streaming_extract_tool_calls_implicit_end(
self, thinking_parser, mock_request
):
text = self._reasoning_then_tool("Let me look up the weather.\n\n")
result = thinking_parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "NYC"}
def test_non_streaming_parse_implicit_end(self, thinking_parser, mock_request):
text = self._reasoning_then_tool("Let me look up the weather.\n\n")
reasoning, content, tool_calls = thinking_parser.parse(text, mock_request)
assert reasoning == "Let me look up the weather."
assert content is None
assert tool_calls is not None
assert len(tool_calls) == 1
assert tool_calls[0].name == "get_weather"
args = json.loads(tool_calls[0].arguments)
assert args == {"location": "NYC"}
def test_streaming_reasoning_implicit_end(self, thinking_parser):
chunks = [
"Let me look up the weather.\n\n",
DSML_TOOL_START,
DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END,
]
reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks)
assert reasoning == "Let me look up the weather."
assert DSML_TOOL_START not in reasoning
assert DSML_INVOKE_PREFIX not in reasoning
def test_streaming_tool_extraction_implicit_end(
self, thinking_parser, mock_request
):
chunks = [
"Let me check.\n\n",
DSML_TOOL_START,
DSML_INVOKE_PREFIX
+ "get_weather"
+ DSML_INVOKE_NAME_END
+ "\n"
+ _param("location", "true", "NYC")
+ "\n"
+ DSML_INVOKE_END,
DSML_TOOL_END,
]
results = simulate_tool_streaming(thinking_parser, mock_request, chunks)
assert collect_function_name(results) == "get_weather"
args = json.loads(collect_tool_arguments(results))
assert args == {"location": "NYC"}
def test_thinking_false_explicit_think_then_tool_call(self, mock_tokenizer):
parser = DeepSeekV4Parser(mock_tokenizer)
chunks = [
DSML_THINK_START,
"Let me check the weather.",
DSML_TOOL_START,
DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END,
]
reasoning, content = simulate_reasoning_streaming(parser, chunks)
assert "Let me check the weather" in reasoning
assert DSML_TOOL_START not in reasoning
assert DSML_THINK_START not in reasoning
def test_non_streaming_parallel_tools_after_implicit_end(
self, thinking_parser, mock_request
):
text = "I need both.\n\n" + _tool_calls(
_invoke("get_weather", ("location", "true", "NYC")),
_invoke("get_time", ("timezone", "true", "EST")),
)
result = thinking_parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "get_weather"
assert result.tool_calls[1].function.name == "get_time"
def test_streaming_implicit_end_trailing_whitespace_stripped(self, thinking_parser):
chunks = [
"Reasoning.\n\n\n",
DSML_TOOL_START,
DSML_INVOKE_PREFIX + "func" + DSML_INVOKE_NAME_END,
]
reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks)
assert reasoning == "Reasoning."
# ── Wrapper argument unwrapping ──────────────────────────────────────
class TestWrapperUnwrapping:
def test_unwrap_arguments_wrapper(self):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
},
},
)
result = _unwrap_wrapper_args(
'{"arguments": {"location": "Beijing"}}',
[tool],
"get_weather",
)
assert json.loads(result) == {"location": "Beijing"}
def test_unwrap_input_wrapper(self):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
},
},
)
result = _unwrap_wrapper_args(
'{"input": {"location": "Beijing"}}',
[tool],
"get_weather",
)
assert json.loads(result) == {"location": "Beijing"}
def test_no_unwrap_when_key_in_schema(self):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "func",
"parameters": {
"type": "object",
"properties": {"arguments": {"type": "string"}},
},
},
)
result = _unwrap_wrapper_args(
'{"arguments": "some value"}',
[tool],
"func",
)
assert json.loads(result) == {"arguments": "some value"}
def test_no_unwrap_when_no_tools(self):
result = _unwrap_wrapper_args(
'{"arguments": {"location": "Beijing"}}',
None,
"get_weather",
)
assert json.loads(result) == {"arguments": {"location": "Beijing"}}
def test_unwrap_json_string_inner(self):
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
},
},
)
result = _unwrap_wrapper_args(
'{"arguments": "{\\"location\\": \\"Beijing\\"}"}',
[tool],
"get_weather",
)
assert json.loads(result) == {"location": "Beijing"}
# ── Parallel tool call wrapper unwrapping ───────────────────────────
def _make_tool(name, properties):
from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E501
ChatCompletionToolsParam,
)
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": {
"type": "object",
"properties": properties,
},
},
)
def _invoke(name, *params):
body = "\n".join(_param(n, s, v) for n, s, v in params)
return (
f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}"
)
def _tool_calls(*invokes):
return DSML_TOOL_START + "\n".join(invokes) + DSML_TOOL_END
class TestParallelUnwrapping:
@pytest.fixture
def weather_tool(self):
return _make_tool(
"get_weather",
{
"location": {"type": "string"},
"unit": {"type": "string"},
},
)
@pytest.fixture
def time_tool(self):
return _make_tool(
"get_time",
{"timezone": {"type": "string"}},
)
@pytest.mark.parametrize(
"weather_args, expected",
[
(
'{"location": "NYC", "unit": "celsius"}',
{"location": "NYC", "unit": "celsius"},
),
('{"location": "NYC"}', {"location": "NYC"}),
],
ids=["all_props", "subset_props"],
)
def test_unwrap_parallel_uses_correct_schema(
self,
mock_tokenizer,
mock_request,
weather_tool,
time_tool,
weather_args,
expected,
):
tools = [weather_tool, time_tool]
parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
mock_request.tools = tools
text = _tool_calls(
_invoke("get_weather", ("arguments", "false", weather_args)),
_invoke("get_time", ("timezone", "true", "EST")),
)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "get_weather"
args0 = json.loads(result.tool_calls[0].function.arguments)
assert args0 == expected
assert result.tool_calls[1].function.name == "get_time"
args1 = json.loads(result.tool_calls[1].function.arguments)
assert args1 == {"timezone": "EST"}
def test_unwrap_parallel_streaming(
self, mock_tokenizer, mock_request, weather_tool, time_tool
):
tools = [weather_tool, time_tool]
parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
mock_request.tools = tools
chunks = [
DSML_TOOL_START,
_invoke(
"get_weather",
("arguments", "false", '{"location": "NYC"}'),
),
_invoke("get_time", ("timezone", "true", "EST")),
DSML_TOOL_END,
]
results = simulate_tool_streaming(parser, mock_request, chunks)
final_delta, _ = results[-1]
finish_delta = parser.finish_streaming()
extracted = parser._build_extracted_result(final_delta, finish_delta)
assert extracted.tools_called is True
assert len(extracted.tool_calls) == 2
args0 = json.loads(extracted.tool_calls[0].function.arguments)
assert args0 == {"location": "NYC"}
args1 = json.loads(extracted.tool_calls[1].function.arguments)
assert args1 == {"timezone": "EST"}
def test_no_unwrap_parallel_when_no_match(
self, mock_tokenizer, mock_request, weather_tool, time_tool
):
tools = [weather_tool, time_tool]
parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
mock_request.tools = tools
text = _tool_calls(
_invoke(
"get_weather",
("arguments", "false", '{"unknown_key": "val"}'),
),
_invoke("get_time", ("timezone", "true", "EST")),
)
result = parser.extract_tool_calls(text, mock_request)
assert len(result.tool_calls) == 2
args0 = json.loads(result.tool_calls[0].function.arguments)
assert args0 == {"arguments": {"unknown_key": "val"}}
args1 = json.loads(result.tool_calls[1].function.arguments)
assert args1 == {"timezone": "EST"}
def test_unwrap_single_tool_still_works(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
tools = [tool]
parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
mock_request.tools = tools
text = _tool_calls(
_invoke(
"get_weather",
("arguments", "false", '{"location": "Beijing"}'),
),
)
result = parser.extract_tool_calls(text, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "Beijing"}
# ── Streaming wrapper consistency ─────────────────────────────────────
class TestStreamingWrapperConsistency:
"""Streamed arg deltas must stay consistent with final extraction
when wrapper params like 'arguments' are unwrapped."""
def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request):
tool = _make_tool("get_weather", {"location": {"type": "string"}})
tools = [tool]
parser = DeepSeekV4Parser(mock_tokenizer, tools=tools)
mock_request.tools = tools
chunks = [
DSML_TOOL_START,
_invoke(
"get_weather",
("arguments", "false", '{"location": "NYC"}'),
),
DSML_TOOL_END,
]
results = simulate_tool_streaming(parser, mock_request, chunks)
streamed_args = collect_tool_arguments(results)
final_delta, _ = results[-1]
finish_delta = parser.finish_streaming()
extracted = parser._build_extracted_result(final_delta, finish_delta)
assert extracted.tools_called is True
assert len(extracted.tool_calls) == 1
final_args = extracted.tool_calls[0].function.arguments
assert json.loads(final_args) == {"location": "NYC"}
assert '"arguments"' not in streamed_args, (
f"Streamed args should not contain wrapper key, got: {streamed_args!r}"
)
assert final_args.startswith(streamed_args), (
f"Extracted args {final_args!r} "
f"should start with streamed args {streamed_args!r}"
)
# ── DelegatingParser: large delta with </think> + tool calls ─────────
_DSV4_FULL_VOCAB = {
DSML_THINK_START: 128821,
DSML_THINK_END: 128822,
DSML_TOOL_START: 128823,
DSML_TOOL_END: 128824,
}
class _DeepSeekV4Delegating(DelegatingParser):
reasoning_parser_cls = DeepSeekV4ParserReasoningAdapter
tool_parser_cls = DeepSeekV4ParserToolAdapter
def _dsv4_tokens(
reasoning: str,
tool_name: str,
params: list[tuple[str, str, str]],
) -> list[tuple[int, str]]:
"""Build a token sequence: reasoning + </think> + DSML tool block."""
tokens: list[tuple[int, str]] = []
tid = 100
for word in reasoning.split(" "):
prefix = " " if tokens else ""
tokens.append((tid, prefix + word))
tid += 1
tokens.append((_DSV4_FULL_VOCAB[DSML_THINK_END], DSML_THINK_END))
tokens.append((tid, "\n\n"))
tid += 1
tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_START], DSML_TOOL_START))
tokens.append((tid, "\n"))
tid += 1
invoke_prefix_text = f"{DSML_INVOKE_PREFIX}{tool_name}{DSML_INVOKE_NAME_END}"
tokens.append((tid, invoke_prefix_text))
tid += 1
tokens.append((tid, "\n"))
tid += 1
for name, is_str, value in params:
param_text = _param(name, is_str, value)
tokens.append((tid, param_text))
tid += 1
tokens.append((tid, "\n"))
tid += 1
tokens.append((tid, DSML_INVOKE_END))
tid += 1
tokens.append((tid, "\n"))
tid += 1
tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_END], DSML_TOOL_END))
return tokens
class TestDelegatingParserLargeDelta:
"""Regression: tool calls lost when </think> + DSML arrive in same delta.
The DelegatingParser used by the serving layer splits reasoning and
tool parsing across two separate engine instances. When </think> and
the entire DSML tool block arrive in a single large streaming delta,
the content transfer from reasoning adapter to tool adapter must
preserve the tool call text.
"""
@pytest.fixture
def dsv4_tokens(self):
return _dsv4_tokens(
reasoning="The user wants the current weather in Berlin.",
tool_name="get_weather",
params=[
("location", "true", "Berlin"),
("units", "true", "celsius"),
],
)
@pytest.fixture
def dsv4_tokenizer(self, dsv4_tokens):
return MockTokenizer(
vocab=dict(_DSV4_FULL_VOCAB),
tokens=dsv4_tokens,
)
@pytest.mark.parametrize(
"chunk_size",
[1, 2, 3, 5, None],
ids=lambda c: f"chunk={c}",
)
def test_tool_calls_extracted_at_all_chunk_sizes(
self, dsv4_tokenizer, dsv4_tokens, chunk_size
):
parser = _DeepSeekV4Delegating(
dsv4_tokenizer,
chat_template_kwargs={"thinking": True},
)
deltas = replay_streaming(
parser,
dsv4_tokens,
chunk_size=chunk_size,
finished_on_last=True,
tools=DUMMY_TOOLS,
)
output = collect_output(deltas)
assert "The user wants" in output.reasoning
assert len(output.tool_calls) == 1, (
f"Expected 1 tool call but got {len(output.tool_calls)}; "
f"reasoning={output.reasoning!r}, content={output.content!r}"
)
assert output.tool_calls[0]["name"] == "get_weather"
args = json.loads(output.tool_calls[0]["arguments"])
assert args == {"location": "Berlin", "units": "celsius"}
def test_eos_drop_token_does_not_swallow_tool_calls(self):
"""Tool calls must survive when an EOS DROP token's ID is in
delta_token_ids but its text is absent from delta_text.
At large stream_interval the EOS token ID arrives in the same
delta as </think> + tool calls but the detokenizer strips the
EOS text. The scanner's _rebuild_from_anchors defers all text
after </think> when it can't find the EOS anchor text. The
reasoning adapter's finish_streaming must flush deferred text
as content (with skip_tool_parsing), not as tool calls.
"""
eos_text = "<|end▁of▁sentence|>"
eos_id = 128801
vocab = {
DSML_THINK_START: 128821,
DSML_THINK_END: 128822,
eos_text: eos_id,
}
reasoning = "The user wants weather."
tool_block = (
"\n\n"
+ DSML_TOOL_START
+ "\n"
+ DSML_INVOKE_PREFIX
+ "get_weather"
+ DSML_INVOKE_NAME_END
+ "\n"
+ _param("location", "true", "Berlin")
+ "\n"
+ DSML_INVOKE_END
+ "\n"
+ DSML_TOOL_END
)
# delta_text does NOT include EOS text (detokenizer strips it)
full_text = reasoning + DSML_THINK_END + tool_block
# Build token list: word-split reasoning, then special tokens,
# then word-split tool block content, then EOS.
# EOS ID is present but its text is NOT in delta_text.
tokens: list[tuple[int, str]] = []
tid = 100
for word in reasoning.split(" "):
pfx = " " if tokens else ""
tokens.append((tid, pfx + word))
tid += 1
tokens.append((128822, DSML_THINK_END))
for ch in tool_block:
tokens.append((tid, ch))
tid += 1
tokens.append((eos_id, eos_text))
all_ids = [t[0] for t in tokens]
tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
request = _test_request(tools=DUMMY_TOOLS)
# All-in-one delta: EOS ID in token_ids but text NOT in
# delta_text (detokenizer strips EOS). This is the scenario
# at large stream_interval.
parser = _DeepSeekV4Delegating(
tokenizer,
chat_template_kwargs={"thinking": True},
)
deltas = [
parser.parse_delta(
full_text,
all_ids,
request,
prompt_token_ids=[],
finished=True,
)
]
output = collect_output(deltas)
assert "The user wants" in output.reasoning
assert len(output.tool_calls) == 1, (
f"Expected 1 tool call but got {len(output.tool_calls)}; "
f"reasoning={output.reasoning!r}, content={output.content!r}"
)
assert output.tool_calls[0]["name"] == "get_weather"
args = json.loads(output.tool_calls[0]["arguments"])
assert args == {"location": "Berlin"}
+15 -12
View File
@@ -20,6 +20,7 @@ import pytest
from tests.parser.engine.replay_harness import (
DUMMY_TOOLS,
MockTokenizer,
Sample,
_test_request,
assert_no_terminal_leakage,
assert_parse_output,
@@ -100,6 +101,14 @@ def _discover_parsers() -> list[_ParserInfo]:
_PARSERS = _discover_parsers()
def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra):
kwargs = dict(extra)
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
return parser_cls(tokenizer, sample.tools, **kwargs)
_ENGINE_PARSERS: dict[str, type[ParserEngine]] = {
f"{p.name}_engine": p.parser_cls for p in _PARSERS
}
@@ -123,7 +132,7 @@ class TestReplayWithHoldback:
def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback):
tokenizer = make_mock_tokenizer(sample)
parser = parser_cls(tokenizer, sample.tools)
parser = _make_parser(parser_cls, tokenizer, sample)
deltas = replay_streaming(
parser,
sample.tokens,
@@ -160,7 +169,7 @@ class TestTextHoldback:
def test_replay(self, parser_cls, sample, terminals, delay):
tokenizer = make_mock_tokenizer(sample)
parser = parser_cls(tokenizer, sample.tools)
parser = _make_parser(parser_cls, tokenizer, sample)
deltas = replay_with_text_holdback(
parser,
sample.tokens,
@@ -190,7 +199,7 @@ class TestReplay:
def test_replay(self, parser_cls, sample, terminals, chunk_size):
tokenizer = make_mock_tokenizer(sample)
parser = parser_cls(tokenizer, sample.tools)
parser = _make_parser(parser_cls, tokenizer, sample)
deltas = replay_streaming(
parser,
sample.tokens,
@@ -227,7 +236,7 @@ class TestDeferralFinish:
def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text):
tokenizer = make_mock_tokenizer(sample)
parser = parser_cls(tokenizer, sample.tools)
parser = _make_parser(parser_cls, tokenizer, sample)
request = _test_request()
@@ -528,10 +537,7 @@ class TestDropTokenReplay:
for sample in parser_info.samples:
injected = _inject_drop_tokens(sample)
tokenizer = make_mock_tokenizer(injected)
parser = parser_info.parser_cls(
tokenizer,
tools=sample.tools,
)
parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
results = replay_streaming(
parser,
@@ -562,10 +568,7 @@ class TestDropTokenNonStreaming:
for sample in parser_info.samples:
injected = _inject_drop_tokens(sample)
tokenizer = make_mock_tokenizer(injected)
parser = parser_info.parser_cls(
tokenizer,
tools=sample.tools,
)
parser = _make_parser(parser_info.parser_cls, tokenizer, sample)
request = _test_request(tools=sample.tools)
output = parse_non_streaming(parser, injected, request)
+145 -3
View File
@@ -29,6 +29,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.engine.registered_adapters import (
DeepSeekV4Parser,
DeepSeekV32Parser,
Gemma4Parser,
Glm47MoeParser,
KimiK2Parser,
@@ -644,6 +646,144 @@ def _build_seed_oss(scenario: Scenario, validate: bool = True) -> Sample:
return sample
# ── DeepSeek V4 (DSML tool format) ──────────────────────────────────
_DSML = "|DSML|"
_DSV4_VOCAB: dict[str, int] = {
"<think>": 128821,
"</think>": 128822,
f"<{_DSML}tool_calls>": 128823,
f"</{_DSML}tool_calls>": 128824,
}
def _dsv4_param_text(key: str, value: Any) -> str:
is_string = isinstance(value, str)
if is_string:
val_str = value
elif isinstance(value, bool):
val_str = "true" if value else "false"
elif isinstance(value, (int, float)):
val_str = str(value)
else:
val_str = json.dumps(value, ensure_ascii=False)
string_attr = "true" if is_string else "false"
return (
f'<{_DSML}parameter name="{key}" string="{string_attr}">'
f"{val_str}</{_DSML}parameter>\n"
)
def _dsv4_tool_text(tc: ToolCallSpec) -> str:
parts = [f'<{_DSML}invoke name="{tc.name}">\n']
for key, value in tc.arguments.items():
parts.append(_dsv4_param_text(key, value))
parts.append(f"</{_DSML}invoke>\n")
return "".join(parts)
def _dsml_tool_segs(
scenario: Scenario,
tag: str,
) -> list[tuple[str, bool]]:
if not scenario.tool_calls:
return []
parts = ["\n"]
for tc in scenario.tool_calls:
parts.append(_dsv4_tool_text(tc))
return [
(f"<{_DSML}{tag}>", True),
("".join(parts), False),
(f"</{_DSML}{tag}>", True),
]
def _dsv4_segments(scenario: Scenario, thinking: bool) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if thinking:
if scenario.reasoning is not None:
segs.append((scenario.reasoning, False))
if scenario.content is not None or scenario.tool_calls:
segs.append(("</think>", True))
else:
if scenario.reasoning is not None:
segs.append(("<think>", True))
segs.append((scenario.reasoning, False))
segs.append(("</think>", True))
if scenario.content is not None:
segs.append((scenario.content, False))
segs.extend(_dsml_tool_segs(scenario, "tool_calls"))
return segs
def _build_deepseek_v4(scenario: Scenario, validate: bool = True) -> Sample:
thinking = scenario.reasoning is not None
chat_kwargs = {"thinking": True} if thinking else None
if thinking:
expected_reasoning: str | None = scenario.reasoning or ""
else:
expected_reasoning = None
sample = _make_sample(
sample_id=f"deepseek_v4-{scenario.id}",
description=scenario.description,
vocab=_DSV4_VOCAB,
segments=_dsv4_segments(scenario, thinking),
expected_reasoning=expected_reasoning,
expected_content=_qwen3_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
chat_template_kwargs=chat_kwargs,
)
if validate:
kwargs = {}
if chat_kwargs:
kwargs["chat_template_kwargs"] = chat_kwargs
_validate_sample(sample, DeepSeekV4Parser, **kwargs)
return sample
# ── DeepSeek V3.2 (DSML tool format, no reasoning) ──────────────────
_DSV32_VOCAB: dict[str, int] = {
f"<{_DSML}function_calls>": 128830,
f"</{_DSML}function_calls>": 128831,
}
def _dsv32_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if scenario.content is not None:
segs.append((scenario.content, False))
segs.extend(_dsml_tool_segs(scenario, "function_calls"))
return segs
def _build_deepseek_v32(scenario: Scenario, validate: bool = True) -> Sample | None:
if scenario.reasoning is not None:
return None
sample = _make_sample(
sample_id=f"deepseek_v32-{scenario.id}",
description=scenario.description,
vocab=_DSV32_VOCAB,
segments=_dsv32_segments(scenario),
expected_reasoning=None,
expected_content=_qwen3_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
)
if validate:
_validate_sample(sample, DeepSeekV32Parser)
return sample
# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ──────────────
_GLM47_MOE_VOCAB: dict[str, int] = {
@@ -811,13 +951,15 @@ _KIMI_K2_SCENARIOS = [
# ── Registry and public API ──────────────────────────────────────────
_BUILDERS: dict[str, Any] = {
"qwen3": _build_qwen3,
"deepseek_v32": _build_deepseek_v32,
"deepseek_v4": _build_deepseek_v4,
"gemma4": _build_gemma4,
"minimax_m2": _build_minimax_m2,
"nemotron_v3": _build_nemotron_v3,
"seed_oss": _build_seed_oss,
"glm47_moe": _build_glm47_moe,
"kimi_k2": _build_kimi_k2,
"qwen3": _build_qwen3,
}
@@ -826,10 +968,10 @@ def build_samples(model: str) -> tuple[Sample, ...]:
"""Build all scenario samples for a model, self-validated."""
builder = _BUILDERS[model]
scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS
return tuple(builder(s) for s in scenarios)
return tuple(s for s in (builder(sc) for sc in scenarios) if s is not None)
def build_sample(model: str, scenario: Scenario) -> Sample:
def build_sample(model: str, scenario: Scenario) -> Sample | None:
"""Build a single sample for one model + scenario."""
return _BUILDERS[model](scenario)