[Bugfix] Stream DeepSeek DSML tool-call argument deltas incrementally (#42879)

Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com>
Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
jack
2026-05-28 17:50:23 +08:00
committed by GitHub
co-authored by QwertyJack Chauncey
parent 2a781756a1
commit b372ad3e90
3 changed files with 445 additions and 63 deletions
@@ -10,6 +10,7 @@ import json
from unittest.mock import MagicMock
import pytest
from openai.types.responses.function_tool import FunctionTool
from tests.tool_parsers.utils import run_tool_extraction_streaming
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -718,6 +719,81 @@ class TestExtractToolCallsStreaming:
"patches": [{"op": "replace", "path": "/schedule", "value": "quiet"}],
}
def test_responses_function_tool_schema_in_streaming(self):
"""Responses API FunctionTool schemas must drive streaming conversion."""
tool = FunctionTool(
type="function",
name="toggle",
parameters={
"type": "object",
"properties": {
"enabled": {"type": "boolean"},
"count": {"type": "integer"},
},
},
)
parser = make_parser(tools=[tool])
full_text = (
f"{FC_START}\n"
f'{INV_START}toggle">\n'
f'{PARAM_START}enabled" string="false">true{PARAM_END}\n'
f'{PARAM_START}count" string="false">42{PARAM_END}\n'
f"{INV_END}\n"
f"{FC_END}"
)
deltas = self._stream(parser, full_text)
args = json.loads(self._reconstruct_args(deltas))
assert args == {"enabled": True, "count": 42}
assert isinstance(args["enabled"], bool)
assert isinstance(args["count"], int)
def test_streaming_matches_non_streaming_conversion_fallbacks(self):
"""Streaming must reuse conversion fallback semantics for string=false."""
tool = ChatCompletionToolsParam(
function=FunctionDefinition(
name="coerce",
parameters={
"type": "object",
"properties": {
"union_value": {"type": ["null", "string"]},
"bad_int": {"type": "integer"},
"nullable_string": {"type": ["null", "string"]},
"null_string": {"type": "string"},
"whole_number": {"type": "number"},
},
},
),
)
parser = make_parser(tools=[tool])
full_text = (
f"{FC_START}\n"
f'{INV_START}coerce">\n'
f'{PARAM_START}union_value" string="false">hello{PARAM_END}\n'
f'{PARAM_START}bad_int" string="false">abc{PARAM_END}\n'
f'{PARAM_START}nullable_string" string="false">null{PARAM_END}\n'
f'{PARAM_START}null_string" string="false">null{PARAM_END}\n'
f'{PARAM_START}whole_number" string="false">3.0{PARAM_END}\n'
f"{INV_END}\n"
f"{FC_END}"
)
non_stream = parser.extract_tool_calls(full_text, None)
non_stream_args = json.loads(non_stream.tool_calls[0].function.arguments)
deltas = self._stream(parser, full_text)
stream_args = json.loads(self._reconstruct_args(deltas))
assert stream_args == non_stream_args
assert stream_args == {
"union_value": "hello",
"bad_int": "abc",
"nullable_string": None,
"null_string": "null",
"whole_number": 3,
}
def test_multiple_tools_streaming(self, parser):
full_text = (
f"{FC_START}\n"
@@ -890,8 +966,8 @@ class TestExtractToolCallsStreaming:
assert json.loads(self._reconstruct_args(deltas, tool_index=0)) == {"p": "v1"}
assert json.loads(self._reconstruct_args(deltas, tool_index=1)) == {"q": "v2"}
def test_no_emission_while_incomplete(self, parser):
"""No tool calls should be emitted until an invoke block completes."""
def test_emits_arguments_before_invoke_completes(self, parser):
"""Argument deltas should stream before the invoke block closes."""
# Stream only a partial invoke (no closing tag)
partial_text = (
f"{FC_START}\n"
@@ -899,8 +975,13 @@ class TestExtractToolCallsStreaming:
f'{PARAM_START}k" string="true">val{PARAM_END}\n'
)
deltas = self._stream(parser, partial_text)
# Should have no tool call deltas yet
assert all(not d.tool_calls for d in deltas)
arg_chunks = [
tc.function.arguments
for delta in deltas
for tc in delta.tool_calls or []
if tc.function and tc.function.arguments is not None
]
assert "".join(arg_chunks) == '{"k":"val"'
def test_no_marker_leak_chunked(self, parser):
"""Chunked streaming must NOT leak DSML start-marker fragments
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
@@ -164,11 +165,57 @@ def test_streaming_extracts_complete_invokes():
for delta in deltas
if delta.tool_calls
for tool_call in delta.tool_calls
if tool_call.function.name
]
assert names == ["search"]
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
def test_streaming_emits_incremental_argument_chunks():
tool = ChatCompletionToolsParam(
function=FunctionDefinition(
name="plan_trip",
parameters={
"type": "object",
"properties": {
"days": {"type": "integer"},
"flexible": {"type": "boolean"},
"cities": {"type": "array", "items": {"type": "string"}},
"notes": {"type": "string"},
},
},
),
)
parser = make_parser(tools=[tool])
full_text = (
f"{TC_START}\n"
f'{INV_START}plan_trip">\n'
f'{PARAM_START}days" string="false">3{PARAM_END}\n'
f'{PARAM_START}flexible" string="false">false{PARAM_END}\n'
f'{PARAM_START}cities" string="false">'
f'["Beijing","Shanghai","Tokyo","New York"]{PARAM_END}\n'
f'{PARAM_START}notes" string="true">靠窗座位{PARAM_END}\n'
f"{INV_END}\n"
f"{TC_END}"
)
deltas = stream(parser, full_text, chunk_size=4)
arg_chunks = [
tool_call.function.arguments
for delta in deltas
for tool_call in delta.tool_calls or []
if tool_call.function and tool_call.function.arguments is not None
]
assert len([chunk for chunk in arg_chunks if chunk]) > 2
assert json.loads("".join(arg_chunks)) == {
"days": 3,
"flexible": False,
"cities": ["Beijing", "Shanghai", "Tokyo", "New York"],
"notes": "靠窗座位",
}
def test_get_vllm_registry_structural_tag_returns_structural_tag(
sample_tools: list[ChatCompletionToolsParam],
) -> None:
+313 -59
View File
@@ -4,7 +4,7 @@
import json
import uuid
from collections.abc import Sequence
from typing import Any
from typing import Any, Literal
import regex as re
@@ -62,6 +62,15 @@ class DeepSeekV32ToolParser(ToolParser):
# Streaming state
self.current_tool_index: int = 0
self._sent_content_idx: int = 0
self._buffer: str = ""
self._in_tool_calls: bool = False
self._active_tool_index: int | None = None
self._active_tool_name: str | None = None
self._active_param_name: str | None = None
self._active_param_string_attr: str | None = None
self._active_param_mode: str | None = None
self._active_param_parts: list[str] = []
self._args_started: list[bool] = []
# Regex patterns for complete parsing
self.tool_call_complete_regex = re.compile(
@@ -77,6 +86,10 @@ class DeepSeekV32ToolParser(ToolParser):
r'<|DSML|parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>(.*?)</|DSML|parameter>',
re.DOTALL,
)
self.invoke_start_regex = re.compile(r'<|DSML|invoke\s+name="([^"]+)"\s*>')
self.parameter_start_regex = re.compile(
r'<|DSML|parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>'
)
if not self.model_tokenizer:
raise ValueError(
@@ -117,7 +130,7 @@ class DeepSeekV32ToolParser(ToolParser):
@staticmethod
def _repair_param_dict(
param_dict: dict[str, Any],
param_config: dict[str, dict],
param_config: dict[str, Any],
) -> dict[str, Any]:
"""Unwrap single 'arguments' / 'input' wrappers when the wrapper
is not part of the requested tool schema and the wrapped object
@@ -154,6 +167,15 @@ class DeepSeekV32ToolParser(ToolParser):
converted[name] = coerce_to_schema_type(value, param_types)
return self._repair_param_dict(converted, param_config)
def _get_param_config(self, function_name: str | None) -> dict[str, Any]:
if not function_name or not self.tools:
return {}
return find_tool_properties(self.tools, function_name)
@staticmethod
def _json_escape_string_content(text: str) -> str:
return json.dumps(text, ensure_ascii=False)[1:-1]
def extract_tool_calls(
self,
model_output: str,
@@ -210,67 +232,295 @@ class DeepSeekV32ToolParser(ToolParser):
"""Reset all streaming state."""
self.current_tool_index = 0
self._sent_content_idx = 0
self._buffer = ""
self._in_tool_calls = False
self._active_tool_index = None
self._active_tool_name = None
self._active_param_name = None
self._active_param_string_attr = None
self._active_param_mode = None
self._active_param_parts.clear()
self.prev_tool_call_arr.clear()
self.streamed_args_for_tool.clear()
self._args_started.clear()
def _extract_delta_tool_calls(
def _add_tool_call_delta(
self,
current_text: str,
request: ChatCompletionRequest | None,
) -> list[DeltaToolCall]:
"""Extract DeltaToolCalls from newly completed <invoke> blocks.
tool_call_deltas: dict[int, DeltaToolCall],
index: int,
*,
call_id: str | None = None,
call_type: Literal["function"] | None = None,
name: str | None = None,
arguments: str | None = None,
) -> None:
if arguments:
self.streamed_args_for_tool[index] += arguments
Tracks progress via ``current_tool_index`` so each block is
extracted exactly once across successive streaming calls.
"""
complete_invokes = self.invoke_complete_regex.findall(current_text)
delta_tool_calls: list[DeltaToolCall] = []
while len(complete_invokes) > self.current_tool_index:
invoke_name, invoke_body = complete_invokes[self.current_tool_index]
param_dict = self._parse_invoke_params(invoke_body)
converted = self._convert_params_with_schema(invoke_name, param_dict)
args_json = json.dumps(converted, ensure_ascii=False)
idx = self.current_tool_index
self.current_tool_index += 1
self.prev_tool_call_arr.append(
{"name": invoke_name, "arguments": converted}
if index not in tool_call_deltas:
tool_call_deltas[index] = DeltaToolCall(
index=index,
id=call_id,
type=call_type,
function=DeltaFunctionCall(name=name, arguments=arguments),
)
self.streamed_args_for_tool.append(args_json)
return
delta_tool_calls.append(
DeltaToolCall(
index=idx,
id=self._generate_tool_call_id(),
function=DeltaFunctionCall(
name=invoke_name,
arguments=args_json,
),
type="function",
)
)
delta = tool_call_deltas[index]
if call_id is not None:
delta.id = call_id
if call_type is not None:
delta.type = call_type
if delta.function is None:
delta.function = DeltaFunctionCall()
if name is not None:
delta.function.name = name
if arguments is not None:
delta.function.arguments = (delta.function.arguments or "") + arguments
return delta_tool_calls
def _begin_streaming_tool_call(
self,
name: str,
tool_call_deltas: dict[int, DeltaToolCall],
) -> None:
index = self.current_tool_index
self.current_tool_index += 1
self._active_tool_index = index
self._active_tool_name = name
self.prev_tool_call_arr.append({"name": name, "arguments": {}})
self.streamed_args_for_tool.append("")
self._args_started.append(False)
self._add_tool_call_delta(
tool_call_deltas,
index,
call_id=self._generate_tool_call_id(),
call_type="function",
name=name,
arguments="",
)
def _extract_content(self, current_text: str) -> str | None:
"""Return unsent non-tool-call text, or None.
def _append_param_prefix(
self,
tool_call_deltas: dict[int, DeltaToolCall],
index: int,
key: str,
*,
as_string: bool,
) -> None:
prefix = "{" if not self._args_started[index] else ","
self._args_started[index] = True
arguments = prefix + json.dumps(key, ensure_ascii=False) + ":"
if as_string:
arguments += '"'
self._add_tool_call_delta(tool_call_deltas, index, arguments=arguments)
Holds back any suffix that could be a partial start marker
so that split markers are never leaked as content.
"""
if self.tool_call_start_token not in current_text:
overlap = partial_tag_overlap(current_text, self.tool_call_start_token)
sendable_idx = len(current_text) - overlap
else:
sendable_idx = current_text.index(self.tool_call_start_token)
def _append_json_param_value(
self,
tool_call_deltas: dict[int, DeltaToolCall],
index: int,
key: str,
value: Any,
) -> None:
self._append_param_prefix(tool_call_deltas, index, key, as_string=False)
self._add_tool_call_delta(
tool_call_deltas,
index,
arguments=json.dumps(value, ensure_ascii=False),
)
if sendable_idx > self._sent_content_idx:
content = current_text[self._sent_content_idx : sendable_idx]
self._sent_content_idx = sendable_idx
return content
return None
def _param_types_for_name(self, name: str) -> list[str]:
param_config = self._get_param_config(self._active_tool_name)
if name in param_config and isinstance(param_config[name], dict):
return extract_types_from_schema(param_config[name])
return ["string"]
@staticmethod
def _can_stream_raw_param(param_types: list[str]) -> bool:
# Scalars and unions need the complete value so streaming and
# non-streaming share the same coercion fallback behavior.
return set(param_types).issubset({"object", "array"})
def _should_buffer_wrapper_param(self, name: str) -> bool:
if (
self._active_tool_index is None
or self._args_started[self._active_tool_index]
):
return False
param_config = self._get_param_config(self._active_tool_name)
return bool(
param_config and name in ("arguments", "input") and name not in param_config
)
def _finish_buffered_param(
self,
tool_call_deltas: dict[int, DeltaToolCall],
index: int,
) -> None:
assert self._active_param_name is not None
assert self._active_param_string_attr is not None
raw_value = "".join(self._active_param_parts)
converted = self._convert_params_with_schema(
self._active_tool_name or "",
{self._active_param_name: (raw_value, self._active_param_string_attr)},
)
for key, value in converted.items():
self._append_json_param_value(tool_call_deltas, index, key, value)
def _close_streaming_tool_call(
self,
tool_call_deltas: dict[int, DeltaToolCall],
) -> None:
index = self._active_tool_index
if index is None:
return
suffix = "}" if self._args_started[index] else "{}"
self._add_tool_call_delta(tool_call_deltas, index, arguments=suffix)
try:
self.prev_tool_call_arr[index] = {
"name": self._active_tool_name,
"arguments": json.loads(self.streamed_args_for_tool[index]),
}
except (json.JSONDecodeError, IndexError):
logger.exception("Failed to finalize DeepSeek DSML streaming tool call")
self._active_tool_index = None
self._active_tool_name = None
self._active_param_name = None
self._active_param_string_attr = None
self._active_param_mode = None
self._active_param_parts.clear()
def _process_streaming_buffer(
self,
content_parts: list[str],
tool_call_deltas: dict[int, DeltaToolCall],
) -> None:
parameter_end_token = "</|DSML|parameter>"
invoke_end_token = "</|DSML|invoke>"
while True:
if not self._in_tool_calls:
start_idx = self._buffer.find(self.tool_call_start_token)
if start_idx == -1:
overlap = partial_tag_overlap(
self._buffer, self.tool_call_start_token
)
sendable_idx = len(self._buffer) - overlap
if sendable_idx > 0:
content_parts.append(self._buffer[:sendable_idx])
self._buffer = self._buffer[sendable_idx:]
return
if start_idx > 0:
content_parts.append(self._buffer[:start_idx])
self._buffer = self._buffer[start_idx:]
continue
self._buffer = self._buffer[len(self.tool_call_start_token) :]
self._in_tool_calls = True
continue
if self._active_tool_index is None:
stripped_len = len(self._buffer) - len(self._buffer.lstrip())
if stripped_len:
self._buffer = self._buffer[stripped_len:]
continue
if self._buffer.startswith(self.tool_call_end_token):
self._buffer = self._buffer[len(self.tool_call_end_token) :]
self._in_tool_calls = False
continue
match = self.invoke_start_regex.match(self._buffer)
if match is None:
return
self._buffer = self._buffer[match.end() :]
self._begin_streaming_tool_call(match.group(1), tool_call_deltas)
continue
index = self._active_tool_index
if self._active_param_mode is not None:
end_pos = self._buffer.find(parameter_end_token)
if end_pos != -1:
raw_content = self._buffer[:end_pos]
self._buffer = self._buffer[end_pos + len(parameter_end_token) :]
if self._active_param_mode in ("wrapper", "buffered"):
self._active_param_parts.append(raw_content)
self._finish_buffered_param(tool_call_deltas, index)
elif self._active_param_mode == "string":
arguments = self._json_escape_string_content(raw_content) + '"'
self._add_tool_call_delta(
tool_call_deltas, index, arguments=arguments
)
else:
self._add_tool_call_delta(
tool_call_deltas, index, arguments=raw_content
)
self._active_param_name = None
self._active_param_string_attr = None
self._active_param_mode = None
self._active_param_parts.clear()
continue
overlap = partial_tag_overlap(self._buffer, parameter_end_token)
safe_len = len(self._buffer) - overlap
if safe_len > 0:
raw_content = self._buffer[:safe_len]
self._buffer = self._buffer[safe_len:]
if self._active_param_mode in ("wrapper", "buffered"):
self._active_param_parts.append(raw_content)
elif self._active_param_mode == "string":
self._add_tool_call_delta(
tool_call_deltas,
index,
arguments=self._json_escape_string_content(raw_content),
)
else:
self._add_tool_call_delta(
tool_call_deltas, index, arguments=raw_content
)
return
stripped_len = len(self._buffer) - len(self._buffer.lstrip())
if stripped_len:
self._buffer = self._buffer[stripped_len:]
continue
if self._buffer.startswith(invoke_end_token):
self._buffer = self._buffer[len(invoke_end_token) :]
self._close_streaming_tool_call(tool_call_deltas)
continue
match = self.parameter_start_regex.match(self._buffer)
if match is None:
return
self._buffer = self._buffer[match.end() :]
name = match.group(1)
string_attr = match.group(2)
self._active_param_name = name
self._active_param_string_attr = string_attr
if self._should_buffer_wrapper_param(name):
self._active_param_mode = "wrapper"
continue
if string_attr == "true":
self._append_param_prefix(tool_call_deltas, index, name, as_string=True)
self._active_param_mode = "string"
continue
param_types = self._param_types_for_name(name)
if not self._can_stream_raw_param(param_types):
self._active_param_mode = "buffered"
continue
self._append_param_prefix(tool_call_deltas, index, name, as_string=False)
self._active_param_mode = "raw"
def extract_tool_calls_streaming(
self,
@@ -284,20 +534,24 @@ class DeepSeekV32ToolParser(ToolParser):
) -> DeltaMessage | None:
"""Extract tool calls from streaming model output.
Uses a buffer-until-complete-invoke strategy: tokens are buffered
until a complete invoke block is available, then parsed and emitted
in one shot.
Buffers DSML markup while streaming tool-call metadata and argument
JSON fragments as soon as they are complete enough to be valid deltas.
"""
# First chunk of a new stream — reset state from prior request.
if not previous_text:
self._reset_streaming_state()
content = self._extract_content(current_text)
delta_tool_calls = self._extract_delta_tool_calls(current_text, request)
self._buffer += delta_text
content_parts: list[str] = []
tool_call_deltas: dict[int, DeltaToolCall] = {}
self._process_streaming_buffer(content_parts, tool_call_deltas)
if delta_tool_calls or content:
return DeltaMessage(content=content, tool_calls=delta_tool_calls)
if content_parts or tool_call_deltas:
content = "".join(content_parts) or None
return DeltaMessage(
content=content, tool_calls=list(tool_call_deltas.values())
)
# Empty delta with token ids means EOS or closing tag; return
# non-None so the serving framework can finalize finish_reason.