From b372ad3e9018f032478619adbc7f7fdcc9318212 Mon Sep 17 00:00:00 2001 From: jack Date: Thu, 28 May 2026 17:50:23 +0800 Subject: [PATCH] [Bugfix] Stream DeepSeek DSML tool-call argument deltas incrementally (#42879) Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey --- .../test_deepseekv32_tool_parser.py | 89 ++++- .../test_deepseekv4_tool_parser.py | 47 +++ vllm/tool_parsers/deepseekv32_tool_parser.py | 372 +++++++++++++++--- 3 files changed, 445 insertions(+), 63 deletions(-) diff --git a/tests/tool_parsers/test_deepseekv32_tool_parser.py b/tests/tool_parsers/test_deepseekv32_tool_parser.py index 5929be09fa..a35976b8bb 100644 --- a/tests/tool_parsers/test_deepseekv32_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv32_tool_parser.py @@ -10,6 +10,7 @@ import json from unittest.mock import MagicMock import pytest +from openai.types.responses.function_tool import FunctionTool from tests.tool_parsers.utils import run_tool_extraction_streaming from vllm.entrypoints.openai.chat_completion.protocol import ( @@ -718,6 +719,81 @@ class TestExtractToolCallsStreaming: "patches": [{"op": "replace", "path": "/schedule", "value": "quiet"}], } + def test_responses_function_tool_schema_in_streaming(self): + """Responses API FunctionTool schemas must drive streaming conversion.""" + tool = FunctionTool( + type="function", + name="toggle", + parameters={ + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "count": {"type": "integer"}, + }, + }, + ) + parser = make_parser(tools=[tool]) + full_text = ( + f"{FC_START}\n" + f'{INV_START}toggle">\n' + f'{PARAM_START}enabled" string="false">true{PARAM_END}\n' + f'{PARAM_START}count" string="false">42{PARAM_END}\n' + f"{INV_END}\n" + f"{FC_END}" + ) + + deltas = self._stream(parser, full_text) + args = json.loads(self._reconstruct_args(deltas)) + + assert args == {"enabled": True, "count": 42} + assert isinstance(args["enabled"], bool) + assert isinstance(args["count"], int) + + def test_streaming_matches_non_streaming_conversion_fallbacks(self): + """Streaming must reuse conversion fallback semantics for string=false.""" + tool = ChatCompletionToolsParam( + function=FunctionDefinition( + name="coerce", + parameters={ + "type": "object", + "properties": { + "union_value": {"type": ["null", "string"]}, + "bad_int": {"type": "integer"}, + "nullable_string": {"type": ["null", "string"]}, + "null_string": {"type": "string"}, + "whole_number": {"type": "number"}, + }, + }, + ), + ) + parser = make_parser(tools=[tool]) + full_text = ( + f"{FC_START}\n" + f'{INV_START}coerce">\n' + f'{PARAM_START}union_value" string="false">hello{PARAM_END}\n' + f'{PARAM_START}bad_int" string="false">abc{PARAM_END}\n' + f'{PARAM_START}nullable_string" string="false">null{PARAM_END}\n' + f'{PARAM_START}null_string" string="false">null{PARAM_END}\n' + f'{PARAM_START}whole_number" string="false">3.0{PARAM_END}\n' + f"{INV_END}\n" + f"{FC_END}" + ) + + non_stream = parser.extract_tool_calls(full_text, None) + non_stream_args = json.loads(non_stream.tool_calls[0].function.arguments) + + deltas = self._stream(parser, full_text) + stream_args = json.loads(self._reconstruct_args(deltas)) + + assert stream_args == non_stream_args + assert stream_args == { + "union_value": "hello", + "bad_int": "abc", + "nullable_string": None, + "null_string": "null", + "whole_number": 3, + } + def test_multiple_tools_streaming(self, parser): full_text = ( f"{FC_START}\n" @@ -890,8 +966,8 @@ class TestExtractToolCallsStreaming: assert json.loads(self._reconstruct_args(deltas, tool_index=0)) == {"p": "v1"} assert json.loads(self._reconstruct_args(deltas, tool_index=1)) == {"q": "v2"} - def test_no_emission_while_incomplete(self, parser): - """No tool calls should be emitted until an invoke block completes.""" + def test_emits_arguments_before_invoke_completes(self, parser): + """Argument deltas should stream before the invoke block closes.""" # Stream only a partial invoke (no closing tag) partial_text = ( f"{FC_START}\n" @@ -899,8 +975,13 @@ class TestExtractToolCallsStreaming: f'{PARAM_START}k" string="true">val{PARAM_END}\n' ) deltas = self._stream(parser, partial_text) - # Should have no tool call deltas yet - assert all(not d.tool_calls for d in deltas) + arg_chunks = [ + tc.function.arguments + for delta in deltas + for tc in delta.tool_calls or [] + if tc.function and tc.function.arguments is not None + ] + assert "".join(arg_chunks) == '{"k":"val"' def test_no_marker_leak_chunked(self, parser): """Chunked streaming must NOT leak DSML start-marker fragments diff --git a/tests/tool_parsers/test_deepseekv4_tool_parser.py b/tests/tool_parsers/test_deepseekv4_tool_parser.py index 2394575bc5..ab66d6e64c 100644 --- a/tests/tool_parsers/test_deepseekv4_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv4_tool_parser.py @@ -14,6 +14,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionToolsParam, + FunctionDefinition, ) from vllm.tool_parsers import ToolParserManager from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser @@ -164,11 +165,57 @@ def test_streaming_extracts_complete_invokes(): for delta in deltas if delta.tool_calls for tool_call in delta.tool_calls + if tool_call.function.name ] assert names == ["search"] assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"} +def test_streaming_emits_incremental_argument_chunks(): + tool = ChatCompletionToolsParam( + function=FunctionDefinition( + name="plan_trip", + parameters={ + "type": "object", + "properties": { + "days": {"type": "integer"}, + "flexible": {"type": "boolean"}, + "cities": {"type": "array", "items": {"type": "string"}}, + "notes": {"type": "string"}, + }, + }, + ), + ) + parser = make_parser(tools=[tool]) + full_text = ( + f"{TC_START}\n" + f'{INV_START}plan_trip">\n' + f'{PARAM_START}days" string="false">3{PARAM_END}\n' + f'{PARAM_START}flexible" string="false">false{PARAM_END}\n' + f'{PARAM_START}cities" string="false">' + f'["Beijing","Shanghai","Tokyo","New York"]{PARAM_END}\n' + f'{PARAM_START}notes" string="true">靠窗座位{PARAM_END}\n' + f"{INV_END}\n" + f"{TC_END}" + ) + + deltas = stream(parser, full_text, chunk_size=4) + arg_chunks = [ + tool_call.function.arguments + for delta in deltas + for tool_call in delta.tool_calls or [] + if tool_call.function and tool_call.function.arguments is not None + ] + + assert len([chunk for chunk in arg_chunks if chunk]) > 2 + assert json.loads("".join(arg_chunks)) == { + "days": 3, + "flexible": False, + "cities": ["Beijing", "Shanghai", "Tokyo", "New York"], + "notes": "靠窗座位", + } + + def test_get_vllm_registry_structural_tag_returns_structural_tag( sample_tools: list[ChatCompletionToolsParam], ) -> None: diff --git a/vllm/tool_parsers/deepseekv32_tool_parser.py b/vllm/tool_parsers/deepseekv32_tool_parser.py index 511b6e7c35..7d5e299be8 100644 --- a/vllm/tool_parsers/deepseekv32_tool_parser.py +++ b/vllm/tool_parsers/deepseekv32_tool_parser.py @@ -4,7 +4,7 @@ import json import uuid from collections.abc import Sequence -from typing import Any +from typing import Any, Literal import regex as re @@ -62,6 +62,15 @@ class DeepSeekV32ToolParser(ToolParser): # Streaming state self.current_tool_index: int = 0 self._sent_content_idx: int = 0 + self._buffer: str = "" + self._in_tool_calls: bool = False + self._active_tool_index: int | None = None + self._active_tool_name: str | None = None + self._active_param_name: str | None = None + self._active_param_string_attr: str | None = None + self._active_param_mode: str | None = None + self._active_param_parts: list[str] = [] + self._args_started: list[bool] = [] # Regex patterns for complete parsing self.tool_call_complete_regex = re.compile( @@ -77,6 +86,10 @@ class DeepSeekV32ToolParser(ToolParser): r'<|DSML|parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>(.*?)', re.DOTALL, ) + self.invoke_start_regex = re.compile(r'<|DSML|invoke\s+name="([^"]+)"\s*>') + self.parameter_start_regex = re.compile( + r'<|DSML|parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>' + ) if not self.model_tokenizer: raise ValueError( @@ -117,7 +130,7 @@ class DeepSeekV32ToolParser(ToolParser): @staticmethod def _repair_param_dict( param_dict: dict[str, Any], - param_config: dict[str, dict], + param_config: dict[str, Any], ) -> dict[str, Any]: """Unwrap single 'arguments' / 'input' wrappers when the wrapper is not part of the requested tool schema and the wrapped object @@ -154,6 +167,15 @@ class DeepSeekV32ToolParser(ToolParser): converted[name] = coerce_to_schema_type(value, param_types) return self._repair_param_dict(converted, param_config) + def _get_param_config(self, function_name: str | None) -> dict[str, Any]: + if not function_name or not self.tools: + return {} + return find_tool_properties(self.tools, function_name) + + @staticmethod + def _json_escape_string_content(text: str) -> str: + return json.dumps(text, ensure_ascii=False)[1:-1] + def extract_tool_calls( self, model_output: str, @@ -210,67 +232,295 @@ class DeepSeekV32ToolParser(ToolParser): """Reset all streaming state.""" self.current_tool_index = 0 self._sent_content_idx = 0 + self._buffer = "" + self._in_tool_calls = False + self._active_tool_index = None + self._active_tool_name = None + self._active_param_name = None + self._active_param_string_attr = None + self._active_param_mode = None + self._active_param_parts.clear() self.prev_tool_call_arr.clear() self.streamed_args_for_tool.clear() + self._args_started.clear() - def _extract_delta_tool_calls( + def _add_tool_call_delta( self, - current_text: str, - request: ChatCompletionRequest | None, - ) -> list[DeltaToolCall]: - """Extract DeltaToolCalls from newly completed blocks. + tool_call_deltas: dict[int, DeltaToolCall], + index: int, + *, + call_id: str | None = None, + call_type: Literal["function"] | None = None, + name: str | None = None, + arguments: str | None = None, + ) -> None: + if arguments: + self.streamed_args_for_tool[index] += arguments - Tracks progress via ``current_tool_index`` so each block is - extracted exactly once across successive streaming calls. - """ - complete_invokes = self.invoke_complete_regex.findall(current_text) - delta_tool_calls: list[DeltaToolCall] = [] - - while len(complete_invokes) > self.current_tool_index: - invoke_name, invoke_body = complete_invokes[self.current_tool_index] - param_dict = self._parse_invoke_params(invoke_body) - - converted = self._convert_params_with_schema(invoke_name, param_dict) - args_json = json.dumps(converted, ensure_ascii=False) - idx = self.current_tool_index - self.current_tool_index += 1 - - self.prev_tool_call_arr.append( - {"name": invoke_name, "arguments": converted} + if index not in tool_call_deltas: + tool_call_deltas[index] = DeltaToolCall( + index=index, + id=call_id, + type=call_type, + function=DeltaFunctionCall(name=name, arguments=arguments), ) - self.streamed_args_for_tool.append(args_json) + return - delta_tool_calls.append( - DeltaToolCall( - index=idx, - id=self._generate_tool_call_id(), - function=DeltaFunctionCall( - name=invoke_name, - arguments=args_json, - ), - type="function", - ) - ) + delta = tool_call_deltas[index] + if call_id is not None: + delta.id = call_id + if call_type is not None: + delta.type = call_type + if delta.function is None: + delta.function = DeltaFunctionCall() + if name is not None: + delta.function.name = name + if arguments is not None: + delta.function.arguments = (delta.function.arguments or "") + arguments - return delta_tool_calls + def _begin_streaming_tool_call( + self, + name: str, + tool_call_deltas: dict[int, DeltaToolCall], + ) -> None: + index = self.current_tool_index + self.current_tool_index += 1 + self._active_tool_index = index + self._active_tool_name = name + self.prev_tool_call_arr.append({"name": name, "arguments": {}}) + self.streamed_args_for_tool.append("") + self._args_started.append(False) + self._add_tool_call_delta( + tool_call_deltas, + index, + call_id=self._generate_tool_call_id(), + call_type="function", + name=name, + arguments="", + ) - def _extract_content(self, current_text: str) -> str | None: - """Return unsent non-tool-call text, or None. + def _append_param_prefix( + self, + tool_call_deltas: dict[int, DeltaToolCall], + index: int, + key: str, + *, + as_string: bool, + ) -> None: + prefix = "{" if not self._args_started[index] else "," + self._args_started[index] = True + arguments = prefix + json.dumps(key, ensure_ascii=False) + ":" + if as_string: + arguments += '"' + self._add_tool_call_delta(tool_call_deltas, index, arguments=arguments) - Holds back any suffix that could be a partial start marker - so that split markers are never leaked as content. - """ - if self.tool_call_start_token not in current_text: - overlap = partial_tag_overlap(current_text, self.tool_call_start_token) - sendable_idx = len(current_text) - overlap - else: - sendable_idx = current_text.index(self.tool_call_start_token) + def _append_json_param_value( + self, + tool_call_deltas: dict[int, DeltaToolCall], + index: int, + key: str, + value: Any, + ) -> None: + self._append_param_prefix(tool_call_deltas, index, key, as_string=False) + self._add_tool_call_delta( + tool_call_deltas, + index, + arguments=json.dumps(value, ensure_ascii=False), + ) - if sendable_idx > self._sent_content_idx: - content = current_text[self._sent_content_idx : sendable_idx] - self._sent_content_idx = sendable_idx - return content - return None + def _param_types_for_name(self, name: str) -> list[str]: + param_config = self._get_param_config(self._active_tool_name) + if name in param_config and isinstance(param_config[name], dict): + return extract_types_from_schema(param_config[name]) + return ["string"] + + @staticmethod + def _can_stream_raw_param(param_types: list[str]) -> bool: + # Scalars and unions need the complete value so streaming and + # non-streaming share the same coercion fallback behavior. + return set(param_types).issubset({"object", "array"}) + + def _should_buffer_wrapper_param(self, name: str) -> bool: + if ( + self._active_tool_index is None + or self._args_started[self._active_tool_index] + ): + return False + param_config = self._get_param_config(self._active_tool_name) + return bool( + param_config and name in ("arguments", "input") and name not in param_config + ) + + def _finish_buffered_param( + self, + tool_call_deltas: dict[int, DeltaToolCall], + index: int, + ) -> None: + assert self._active_param_name is not None + assert self._active_param_string_attr is not None + raw_value = "".join(self._active_param_parts) + converted = self._convert_params_with_schema( + self._active_tool_name or "", + {self._active_param_name: (raw_value, self._active_param_string_attr)}, + ) + for key, value in converted.items(): + self._append_json_param_value(tool_call_deltas, index, key, value) + + def _close_streaming_tool_call( + self, + tool_call_deltas: dict[int, DeltaToolCall], + ) -> None: + index = self._active_tool_index + if index is None: + return + + suffix = "}" if self._args_started[index] else "{}" + self._add_tool_call_delta(tool_call_deltas, index, arguments=suffix) + try: + self.prev_tool_call_arr[index] = { + "name": self._active_tool_name, + "arguments": json.loads(self.streamed_args_for_tool[index]), + } + except (json.JSONDecodeError, IndexError): + logger.exception("Failed to finalize DeepSeek DSML streaming tool call") + + self._active_tool_index = None + self._active_tool_name = None + self._active_param_name = None + self._active_param_string_attr = None + self._active_param_mode = None + self._active_param_parts.clear() + + def _process_streaming_buffer( + self, + content_parts: list[str], + tool_call_deltas: dict[int, DeltaToolCall], + ) -> None: + parameter_end_token = "" + invoke_end_token = "" + + while True: + if not self._in_tool_calls: + start_idx = self._buffer.find(self.tool_call_start_token) + if start_idx == -1: + overlap = partial_tag_overlap( + self._buffer, self.tool_call_start_token + ) + sendable_idx = len(self._buffer) - overlap + if sendable_idx > 0: + content_parts.append(self._buffer[:sendable_idx]) + self._buffer = self._buffer[sendable_idx:] + return + + if start_idx > 0: + content_parts.append(self._buffer[:start_idx]) + self._buffer = self._buffer[start_idx:] + continue + + self._buffer = self._buffer[len(self.tool_call_start_token) :] + self._in_tool_calls = True + continue + + if self._active_tool_index is None: + stripped_len = len(self._buffer) - len(self._buffer.lstrip()) + if stripped_len: + self._buffer = self._buffer[stripped_len:] + continue + + if self._buffer.startswith(self.tool_call_end_token): + self._buffer = self._buffer[len(self.tool_call_end_token) :] + self._in_tool_calls = False + continue + + match = self.invoke_start_regex.match(self._buffer) + if match is None: + return + + self._buffer = self._buffer[match.end() :] + self._begin_streaming_tool_call(match.group(1), tool_call_deltas) + continue + + index = self._active_tool_index + + if self._active_param_mode is not None: + end_pos = self._buffer.find(parameter_end_token) + if end_pos != -1: + raw_content = self._buffer[:end_pos] + self._buffer = self._buffer[end_pos + len(parameter_end_token) :] + if self._active_param_mode in ("wrapper", "buffered"): + self._active_param_parts.append(raw_content) + self._finish_buffered_param(tool_call_deltas, index) + elif self._active_param_mode == "string": + arguments = self._json_escape_string_content(raw_content) + '"' + self._add_tool_call_delta( + tool_call_deltas, index, arguments=arguments + ) + else: + self._add_tool_call_delta( + tool_call_deltas, index, arguments=raw_content + ) + + self._active_param_name = None + self._active_param_string_attr = None + self._active_param_mode = None + self._active_param_parts.clear() + continue + + overlap = partial_tag_overlap(self._buffer, parameter_end_token) + safe_len = len(self._buffer) - overlap + if safe_len > 0: + raw_content = self._buffer[:safe_len] + self._buffer = self._buffer[safe_len:] + if self._active_param_mode in ("wrapper", "buffered"): + self._active_param_parts.append(raw_content) + elif self._active_param_mode == "string": + self._add_tool_call_delta( + tool_call_deltas, + index, + arguments=self._json_escape_string_content(raw_content), + ) + else: + self._add_tool_call_delta( + tool_call_deltas, index, arguments=raw_content + ) + return + + stripped_len = len(self._buffer) - len(self._buffer.lstrip()) + if stripped_len: + self._buffer = self._buffer[stripped_len:] + continue + + if self._buffer.startswith(invoke_end_token): + self._buffer = self._buffer[len(invoke_end_token) :] + self._close_streaming_tool_call(tool_call_deltas) + continue + + match = self.parameter_start_regex.match(self._buffer) + if match is None: + return + + self._buffer = self._buffer[match.end() :] + name = match.group(1) + string_attr = match.group(2) + self._active_param_name = name + self._active_param_string_attr = string_attr + + if self._should_buffer_wrapper_param(name): + self._active_param_mode = "wrapper" + continue + + if string_attr == "true": + self._append_param_prefix(tool_call_deltas, index, name, as_string=True) + self._active_param_mode = "string" + continue + + param_types = self._param_types_for_name(name) + if not self._can_stream_raw_param(param_types): + self._active_param_mode = "buffered" + continue + + self._append_param_prefix(tool_call_deltas, index, name, as_string=False) + self._active_param_mode = "raw" def extract_tool_calls_streaming( self, @@ -284,20 +534,24 @@ class DeepSeekV32ToolParser(ToolParser): ) -> DeltaMessage | None: """Extract tool calls from streaming model output. - Uses a buffer-until-complete-invoke strategy: tokens are buffered - until a complete invoke block is available, then parsed and emitted - in one shot. + Buffers DSML markup while streaming tool-call metadata and argument + JSON fragments as soon as they are complete enough to be valid deltas. """ # First chunk of a new stream — reset state from prior request. if not previous_text: self._reset_streaming_state() - content = self._extract_content(current_text) - delta_tool_calls = self._extract_delta_tool_calls(current_text, request) + self._buffer += delta_text + content_parts: list[str] = [] + tool_call_deltas: dict[int, DeltaToolCall] = {} + self._process_streaming_buffer(content_parts, tool_call_deltas) - if delta_tool_calls or content: - return DeltaMessage(content=content, tool_calls=delta_tool_calls) + if content_parts or tool_call_deltas: + content = "".join(content_parts) or None + return DeltaMessage( + content=content, tool_calls=list(tool_call_deltas.values()) + ) # Empty delta with token ids means EOS or closing tag; return # non-None so the serving framework can finalize finish_reason.