[Bugfix] Stream DeepSeek DSML tool-call argument deltas incrementally (#42879)
Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
QwertyJack
Chauncey
parent
2a781756a1
commit
b372ad3e90
@@ -10,6 +10,7 @@ import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from openai.types.responses.function_tool import FunctionTool
|
||||
|
||||
from tests.tool_parsers.utils import run_tool_extraction_streaming
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -718,6 +719,81 @@ class TestExtractToolCallsStreaming:
|
||||
"patches": [{"op": "replace", "path": "/schedule", "value": "quiet"}],
|
||||
}
|
||||
|
||||
def test_responses_function_tool_schema_in_streaming(self):
|
||||
"""Responses API FunctionTool schemas must drive streaming conversion."""
|
||||
tool = FunctionTool(
|
||||
type="function",
|
||||
name="toggle",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"count": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{FC_START}\n"
|
||||
f'{INV_START}toggle">\n'
|
||||
f'{PARAM_START}enabled" string="false">true{PARAM_END}\n'
|
||||
f'{PARAM_START}count" string="false">42{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{FC_END}"
|
||||
)
|
||||
|
||||
deltas = self._stream(parser, full_text)
|
||||
args = json.loads(self._reconstruct_args(deltas))
|
||||
|
||||
assert args == {"enabled": True, "count": 42}
|
||||
assert isinstance(args["enabled"], bool)
|
||||
assert isinstance(args["count"], int)
|
||||
|
||||
def test_streaming_matches_non_streaming_conversion_fallbacks(self):
|
||||
"""Streaming must reuse conversion fallback semantics for string=false."""
|
||||
tool = ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="coerce",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"union_value": {"type": ["null", "string"]},
|
||||
"bad_int": {"type": "integer"},
|
||||
"nullable_string": {"type": ["null", "string"]},
|
||||
"null_string": {"type": "string"},
|
||||
"whole_number": {"type": "number"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{FC_START}\n"
|
||||
f'{INV_START}coerce">\n'
|
||||
f'{PARAM_START}union_value" string="false">hello{PARAM_END}\n'
|
||||
f'{PARAM_START}bad_int" string="false">abc{PARAM_END}\n'
|
||||
f'{PARAM_START}nullable_string" string="false">null{PARAM_END}\n'
|
||||
f'{PARAM_START}null_string" string="false">null{PARAM_END}\n'
|
||||
f'{PARAM_START}whole_number" string="false">3.0{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{FC_END}"
|
||||
)
|
||||
|
||||
non_stream = parser.extract_tool_calls(full_text, None)
|
||||
non_stream_args = json.loads(non_stream.tool_calls[0].function.arguments)
|
||||
|
||||
deltas = self._stream(parser, full_text)
|
||||
stream_args = json.loads(self._reconstruct_args(deltas))
|
||||
|
||||
assert stream_args == non_stream_args
|
||||
assert stream_args == {
|
||||
"union_value": "hello",
|
||||
"bad_int": "abc",
|
||||
"nullable_string": None,
|
||||
"null_string": "null",
|
||||
"whole_number": 3,
|
||||
}
|
||||
|
||||
def test_multiple_tools_streaming(self, parser):
|
||||
full_text = (
|
||||
f"{FC_START}\n"
|
||||
@@ -890,8 +966,8 @@ class TestExtractToolCallsStreaming:
|
||||
assert json.loads(self._reconstruct_args(deltas, tool_index=0)) == {"p": "v1"}
|
||||
assert json.loads(self._reconstruct_args(deltas, tool_index=1)) == {"q": "v2"}
|
||||
|
||||
def test_no_emission_while_incomplete(self, parser):
|
||||
"""No tool calls should be emitted until an invoke block completes."""
|
||||
def test_emits_arguments_before_invoke_completes(self, parser):
|
||||
"""Argument deltas should stream before the invoke block closes."""
|
||||
# Stream only a partial invoke (no closing tag)
|
||||
partial_text = (
|
||||
f"{FC_START}\n"
|
||||
@@ -899,8 +975,13 @@ class TestExtractToolCallsStreaming:
|
||||
f'{PARAM_START}k" string="true">val{PARAM_END}\n'
|
||||
)
|
||||
deltas = self._stream(parser, partial_text)
|
||||
# Should have no tool call deltas yet
|
||||
assert all(not d.tool_calls for d in deltas)
|
||||
arg_chunks = [
|
||||
tc.function.arguments
|
||||
for delta in deltas
|
||||
for tc in delta.tool_calls or []
|
||||
if tc.function and tc.function.arguments is not None
|
||||
]
|
||||
assert "".join(arg_chunks) == '{"k":"val"'
|
||||
|
||||
def test_no_marker_leak_chunked(self, parser):
|
||||
"""Chunked streaming must NOT leak DSML start-marker fragments
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
|
||||
@@ -164,11 +165,57 @@ def test_streaming_extracts_complete_invokes():
|
||||
for delta in deltas
|
||||
if delta.tool_calls
|
||||
for tool_call in delta.tool_calls
|
||||
if tool_call.function.name
|
||||
]
|
||||
assert names == ["search"]
|
||||
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
|
||||
|
||||
|
||||
def test_streaming_emits_incremental_argument_chunks():
|
||||
tool = ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="plan_trip",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"days": {"type": "integer"},
|
||||
"flexible": {"type": "boolean"},
|
||||
"cities": {"type": "array", "items": {"type": "string"}},
|
||||
"notes": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{TC_START}\n"
|
||||
f'{INV_START}plan_trip">\n'
|
||||
f'{PARAM_START}days" string="false">3{PARAM_END}\n'
|
||||
f'{PARAM_START}flexible" string="false">false{PARAM_END}\n'
|
||||
f'{PARAM_START}cities" string="false">'
|
||||
f'["Beijing","Shanghai","Tokyo","New York"]{PARAM_END}\n'
|
||||
f'{PARAM_START}notes" string="true">靠窗座位{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{TC_END}"
|
||||
)
|
||||
|
||||
deltas = stream(parser, full_text, chunk_size=4)
|
||||
arg_chunks = [
|
||||
tool_call.function.arguments
|
||||
for delta in deltas
|
||||
for tool_call in delta.tool_calls or []
|
||||
if tool_call.function and tool_call.function.arguments is not None
|
||||
]
|
||||
|
||||
assert len([chunk for chunk in arg_chunks if chunk]) > 2
|
||||
assert json.loads("".join(arg_chunks)) == {
|
||||
"days": 3,
|
||||
"flexible": False,
|
||||
"cities": ["Beijing", "Shanghai", "Tokyo", "New York"],
|
||||
"notes": "靠窗座位",
|
||||
}
|
||||
|
||||
|
||||
def test_get_vllm_registry_structural_tag_returns_structural_tag(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
) -> None:
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
from typing import Any, Literal
|
||||
|
||||
import regex as re
|
||||
|
||||
@@ -62,6 +62,15 @@ class DeepSeekV32ToolParser(ToolParser):
|
||||
# Streaming state
|
||||
self.current_tool_index: int = 0
|
||||
self._sent_content_idx: int = 0
|
||||
self._buffer: str = ""
|
||||
self._in_tool_calls: bool = False
|
||||
self._active_tool_index: int | None = None
|
||||
self._active_tool_name: str | None = None
|
||||
self._active_param_name: str | None = None
|
||||
self._active_param_string_attr: str | None = None
|
||||
self._active_param_mode: str | None = None
|
||||
self._active_param_parts: list[str] = []
|
||||
self._args_started: list[bool] = []
|
||||
|
||||
# Regex patterns for complete parsing
|
||||
self.tool_call_complete_regex = re.compile(
|
||||
@@ -77,6 +86,10 @@ class DeepSeekV32ToolParser(ToolParser):
|
||||
r'<|DSML|parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>(.*?)</|DSML|parameter>',
|
||||
re.DOTALL,
|
||||
)
|
||||
self.invoke_start_regex = re.compile(r'<|DSML|invoke\s+name="([^"]+)"\s*>')
|
||||
self.parameter_start_regex = re.compile(
|
||||
r'<|DSML|parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>'
|
||||
)
|
||||
|
||||
if not self.model_tokenizer:
|
||||
raise ValueError(
|
||||
@@ -117,7 +130,7 @@ class DeepSeekV32ToolParser(ToolParser):
|
||||
@staticmethod
|
||||
def _repair_param_dict(
|
||||
param_dict: dict[str, Any],
|
||||
param_config: dict[str, dict],
|
||||
param_config: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Unwrap single 'arguments' / 'input' wrappers when the wrapper
|
||||
is not part of the requested tool schema and the wrapped object
|
||||
@@ -154,6 +167,15 @@ class DeepSeekV32ToolParser(ToolParser):
|
||||
converted[name] = coerce_to_schema_type(value, param_types)
|
||||
return self._repair_param_dict(converted, param_config)
|
||||
|
||||
def _get_param_config(self, function_name: str | None) -> dict[str, Any]:
|
||||
if not function_name or not self.tools:
|
||||
return {}
|
||||
return find_tool_properties(self.tools, function_name)
|
||||
|
||||
@staticmethod
|
||||
def _json_escape_string_content(text: str) -> str:
|
||||
return json.dumps(text, ensure_ascii=False)[1:-1]
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
@@ -210,67 +232,295 @@ class DeepSeekV32ToolParser(ToolParser):
|
||||
"""Reset all streaming state."""
|
||||
self.current_tool_index = 0
|
||||
self._sent_content_idx = 0
|
||||
self._buffer = ""
|
||||
self._in_tool_calls = False
|
||||
self._active_tool_index = None
|
||||
self._active_tool_name = None
|
||||
self._active_param_name = None
|
||||
self._active_param_string_attr = None
|
||||
self._active_param_mode = None
|
||||
self._active_param_parts.clear()
|
||||
self.prev_tool_call_arr.clear()
|
||||
self.streamed_args_for_tool.clear()
|
||||
self._args_started.clear()
|
||||
|
||||
def _extract_delta_tool_calls(
|
||||
def _add_tool_call_delta(
|
||||
self,
|
||||
current_text: str,
|
||||
request: ChatCompletionRequest | None,
|
||||
) -> list[DeltaToolCall]:
|
||||
"""Extract DeltaToolCalls from newly completed <invoke> blocks.
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
index: int,
|
||||
*,
|
||||
call_id: str | None = None,
|
||||
call_type: Literal["function"] | None = None,
|
||||
name: str | None = None,
|
||||
arguments: str | None = None,
|
||||
) -> None:
|
||||
if arguments:
|
||||
self.streamed_args_for_tool[index] += arguments
|
||||
|
||||
Tracks progress via ``current_tool_index`` so each block is
|
||||
extracted exactly once across successive streaming calls.
|
||||
"""
|
||||
complete_invokes = self.invoke_complete_regex.findall(current_text)
|
||||
delta_tool_calls: list[DeltaToolCall] = []
|
||||
|
||||
while len(complete_invokes) > self.current_tool_index:
|
||||
invoke_name, invoke_body = complete_invokes[self.current_tool_index]
|
||||
param_dict = self._parse_invoke_params(invoke_body)
|
||||
|
||||
converted = self._convert_params_with_schema(invoke_name, param_dict)
|
||||
args_json = json.dumps(converted, ensure_ascii=False)
|
||||
idx = self.current_tool_index
|
||||
self.current_tool_index += 1
|
||||
|
||||
self.prev_tool_call_arr.append(
|
||||
{"name": invoke_name, "arguments": converted}
|
||||
if index not in tool_call_deltas:
|
||||
tool_call_deltas[index] = DeltaToolCall(
|
||||
index=index,
|
||||
id=call_id,
|
||||
type=call_type,
|
||||
function=DeltaFunctionCall(name=name, arguments=arguments),
|
||||
)
|
||||
self.streamed_args_for_tool.append(args_json)
|
||||
return
|
||||
|
||||
delta_tool_calls.append(
|
||||
DeltaToolCall(
|
||||
index=idx,
|
||||
id=self._generate_tool_call_id(),
|
||||
function=DeltaFunctionCall(
|
||||
name=invoke_name,
|
||||
arguments=args_json,
|
||||
),
|
||||
type="function",
|
||||
)
|
||||
)
|
||||
delta = tool_call_deltas[index]
|
||||
if call_id is not None:
|
||||
delta.id = call_id
|
||||
if call_type is not None:
|
||||
delta.type = call_type
|
||||
if delta.function is None:
|
||||
delta.function = DeltaFunctionCall()
|
||||
if name is not None:
|
||||
delta.function.name = name
|
||||
if arguments is not None:
|
||||
delta.function.arguments = (delta.function.arguments or "") + arguments
|
||||
|
||||
return delta_tool_calls
|
||||
def _begin_streaming_tool_call(
|
||||
self,
|
||||
name: str,
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
) -> None:
|
||||
index = self.current_tool_index
|
||||
self.current_tool_index += 1
|
||||
self._active_tool_index = index
|
||||
self._active_tool_name = name
|
||||
self.prev_tool_call_arr.append({"name": name, "arguments": {}})
|
||||
self.streamed_args_for_tool.append("")
|
||||
self._args_started.append(False)
|
||||
self._add_tool_call_delta(
|
||||
tool_call_deltas,
|
||||
index,
|
||||
call_id=self._generate_tool_call_id(),
|
||||
call_type="function",
|
||||
name=name,
|
||||
arguments="",
|
||||
)
|
||||
|
||||
def _extract_content(self, current_text: str) -> str | None:
|
||||
"""Return unsent non-tool-call text, or None.
|
||||
def _append_param_prefix(
|
||||
self,
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
index: int,
|
||||
key: str,
|
||||
*,
|
||||
as_string: bool,
|
||||
) -> None:
|
||||
prefix = "{" if not self._args_started[index] else ","
|
||||
self._args_started[index] = True
|
||||
arguments = prefix + json.dumps(key, ensure_ascii=False) + ":"
|
||||
if as_string:
|
||||
arguments += '"'
|
||||
self._add_tool_call_delta(tool_call_deltas, index, arguments=arguments)
|
||||
|
||||
Holds back any suffix that could be a partial start marker
|
||||
so that split markers are never leaked as content.
|
||||
"""
|
||||
if self.tool_call_start_token not in current_text:
|
||||
overlap = partial_tag_overlap(current_text, self.tool_call_start_token)
|
||||
sendable_idx = len(current_text) - overlap
|
||||
else:
|
||||
sendable_idx = current_text.index(self.tool_call_start_token)
|
||||
def _append_json_param_value(
|
||||
self,
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
index: int,
|
||||
key: str,
|
||||
value: Any,
|
||||
) -> None:
|
||||
self._append_param_prefix(tool_call_deltas, index, key, as_string=False)
|
||||
self._add_tool_call_delta(
|
||||
tool_call_deltas,
|
||||
index,
|
||||
arguments=json.dumps(value, ensure_ascii=False),
|
||||
)
|
||||
|
||||
if sendable_idx > self._sent_content_idx:
|
||||
content = current_text[self._sent_content_idx : sendable_idx]
|
||||
self._sent_content_idx = sendable_idx
|
||||
return content
|
||||
return None
|
||||
def _param_types_for_name(self, name: str) -> list[str]:
|
||||
param_config = self._get_param_config(self._active_tool_name)
|
||||
if name in param_config and isinstance(param_config[name], dict):
|
||||
return extract_types_from_schema(param_config[name])
|
||||
return ["string"]
|
||||
|
||||
@staticmethod
|
||||
def _can_stream_raw_param(param_types: list[str]) -> bool:
|
||||
# Scalars and unions need the complete value so streaming and
|
||||
# non-streaming share the same coercion fallback behavior.
|
||||
return set(param_types).issubset({"object", "array"})
|
||||
|
||||
def _should_buffer_wrapper_param(self, name: str) -> bool:
|
||||
if (
|
||||
self._active_tool_index is None
|
||||
or self._args_started[self._active_tool_index]
|
||||
):
|
||||
return False
|
||||
param_config = self._get_param_config(self._active_tool_name)
|
||||
return bool(
|
||||
param_config and name in ("arguments", "input") and name not in param_config
|
||||
)
|
||||
|
||||
def _finish_buffered_param(
|
||||
self,
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
index: int,
|
||||
) -> None:
|
||||
assert self._active_param_name is not None
|
||||
assert self._active_param_string_attr is not None
|
||||
raw_value = "".join(self._active_param_parts)
|
||||
converted = self._convert_params_with_schema(
|
||||
self._active_tool_name or "",
|
||||
{self._active_param_name: (raw_value, self._active_param_string_attr)},
|
||||
)
|
||||
for key, value in converted.items():
|
||||
self._append_json_param_value(tool_call_deltas, index, key, value)
|
||||
|
||||
def _close_streaming_tool_call(
|
||||
self,
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
) -> None:
|
||||
index = self._active_tool_index
|
||||
if index is None:
|
||||
return
|
||||
|
||||
suffix = "}" if self._args_started[index] else "{}"
|
||||
self._add_tool_call_delta(tool_call_deltas, index, arguments=suffix)
|
||||
try:
|
||||
self.prev_tool_call_arr[index] = {
|
||||
"name": self._active_tool_name,
|
||||
"arguments": json.loads(self.streamed_args_for_tool[index]),
|
||||
}
|
||||
except (json.JSONDecodeError, IndexError):
|
||||
logger.exception("Failed to finalize DeepSeek DSML streaming tool call")
|
||||
|
||||
self._active_tool_index = None
|
||||
self._active_tool_name = None
|
||||
self._active_param_name = None
|
||||
self._active_param_string_attr = None
|
||||
self._active_param_mode = None
|
||||
self._active_param_parts.clear()
|
||||
|
||||
def _process_streaming_buffer(
|
||||
self,
|
||||
content_parts: list[str],
|
||||
tool_call_deltas: dict[int, DeltaToolCall],
|
||||
) -> None:
|
||||
parameter_end_token = "</|DSML|parameter>"
|
||||
invoke_end_token = "</|DSML|invoke>"
|
||||
|
||||
while True:
|
||||
if not self._in_tool_calls:
|
||||
start_idx = self._buffer.find(self.tool_call_start_token)
|
||||
if start_idx == -1:
|
||||
overlap = partial_tag_overlap(
|
||||
self._buffer, self.tool_call_start_token
|
||||
)
|
||||
sendable_idx = len(self._buffer) - overlap
|
||||
if sendable_idx > 0:
|
||||
content_parts.append(self._buffer[:sendable_idx])
|
||||
self._buffer = self._buffer[sendable_idx:]
|
||||
return
|
||||
|
||||
if start_idx > 0:
|
||||
content_parts.append(self._buffer[:start_idx])
|
||||
self._buffer = self._buffer[start_idx:]
|
||||
continue
|
||||
|
||||
self._buffer = self._buffer[len(self.tool_call_start_token) :]
|
||||
self._in_tool_calls = True
|
||||
continue
|
||||
|
||||
if self._active_tool_index is None:
|
||||
stripped_len = len(self._buffer) - len(self._buffer.lstrip())
|
||||
if stripped_len:
|
||||
self._buffer = self._buffer[stripped_len:]
|
||||
continue
|
||||
|
||||
if self._buffer.startswith(self.tool_call_end_token):
|
||||
self._buffer = self._buffer[len(self.tool_call_end_token) :]
|
||||
self._in_tool_calls = False
|
||||
continue
|
||||
|
||||
match = self.invoke_start_regex.match(self._buffer)
|
||||
if match is None:
|
||||
return
|
||||
|
||||
self._buffer = self._buffer[match.end() :]
|
||||
self._begin_streaming_tool_call(match.group(1), tool_call_deltas)
|
||||
continue
|
||||
|
||||
index = self._active_tool_index
|
||||
|
||||
if self._active_param_mode is not None:
|
||||
end_pos = self._buffer.find(parameter_end_token)
|
||||
if end_pos != -1:
|
||||
raw_content = self._buffer[:end_pos]
|
||||
self._buffer = self._buffer[end_pos + len(parameter_end_token) :]
|
||||
if self._active_param_mode in ("wrapper", "buffered"):
|
||||
self._active_param_parts.append(raw_content)
|
||||
self._finish_buffered_param(tool_call_deltas, index)
|
||||
elif self._active_param_mode == "string":
|
||||
arguments = self._json_escape_string_content(raw_content) + '"'
|
||||
self._add_tool_call_delta(
|
||||
tool_call_deltas, index, arguments=arguments
|
||||
)
|
||||
else:
|
||||
self._add_tool_call_delta(
|
||||
tool_call_deltas, index, arguments=raw_content
|
||||
)
|
||||
|
||||
self._active_param_name = None
|
||||
self._active_param_string_attr = None
|
||||
self._active_param_mode = None
|
||||
self._active_param_parts.clear()
|
||||
continue
|
||||
|
||||
overlap = partial_tag_overlap(self._buffer, parameter_end_token)
|
||||
safe_len = len(self._buffer) - overlap
|
||||
if safe_len > 0:
|
||||
raw_content = self._buffer[:safe_len]
|
||||
self._buffer = self._buffer[safe_len:]
|
||||
if self._active_param_mode in ("wrapper", "buffered"):
|
||||
self._active_param_parts.append(raw_content)
|
||||
elif self._active_param_mode == "string":
|
||||
self._add_tool_call_delta(
|
||||
tool_call_deltas,
|
||||
index,
|
||||
arguments=self._json_escape_string_content(raw_content),
|
||||
)
|
||||
else:
|
||||
self._add_tool_call_delta(
|
||||
tool_call_deltas, index, arguments=raw_content
|
||||
)
|
||||
return
|
||||
|
||||
stripped_len = len(self._buffer) - len(self._buffer.lstrip())
|
||||
if stripped_len:
|
||||
self._buffer = self._buffer[stripped_len:]
|
||||
continue
|
||||
|
||||
if self._buffer.startswith(invoke_end_token):
|
||||
self._buffer = self._buffer[len(invoke_end_token) :]
|
||||
self._close_streaming_tool_call(tool_call_deltas)
|
||||
continue
|
||||
|
||||
match = self.parameter_start_regex.match(self._buffer)
|
||||
if match is None:
|
||||
return
|
||||
|
||||
self._buffer = self._buffer[match.end() :]
|
||||
name = match.group(1)
|
||||
string_attr = match.group(2)
|
||||
self._active_param_name = name
|
||||
self._active_param_string_attr = string_attr
|
||||
|
||||
if self._should_buffer_wrapper_param(name):
|
||||
self._active_param_mode = "wrapper"
|
||||
continue
|
||||
|
||||
if string_attr == "true":
|
||||
self._append_param_prefix(tool_call_deltas, index, name, as_string=True)
|
||||
self._active_param_mode = "string"
|
||||
continue
|
||||
|
||||
param_types = self._param_types_for_name(name)
|
||||
if not self._can_stream_raw_param(param_types):
|
||||
self._active_param_mode = "buffered"
|
||||
continue
|
||||
|
||||
self._append_param_prefix(tool_call_deltas, index, name, as_string=False)
|
||||
self._active_param_mode = "raw"
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
@@ -284,20 +534,24 @@ class DeepSeekV32ToolParser(ToolParser):
|
||||
) -> DeltaMessage | None:
|
||||
"""Extract tool calls from streaming model output.
|
||||
|
||||
Uses a buffer-until-complete-invoke strategy: tokens are buffered
|
||||
until a complete invoke block is available, then parsed and emitted
|
||||
in one shot.
|
||||
Buffers DSML markup while streaming tool-call metadata and argument
|
||||
JSON fragments as soon as they are complete enough to be valid deltas.
|
||||
"""
|
||||
|
||||
# First chunk of a new stream — reset state from prior request.
|
||||
if not previous_text:
|
||||
self._reset_streaming_state()
|
||||
|
||||
content = self._extract_content(current_text)
|
||||
delta_tool_calls = self._extract_delta_tool_calls(current_text, request)
|
||||
self._buffer += delta_text
|
||||
content_parts: list[str] = []
|
||||
tool_call_deltas: dict[int, DeltaToolCall] = {}
|
||||
self._process_streaming_buffer(content_parts, tool_call_deltas)
|
||||
|
||||
if delta_tool_calls or content:
|
||||
return DeltaMessage(content=content, tool_calls=delta_tool_calls)
|
||||
if content_parts or tool_call_deltas:
|
||||
content = "".join(content_parts) or None
|
||||
return DeltaMessage(
|
||||
content=content, tool_calls=list(tool_call_deltas.values())
|
||||
)
|
||||
|
||||
# Empty delta with token ids means EOS or closing tag; return
|
||||
# non-None so the serving framework can finalize finish_reason.
|
||||
|
||||
Reference in New Issue
Block a user