[Bugfix] Stream DeepSeek DSML tool-call argument deltas incrementally (#42879)
Signed-off-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: QwertyJack <7554089+QwertyJack@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
QwertyJack
Chauncey
parent
2a781756a1
commit
b372ad3e90
@@ -10,6 +10,7 @@ import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from openai.types.responses.function_tool import FunctionTool
|
||||
|
||||
from tests.tool_parsers.utils import run_tool_extraction_streaming
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
@@ -718,6 +719,81 @@ class TestExtractToolCallsStreaming:
|
||||
"patches": [{"op": "replace", "path": "/schedule", "value": "quiet"}],
|
||||
}
|
||||
|
||||
def test_responses_function_tool_schema_in_streaming(self):
|
||||
"""Responses API FunctionTool schemas must drive streaming conversion."""
|
||||
tool = FunctionTool(
|
||||
type="function",
|
||||
name="toggle",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"enabled": {"type": "boolean"},
|
||||
"count": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{FC_START}\n"
|
||||
f'{INV_START}toggle">\n'
|
||||
f'{PARAM_START}enabled" string="false">true{PARAM_END}\n'
|
||||
f'{PARAM_START}count" string="false">42{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{FC_END}"
|
||||
)
|
||||
|
||||
deltas = self._stream(parser, full_text)
|
||||
args = json.loads(self._reconstruct_args(deltas))
|
||||
|
||||
assert args == {"enabled": True, "count": 42}
|
||||
assert isinstance(args["enabled"], bool)
|
||||
assert isinstance(args["count"], int)
|
||||
|
||||
def test_streaming_matches_non_streaming_conversion_fallbacks(self):
|
||||
"""Streaming must reuse conversion fallback semantics for string=false."""
|
||||
tool = ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="coerce",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"union_value": {"type": ["null", "string"]},
|
||||
"bad_int": {"type": "integer"},
|
||||
"nullable_string": {"type": ["null", "string"]},
|
||||
"null_string": {"type": "string"},
|
||||
"whole_number": {"type": "number"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{FC_START}\n"
|
||||
f'{INV_START}coerce">\n'
|
||||
f'{PARAM_START}union_value" string="false">hello{PARAM_END}\n'
|
||||
f'{PARAM_START}bad_int" string="false">abc{PARAM_END}\n'
|
||||
f'{PARAM_START}nullable_string" string="false">null{PARAM_END}\n'
|
||||
f'{PARAM_START}null_string" string="false">null{PARAM_END}\n'
|
||||
f'{PARAM_START}whole_number" string="false">3.0{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{FC_END}"
|
||||
)
|
||||
|
||||
non_stream = parser.extract_tool_calls(full_text, None)
|
||||
non_stream_args = json.loads(non_stream.tool_calls[0].function.arguments)
|
||||
|
||||
deltas = self._stream(parser, full_text)
|
||||
stream_args = json.loads(self._reconstruct_args(deltas))
|
||||
|
||||
assert stream_args == non_stream_args
|
||||
assert stream_args == {
|
||||
"union_value": "hello",
|
||||
"bad_int": "abc",
|
||||
"nullable_string": None,
|
||||
"null_string": "null",
|
||||
"whole_number": 3,
|
||||
}
|
||||
|
||||
def test_multiple_tools_streaming(self, parser):
|
||||
full_text = (
|
||||
f"{FC_START}\n"
|
||||
@@ -890,8 +966,8 @@ class TestExtractToolCallsStreaming:
|
||||
assert json.loads(self._reconstruct_args(deltas, tool_index=0)) == {"p": "v1"}
|
||||
assert json.loads(self._reconstruct_args(deltas, tool_index=1)) == {"q": "v2"}
|
||||
|
||||
def test_no_emission_while_incomplete(self, parser):
|
||||
"""No tool calls should be emitted until an invoke block completes."""
|
||||
def test_emits_arguments_before_invoke_completes(self, parser):
|
||||
"""Argument deltas should stream before the invoke block closes."""
|
||||
# Stream only a partial invoke (no closing tag)
|
||||
partial_text = (
|
||||
f"{FC_START}\n"
|
||||
@@ -899,8 +975,13 @@ class TestExtractToolCallsStreaming:
|
||||
f'{PARAM_START}k" string="true">val{PARAM_END}\n'
|
||||
)
|
||||
deltas = self._stream(parser, partial_text)
|
||||
# Should have no tool call deltas yet
|
||||
assert all(not d.tool_calls for d in deltas)
|
||||
arg_chunks = [
|
||||
tc.function.arguments
|
||||
for delta in deltas
|
||||
for tc in delta.tool_calls or []
|
||||
if tc.function and tc.function.arguments is not None
|
||||
]
|
||||
assert "".join(arg_chunks) == '{"k":"val"'
|
||||
|
||||
def test_no_marker_leak_chunked(self, parser):
|
||||
"""Chunked streaming must NOT leak DSML start-marker fragments
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
|
||||
@@ -164,11 +165,57 @@ def test_streaming_extracts_complete_invokes():
|
||||
for delta in deltas
|
||||
if delta.tool_calls
|
||||
for tool_call in delta.tool_calls
|
||||
if tool_call.function.name
|
||||
]
|
||||
assert names == ["search"]
|
||||
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
|
||||
|
||||
|
||||
def test_streaming_emits_incremental_argument_chunks():
|
||||
tool = ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="plan_trip",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"days": {"type": "integer"},
|
||||
"flexible": {"type": "boolean"},
|
||||
"cities": {"type": "array", "items": {"type": "string"}},
|
||||
"notes": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{TC_START}\n"
|
||||
f'{INV_START}plan_trip">\n'
|
||||
f'{PARAM_START}days" string="false">3{PARAM_END}\n'
|
||||
f'{PARAM_START}flexible" string="false">false{PARAM_END}\n'
|
||||
f'{PARAM_START}cities" string="false">'
|
||||
f'["Beijing","Shanghai","Tokyo","New York"]{PARAM_END}\n'
|
||||
f'{PARAM_START}notes" string="true">靠窗座位{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{TC_END}"
|
||||
)
|
||||
|
||||
deltas = stream(parser, full_text, chunk_size=4)
|
||||
arg_chunks = [
|
||||
tool_call.function.arguments
|
||||
for delta in deltas
|
||||
for tool_call in delta.tool_calls or []
|
||||
if tool_call.function and tool_call.function.arguments is not None
|
||||
]
|
||||
|
||||
assert len([chunk for chunk in arg_chunks if chunk]) > 2
|
||||
assert json.loads("".join(arg_chunks)) == {
|
||||
"days": 3,
|
||||
"flexible": False,
|
||||
"cities": ["Beijing", "Shanghai", "Tokyo", "New York"],
|
||||
"notes": "靠窗座位",
|
||||
}
|
||||
|
||||
|
||||
def test_get_vllm_registry_structural_tag_returns_structural_tag(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user