[Bugfix][ToolParser] Stream every tool call when one delta completes several (#60354)

Signed-off-by: Junyi Yao <j.yao@wustl.edu>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Junyi Yao
2026-10-08 01:07:35 +00:00
committed by GitHub
co-authored by sfeng33
parent 2dd887426e
commit fdfc171a74
4 changed files with 338 additions and 202 deletions
+110 -42
View File
@@ -295,45 +295,61 @@ def test_structured_outputs_json_without_parameters(
)
def _collect_required_tool_streaming_json(output_json: str, delta_len: int) -> str:
def _stream_required_tool_calls(
output_json: str,
deltas: list[str],
tool_call_id_type: str = "random",
tool_call_idx: int | None = None,
) -> list[dict]:
"""Feed ``deltas`` (which concatenate to ``output_json``) through the
required-tool streaming helper and rebuild the calls per array index,
checking that every index gets exactly one id/name chunk first."""
assert "".join(deltas) == output_json
previous_text = ""
function_name_returned = False
messages = []
for i in range(0, len(output_json), delta_len):
delta_text = output_json[i : i + delta_len]
calls: dict[int, dict] = {}
for delta_text in deltas:
current_text = previous_text + delta_text
delta_message, function_name_returned = extract_required_tool_call_streaming(
delta_message, _ = extract_required_tool_call_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
function_name_returned=function_name_returned,
tool_call_idx=None,
tool_call_id_type="random",
tool_call_idx=tool_call_idx,
tool_call_id_type=tool_call_id_type,
)
if delta_message:
messages.append(delta_message)
previous_text = current_text
if delta_message is None:
continue
assert delta_message.tool_calls
for tc in delta_message.tool_calls:
assert tc.function is not None
if tc.id is not None:
assert tc.index not in calls, "id/name sent twice for one index"
assert tc.type == "function"
assert tc.function.name
calls[tc.index] = {
"id": tc.id,
"name": tc.function.name,
"arguments": tc.function.arguments or "",
}
else:
assert tc.index in calls, "arguments before id/name"
assert tc.function.name is None
assert tc.function.arguments
calls[tc.index]["arguments"] += tc.function.arguments
assert list(calls) == list(range(len(calls)))
return [calls[i] for i in range(len(calls))]
assert len(messages) > 0
combined_messages = "["
for message in messages:
fn = message.tool_calls[0].function
assert fn is not None
if fn.name:
if len(combined_messages) > 1:
combined_messages += "},"
def _fixed_len_deltas(text: str, delta_len: int) -> list[str]:
return [text[i : i + delta_len] for i in range(0, len(text), delta_len)]
combined_messages += (
'{"name": "' + fn.name + '", "parameters": ' + (fn.arguments or "")
)
else:
combined_messages += fn.arguments or ""
combined_messages += "}]"
return combined_messages
def _assert_streams_to(output: list[dict], deltas: list[str]) -> None:
calls = _stream_required_tool_calls(json.dumps(output), deltas)
assert [
{"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls
] == output
for call, expected in zip(calls, output):
assert call["arguments"] == json.dumps(expected["parameters"])
@pytest.mark.parametrize("output", VALID_TOOLS)
@@ -343,11 +359,7 @@ def test_streaming_output_valid(output, empty_params, delta_len):
output = deepcopy(output)
if empty_params:
output = [{"name": o["name"], "parameters": {}} for o in output]
output_json = json.dumps(output)
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len)
assert json.loads(combined_messages) == output
assert json.dumps(json.loads(combined_messages)) == output_json
_assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len))
@pytest.mark.parametrize(
@@ -358,22 +370,78 @@ def test_streaming_output_valid(output, empty_params, delta_len):
"a }} b",
'a " } b',
r"a \ } b",
# Text that looks like the end of one call and the start of another.
'"}}, {"name": "get_forecast", "parameters": {"city": "x',
],
)
@pytest.mark.parametrize("delta_len", [1, 2, 3, 8, 9999])
def test_streaming_output_valid_with_braces_in_string(city, delta_len):
output = [{"name": "get_current_weather", "parameters": {"city": city}}]
output_json = json.dumps(output)
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len)
assert json.loads(combined_messages) == output
assert json.dumps(json.loads(combined_messages)) == output_json
output = [
{"name": "get_current_weather", "parameters": {"city": city}},
{"name": "get_forecast", "parameters": {"city": city, "days": 1}},
]
_assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len))
def test_streaming_output_valid_with_trailing_extra_data():
output = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}]
output_json = json.dumps(output) + "\nDONE"
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len=3)
assert json.loads(combined_messages) == output
calls = _stream_required_tool_calls(output_json, _fixed_len_deltas(output_json, 3))
assert [
{"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls
] == output
TWO_CALLS = [
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 3}},
]
def _split_at(text: str, cuts: list[int]) -> list[str]:
bounds = [0, *cuts, len(text)]
return [text[a:b] for a, b in zip(bounds, bounds[1:])]
def test_streaming_multiple_calls_completed_in_one_delta():
"""Regression test for #60351: a delta that completes more than one
call must stream every call, not only the last one."""
text = json.dumps(TWO_CALLS)
_assert_streams_to(TWO_CALLS, [text[:1], text[1:]])
_assert_streams_to(TWO_CALLS, [text])
def test_streaming_delta_crossing_call_boundary():
"""A delta that closes one call and opens the next must attribute the
closing argument text to the first call and still announce the next."""
text = json.dumps(TWO_CALLS)
boundary = text.index("}, {") + 1
name_end = text.index('"get_forecast"') + 6
for cuts in (
[boundary - 1, name_end], # '}, {"name": "get_f' in one delta
[boundary - 2, boundary + 2], # '"}}, {' then the rest
[boundary + 2], # opening brace of the next call ends a delta
):
_assert_streams_to(TWO_CALLS, _split_at(text, cuts))
def test_streaming_invalid_name_string_is_never_announced():
"""A closed name that is not a valid JSON string must not be streamed
(the non-streaming path rejects such an array as well)."""
text = '[{"name": "bad\\x", "parameters": {"city": "x"}}]'
assert _stream_required_tool_calls(text, _fixed_len_deltas(text, 4)) == []
def test_streaming_tool_call_idx_increments_per_started_call():
"""kimi_k2 ids number the calls started in one delta consecutively."""
text = json.dumps(TWO_CALLS)
calls = _stream_required_tool_calls(
text, [text], tool_call_id_type="kimi_k2", tool_call_idx=5
)
assert [c["id"] for c in calls] == [
"functions.get_current_weather:5",
"functions.get_forecast:6",
]
FUNCTION_TOOL = FunctionTool(
+7 -8
View File
@@ -741,8 +741,6 @@ class DelegatingParser(Parser):
extract_required_tool_call_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
function_name_returned=function_name_returned,
tool_call_idx=tool_call_idx,
tool_call_id_type=tool_call_id_type,
)
@@ -952,12 +950,13 @@ class DelegatingParser(Parser):
elif not delta_message.reasoning:
delta_message.reasoning = reasoning_from_this_batch
if (
delta_message
and delta_message.tool_calls
and delta_message.tool_calls[0].id is not None
):
state.history_tool_call_cnt += 1
if delta_message and delta_message.tool_calls:
# A delta may start several tool calls at once (e.g. the
# "required" path when one delta completes more than one
# call), so count every id rather than only the first.
state.history_tool_call_cnt += sum(
tc.id is not None for tc in delta_message.tool_calls
)
# No phase active: pass through as content.
# Skip when reasoning just ended in this delta — the engine already
-6
View File
@@ -640,12 +640,6 @@ class OnlineDerenderer:
if tc.index < len(last_tool_call_ids):
# Pin: reuse the ID already recorded for this index
# rather than one a from scratch replay regenerated.
# Real trigger not just defensive with
# tool_choice="required",
# extract_required_tool_call_streaming resets
# function_name_returned to False whenever the
# partial JSON transiently fails to parse which
# re-emits id+name for the same index on replay.
tc.id = last_tool_call_ids[tc.index]
else:
last_tool_call_ids.append(tc.id)
+221 -146
View File
@@ -2,19 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass
from typing import TYPE_CHECKING
import partial_json_parser
import regex as re
from partial_json_parser.core.options import Allow
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
)
from vllm.tool_parsers.utils import partial_json_loads
from vllm.utils.mistral import is_mistral_tokenizer
if TYPE_CHECKING:
@@ -23,63 +19,6 @@ else:
TokenizerLike = object
def _bracket_level_state(
s: str, opening: str = "{", closing: str = "}"
) -> tuple[int, bool, bool]:
level = 0
in_string = False
escaped = False
for char in s:
if escaped:
escaped = False
continue
if in_string and char == "\\":
escaped = True
continue
if char == '"':
in_string = not in_string
continue
if not in_string:
if char == opening:
level += 1
elif char == closing:
level -= 1
return level, in_string, escaped
def filter_delta_text(
delta_text: str,
previous_text: str,
) -> tuple[str, bool]:
"""Trim trailing tool-list delimiters from required-tool streaming text."""
bracket_level, in_string, escaped = _bracket_level_state(previous_text)
updated_delta = ""
passed_zero = False
for char in delta_text:
if escaped:
escaped = False
elif in_string:
if char == "\\":
escaped = True
elif char == '"':
in_string = False
elif char == '"':
in_string = True
elif char == "{":
bracket_level += 1
passed_zero = bracket_level == 0
elif char == "}":
bracket_level -= 1
passed_zero = bracket_level == 0
if bracket_level != 0:
updated_delta += char
else:
if not in_string and char == ",":
break
return updated_delta, passed_zero
def extract_named_tool_call_streaming(
*,
delta_text: str,
@@ -124,100 +63,236 @@ def extract_named_tool_call_streaming(
)
@dataclass
class _RequiredToolCallSpan:
"""Where one element of a ``tool_choice="required"`` array sits in the text.
The model output for ``tool_choice="required"`` is a JSON array of
``{"name": ..., "parameters": ...}`` objects. ``name`` is only set once
its string value is closed and decodes as JSON; ``name_end`` is the
offset just past that closing quote. ``args_start``/``args_end`` are the
character offsets of the ``parameters`` value (``args_end`` is ``None``
while the value is still being generated).
All offsets are prefix-stable: scanning any prefix of the text yields the
same spans truncated to that prefix. That is what lets one scan of the
current text tell what an earlier, shorter text had already revealed.
"""
name: str | None = None
name_end: int | None = None
args_start: int | None = None
args_end: int | None = None
def ready_at(self, length: int) -> bool:
"""Whether the first ``length`` characters of the text already made
this call streamable: its name was complete and its parameters value
had started."""
return (
self.name is not None
and self.name_end is not None
and self.name_end <= length
and self.args_start is not None
and self.args_start < length
)
@property
def ready(self) -> bool:
return self.name is not None and self.args_start is not None
# Parse states for one element object of the required-tool-call array.
_EXPECT_KEY = 0
_EXPECT_COLON = 1
_EXPECT_VALUE = 2
_IN_STRING_VALUE = 3
_IN_CONTAINER_VALUE = 4
_IN_SCALAR_VALUE = 5
_AFTER_VALUE = 6
def _scan_required_tool_calls(text: str) -> list[_RequiredToolCallSpan]:
"""Locate every tool call of a (possibly partial) required-tool-call array.
A single string- and nesting-aware pass over ``text`` that records, for
each top-level array element, its ``name`` (once complete) and the
character span of its ``parameters`` value. Text after the closing ``]``
of the array is ignored. Only the element level of the array is
interpreted, so braces, brackets and quotes inside nested values and
strings cannot confuse it.
"""
spans: list[_RequiredToolCallSpan] = []
depth = 0 # nesting depth of ``{}`` and ``[]``; element objects sit at 2
in_string = False
escaped = False
string_start = 0
cur: _RequiredToolCallSpan | None = None
expect = _EXPECT_KEY
key: str | None = None
value_start = 0
def end_value(pos: int) -> None:
nonlocal expect
assert cur is not None
if key == "parameters":
cur.args_end = pos
elif key == "name" and expect == _IN_STRING_VALUE:
try:
cur.name = json.loads(text[value_start:pos])
cur.name_end = pos
except json.JSONDecodeError:
# Not a valid JSON string: the call can never become ready,
# which matches the non-streaming path rejecting the array.
cur.name = None
expect = _AFTER_VALUE
def start_value(pos: int, state: int) -> None:
nonlocal expect, value_start
assert cur is not None
value_start = pos
if key == "parameters":
cur.args_start = pos
expect = state
for i, ch in enumerate(text):
if in_string:
if escaped:
escaped = False
elif ch == "\\":
escaped = True
elif ch == '"':
in_string = False
if depth == 2 and cur is not None:
if expect == _EXPECT_KEY:
key = text[string_start + 1 : i]
expect = _EXPECT_COLON
elif expect == _IN_STRING_VALUE:
end_value(i + 1)
continue
if ch == '"':
in_string = True
string_start = i
if depth == 2 and cur is not None and expect == _EXPECT_VALUE:
start_value(i, _IN_STRING_VALUE)
continue
if ch in "{[":
if depth == 0:
if ch != "[":
return spans
elif depth == 1:
if ch != "{":
return spans
cur = _RequiredToolCallSpan()
spans.append(cur)
expect = _EXPECT_KEY
key = None
elif depth == 2 and cur is not None and expect == _EXPECT_VALUE:
start_value(i, _IN_CONTAINER_VALUE)
depth += 1
continue
if ch in "}]":
if depth == 2 and cur is not None and expect == _IN_SCALAR_VALUE:
end_value(i)
depth -= 1
if depth == 2 and cur is not None and expect == _IN_CONTAINER_VALUE:
end_value(i + 1)
elif depth == 1:
cur = None
elif depth <= 0:
return spans
continue
if depth != 2 or cur is None:
continue
if ch == ":":
if expect == _EXPECT_COLON:
expect = _EXPECT_VALUE
elif ch == ",":
if expect == _IN_SCALAR_VALUE:
end_value(i)
expect = _EXPECT_KEY
key = None
elif ch.isspace():
if expect == _IN_SCALAR_VALUE:
end_value(i)
elif expect == _EXPECT_VALUE:
start_value(i, _IN_SCALAR_VALUE)
return spans
def extract_required_tool_call_streaming(
*,
previous_text: str,
current_text: str | None,
delta_text: str,
function_name_returned: bool,
tool_call_idx: int | None,
tool_call_id_type: str,
) -> tuple[DeltaMessage | None, bool]:
if current_text is None or current_text == "":
# if the current text is empty, we cannot parse it
return None, function_name_returned
try:
flags = Allow.ALL
obj, _ = partial_json_loads(current_text, flags)
except (
partial_json_parser.core.exceptions.MalformedJSON,
json.JSONDecodeError,
):
obj = None
"""Stream the tool calls of a ``tool_choice="required"`` JSON array.
# check if the current text is a valid array
# containing a partial tool calling object
# if not repeat
if obj is None or not isinstance(obj, list) or not len(obj) > 0:
function_name_returned = False
delta_message = None
else:
_, finishes_previous_tool = filter_delta_text(delta_text, previous_text)
# take the last tool call from the generated list
current_tool_call = obj[-1]
``previous_text`` must be a prefix of ``current_text`` (the accumulated
text before and after this delta). Every array element that has something
not yet sent is emitted, in array order, so a delta may carry several
``DeltaToolCall`` entries: the first chunk of a call carries its id, name
and the arguments generated so far; later chunks carry only new argument
text. What was already sent is read off the same scan, since the spans
are prefix-stable, so no state is needed beyond the two texts.
# once parameters have been generated the name is complete as well
if not finishes_previous_tool and (
"name" not in current_tool_call or "parameters" not in current_tool_call
):
function_name_returned = False
delta_message = None
else:
if not function_name_returned:
# get partly generated arguments from the latest tool call
param_match = re.search(
r'.*"parameters":\s*(.*)', current_text, re.DOTALL
)
if param_match:
arguments = param_match.group(1)
arguments_prefix = current_text[: param_match.start(1)]
arguments, _ = filter_delta_text(arguments, arguments_prefix)
else:
arguments = ""
Returns the delta (``None`` when nothing is new) and whether at least one
call has been announced so far.
"""
if not current_text:
return None, False
# if this iteration finishes a previous tool call but a
# new incomplete tool is already generated, take the
# previous from the list
if finishes_previous_tool and "parameters" not in current_tool_call:
current_tool_call = obj[-2]
spans = _scan_required_tool_calls(current_text)
sent_len = len(previous_text)
function_name_returned = True
tool_call_id = make_tool_call_id(
id_type=tool_call_id_type,
func_name=current_tool_call["name"],
idx=tool_call_idx,
)
delta_message = DeltaMessage(
tool_calls=[
DeltaToolCall(
id=tool_call_id,
function=DeltaFunctionCall(
name=current_tool_call["name"], arguments=arguments
),
index=len(obj) - 1,
type="function",
)
]
)
else:
delta_text, _ = filter_delta_text(delta_text, previous_text)
if delta_text != "":
delta_message = DeltaMessage(
tool_calls=[
DeltaToolCall(
function=DeltaFunctionCall(
# OpenAI API returns None
# instead of name every time
name=None,
arguments=delta_text,
),
index=len(obj) - 1,
)
]
tool_calls: list[DeltaToolCall] = []
started = 0
for index, span in enumerate(spans):
if not span.ready:
# Elements are generated in order: nothing later can be ready.
break
assert span.args_start is not None
end = span.args_end if span.args_end is not None else len(current_text)
if span.ready_at(sent_len):
# Already announced: send only the new argument text.
sent_end = min(end, sent_len)
new_arguments = current_text[sent_end:end]
if new_arguments:
tool_calls.append(
DeltaToolCall(
index=index,
function=DeltaFunctionCall(
# OpenAI API returns None instead of name every time
name=None,
arguments=new_arguments,
),
)
else:
delta_message = None
)
continue
return delta_message, function_name_returned
idx = None if tool_call_idx is None else tool_call_idx + started
started += 1
tool_calls.append(
DeltaToolCall(
id=make_tool_call_id(
id_type=tool_call_id_type, func_name=span.name, idx=idx
),
type="function",
index=index,
function=DeltaFunctionCall(
name=span.name,
arguments=current_text[span.args_start : end],
),
)
)
function_name_returned = any(span.ready for span in spans)
if not tool_calls:
return None, function_name_returned
return DeltaMessage(tool_calls=tool_calls), function_name_returned