[Bugfix][ToolParser] Stream every tool call when one delta completes several (#60354)

Signed-off-by: Junyi Yao <j.yao@wustl.edu>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Junyi Yao
2026-10-08 01:07:35 +00:00
committed by GitHub
co-authored by sfeng33
parent 2dd887426e
commit fdfc171a74
4 changed files with 338 additions and 202 deletions
+110 -42
View File
@@ -295,45 +295,61 @@ def test_structured_outputs_json_without_parameters(
)
def _collect_required_tool_streaming_json(output_json: str, delta_len: int) -> str:
def _stream_required_tool_calls(
output_json: str,
deltas: list[str],
tool_call_id_type: str = "random",
tool_call_idx: int | None = None,
) -> list[dict]:
"""Feed ``deltas`` (which concatenate to ``output_json``) through the
required-tool streaming helper and rebuild the calls per array index,
checking that every index gets exactly one id/name chunk first."""
assert "".join(deltas) == output_json
previous_text = ""
function_name_returned = False
messages = []
for i in range(0, len(output_json), delta_len):
delta_text = output_json[i : i + delta_len]
calls: dict[int, dict] = {}
for delta_text in deltas:
current_text = previous_text + delta_text
delta_message, function_name_returned = extract_required_tool_call_streaming(
delta_message, _ = extract_required_tool_call_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
function_name_returned=function_name_returned,
tool_call_idx=None,
tool_call_id_type="random",
tool_call_idx=tool_call_idx,
tool_call_id_type=tool_call_id_type,
)
if delta_message:
messages.append(delta_message)
previous_text = current_text
if delta_message is None:
continue
assert delta_message.tool_calls
for tc in delta_message.tool_calls:
assert tc.function is not None
if tc.id is not None:
assert tc.index not in calls, "id/name sent twice for one index"
assert tc.type == "function"
assert tc.function.name
calls[tc.index] = {
"id": tc.id,
"name": tc.function.name,
"arguments": tc.function.arguments or "",
}
else:
assert tc.index in calls, "arguments before id/name"
assert tc.function.name is None
assert tc.function.arguments
calls[tc.index]["arguments"] += tc.function.arguments
assert list(calls) == list(range(len(calls)))
return [calls[i] for i in range(len(calls))]
assert len(messages) > 0
combined_messages = "["
for message in messages:
fn = message.tool_calls[0].function
assert fn is not None
if fn.name:
if len(combined_messages) > 1:
combined_messages += "},"
def _fixed_len_deltas(text: str, delta_len: int) -> list[str]:
return [text[i : i + delta_len] for i in range(0, len(text), delta_len)]
combined_messages += (
'{"name": "' + fn.name + '", "parameters": ' + (fn.arguments or "")
)
else:
combined_messages += fn.arguments or ""
combined_messages += "}]"
return combined_messages
def _assert_streams_to(output: list[dict], deltas: list[str]) -> None:
calls = _stream_required_tool_calls(json.dumps(output), deltas)
assert [
{"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls
] == output
for call, expected in zip(calls, output):
assert call["arguments"] == json.dumps(expected["parameters"])
@pytest.mark.parametrize("output", VALID_TOOLS)
@@ -343,11 +359,7 @@ def test_streaming_output_valid(output, empty_params, delta_len):
output = deepcopy(output)
if empty_params:
output = [{"name": o["name"], "parameters": {}} for o in output]
output_json = json.dumps(output)
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len)
assert json.loads(combined_messages) == output
assert json.dumps(json.loads(combined_messages)) == output_json
_assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len))
@pytest.mark.parametrize(
@@ -358,22 +370,78 @@ def test_streaming_output_valid(output, empty_params, delta_len):
"a }} b",
'a " } b',
r"a \ } b",
# Text that looks like the end of one call and the start of another.
'"}}, {"name": "get_forecast", "parameters": {"city": "x',
],
)
@pytest.mark.parametrize("delta_len", [1, 2, 3, 8, 9999])
def test_streaming_output_valid_with_braces_in_string(city, delta_len):
output = [{"name": "get_current_weather", "parameters": {"city": city}}]
output_json = json.dumps(output)
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len)
assert json.loads(combined_messages) == output
assert json.dumps(json.loads(combined_messages)) == output_json
output = [
{"name": "get_current_weather", "parameters": {"city": city}},
{"name": "get_forecast", "parameters": {"city": city, "days": 1}},
]
_assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len))
def test_streaming_output_valid_with_trailing_extra_data():
output = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}]
output_json = json.dumps(output) + "\nDONE"
combined_messages = _collect_required_tool_streaming_json(output_json, delta_len=3)
assert json.loads(combined_messages) == output
calls = _stream_required_tool_calls(output_json, _fixed_len_deltas(output_json, 3))
assert [
{"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls
] == output
TWO_CALLS = [
{"name": "get_current_weather", "parameters": {"city": "Dallas"}},
{"name": "get_forecast", "parameters": {"city": "Dallas", "days": 3}},
]
def _split_at(text: str, cuts: list[int]) -> list[str]:
bounds = [0, *cuts, len(text)]
return [text[a:b] for a, b in zip(bounds, bounds[1:])]
def test_streaming_multiple_calls_completed_in_one_delta():
"""Regression test for #60351: a delta that completes more than one
call must stream every call, not only the last one."""
text = json.dumps(TWO_CALLS)
_assert_streams_to(TWO_CALLS, [text[:1], text[1:]])
_assert_streams_to(TWO_CALLS, [text])
def test_streaming_delta_crossing_call_boundary():
"""A delta that closes one call and opens the next must attribute the
closing argument text to the first call and still announce the next."""
text = json.dumps(TWO_CALLS)
boundary = text.index("}, {") + 1
name_end = text.index('"get_forecast"') + 6
for cuts in (
[boundary - 1, name_end], # '}, {"name": "get_f' in one delta
[boundary - 2, boundary + 2], # '"}}, {' then the rest
[boundary + 2], # opening brace of the next call ends a delta
):
_assert_streams_to(TWO_CALLS, _split_at(text, cuts))
def test_streaming_invalid_name_string_is_never_announced():
"""A closed name that is not a valid JSON string must not be streamed
(the non-streaming path rejects such an array as well)."""
text = '[{"name": "bad\\x", "parameters": {"city": "x"}}]'
assert _stream_required_tool_calls(text, _fixed_len_deltas(text, 4)) == []
def test_streaming_tool_call_idx_increments_per_started_call():
"""kimi_k2 ids number the calls started in one delta consecutively."""
text = json.dumps(TWO_CALLS)
calls = _stream_required_tool_calls(
text, [text], tool_call_id_type="kimi_k2", tool_call_idx=5
)
assert [c["id"] for c in calls] == [
"functions.get_current_weather:5",
"functions.get_forecast:6",
]
FUNCTION_TOOL = FunctionTool(