From fdfc171a7495bed2997609a0758d7e69c69440c0 Mon Sep 17 00:00:00 2001 From: Junyi Yao Date: Wed, 7 Oct 2026 18:07:35 -0700 Subject: [PATCH] [Bugfix][ToolParser] Stream every tool call when one delta completes several (#60354) Signed-off-by: Junyi Yao Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com> --- tests/tool_use/test_tool_choice_required.py | 152 +++++--- vllm/parser/abstract_parser.py | 15 +- vllm/renderers/online_derenderer.py | 6 - vllm/tool_parsers/streaming.py | 367 ++++++++++++-------- 4 files changed, 338 insertions(+), 202 deletions(-) diff --git a/tests/tool_use/test_tool_choice_required.py b/tests/tool_use/test_tool_choice_required.py index ebdea5aa0b..be58ffa83a 100644 --- a/tests/tool_use/test_tool_choice_required.py +++ b/tests/tool_use/test_tool_choice_required.py @@ -295,45 +295,61 @@ def test_structured_outputs_json_without_parameters( ) -def _collect_required_tool_streaming_json(output_json: str, delta_len: int) -> str: +def _stream_required_tool_calls( + output_json: str, + deltas: list[str], + tool_call_id_type: str = "random", + tool_call_idx: int | None = None, +) -> list[dict]: + """Feed ``deltas`` (which concatenate to ``output_json``) through the + required-tool streaming helper and rebuild the calls per array index, + checking that every index gets exactly one id/name chunk first.""" + assert "".join(deltas) == output_json previous_text = "" - function_name_returned = False - messages = [] - for i in range(0, len(output_json), delta_len): - delta_text = output_json[i : i + delta_len] + calls: dict[int, dict] = {} + for delta_text in deltas: current_text = previous_text + delta_text - - delta_message, function_name_returned = extract_required_tool_call_streaming( + delta_message, _ = extract_required_tool_call_streaming( previous_text=previous_text, current_text=current_text, - delta_text=delta_text, - function_name_returned=function_name_returned, - tool_call_idx=None, - tool_call_id_type="random", + tool_call_idx=tool_call_idx, + tool_call_id_type=tool_call_id_type, ) - - if delta_message: - messages.append(delta_message) - previous_text = current_text + if delta_message is None: + continue + assert delta_message.tool_calls + for tc in delta_message.tool_calls: + assert tc.function is not None + if tc.id is not None: + assert tc.index not in calls, "id/name sent twice for one index" + assert tc.type == "function" + assert tc.function.name + calls[tc.index] = { + "id": tc.id, + "name": tc.function.name, + "arguments": tc.function.arguments or "", + } + else: + assert tc.index in calls, "arguments before id/name" + assert tc.function.name is None + assert tc.function.arguments + calls[tc.index]["arguments"] += tc.function.arguments + assert list(calls) == list(range(len(calls))) + return [calls[i] for i in range(len(calls))] - assert len(messages) > 0 - combined_messages = "[" - for message in messages: - fn = message.tool_calls[0].function - assert fn is not None - if fn.name: - if len(combined_messages) > 1: - combined_messages += "}," +def _fixed_len_deltas(text: str, delta_len: int) -> list[str]: + return [text[i : i + delta_len] for i in range(0, len(text), delta_len)] - combined_messages += ( - '{"name": "' + fn.name + '", "parameters": ' + (fn.arguments or "") - ) - else: - combined_messages += fn.arguments or "" - combined_messages += "}]" - return combined_messages + +def _assert_streams_to(output: list[dict], deltas: list[str]) -> None: + calls = _stream_required_tool_calls(json.dumps(output), deltas) + assert [ + {"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls + ] == output + for call, expected in zip(calls, output): + assert call["arguments"] == json.dumps(expected["parameters"]) @pytest.mark.parametrize("output", VALID_TOOLS) @@ -343,11 +359,7 @@ def test_streaming_output_valid(output, empty_params, delta_len): output = deepcopy(output) if empty_params: output = [{"name": o["name"], "parameters": {}} for o in output] - output_json = json.dumps(output) - - combined_messages = _collect_required_tool_streaming_json(output_json, delta_len) - assert json.loads(combined_messages) == output - assert json.dumps(json.loads(combined_messages)) == output_json + _assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len)) @pytest.mark.parametrize( @@ -358,22 +370,78 @@ def test_streaming_output_valid(output, empty_params, delta_len): "a }} b", 'a " } b', r"a \ } b", + # Text that looks like the end of one call and the start of another. + '"}}, {"name": "get_forecast", "parameters": {"city": "x', ], ) @pytest.mark.parametrize("delta_len", [1, 2, 3, 8, 9999]) def test_streaming_output_valid_with_braces_in_string(city, delta_len): - output = [{"name": "get_current_weather", "parameters": {"city": city}}] - output_json = json.dumps(output) - combined_messages = _collect_required_tool_streaming_json(output_json, delta_len) - assert json.loads(combined_messages) == output - assert json.dumps(json.loads(combined_messages)) == output_json + output = [ + {"name": "get_current_weather", "parameters": {"city": city}}, + {"name": "get_forecast", "parameters": {"city": city, "days": 1}}, + ] + _assert_streams_to(output, _fixed_len_deltas(json.dumps(output), delta_len)) def test_streaming_output_valid_with_trailing_extra_data(): output = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}] output_json = json.dumps(output) + "\nDONE" - combined_messages = _collect_required_tool_streaming_json(output_json, delta_len=3) - assert json.loads(combined_messages) == output + calls = _stream_required_tool_calls(output_json, _fixed_len_deltas(output_json, 3)) + assert [ + {"name": c["name"], "parameters": json.loads(c["arguments"])} for c in calls + ] == output + + +TWO_CALLS = [ + {"name": "get_current_weather", "parameters": {"city": "Dallas"}}, + {"name": "get_forecast", "parameters": {"city": "Dallas", "days": 3}}, +] + + +def _split_at(text: str, cuts: list[int]) -> list[str]: + bounds = [0, *cuts, len(text)] + return [text[a:b] for a, b in zip(bounds, bounds[1:])] + + +def test_streaming_multiple_calls_completed_in_one_delta(): + """Regression test for #60351: a delta that completes more than one + call must stream every call, not only the last one.""" + text = json.dumps(TWO_CALLS) + _assert_streams_to(TWO_CALLS, [text[:1], text[1:]]) + _assert_streams_to(TWO_CALLS, [text]) + + +def test_streaming_delta_crossing_call_boundary(): + """A delta that closes one call and opens the next must attribute the + closing argument text to the first call and still announce the next.""" + text = json.dumps(TWO_CALLS) + boundary = text.index("}, {") + 1 + name_end = text.index('"get_forecast"') + 6 + for cuts in ( + [boundary - 1, name_end], # '}, {"name": "get_f' in one delta + [boundary - 2, boundary + 2], # '"}}, {' then the rest + [boundary + 2], # opening brace of the next call ends a delta + ): + _assert_streams_to(TWO_CALLS, _split_at(text, cuts)) + + +def test_streaming_invalid_name_string_is_never_announced(): + """A closed name that is not a valid JSON string must not be streamed + (the non-streaming path rejects such an array as well).""" + text = '[{"name": "bad\\x", "parameters": {"city": "x"}}]' + assert _stream_required_tool_calls(text, _fixed_len_deltas(text, 4)) == [] + + +def test_streaming_tool_call_idx_increments_per_started_call(): + """kimi_k2 ids number the calls started in one delta consecutively.""" + text = json.dumps(TWO_CALLS) + calls = _stream_required_tool_calls( + text, [text], tool_call_id_type="kimi_k2", tool_call_idx=5 + ) + assert [c["id"] for c in calls] == [ + "functions.get_current_weather:5", + "functions.get_forecast:6", + ] FUNCTION_TOOL = FunctionTool( diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index f75d7ce902..4af133c988 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -741,8 +741,6 @@ class DelegatingParser(Parser): extract_required_tool_call_streaming( previous_text=previous_text, current_text=current_text, - delta_text=delta_text, - function_name_returned=function_name_returned, tool_call_idx=tool_call_idx, tool_call_id_type=tool_call_id_type, ) @@ -952,12 +950,13 @@ class DelegatingParser(Parser): elif not delta_message.reasoning: delta_message.reasoning = reasoning_from_this_batch - if ( - delta_message - and delta_message.tool_calls - and delta_message.tool_calls[0].id is not None - ): - state.history_tool_call_cnt += 1 + if delta_message and delta_message.tool_calls: + # A delta may start several tool calls at once (e.g. the + # "required" path when one delta completes more than one + # call), so count every id rather than only the first. + state.history_tool_call_cnt += sum( + tc.id is not None for tc in delta_message.tool_calls + ) # No phase active: pass through as content. # Skip when reasoning just ended in this delta — the engine already diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index bd5bf88fbe..66b73fb491 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -640,12 +640,6 @@ class OnlineDerenderer: if tc.index < len(last_tool_call_ids): # Pin: reuse the ID already recorded for this index # rather than one a from scratch replay regenerated. - # Real trigger not just defensive with - # tool_choice="required", - # extract_required_tool_call_streaming resets - # function_name_returned to False whenever the - # partial JSON transiently fails to parse which - # re-emits id+name for the same index on replay. tc.id = last_tool_call_ids[tc.index] else: last_tool_call_ids.append(tc.id) diff --git a/vllm/tool_parsers/streaming.py b/vllm/tool_parsers/streaming.py index 70da7f3cc6..07c1a2e031 100644 --- a/vllm/tool_parsers/streaming.py +++ b/vllm/tool_parsers/streaming.py @@ -2,19 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +from dataclasses import dataclass from typing import TYPE_CHECKING -import partial_json_parser -import regex as re -from partial_json_parser.core.options import Allow - from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, DeltaToolCall, ) -from vllm.tool_parsers.utils import partial_json_loads from vllm.utils.mistral import is_mistral_tokenizer if TYPE_CHECKING: @@ -23,63 +19,6 @@ else: TokenizerLike = object -def _bracket_level_state( - s: str, opening: str = "{", closing: str = "}" -) -> tuple[int, bool, bool]: - level = 0 - in_string = False - escaped = False - for char in s: - if escaped: - escaped = False - continue - if in_string and char == "\\": - escaped = True - continue - if char == '"': - in_string = not in_string - continue - if not in_string: - if char == opening: - level += 1 - elif char == closing: - level -= 1 - return level, in_string, escaped - - -def filter_delta_text( - delta_text: str, - previous_text: str, -) -> tuple[str, bool]: - """Trim trailing tool-list delimiters from required-tool streaming text.""" - bracket_level, in_string, escaped = _bracket_level_state(previous_text) - updated_delta = "" - passed_zero = False - for char in delta_text: - if escaped: - escaped = False - elif in_string: - if char == "\\": - escaped = True - elif char == '"': - in_string = False - elif char == '"': - in_string = True - elif char == "{": - bracket_level += 1 - passed_zero = bracket_level == 0 - elif char == "}": - bracket_level -= 1 - passed_zero = bracket_level == 0 - - if bracket_level != 0: - updated_delta += char - else: - if not in_string and char == ",": - break - return updated_delta, passed_zero - - def extract_named_tool_call_streaming( *, delta_text: str, @@ -124,100 +63,236 @@ def extract_named_tool_call_streaming( ) +@dataclass +class _RequiredToolCallSpan: + """Where one element of a ``tool_choice="required"`` array sits in the text. + + The model output for ``tool_choice="required"`` is a JSON array of + ``{"name": ..., "parameters": ...}`` objects. ``name`` is only set once + its string value is closed and decodes as JSON; ``name_end`` is the + offset just past that closing quote. ``args_start``/``args_end`` are the + character offsets of the ``parameters`` value (``args_end`` is ``None`` + while the value is still being generated). + + All offsets are prefix-stable: scanning any prefix of the text yields the + same spans truncated to that prefix. That is what lets one scan of the + current text tell what an earlier, shorter text had already revealed. + """ + + name: str | None = None + name_end: int | None = None + args_start: int | None = None + args_end: int | None = None + + def ready_at(self, length: int) -> bool: + """Whether the first ``length`` characters of the text already made + this call streamable: its name was complete and its parameters value + had started.""" + return ( + self.name is not None + and self.name_end is not None + and self.name_end <= length + and self.args_start is not None + and self.args_start < length + ) + + @property + def ready(self) -> bool: + return self.name is not None and self.args_start is not None + + +# Parse states for one element object of the required-tool-call array. +_EXPECT_KEY = 0 +_EXPECT_COLON = 1 +_EXPECT_VALUE = 2 +_IN_STRING_VALUE = 3 +_IN_CONTAINER_VALUE = 4 +_IN_SCALAR_VALUE = 5 +_AFTER_VALUE = 6 + + +def _scan_required_tool_calls(text: str) -> list[_RequiredToolCallSpan]: + """Locate every tool call of a (possibly partial) required-tool-call array. + + A single string- and nesting-aware pass over ``text`` that records, for + each top-level array element, its ``name`` (once complete) and the + character span of its ``parameters`` value. Text after the closing ``]`` + of the array is ignored. Only the element level of the array is + interpreted, so braces, brackets and quotes inside nested values and + strings cannot confuse it. + """ + spans: list[_RequiredToolCallSpan] = [] + depth = 0 # nesting depth of ``{}`` and ``[]``; element objects sit at 2 + in_string = False + escaped = False + string_start = 0 + cur: _RequiredToolCallSpan | None = None + expect = _EXPECT_KEY + key: str | None = None + value_start = 0 + + def end_value(pos: int) -> None: + nonlocal expect + assert cur is not None + if key == "parameters": + cur.args_end = pos + elif key == "name" and expect == _IN_STRING_VALUE: + try: + cur.name = json.loads(text[value_start:pos]) + cur.name_end = pos + except json.JSONDecodeError: + # Not a valid JSON string: the call can never become ready, + # which matches the non-streaming path rejecting the array. + cur.name = None + expect = _AFTER_VALUE + + def start_value(pos: int, state: int) -> None: + nonlocal expect, value_start + assert cur is not None + value_start = pos + if key == "parameters": + cur.args_start = pos + expect = state + + for i, ch in enumerate(text): + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + if depth == 2 and cur is not None: + if expect == _EXPECT_KEY: + key = text[string_start + 1 : i] + expect = _EXPECT_COLON + elif expect == _IN_STRING_VALUE: + end_value(i + 1) + continue + + if ch == '"': + in_string = True + string_start = i + if depth == 2 and cur is not None and expect == _EXPECT_VALUE: + start_value(i, _IN_STRING_VALUE) + continue + + if ch in "{[": + if depth == 0: + if ch != "[": + return spans + elif depth == 1: + if ch != "{": + return spans + cur = _RequiredToolCallSpan() + spans.append(cur) + expect = _EXPECT_KEY + key = None + elif depth == 2 and cur is not None and expect == _EXPECT_VALUE: + start_value(i, _IN_CONTAINER_VALUE) + depth += 1 + continue + + if ch in "}]": + if depth == 2 and cur is not None and expect == _IN_SCALAR_VALUE: + end_value(i) + depth -= 1 + if depth == 2 and cur is not None and expect == _IN_CONTAINER_VALUE: + end_value(i + 1) + elif depth == 1: + cur = None + elif depth <= 0: + return spans + continue + + if depth != 2 or cur is None: + continue + + if ch == ":": + if expect == _EXPECT_COLON: + expect = _EXPECT_VALUE + elif ch == ",": + if expect == _IN_SCALAR_VALUE: + end_value(i) + expect = _EXPECT_KEY + key = None + elif ch.isspace(): + if expect == _IN_SCALAR_VALUE: + end_value(i) + elif expect == _EXPECT_VALUE: + start_value(i, _IN_SCALAR_VALUE) + + return spans + + def extract_required_tool_call_streaming( *, previous_text: str, current_text: str | None, - delta_text: str, - function_name_returned: bool, tool_call_idx: int | None, tool_call_id_type: str, ) -> tuple[DeltaMessage | None, bool]: - if current_text is None or current_text == "": - # if the current text is empty, we cannot parse it - return None, function_name_returned - try: - flags = Allow.ALL - obj, _ = partial_json_loads(current_text, flags) - except ( - partial_json_parser.core.exceptions.MalformedJSON, - json.JSONDecodeError, - ): - obj = None + """Stream the tool calls of a ``tool_choice="required"`` JSON array. - # check if the current text is a valid array - # containing a partial tool calling object - # if not repeat - if obj is None or not isinstance(obj, list) or not len(obj) > 0: - function_name_returned = False - delta_message = None - else: - _, finishes_previous_tool = filter_delta_text(delta_text, previous_text) - # take the last tool call from the generated list - current_tool_call = obj[-1] + ``previous_text`` must be a prefix of ``current_text`` (the accumulated + text before and after this delta). Every array element that has something + not yet sent is emitted, in array order, so a delta may carry several + ``DeltaToolCall`` entries: the first chunk of a call carries its id, name + and the arguments generated so far; later chunks carry only new argument + text. What was already sent is read off the same scan, since the spans + are prefix-stable, so no state is needed beyond the two texts. - # once parameters have been generated the name is complete as well - if not finishes_previous_tool and ( - "name" not in current_tool_call or "parameters" not in current_tool_call - ): - function_name_returned = False - delta_message = None - else: - if not function_name_returned: - # get partly generated arguments from the latest tool call - param_match = re.search( - r'.*"parameters":\s*(.*)', current_text, re.DOTALL - ) - if param_match: - arguments = param_match.group(1) - arguments_prefix = current_text[: param_match.start(1)] - arguments, _ = filter_delta_text(arguments, arguments_prefix) - else: - arguments = "" + Returns the delta (``None`` when nothing is new) and whether at least one + call has been announced so far. + """ + if not current_text: + return None, False - # if this iteration finishes a previous tool call but a - # new incomplete tool is already generated, take the - # previous from the list - if finishes_previous_tool and "parameters" not in current_tool_call: - current_tool_call = obj[-2] + spans = _scan_required_tool_calls(current_text) + sent_len = len(previous_text) - function_name_returned = True - tool_call_id = make_tool_call_id( - id_type=tool_call_id_type, - func_name=current_tool_call["name"], - idx=tool_call_idx, - ) - delta_message = DeltaMessage( - tool_calls=[ - DeltaToolCall( - id=tool_call_id, - function=DeltaFunctionCall( - name=current_tool_call["name"], arguments=arguments - ), - index=len(obj) - 1, - type="function", - ) - ] - ) - - else: - delta_text, _ = filter_delta_text(delta_text, previous_text) - - if delta_text != "": - delta_message = DeltaMessage( - tool_calls=[ - DeltaToolCall( - function=DeltaFunctionCall( - # OpenAI API returns None - # instead of name every time - name=None, - arguments=delta_text, - ), - index=len(obj) - 1, - ) - ] + tool_calls: list[DeltaToolCall] = [] + started = 0 + for index, span in enumerate(spans): + if not span.ready: + # Elements are generated in order: nothing later can be ready. + break + assert span.args_start is not None + end = span.args_end if span.args_end is not None else len(current_text) + if span.ready_at(sent_len): + # Already announced: send only the new argument text. + sent_end = min(end, sent_len) + new_arguments = current_text[sent_end:end] + if new_arguments: + tool_calls.append( + DeltaToolCall( + index=index, + function=DeltaFunctionCall( + # OpenAI API returns None instead of name every time + name=None, + arguments=new_arguments, + ), ) - else: - delta_message = None + ) + continue - return delta_message, function_name_returned + idx = None if tool_call_idx is None else tool_call_idx + started + started += 1 + tool_calls.append( + DeltaToolCall( + id=make_tool_call_id( + id_type=tool_call_id_type, func_name=span.name, idx=idx + ), + type="function", + index=index, + function=DeltaFunctionCall( + name=span.name, + arguments=current_text[span.args_start : end], + ), + ) + ) + + function_name_returned = any(span.ready for span in spans) + if not tool_calls: + return None, function_name_returned + return DeltaMessage(tool_calls=tool_calls), function_name_returned