From 899d72a58c118035be5e9420a15cfa1110c64f47 Mon Sep 17 00:00:00 2001 From: Ting SUN Date: Wed, 24 Jun 2026 03:29:34 +0700 Subject: [PATCH] [Bugfix][ToolParser] Handle braces in required tool streaming strings (#45389) Signed-off-by: Ting Sun Co-authored-by: Flora Feng <4florafeng@gmail.com> --- tests/tool_use/test_tool_choice_required.py | 68 +++++++++++---------- vllm/tool_parsers/streaming.py | 54 ++++++++++++---- 2 files changed, 78 insertions(+), 44 deletions(-) diff --git a/tests/tool_use/test_tool_choice_required.py b/tests/tool_use/test_tool_choice_required.py index 929bb33da0..f37a3c9681 100644 --- a/tests/tool_use/test_tool_choice_required.py +++ b/tests/tool_use/test_tool_choice_required.py @@ -280,15 +280,7 @@ def test_structured_outputs_json_without_parameters( ) -@pytest.mark.parametrize("output", VALID_TOOLS) -@pytest.mark.parametrize("empty_params", [False, True]) -@pytest.mark.parametrize("delta_len", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) -def test_streaming_output_valid(output, empty_params, delta_len): - output = deepcopy(output) - if empty_params: - output = [{"name": o["name"], "parameters": {}} for o in output] - output_json = json.dumps(output) - +def _collect_required_tool_streaming_json(output_json: str, delta_len: int) -> str: previous_text = "" function_name_returned = False messages = [] @@ -327,6 +319,38 @@ def test_streaming_output_valid(output, empty_params, delta_len): else: combined_messages += message.tool_calls[0].function.arguments combined_messages += "}]" + return combined_messages + + +@pytest.mark.parametrize("output", VALID_TOOLS) +@pytest.mark.parametrize("empty_params", [False, True]) +@pytest.mark.parametrize("delta_len", [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) +def test_streaming_output_valid(output, empty_params, delta_len): + output = deepcopy(output) + if empty_params: + output = [{"name": o["name"], "parameters": {}} for o in output] + output_json = json.dumps(output) + + combined_messages = _collect_required_tool_streaming_json(output_json, delta_len) + assert json.loads(combined_messages) == output + assert json.dumps(json.loads(combined_messages)) == output_json + + +@pytest.mark.parametrize( + "city", + [ + "a { b", + "a } b", + "a }} b", + 'a " } b', + r"a \ } b", + ], +) +@pytest.mark.parametrize("delta_len", [1, 2, 3, 8, 9999]) +def test_streaming_output_valid_with_braces_in_string(city, delta_len): + output = [{"name": "get_current_weather", "parameters": {"city": city}}] + output_json = json.dumps(output) + combined_messages = _collect_required_tool_streaming_json(output_json, delta_len) assert json.loads(combined_messages) == output assert json.dumps(json.loads(combined_messages)) == output_json @@ -334,30 +358,8 @@ def test_streaming_output_valid(output, empty_params, delta_len): def test_streaming_output_valid_with_trailing_extra_data(): output = [{"name": "get_current_weather", "parameters": {"city": "Vienna"}}] output_json = json.dumps(output) + "\nDONE" - - previous_text = "" - function_name_returned = False - messages = [] - delta_len = 3 - for i in range(0, len(output_json), delta_len): - delta_text = output_json[i : i + delta_len] - current_text = previous_text + delta_text - - delta_message, function_name_returned = extract_required_tool_call_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_text, - function_name_returned=function_name_returned, - tool_call_idx=None, - tool_call_id_type="random", - ) - - if delta_message: - messages.append(delta_message) - - previous_text = current_text - - assert len(messages) > 0 + combined_messages = _collect_required_tool_streaming_json(output_json, delta_len=3) + assert json.loads(combined_messages) == output FUNCTION_TOOL = FunctionTool( diff --git a/vllm/tool_parsers/streaming.py b/vllm/tool_parsers/streaming.py index 53b3f06bb8..5ee7c6f6c2 100644 --- a/vllm/tool_parsers/streaming.py +++ b/vllm/tool_parsers/streaming.py @@ -23,14 +23,33 @@ else: TokenizerLike = object +def _bracket_level_state( + s: str, opening: str = "{", closing: str = "}" +) -> tuple[int, bool, bool]: + level = 0 + in_string = False + escaped = False + for char in s: + if escaped: + escaped = False + continue + if in_string and char == "\\": + escaped = True + continue + if char == '"': + in_string = not in_string + continue + if not in_string: + if char == opening: + level += 1 + elif char == closing: + level -= 1 + return level, in_string, escaped + + def _bracket_level(s: str, opening: str = "{", closing: str = "}") -> int: """Calculate the current level of nested brackets in a string.""" - level = 0 - for char in s: - if char == opening: - level += 1 - elif char == closing: - level -= 1 + level, _, _ = _bracket_level_state(s, opening, closing) return level @@ -39,11 +58,20 @@ def filter_delta_text( previous_text: str, ) -> tuple[str, bool]: """Trim trailing tool-list delimiters from required-tool streaming text.""" - bracket_level = _bracket_level(previous_text) + bracket_level, in_string, escaped = _bracket_level_state(previous_text) updated_delta = "" passed_zero = False for char in delta_text: - if char == "{": + if escaped: + escaped = False + elif in_string: + if char == "\\": + escaped = True + elif char == '"': + in_string = False + elif char == '"': + in_string = True + elif char == "{": bracket_level += 1 passed_zero = bracket_level == 0 elif char == "}": @@ -53,7 +81,7 @@ def filter_delta_text( if bracket_level != 0: updated_delta += char else: - if char == ",": + if not in_string and char == ",": break return updated_delta, passed_zero @@ -146,8 +174,12 @@ def extract_required_tool_call_streaming( param_match = re.search( r'.*"parameters":\s*(.*)', current_text, re.DOTALL ) - arguments = param_match.group(1) if param_match else "" - arguments, _ = filter_delta_text(arguments, previous_text) + if param_match: + arguments = param_match.group(1) + arguments_prefix = current_text[: param_match.start(1)] + arguments, _ = filter_delta_text(arguments, arguments_prefix) + else: + arguments = "" # if this iteration finishes a previous tool call but a # new incomplete tool is already generated, take the