From 03ce1c6ed908788cf508219b666720783a723123 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Sun, 19 Apr 2026 04:30:27 -0400 Subject: [PATCH] [Bugfix] Kimi-K2 tool parser streaming - fix token leakage, argument truncation, and content dropping (#38579) Signed-off-by: sfeng33 <4florafeng@gmail.com> --- .../tool_parsers/test_kimi_k2_tool_parser.py | 1468 ++++++----------- vllm/tool_parsers/kimi_k2_tool_parser.py | 637 ++----- 2 files changed, 692 insertions(+), 1413 deletions(-) diff --git a/tests/tool_parsers/test_kimi_k2_tool_parser.py b/tests/tool_parsers/test_kimi_k2_tool_parser.py index 09c1c461c1..b56032b91c 100644 --- a/tests/tool_parsers/test_kimi_k2_tool_parser.py +++ b/tests/tool_parsers/test_kimi_k2_tool_parser.py @@ -3,14 +3,20 @@ # ruff: noqa: E501 import json +from unittest.mock import MagicMock import pytest -from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall +from tests.tool_parsers.utils import ( + run_tool_extraction, + run_tool_extraction_streaming, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) from vllm.tokenizers import get_tokenizer from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser -# Use a common model that is likely to be available MODEL = "moonshotai/Kimi-K2-Instruct" @@ -20,959 +26,557 @@ def kimi_k2_tokenizer(): @pytest.fixture -def kimi_k2_tool_parser(kimi_k2_tokenizer): +def parser(kimi_k2_tokenizer): return KimiK2ToolParser(kimi_k2_tokenizer) -def assert_tool_calls( - actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall] -): - assert len(actual_tool_calls) == len(expected_tool_calls) +SECTION_BEGIN = "<|tool_calls_section_begin|>" +SECTION_END = "<|tool_calls_section_end|>" +TOOL_BEGIN = "<|tool_call_begin|>" +TOOL_END = "<|tool_call_end|>" +ARG_BEGIN = "<|tool_call_argument_begin|>" - for actual_tool_call, expected_tool_call in zip( - actual_tool_calls, expected_tool_calls + +def _tool(tool_id: str, args: str) -> str: + return f"{TOOL_BEGIN}{tool_id} {ARG_BEGIN}{args}{TOOL_END}" + + +def _wrap(*tool_strs: str) -> str: + return SECTION_BEGIN + "".join(tool_strs) + SECTION_END + + +class TestExtractToolCalls: + def test_no_tools(self, parser): + content, tool_calls = run_tool_extraction( + parser, "This is a test", streaming=False + ) + assert content == "This is a test" + assert tool_calls == [] + + @pytest.mark.parametrize( + "model_output, expected_names, expected_args_list, expected_content", + [ + pytest.param( + "I'll check. " + + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')), + ["get_weather"], + [{"city": "Beijing"}], + "I'll check. ", + id="single_tool_call", + ), + pytest.param( + "Compare weather. " + + _wrap( + _tool("functions.get_weather:0", '{"city": "Beijing"}'), + _tool("functions.get_weather:1", '{"city": "Shanghai"}'), + ), + ["get_weather", "get_weather"], + [{"city": "Beijing"}, {"city": "Shanghai"}], + "Compare weather. ", + id="parallel_tool_calls", + ), + pytest.param( + "Multiple tasks. " + + _wrap( + _tool("functions.get_weather:0", '{"city": "New York"}'), + _tool("functions.get_news:1", '{"topic": "technology"}'), + _tool( + "functions.send_email:2", + '{"to": "user@example.com", "subject": "Daily Update"}', + ), + ), + ["get_weather", "get_news", "send_email"], + [ + {"city": "New York"}, + {"topic": "technology"}, + {"to": "user@example.com", "subject": "Daily Update"}, + ], + "Multiple tasks. ", + id="three_tool_calls", + ), + pytest.param( + "Process HTML. " + + _wrap( + _tool("functions.process_html:0", '{"html": "
content
"}') + ), + ["process_html"], + [{"html": "
content
"}], + "Process HTML. ", + id="angle_brackets_in_json", + ), + pytest.param( + "Formatted. " + + _wrap( + _tool( + "functions.process_data:0", + '{\n "name": "test",\n "value": 123\n}', + ) + ), + ["process_data"], + [{"name": "test", "value": 123}], + "Formatted. ", + id="multiline_json", + ), + pytest.param( + "No prefix. " + _wrap(_tool("get_weather:0", '{"city": "Tokyo"}')), + ["get_weather"], + [{"city": "Tokyo"}], + "No prefix. ", + id="no_functions_prefix", + ), + pytest.param( + "Empty args. " + _wrap(_tool("functions.test:0", "{}")), + ["test"], + [{}], + "Empty args. ", + id="empty_arguments", + ), + ], + ) + def test_extract_tool_calls( + self, parser, model_output, expected_names, expected_args_list, expected_content ): - assert actual_tool_call.type == "function" - assert actual_tool_call.function == expected_tool_call.function + content, tool_calls = run_tool_extraction(parser, model_output, streaming=False) + assert content == expected_content + assert len(tool_calls) == len(expected_names) + for tc, name, expected_args in zip( + tool_calls, expected_names, expected_args_list + ): + assert tc.type == "function" + assert tc.function.name == name + assert json.loads(tc.function.arguments) == expected_args + # id format: "something:digit" + assert tc.id.split(":")[-1].isdigit() - # assert tool call id format: should contain function name and numeric index - # Format can be either "functions.func_name:0" or "func_name:0" - assert actual_tool_call.id.split(":")[-1].isdigit() - assert ( - actual_tool_call.id.split(":")[0].split(".")[-1] - == expected_tool_call.function.name + def test_invalid_json_still_extracted(self, parser): + """Tool calls with invalid JSON are still returned (arguments as-is).""" + model_output = ( + "Help. " + + SECTION_BEGIN + + _tool("functions.bad:0", '{"city": "Beijing"') + + _tool("functions.good:1", '{"city": "Shanghai"}') + + SECTION_END + ) + content, tool_calls = run_tool_extraction(parser, model_output, streaming=False) + assert len(tool_calls) == 2 + assert tool_calls[0].function.name == "bad" + assert tool_calls[1].function.name == "good" + + def test_invalid_funcall_id_skipped(self, parser): + """Tool calls with malformed id (no colon+digit) are skipped.""" + model_output = ( + "Help. " + + SECTION_BEGIN + + _tool("functions.invalid.0", '{"city": "Beijing"}') + + _tool("functions.valid:1", '{"city": "Shanghai"}') + + SECTION_END + ) + content, tool_calls = run_tool_extraction(parser, model_output, streaming=False) + assert len(tool_calls) == 1 + assert tool_calls[0].function.name == "valid" + + def test_native_id_extracted(self, parser): + """Regression: parser extracts native ID onto ToolCall (PR #32768).""" + model_output = "Checking weather. " + _wrap( + _tool("functions.get_weather:0", '{"city": "Tokyo"}') + ) + content, tool_calls = run_tool_extraction(parser, model_output, streaming=False) + assert len(tool_calls) == 1 + assert tool_calls[0].id == "functions.get_weather:0" + assert tool_calls[0].function.name == "get_weather" + assert json.loads(tool_calls[0].function.arguments) == {"city": "Tokyo"} + + def test_multi_turn_native_id_continuity(self, kimi_k2_tokenizer): + """Regression: native IDs from turn 1 preserved across turns (PR #32768).""" + turn1_parser = KimiK2ToolParser(kimi_k2_tokenizer) + turn1_output = "Let me check. " + _wrap( + _tool("functions.get_weather:0", '{"city": "Beijing"}') + ) + _, turn1_tools = run_tool_extraction( + turn1_parser, turn1_output, streaming=False + ) + assert len(turn1_tools) == 1 + assert turn1_tools[0].id == "functions.get_weather:0" + + # Fresh parser for turn 2 + turn2_parser = KimiK2ToolParser(kimi_k2_tokenizer) + turn2_output = "Now let me get news. " + _wrap( + _tool("functions.get_news:0", '{"topic": "weather in Beijing"}') + ) + _, turn2_tools = run_tool_extraction( + turn2_parser, turn2_output, streaming=False + ) + assert len(turn2_tools) == 1 + assert turn2_tools[0].id == "functions.get_news:0" + + +def _split_tool_output_to_deltas( + content: str, tool_strs: list[tuple[str, str]] +) -> list[str]: + """Build a list of string deltas with special tokens as separate chunks. + + Args: + content: text before tool section + tool_strs: list of (tool_id, args_json) + """ + deltas = [content, SECTION_BEGIN] + for tool_id, args_json in tool_strs: + deltas.extend( + [ + TOOL_BEGIN, + f"{tool_id} ", + ARG_BEGIN, + f"{args_json} ", + TOOL_END, + ] + ) + deltas.append(SECTION_END) + return deltas + + +class TestStreamingHappyPath: + def test_single_tool_call(self, parser): + """Verify DeltaToolCall output: name, id, arguments for one tool.""" + deltas = _split_tool_output_to_deltas( + "I'll help. ", + [("functions.get_weather:0", '{"city": "Beijing"}')], + ) + rec = run_tool_extraction_streaming(parser, deltas) + + assert len(rec.tool_calls) == 1 + tc = rec.tool_calls[0] + assert tc.function.name == "get_weather" + assert tc.id == "functions.get_weather:0" + assert json.loads(tc.function.arguments) == {"city": "Beijing"} + + def test_multiple_tool_calls(self, parser): + """Two tool calls emitted with correct indices, names, arguments.""" + deltas = _split_tool_output_to_deltas( + "Compare weather. ", + [ + ("functions.get_weather:0", '{"city": "Tokyo"}'), + ("functions.get_weather:1", '{"city": "NYC"}'), + ], + ) + rec = run_tool_extraction_streaming(parser, deltas) + + assert len(rec.tool_calls) == 2 + assert rec.tool_calls[0].function.name == "get_weather" + assert rec.tool_calls[0].id == "functions.get_weather:0" + assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"} + + assert rec.tool_calls[1].function.name == "get_weather" + assert rec.tool_calls[1].id == "functions.get_weather:1" + assert json.loads(rec.tool_calls[1].function.arguments) == {"city": "NYC"} + + def test_content_before_tools(self, parser): + """Content before section is streamed; markers/args don't leak.""" + deltas = _split_tool_output_to_deltas( + "I'll check the weather. ", + [("functions.get_weather:0", '{"city": "Tokyo"}')], + ) + rec = run_tool_extraction_streaming(parser, deltas) + + assert "check the weather" in rec.other_content + # No markers or tool content leaked + for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]: + assert marker not in rec.other_content + assert "get_weather" not in rec.other_content + assert "Tokyo" not in rec.other_content + + def test_no_tool_calls(self, parser): + """Plain text streaming returns content only.""" + deltas = ["This is just ", "regular text ", "without tools."] + rec = run_tool_extraction_streaming(parser, deltas) + + assert rec.other_content == "This is just regular text without tools." + assert rec.tool_calls == [] + + def test_incremental_arguments(self, parser): + """Arguments split across small chunks accumulate correctly.""" + deltas = [ + "Help. ", + SECTION_BEGIN, + TOOL_BEGIN, + "functions.get_weather:0 ", + ARG_BEGIN, + '{"ci', + 'ty": "Be', + 'ijing"}', + " ", + TOOL_END, + SECTION_END, + ] + rec = run_tool_extraction_streaming(parser, deltas) + + assert len(rec.tool_calls) == 1 + assert rec.tool_calls[0].function.name == "get_weather" + assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"} + + @pytest.mark.parametrize( + "model_output", + [ + pytest.param( + "Single. " + + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')), + id="single_tool", + ), + pytest.param( + "Multi. " + + _wrap( + _tool("functions.get_weather:0", '{"city": "Tokyo"}'), + _tool("functions.get_news:1", '{"topic": "tech"}'), + ), + id="parallel_tools", + ), + pytest.param( + "No prefix id. " + _wrap(_tool("get_weather:0", '{"city": "NYC"}')), + id="no_functions_prefix", + ), + ], + ) + def test_streaming_matches_nonstreaming(self, parser, model_output): + """Streaming reconstruction matches non-streaming extraction.""" + content_non, tools_non = run_tool_extraction( + parser, model_output, streaming=False + ) + content_stream, tools_stream = run_tool_extraction( + parser, model_output, streaming=True ) + assert len(tools_non) == len(tools_stream) + for tc_non, tc_stream in zip(tools_non, tools_stream): + assert tc_non.function.name == tc_stream.function.name + assert json.loads(tc_non.function.arguments) == json.loads( + tc_stream.function.arguments + ) -def run_streaming_sequence(parser, deltas): - """Helper to simulate a streaming sequence and return results.""" - previous_text = "" - previous_token_ids: list[int] = [] - results = [] - for delta_text, delta_token_ids in deltas: - current_text = previous_text + delta_text - current_token_ids = previous_token_ids + delta_token_ids - - result = parser.extract_tool_calls_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_text, - previous_token_ids=previous_token_ids, - current_token_ids=current_token_ids, - delta_token_ids=delta_token_ids, - request=None, +class TestStreamingEdgeCases: + def test_marker_suppression(self, parser): + """No special-token markers appear in reconstructed content.""" + deltas = _split_tool_output_to_deltas( + "I'll check. ", + [("functions.get_weather:0", '{"city": "Tokyo"}')], ) - results.append(result) + rec = run_tool_extraction_streaming(parser, deltas) - previous_text = current_text - previous_token_ids = current_token_ids + forbidden = [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN] + for marker in forbidden: + assert marker not in rec.other_content, ( + f"Marker leaked: {marker!r} in {rec.other_content!r}" + ) - return results + def test_noise_between_markers_suppressed(self, parser): + """Text between section_begin and tool_call_begin doesn't leak.""" + deltas = [ + "Reasoning. ", + SECTION_BEGIN, + " spurious noise ", + TOOL_BEGIN, + "functions.test:0 ", + ARG_BEGIN, + '{"k": "v"} ', + TOOL_END, + SECTION_END, + ] + rec = run_tool_extraction_streaming(parser, deltas) + assert "spurious" not in rec.other_content + assert "noise" not in rec.other_content -def test_extract_tool_calls_no_tools(kimi_k2_tool_parser): - model_output = "This is a test" - extracted_tool_calls = kimi_k2_tool_parser.extract_tool_calls( - model_output, request=None - ) # type: ignore[arg-type] - assert not extracted_tool_calls.tools_called - assert extracted_tool_calls.tool_calls == [] - assert extracted_tool_calls.content == model_output + def test_empty_tool_section(self, parser): + """Empty section (begin immediately followed by end) doesn't crash.""" + deltas = ["Reasoning. ", SECTION_BEGIN, SECTION_END] + rec = run_tool_extraction_streaming(parser, deltas) + assert rec.tool_calls == [] -@pytest.mark.parametrize( - ids=[ - "tool_call_with_content_before", - "multi_tool_call_with_content_before", - "concatenated_tool_calls_bug_fix", - "three_concatenated_tool_calls", - "mixed_spacing_tool_calls", - "angle_brackets_in_json", - "newlines_in_json", - ], - argnames=["model_output", "expected_tool_calls", "expected_content"], - argvalues=[ - ( - """I'll help you check the weather. <|tool_calls_section_begin|> <|tool_call_begin|> -functions.get_weather:0 <|tool_call_argument_begin|> {"city": "Beijing"} <|tool_call_end|> <|tool_calls_section_end|>""", + def test_three_different_tools(self, parser): + """Three tool calls with different functions stream correctly.""" + deltas = _split_tool_output_to_deltas( + "Multiple tasks. ", [ - ToolCall( - id="functions.get_weather:0", - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "city": "Beijing", - }, - ), - ), - type="function", - ) + ("functions.get_weather:0", '{"city": "NYC"}'), + ("functions.get_news:1", '{"topic": "tech"}'), + ("functions.send_email:2", '{"to": "a@b.com"}'), ], - "I'll help you check the weather. ", - ), - ( - """I'll help you check the weather. <|tool_calls_section_begin|> <|tool_call_begin|> -functions.get_weather:0 <|tool_call_argument_begin|> {"city": "Beijing"} <|tool_call_end|> <|tool_call_begin|> -functions.get_weather:1 <|tool_call_argument_begin|> {"city": "Shanghai"} <|tool_call_end|> <|tool_calls_section_end|>""", - [ - ToolCall( - id="functions.get_weather:0", - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "city": "Beijing", - }, - ), - ), - type="function", - ), - ToolCall( - id="functions.get_weather:1", - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "city": "Shanghai", - }, - ), - ), - type="function", - ), - ], - "I'll help you check the weather. ", - ), - ( - """I'll get the weather and news for LA today. First, let me get the weather using Los Angeles coordinates, and then get the latest news. <|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"latitude": 34.0522, "longitude": -118.2437}<|tool_call_end|><|tool_call_begin|>functions.get_news:1<|tool_call_argument_begin|>{"content": "Los Angeles today"}<|tool_call_end|><|tool_calls_section_end|>""", - [ - ToolCall( - id="functions.get_weather:0", - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - {"latitude": 34.0522, "longitude": -118.2437} - ), - ), - type="function", - ), - ToolCall( - id="functions.get_news:1", - function=FunctionCall( - name="get_news", - arguments=json.dumps({"content": "Los Angeles today"}), - ), - type="function", - ), - ], - "I'll get the weather and news for LA today. First, let me get the weather using Los Angeles coordinates, and then get the latest news. ", - ), - ( - """I'll help you with multiple tasks. <|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{"city": "New York"}<|tool_call_end|><|tool_call_begin|>functions.get_news:1<|tool_call_argument_begin|>{"topic": "technology"}<|tool_call_end|><|tool_call_begin|>functions.send_email:2<|tool_call_argument_begin|>{"to": "user@example.com", "subject": "Daily Update"}<|tool_call_end|><|tool_calls_section_end|>""", - [ - ToolCall( - id="functions.get_weather:0", - function=FunctionCall( - name="get_weather", - arguments=json.dumps({"city": "New York"}), - ), - type="function", - ), - ToolCall( - id="functions.get_news:1", - function=FunctionCall( - name="get_news", - arguments=json.dumps({"topic": "technology"}), - ), - type="function", - ), - ToolCall( - id="functions.send_email:2", - function=FunctionCall( - name="send_email", - arguments=json.dumps( - {"to": "user@example.com", "subject": "Daily Update"} - ), - ), - type="function", - ), - ], - "I'll help you with multiple tasks. ", - ), - ( - """Mixed spacing test. <|tool_calls_section_begin|> <|tool_call_begin|> functions.test:0 <|tool_call_argument_begin|> {} <|tool_call_end|><|tool_call_begin|>functions.test2:1<|tool_call_argument_begin|>{}<|tool_call_end|> <|tool_calls_section_end|>""", - [ - ToolCall( - id="functions.test:0", - function=FunctionCall( - name="test", - arguments=json.dumps({}), - ), - type="function", - ), - ToolCall( - id="functions.test2:1", - function=FunctionCall( - name="test2", - arguments=json.dumps({}), - ), - type="function", - ), - ], - "Mixed spacing test. ", - ), - ( - """I need to process HTML content. <|tool_calls_section_begin|><|tool_call_begin|>functions.process_html:0<|tool_call_argument_begin|>{"html": "
content
", "text": "normal text"}<|tool_call_end|><|tool_calls_section_end|>""", - [ - ToolCall( - id="functions.process_html:0", - function=FunctionCall( - name="process_html", - arguments=json.dumps( - {"html": "
content
", "text": "normal text"} - ), - ), - type="function", - ) - ], - "I need to process HTML content. ", - ), - ( - """I need to process formatted JSON. <|tool_calls_section_begin|><|tool_call_begin|>functions.process_data:0<|tool_call_argument_begin|>{ - "name": "test", - "value": 123, - "nested": { - "key": "value" - } -}<|tool_call_end|><|tool_calls_section_end|>""", - [ - ToolCall( - id="functions.process_data:0", - function=FunctionCall( - name="process_data", - arguments=json.dumps( - {"name": "test", "value": 123, "nested": {"key": "value"}}, - indent=2, - ), - ), - type="function", - ) - ], - "I need to process formatted JSON. ", - ), - ], -) -def test_extract_tool_calls( - kimi_k2_tool_parser, model_output, expected_tool_calls, expected_content -): - extracted_tool_calls = kimi_k2_tool_parser.extract_tool_calls( - model_output, request=None - ) # type: ignore[arg-type] - assert extracted_tool_calls.tools_called - - assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) - - assert extracted_tool_calls.content == expected_content - - -def test_extract_tool_calls_invalid_json(kimi_k2_tool_parser): - """we'll return every funcall result""" - model_output = """I'll help you check the weather. <|tool_calls_section_begin|> <|tool_call_begin|> -functions.invalid_get_weather:0 <|tool_call_argument_begin|> {"city": "Beijing" <|tool_call_end|> <|tool_call_begin|> -functions.valid_get_weather:1 <|tool_call_argument_begin|> {"city": "Shanghai"} <|tool_call_end|> <|tool_calls_section_end|>""" - - extracted_tool_calls = kimi_k2_tool_parser.extract_tool_calls( - model_output, request=None - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - # Should extract only the valid JSON tool calls - assert len(extracted_tool_calls.tool_calls) == 2 - assert extracted_tool_calls.tool_calls[0].function.name == "invalid_get_weather" - assert extracted_tool_calls.tool_calls[1].function.name == "valid_get_weather" - - -def test_extract_tool_calls_invalid_funcall(kimi_k2_tool_parser): - """we'll return every funcall result""" - model_output = """I'll help you check the weather. <|tool_calls_section_begin|> <|tool_call_begin|> -functions.invalid_get_weather.0 <|tool_call_argument_begin|> {"city": "Beijing"} <|tool_call_end|> <|tool_call_begin|> -functions.valid_get_weather:1 <|tool_call_argument_begin|> {"city": "Shanghai"} <|tool_call_end|> <|tool_calls_section_end|>""" - - extracted_tool_calls = kimi_k2_tool_parser.extract_tool_calls( - model_output, request=None - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - # Should extract only the valid JSON tool calls - assert len(extracted_tool_calls.tool_calls) == 1 - assert extracted_tool_calls.tool_calls[0].function.name == "valid_get_weather" - - -def test_streaming_basic_functionality(kimi_k2_tool_parser): - """Test basic streaming functionality.""" - # Reset streaming state - kimi_k2_tool_parser.current_tool_name_sent = False - kimi_k2_tool_parser.prev_tool_call_arr = [] - kimi_k2_tool_parser.current_tool_id = -1 - kimi_k2_tool_parser.streamed_args_for_tool = [] - - # Test with a simple tool call - current_text = """ check the weather. <|tool_calls_section_begin|> <|tool_call_begin|> -functions.get_weather:0 <|tool_call_argument_begin|> {"city": "Beijing"} <|tool_call_end|> <|tool_calls_section_end|>""" - - # First call should handle the initial setup - result = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="I'll help you", - current_text=current_text, - delta_text="<|tool_calls_section_end|>", - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=None, - ) - - # The result might be None or contain tool call information - # This depends on the internal state management - if result is not None and hasattr(result, "tool_calls") and result.tool_calls: - assert len(result.tool_calls) >= 0 - - -def test_streaming_no_tool_calls(kimi_k2_tool_parser): - """Test streaming when there are no tool calls.""" - current_text = "This is just regular text without any tool calls." - - result = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="This is just regular text", - current_text=current_text, - delta_text=" without any tool calls.", - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=None, - ) - - # Should return the delta text as content - assert result is not None - assert hasattr(result, "content") - assert result.content == " without any tool calls." - - -def test_token_leak_between_section_and_tool_begin(kimi_k2_tool_parser): - """ - Test that text between <|tool_calls_section_begin|> and <|tool_call_begin|> - is suppressed and does not leak into reasoning_delta. - This is the main vulnerability being fixed. - """ - kimi_k2_tool_parser.reset_streaming_state() - - # Get token IDs for the markers - section_begin_token_id = kimi_k2_tool_parser.vocab.get( - "<|tool_calls_section_begin|>" - ) - tool_call_begin_token_id = kimi_k2_tool_parser.vocab.get("<|tool_call_begin|>") - - # Simulate streaming sequence: - deltas = [ - ("I'll help you with that. ", [1, 2, 3]), - ("<|tool_calls_section_begin|>", [section_begin_token_id]), - (" spurious text ", [4, 5]), - ("<|tool_call_begin|>", [tool_call_begin_token_id]), - ] - - results = run_streaming_sequence(kimi_k2_tool_parser, deltas) - - # Delta 1: "I'll help you with that. " - assert results[0] is not None - assert results[0].content == "I'll help you with that. " - - # Delta 2: "<|tool_calls_section_begin|>" - # Section marker should be stripped and suppressed - assert results[1] is None or ( - results[1].content is None or results[1].content == "" - ) - - # Delta 3: " spurious text or tokens " (THE LEAK SCENARIO) - # CRITICAL: This text should be suppressed, NOT returned as reasoning_delta - assert results[2] is None or ( - results[2].content is None or results[2].content == "" - ) - - # Delta 4: "<|tool_call_begin|>..." - # Now we're in tool call mode, result depends on internal state - # The key is that the spurious text from Delta 3 was not leaked - - -def test_split_markers_across_deltas(kimi_k2_tool_parser): - """ - Test that markers split across delta chunks are correctly detected - via the rolling buffer mechanism. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_token_id = kimi_k2_tool_parser.vocab.get( - "<|tool_calls_section_begin|>" - ) - - # Delta 1: partial token, Delta 2: complete marker - deltas = [ - ("<|tool_calls_sec", [3]), - ("tion_begin|> ", [section_begin_token_id, 4]), - ] - - _results = run_streaming_sequence(kimi_k2_tool_parser, deltas) - - # Now the complete marker should be detected via buffer - assert kimi_k2_tool_parser.in_tool_section is True - - -def test_marker_variants(kimi_k2_tool_parser): - """Test that both singular and plural marker variants are recognized.""" - kimi_k2_tool_parser.reset_streaming_state() - - # Test singular variant: <|tool_call_section_begin|> (note: singular "call") - singular_token_id = kimi_k2_tool_parser.vocab.get("<|tool_call_section_begin|>") - - if singular_token_id is not None: # Only test if tokenizer supports it - _result = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Reasoning ", - current_text="Reasoning <|tool_call_section_begin|>", - delta_text="<|tool_call_section_begin|>", - previous_token_ids=[1, 2], - current_token_ids=[1, 2, singular_token_id], - delta_token_ids=[singular_token_id], - request=None, ) - # Should enter tool section mode with singular variant too - assert kimi_k2_tool_parser.in_tool_section is True - - -def test_reentry_to_reasoning_after_tool_section(kimi_k2_tool_parser): - """ - Test that after exiting a tool section with <|tool_calls_section_end|>, - subsequent text is correctly returned as reasoning content. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - - deltas = [ - ("<|tool_calls_section_begin|>", [section_begin_id]), - ("<|tool_calls_section_end|>", [section_end_id]), - (" More reasoning", [10, 11]), - ] - - results = run_streaming_sequence(kimi_k2_tool_parser, deltas) - - assert kimi_k2_tool_parser.in_tool_section is False - assert results[2] is not None - assert results[2].content == " More reasoning" - - -def test_empty_tool_section(kimi_k2_tool_parser): - """Test an empty tool section (begin immediately followed by end).""" - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - - # Section begin - _result1 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Reasoning ", - current_text="Reasoning <|tool_calls_section_begin|>", - delta_text="<|tool_calls_section_begin|>", - previous_token_ids=[1], - current_token_ids=[1, section_begin_id], - delta_token_ids=[section_begin_id], - request=None, - ) - - # Immediate section end - _result2 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Reasoning <|tool_calls_section_begin|>", - current_text="Reasoning <|tool_calls_section_begin|><|tool_calls_section_end|>", - delta_text="<|tool_calls_section_end|>", - previous_token_ids=[1, section_begin_id], - current_token_ids=[1, section_begin_id, section_end_id], - delta_token_ids=[section_end_id], - request=None, - ) - # Should exit cleanly without errors - assert kimi_k2_tool_parser.in_tool_section is False - - -def test_malformed_tool_section_recovery(kimi_k2_tool_parser): - """ - Test that the parser recovers from a malformed tool section - that never closes properly. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - - # Enter tool section - _result1 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text="<|tool_calls_section_begin|>", - delta_text="<|tool_calls_section_begin|>", - previous_token_ids=[], - current_token_ids=[section_begin_id], - delta_token_ids=[section_begin_id], - request=None, - ) - assert kimi_k2_tool_parser.in_tool_section is True - - # Simulate a lot of text without proper tool calls or section end - # This should trigger the error recovery mechanism - large_text = "x" * 10000 # Exceeds max_section_chars - - result2 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="<|tool_calls_section_begin|>", - current_text="<|tool_calls_section_begin|>" + large_text, - delta_text=large_text, - previous_token_ids=[section_begin_id], - current_token_ids=[section_begin_id] + list(range(100, 100 + len(large_text))), - delta_token_ids=list(range(100, 100 + len(large_text))), - request=None, - ) - - # Parser should have force-exited the tool section - assert kimi_k2_tool_parser.in_tool_section is False - # And returned the content as reasoning - assert result2 is not None - assert result2.content == large_text - - -def test_state_reset(kimi_k2_tool_parser): - """Test that reset_streaming_state() properly clears all state.""" - # Put parser in a complex state - kimi_k2_tool_parser.in_tool_section = True - kimi_k2_tool_parser.token_buffer = "some buffer" - kimi_k2_tool_parser.current_tool_id = 5 - kimi_k2_tool_parser.prev_tool_call_arr = [{"id": "test"}] - kimi_k2_tool_parser.section_char_count = 1000 - - # Reset - kimi_k2_tool_parser.reset_streaming_state() - - # Verify all state is cleared - assert kimi_k2_tool_parser.in_tool_section is False - assert kimi_k2_tool_parser.token_buffer == "" - assert kimi_k2_tool_parser.current_tool_id == -1 - assert kimi_k2_tool_parser.prev_tool_call_arr == [] - assert kimi_k2_tool_parser.section_char_count == 0 - assert kimi_k2_tool_parser.current_tool_name_sent is False - assert kimi_k2_tool_parser.streamed_args_for_tool == [] - - -def test_section_begin_noise_tool_begin_same_chunk(kimi_k2_tool_parser): - """ - Test that begin→noise→tool_begin within the SAME chunk suppresses - the noise text correctly (not just across chunks). - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - tool_call_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_call_begin|>") - - # Single delta containing: section_begin + spurious text + tool_call_begin - combined_text = "<|tool_calls_section_begin|> noise text <|tool_call_begin|>" - - result = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Reasoning ", - current_text="Reasoning " + combined_text, - delta_text=combined_text, - previous_token_ids=[1, 2], - current_token_ids=[1, 2, section_begin_id, 3, 4, tool_call_begin_id], - delta_token_ids=[section_begin_id, 3, 4, tool_call_begin_id], - request=None, - ) - - # The noise text should NOT leak into content - # Result should either be None/empty or start tool call parsing - if result is not None and result.content is not None: - # If content is returned, it should not contain the noise - assert "noise text" not in result.content - assert result.content == "" or result.content.strip() == "" - - -def test_stream_ends_without_section_end_marker(kimi_k2_tool_parser): - """ - Test that if the stream ends (EOF) without a proper section end marker, - the parser doesn't leak text, doesn't crash, and resets state cleanly. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - - # Enter tool section - _result1 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text="<|tool_calls_section_begin|>", - delta_text="<|tool_calls_section_begin|>", - previous_token_ids=[], - current_token_ids=[section_begin_id], - delta_token_ids=[section_begin_id], - request=None, - ) - assert kimi_k2_tool_parser.in_tool_section is True - - # Some content in tool section - result2 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="<|tool_calls_section_begin|>", - current_text="<|tool_calls_section_begin|> partial content", - delta_text=" partial content", - previous_token_ids=[section_begin_id], - current_token_ids=[section_begin_id, 10, 11], - delta_token_ids=[10, 11], - request=None, - ) - # Content should be suppressed - assert result2.content == "" or result2.content is None - - # Stream ends (EOF) - no more deltas, no section_end marker - # Simulate this by manually checking state and resetting - # (In real usage, the request handler would call reset_streaming_state) - assert kimi_k2_tool_parser.in_tool_section is True # Still in section - - # Reset state (as would happen between requests) - kimi_k2_tool_parser.reset_streaming_state() - - # Verify clean slate - assert kimi_k2_tool_parser.in_tool_section is False - assert kimi_k2_tool_parser.token_buffer == "" - - # Next request should work normally - result3 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text="New reasoning", - delta_text="New reasoning", - previous_token_ids=[], - current_token_ids=[20, 21], - delta_token_ids=[20, 21], - request=None, - ) - assert result3 is not None - assert result3.content == "New reasoning" - - -def test_same_chunk_begin_and_end_markers(kimi_k2_tool_parser): - """ - CRITICAL TEST: Verify that when both section_begin and section_end - markers appear in the SAME chunk, the parser correctly: - 1. Enters the tool section - 2. Immediately exits the tool section - 3. Does NOT get stuck in in_tool_section=True state - - This tests the bug fix where elif was changed to if to handle - both state transitions in a single delta. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - - # Single chunk with both markers (e.g., empty tool section) - combined_delta = "<|tool_calls_section_begin|><|tool_calls_section_end|>" - - result = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Some reasoning ", - current_text="Some reasoning " + combined_delta, - delta_text=combined_delta, - previous_token_ids=[1, 2], - current_token_ids=[1, 2, section_begin_id, section_end_id], - delta_token_ids=[section_begin_id, section_end_id], - request=None, - ) - - # CRITICAL: Parser should NOT be stuck in tool section - assert kimi_k2_tool_parser.in_tool_section is False, ( - "Parser stuck in tool section after processing both begin/end in same chunk. " - "This indicates the elif bug was not fixed." - ) - - # Result should be empty or contain only stripped content - assert result is not None - assert result.content == "" or result.content is None - - # Verify subsequent content streams correctly (not suppressed) - result2 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Some reasoning " + combined_delta, - current_text="Some reasoning " + combined_delta + " More reasoning", - delta_text=" More reasoning", - previous_token_ids=[1, 2, section_begin_id, section_end_id], - current_token_ids=[1, 2, section_begin_id, section_end_id, 10, 11], - delta_token_ids=[10, 11], - request=None, - ) - - # This content should NOT be suppressed (we're out of tool section) - assert result2 is not None - assert result2.content == " More reasoning" - - -def test_same_chunk_begin_content_end_markers(kimi_k2_tool_parser): - """ - Test the same-chunk scenario with actual content between markers. - Example: <|tool_calls_section_begin|> text <|tool_calls_section_end|> - all arriving in one delta. The key is that the state machine correctly - transitions in and out within the same chunk. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - - # Chunk with begin, some whitespace/noise, and end all together - # This simulates a tool section that opens and closes in the same chunk - combined_delta = "<|tool_calls_section_begin|> <|tool_calls_section_end|>" - - _result = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Reasoning ", - current_text="Reasoning " + combined_delta, - delta_text=combined_delta, - previous_token_ids=[1], - current_token_ids=[1, section_begin_id, 100, section_end_id], - delta_token_ids=[section_begin_id, 100, section_end_id], - request=None, - ) - - # Parser should exit cleanly (not stuck in tool section) - assert kimi_k2_tool_parser.in_tool_section is False - - # Verify the fix: next content should stream normally, not be suppressed - result2 = kimi_k2_tool_parser.extract_tool_calls_streaming( - previous_text="Reasoning " + combined_delta, - current_text="Reasoning " + combined_delta + " Done", - delta_text=" Done", - previous_token_ids=[1, section_begin_id, 100, section_end_id], - current_token_ids=[1, section_begin_id, 100, section_end_id, 200], - delta_token_ids=[200], - request=None, - ) - - # Content after section should be returned (not suppressed) - assert result2 is not None - assert result2.content == " Done" - - -def test_tool_call_end_and_section_end_same_chunk(kimi_k2_tool_parser): - """ - CRITICAL TEST (P1): Verify that when both <|tool_call_end|> and - <|tool_calls_section_end|> appear in the SAME chunk, the parser: - 1. Processes the tool_call_end first (emits final arguments) - 2. THEN exits the section - 3. Does NOT drop the final tool call update - 4. Does NOT leak special tokens into reasoning - - This tests the deferred section exit fix. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - tool_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_call_begin|>") - tool_end_id = kimi_k2_tool_parser.vocab.get("<|tool_call_end|>") - - # Simulate a streaming sequence for a SHORT tool call (all in one chunk): - combined = ( - '<|tool_call_begin|>get_weather:0 <|tool_call_argument_begin|> {"city": "Paris"} ' - "<|tool_call_end|><|tool_calls_section_end|>" - ) - - deltas = [ - ("Let me help. ", [1, 2]), - ("<|tool_calls_section_begin|>", [section_begin_id]), - (combined, [tool_begin_id, 10, 11, 12, tool_end_id, section_end_id]), - (" Done", [20]), - ] - - results = run_streaming_sequence(kimi_k2_tool_parser, deltas) - - # CRITICAL: Parser should have exited section AFTER processing tool - assert kimi_k2_tool_parser.in_tool_section is False - - # Tool call should have been emitted (not dropped) - if results[2] is not None and results[2].content is not None: - # Verify no special tokens leaked into content - assert "<|tool_call_end|>" not in results[2].content - assert "<|tool_calls_section_end|>" not in results[2].content - - # Content after tool section should stream normally - assert results[3] is not None - assert results[3].content == " Done" - - -def test_streaming_tool_call_markers_not_leaked(kimi_k2_tool_parser): - """ - CRITICAL TEST: Verify that tool call markers (<|tool_call_begin|>, - <|tool_call_end|>, <|tool_call_argument_begin|>) are NOT leaked - into the content field during streaming. - - This reproduces the AWS Bedrock bug where tool call markers appeared - in the 'text' field of responses. - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - tool_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_call_begin|>") - tool_end_id = kimi_k2_tool_parser.vocab.get("<|tool_call_end|>") - - # List of markers that should NEVER appear in content - forbidden_markers = [ - "<|tool_call_begin|>", - "<|tool_call_end|>", - "<|tool_call_argument_begin|>", - "<|tool_calls_section_begin|>", - "<|tool_calls_section_end|>", - ] - - all_content = [] - - # Steps: reasoning, section begin, tool call, section end, more reasoning - tool_chunk = ( - "<|tool_call_begin|> functions.get_weather:0 " - '<|tool_call_argument_begin|> {"city": "Tokyo"} <|tool_call_end|>' - ) - deltas = [ - ("I'll check the weather. ", [1, 2, 3]), - ("<|tool_calls_section_begin|>", [section_begin_id]), - (tool_chunk, [tool_begin_id, 10, 11, tool_end_id]), - ("<|tool_calls_section_end|>", [section_end_id]), - (" Here's the result.", [20, 21]), - ] - - results = run_streaming_sequence(kimi_k2_tool_parser, deltas) - - for res in results: - if res and res.content: - all_content.append(res.content) - - # CRITICAL ASSERTIONS: No forbidden markers in any content - full_content = "".join(all_content) - for marker in forbidden_markers: - assert marker not in full_content, ( - f"MARKER LEAK DETECTED: '{marker}' found in content. " - f"Full content: {repr(full_content)}" + rec = run_tool_extraction_streaming(parser, deltas) + + assert len(rec.tool_calls) == 3 + names = [tc.function.name for tc in rec.tool_calls] + assert names == ["get_weather", "get_news", "send_email"] + ids = [tc.id for tc in rec.tool_calls] + assert len(set(ids)) == 3 # unique ids + + def test_truncated_tool_call_no_end_marker(self, parser): + """Stream ending mid-tool-call (max_tokens) doesn't crash.""" + deltas = [ + "I'll check. ", + SECTION_BEGIN, + TOOL_BEGIN, + "functions.get_weather:0 ", + ARG_BEGIN, + '{"city": "Bei', + # Stream ends here — no TOOL_END, no SECTION_END + ] + rec = run_tool_extraction_streaming(parser, deltas) + + # Should not crash; tool name and partial args extracted + assert len(rec.tool_calls) == 1 + assert rec.tool_calls[0].function.name == "get_weather" + assert rec.tool_calls[0].id == "functions.get_weather:0" + assert rec.tool_calls[0].function.arguments == '{"city": "Bei' + # No markers leaked into content + for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]: + assert marker not in rec.other_content + + def test_content_after_tool_section(self, parser): + """Trailing text after section_end doesn't crash or leak markers.""" + deltas = [ + "Before. ", + SECTION_BEGIN, + TOOL_BEGIN, + "functions.get_weather:0 ", + ARG_BEGIN, + '{"city": "Tokyo"} ', + TOOL_END, + SECTION_END, + " After tools.", + ] + rec = run_tool_extraction_streaming(parser, deltas) + + # Tool call extracted correctly + assert len(rec.tool_calls) == 1 + assert rec.tool_calls[0].function.name == "get_weather" + assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"} + # Trailing content after tool section is dropped + assert "After tools." not in rec.other_content + # No markers leaked into content + for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]: + assert marker not in rec.other_content + + +class TestAdjustRequest: + def test_sets_skip_special_tokens_false(self, parser): + request = MagicMock(spec=ChatCompletionRequest) + request.tools = [{"type": "function", "function": {"name": "test"}}] + request.tool_choice = "auto" + request.skip_special_tokens = True + + result = parser.adjust_request(request) + assert result.skip_special_tokens is False + + def test_no_change_when_tool_choice_none(self, parser): + request = MagicMock(spec=ChatCompletionRequest) + request.tools = [{"type": "function", "function": {"name": "test"}}] + request.tool_choice = "none" + request.skip_special_tokens = True + + result = parser.adjust_request(request) + assert result.skip_special_tokens is True + + def test_no_change_when_no_tools(self, parser): + request = MagicMock(spec=ChatCompletionRequest) + request.tools = None + request.tool_choice = "auto" + request.skip_special_tokens = True + + result = parser.adjust_request(request) + assert result.skip_special_tokens is True + + +def _chunk_tokenized_deltas(tokenizer, text: str, stream_interval: int) -> list[str]: + """Encode text, group tokens into chunks of stream_interval, decode each.""" + token_ids = tokenizer.encode(text, add_special_tokens=False) + deltas = [] + prev = "" + for i in range(0, len(token_ids), stream_interval): + decoded = tokenizer.decode( + token_ids[: i + stream_interval], skip_special_tokens=False + ) + deltas.append(decoded[len(prev) :]) + prev = decoded + return deltas + + +class TestStreamingIntervals: + """Test streaming at various token-chunk sizes to catch boundary bugs.""" + + @pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) + def test_single_tool_call_at_interval(self, kimi_k2_tokenizer, stream_interval): + text = "Help. " + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')) + deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval) + parser = KimiK2ToolParser(kimi_k2_tokenizer) + rec = run_tool_extraction_streaming( + parser, deltas, assert_one_tool_per_delta=False ) - # Also check that tool call content (function name, arguments) is not leaked - assert "get_weather" not in full_content, ( - f"TOOL CALL CONTENT LEAKED: 'get_weather' found in content. " - f"Full content: {repr(full_content)}" - ) - assert "Tokyo" not in full_content, ( - f"TOOL CALL CONTENT LEAKED: 'Tokyo' found in content. " - f"Full content: {repr(full_content)}" - ) + assert len(rec.tool_calls) == 1 + assert rec.tool_calls[0].function.name == "get_weather" + assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"} - # Verify that legitimate content was preserved - assert "I'll check the weather." in full_content or len(all_content) > 0 - - -def test_native_id_extracted_and_placed_on_tool_call(kimi_k2_tool_parser): - """Regression: parser extracts native ID onto ToolCall (PR #32768).""" - model_output = ( - "Checking weather. " - "<|tool_calls_section_begin|>" - "<|tool_call_begin|>functions.get_weather:0" - '<|tool_call_argument_begin|>{"city": "Tokyo"}' - "<|tool_call_end|>" - "<|tool_calls_section_end|>" - ) - - result = kimi_k2_tool_parser.extract_tool_calls(model_output, request=None) - assert result.tools_called - assert len(result.tool_calls) == 1 - - tc = result.tool_calls[0] - # Native ID from model output must be used as the tool call ID - assert tc.id == "functions.get_weather:0" - assert tc.function.name == "get_weather" - assert json.loads(tc.function.arguments) == {"city": "Tokyo"} - - -def test_multi_turn_native_id_continuity(kimi_k2_tool_parser, kimi_k2_tokenizer): - """Regression: native IDs from turn 1 preserved across turns (PR #32768).""" - turn1_output = ( - "Let me check. " - "<|tool_calls_section_begin|>" - "<|tool_call_begin|>functions.get_weather:0" - '<|tool_call_argument_begin|>{"city": "Beijing"}' - "<|tool_call_end|>" - "<|tool_calls_section_end|>" - ) - - turn1_result = kimi_k2_tool_parser.extract_tool_calls(turn1_output, request=None) - assert turn1_result.tools_called - assert turn1_result.tool_calls[0].id == "functions.get_weather:0" - - # Fresh parser for turn 2 - turn2_parser = KimiK2ToolParser(kimi_k2_tokenizer) - turn2_output = ( - "Now let me get news. " - "<|tool_calls_section_begin|>" - "<|tool_call_begin|>functions.get_news:0" - '<|tool_call_argument_begin|>{"topic": "weather in Beijing"}' - "<|tool_call_end|>" - "<|tool_calls_section_end|>" - ) - - turn2_result = turn2_parser.extract_tool_calls(turn2_output, request=None) - assert turn2_result.tools_called - assert turn2_result.tool_calls[0].id == "functions.get_news:0" - - -def test_streaming_multiple_tool_calls_not_leaked(kimi_k2_tool_parser): - """ - Test that MULTIPLE tool calls in streaming mode do not leak into content. - This reproduces the AWS Bedrock scenario: "Compare weather in Tokyo and NYC". - """ - kimi_k2_tool_parser.reset_streaming_state() - - section_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_begin|>") - section_end_id = kimi_k2_tool_parser.vocab.get("<|tool_calls_section_end|>") - tool_begin_id = kimi_k2_tool_parser.vocab.get("<|tool_call_begin|>") - tool_end_id = kimi_k2_tool_parser.vocab.get("<|tool_call_end|>") - - all_content = [] - - tool1 = '<|tool_call_begin|> get_weather:0 <|tool_call_argument_begin|> {"city": "Tokyo"} <|tool_call_end|>' - tool2 = ' <|tool_call_begin|> get_weather:1 <|tool_call_argument_begin|> {"city": "New York"} <|tool_call_end|>' - - deltas = [ - ("I'll compare the weather. ", [1, 2, 3]), - ("<|tool_calls_section_begin|>", [section_begin_id]), - (tool1, [tool_begin_id, 10, tool_end_id]), - (tool2, [tool_begin_id, 20, tool_end_id]), - ("<|tool_calls_section_end|>", [section_end_id]), - (" Here's the comparison.", [30]), - ] - - results = run_streaming_sequence(kimi_k2_tool_parser, deltas) - - for res in results: - if res and res.content: - all_content.append(res.content) - - # Assertions - full_content = "".join(all_content) - - # Check no markers leaked - forbidden = ["<|tool_call", "<|tool_calls_section"] - for marker in forbidden: - assert marker not in full_content, ( - f"MARKER LEAKED: {marker} in {repr(full_content)}" + @pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) + def test_content_then_tool_call_at_interval( + self, kimi_k2_tokenizer, stream_interval + ): + text = "Sure, let me check. " + _wrap( + _tool("functions.get_weather:0", '{"city": "Tokyo"}') + ) + deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval) + parser = KimiK2ToolParser(kimi_k2_tokenizer) + rec = run_tool_extraction_streaming( + parser, deltas, assert_one_tool_per_delta=False ) - # Check no tool call content leaked (both tools) - assert "get_weather" not in full_content, f"TOOL NAME LEAKED: {repr(full_content)}" - assert "Tokyo" not in full_content, f"TOOL ARG LEAKED (Tokyo): {repr(full_content)}" - assert "New York" not in full_content, ( - f"TOOL ARG LEAKED (NYC): {repr(full_content)}" - ) + assert "let me check" in rec.other_content + assert "get_weather" not in rec.other_content + assert len(rec.tool_calls) == 1 + assert rec.tool_calls[0].function.name == "get_weather" + assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"} - # Legitimate content preserved - assert "compare" in full_content.lower() or len(all_content) > 0 + @pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) + def test_multiple_tool_calls_at_interval(self, kimi_k2_tokenizer, stream_interval): + text = "Compare. " + _wrap( + _tool("functions.search:0", '{"q": "cats"}'), + _tool("functions.search:1", '{"q": "dogs"}'), + ) + deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval) + parser = KimiK2ToolParser(kimi_k2_tokenizer) + rec = run_tool_extraction_streaming( + parser, deltas, assert_one_tool_per_delta=False + ) + + assert len(rec.tool_calls) == 2 + assert rec.tool_calls[0].function.name == "search" + assert json.loads(rec.tool_calls[0].function.arguments) == {"q": "cats"} + assert rec.tool_calls[1].function.name == "search" + assert json.loads(rec.tool_calls[1].function.arguments) == {"q": "dogs"} + + @pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) + def test_plain_text_at_interval(self, kimi_k2_tokenizer, stream_interval): + text = "This is plain text with no tool calling involved." + deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval) + parser = KimiK2ToolParser(kimi_k2_tokenizer) + rec = run_tool_extraction_streaming( + parser, deltas, assert_one_tool_per_delta=False + ) + + assert rec.other_content == text + assert rec.tool_calls == [] + + def test_content_and_tool_call_in_single_chunk(self, kimi_k2_tokenizer): + """Content + complete tool call in one chunk must both be emitted.""" + text = "Hi! " + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')) + deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval=9999) + parser = KimiK2ToolParser(kimi_k2_tokenizer) + rec = run_tool_extraction_streaming( + parser, deltas, assert_one_tool_per_delta=False + ) + + assert "Hi!" in rec.other_content + assert "get_weather" not in rec.other_content + assert len(rec.tool_calls) == 1 + assert rec.tool_calls[0].function.name == "get_weather" + assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"} diff --git a/vllm/tool_parsers/kimi_k2_tool_parser.py b/vllm/tool_parsers/kimi_k2_tool_parser.py index bc995319e5..7ddd8fa7a8 100644 --- a/vllm/tool_parsers/kimi_k2_tool_parser.py +++ b/vllm/tool_parsers/kimi_k2_tool_parser.py @@ -1,6 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# code modified from deepseekv3_tool_parser.py from collections.abc import Sequence @@ -17,12 +16,14 @@ from vllm.entrypoints.openai.engine.protocol import ( FunctionCall, ToolCall, ) +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import ( Tool, ToolParser, ) +from vllm.tool_parsers.utils import partial_tag_overlap logger = init_logger(__name__) @@ -30,124 +31,44 @@ logger = init_logger(__name__) class KimiK2ToolParser(ToolParser): def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): super().__init__(tokenizer, tools) - self.current_tool_name_sent: bool = False + + # Streaming state + self._sent_content_idx: int = 0 self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[ - str - ] = [] # map what has been streamed for each tool so far to a list + self.streamed_args_for_tool: list[str] = [] - # Section-level state management to prevent token leakage - self.in_tool_section: bool = False - self.token_buffer: str = "" - # Buffer size: empirical worst-case for longest marker (~30 chars) * 2 - # + safety margin for unicode + partial overlap. Prevents unbounded growth. - self.buffer_max_size: int = 1024 - self.section_char_count: int = 0 # Track characters processed in tool section - self.max_section_chars: int = 8192 # Force exit if section exceeds this - self._buffer_overflow_logged: bool = False # Log overflow once per session - - # Support both singular and plural variants + # Section marker self.tool_calls_start_token: str = "<|tool_calls_section_begin|>" - self.tool_calls_end_token: str = "<|tool_calls_section_end|>" - self.tool_calls_start_token_variants: list[str] = [ - "<|tool_calls_section_begin|>", - "<|tool_call_section_begin|>", # singular variant - ] - self.tool_calls_end_token_variants: list[str] = [ - "<|tool_calls_section_end|>", - "<|tool_call_section_end|>", # singular variant - ] + # Individual tool call markers self.tool_call_start_token: str = "<|tool_call_begin|>" self.tool_call_end_token: str = "<|tool_call_end|>" + self.tool_call_arg_token: str = "<|tool_call_argument_begin|>" + # Regex for non-streaming extraction self.tool_call_regex = re.compile( - r"<\|tool_call_begin\|>\s*(?P[^<]+:\d+)\s*<\|tool_call_argument_begin\|>\s*(?P(?:(?!<\|tool_call_begin\|>).)*?)\s*<\|tool_call_end\|>", + r"<\|tool_call_begin\|>\s*(?P[^<]+:\d+)\s*" + r"<\|tool_call_argument_begin\|>\s*" + r"(?P(?:(?!<\|tool_call_begin\|>).)*?)\s*" + r"<\|tool_call_end\|>", re.DOTALL, ) - self.stream_tool_call_portion_regex = re.compile( - r"(?P.+:\d+)\s*<\|tool_call_argument_begin\|>\s*(?P.*)" - ) - - self.stream_tool_call_name_regex = re.compile(r"(?P.+:\d+)\s*") - if not self.model_tokenizer: raise ValueError( "The model tokenizer must be passed to the ToolParser " "constructor during construction." ) - self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) - self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) - # Get token IDs for all variants - self.tool_calls_start_token_ids: list[int] = [ - tid - for variant in self.tool_calls_start_token_variants - if (tid := self.vocab.get(variant)) is not None - ] - self.tool_calls_end_token_ids: list[int] = [ - tid - for variant in self.tool_calls_end_token_variants - if (tid := self.vocab.get(variant)) is not None - ] - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - - if ( - self.tool_calls_start_token_id is None - or self.tool_calls_end_token_id is None - ): - raise RuntimeError( - "Kimi-K2 Tool parser could not locate tool call start/end " - "tokens in the tokenizer!" - ) - - def _check_and_strip_markers(self, text: str) -> tuple[str, bool, bool]: - """ - Check for section begin/end markers in text and strip them. - Returns: (cleaned_text, found_section_begin, found_section_end) - """ - found_begin = False - found_end = False - cleaned = text - - # Check for section begin markers (any variant) - for variant in self.tool_calls_start_token_variants: - if variant in cleaned: - cleaned = cleaned.replace(variant, "") - found_begin = True - - # Check for section end markers (any variant) - for variant in self.tool_calls_end_token_variants: - if variant in cleaned: - cleaned = cleaned.replace(variant, "") - found_end = True - return cleaned, found_begin, found_end - - def _reset_section_state(self) -> None: - """Reset state when exiting tool section.""" - self.in_tool_section = False - self.token_buffer = "" - self.section_char_count = 0 - - def reset_streaming_state(self) -> None: - """ - Reset all streaming state. Call this between requests to prevent - state leakage when parser instance is reused. - """ - # Reset section state - self._reset_section_state() - - # Reset parent class state - self.current_tool_name_sent = False - self.prev_tool_call_arr = [] - self.current_tool_id = -1 - self.streamed_args_for_tool = [] - - logger.debug("Streaming state reset") + def adjust_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + request = super().adjust_request(request) + if request.tools and request.tool_choice != "none": + # Ensure special-token markers appear as literal text in + # current_text so we can do pure text-based parsing. + request.skip_special_tokens = False + return request def extract_tool_calls( self, @@ -198,6 +119,95 @@ class KimiK2ToolParser(ToolParser): tools_called=False, tool_calls=[], content=model_output ) + def _extract_content(self, current_text: str) -> str | None: + """Return unsent content before the tool-calls section, or None. + + Holds back any trailing suffix that partially matches + ``<|tool_calls_section_begin|>`` to avoid leaking marker bytes. + """ + if self.tool_calls_start_token not in current_text: + overlap = partial_tag_overlap(current_text, self.tool_calls_start_token) + sendable_idx = len(current_text) - overlap + else: + sendable_idx = current_text.index(self.tool_calls_start_token) + + if sendable_idx > self._sent_content_idx: + content = current_text[self._sent_content_idx : sendable_idx] + self._sent_content_idx = sendable_idx + return content + return None + + def _extract_tool_calls(self, current_text: str) -> list[str]: + """Extract raw bodies from ``<|tool_call_begin|>…<|tool_call_end|>`` blocks.""" + if self.tool_calls_start_token not in current_text: + return [] + + results: list[str] = [] + pos = current_text.index(self.tool_calls_start_token) + while True: + start = current_text.find(self.tool_call_start_token, pos) + if start == -1: + break + tc_start = start + len(self.tool_call_start_token) + end = current_text.find(self.tool_call_end_token, tc_start) + + if end != -1: + tool_call = current_text[tc_start:end] + pos = end + len(self.tool_call_end_token) + else: + tool_call = current_text[tc_start:] + overlap = partial_tag_overlap(tool_call, self.tool_call_end_token) + if overlap: + tool_call = tool_call[:-overlap] + + results.append(tool_call) + + if end == -1: + break + return results + + @staticmethod + def _extract_tool_id_and_name( + header: str | None, + ) -> tuple[str | None, str | None]: + """Parse ``(tool_id, tool_name)`` from a header + like ``"functions.get_weather:0"``.""" + if header is None: + return None, None + match = re.match(r"(.+:\d+)", header) + if not match: + return None, None + + tool_id = match.group(1).strip() + tool_name = tool_id.split(":")[0].split(".")[-1] + return tool_id, tool_name + + def _split_tool_call(self, tool_call: str) -> tuple[str | None, str | None]: + """Split a tool-call body into ``(header, arguments)`` at the argument marker. + + Example:: + 'get_weather:0 <|tool_call_argument_begin|>{"c' + -> ("get_weather:0", '{"c') + """ + arg_pos = tool_call.find(self.tool_call_arg_token) + if arg_pos == -1: + return None, None + header = tool_call[:arg_pos].strip() + tool_args = tool_call[arg_pos + len(self.tool_call_arg_token) :] + return header, tool_args + + def _compute_args_diff(self, index: int, tool_args: str | None) -> str | None: + """Return new argument text not yet sent for tool `index`, or None.""" + if tool_args is None: + return None + prev = self.streamed_args_for_tool[index] + if len(tool_args) <= len(prev): + return None + diff = tool_args[len(prev) :] + self.streamed_args_for_tool[index] = tool_args + self.prev_tool_call_arr[index]["arguments"] = tool_args + return diff + def extract_tool_calls_streaming( self, previous_text: str, @@ -208,394 +218,59 @@ class KimiK2ToolParser(ToolParser): delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> DeltaMessage | None: - logger.debug("delta_text: %s", delta_text) - logger.debug("delta_token_ids: %s", delta_token_ids) - - # Flag to defer section exit until after tool parsing completes - deferred_section_exit = False - - # Add delta to buffer for split marker detection - self.token_buffer += delta_text - - # Enforce buffer size limit to prevent memory issues - if len(self.token_buffer) > self.buffer_max_size: - if not self._buffer_overflow_logged: - logger.warning( - "Token buffer exceeded max size (%d bytes), flushing excess. " - "This may indicate very long markers or unusual tokenization.", - self.buffer_max_size, - ) - self._buffer_overflow_logged = True - # Keep only the most recent content that might contain partial markers - self.token_buffer = self.token_buffer[-self.buffer_max_size // 2 :] - - # Check buffer for section markers (handles split tokens) - buffered_text, found_section_begin, found_section_end = ( - self._check_and_strip_markers(self.token_buffer) - ) - - # Track section state transitions - if found_section_begin and not self.in_tool_section: - logger.debug("Entering tool section") - self.in_tool_section = True - self.token_buffer = buffered_text # Use cleaned buffer - self.section_char_count = 0 # Reset counter for new section - - if found_section_end and self.in_tool_section: - logger.debug("Detected section end marker") - # CRITICAL: Don't exit early if tool_call_end is in this chunk. - # Tool parser must emit final arguments/close first to avoid dropping - # the final tool update and leaking tokens into reasoning channel. - has_tool_end = self.tool_call_end_token_id in delta_token_ids - if has_tool_end: - # Defer exit until after tool parsing completes - deferred_section_exit = True - logger.debug("Deferring section exit: tool_call_end in same chunk") - self.token_buffer = buffered_text - else: - # No tool call ending, safe to exit immediately - logger.debug("Exiting tool section") - self._reset_section_state() - # Extract any content AFTER the section end marker in delta_text - # (don't use buffered_text as it contains tool call data) - post_section_content = "" - for variant in self.tool_calls_end_token_variants: - if variant in delta_text: - parts = delta_text.split(variant, 1) - if len(parts) > 1: - post_section_content = parts[1] - break - if post_section_content.strip(): - return DeltaMessage(content=post_section_content) - return DeltaMessage(content="") - else: - self.token_buffer = buffered_text - - # Check if any variant of section start token is in current_token_ids - has_section_token = any( - tid in current_token_ids for tid in self.tool_calls_start_token_ids - ) - - # Early return: if no section token detected yet, return as reasoning content - if not has_section_token and not self.in_tool_section: - logger.debug("No tool call tokens found!") - # Don't clear buffer - it needs to accumulate partial markers across deltas - # Buffer overflow is already protected by lines 215-224 - return DeltaMessage(content=delta_text) - - # Strip section markers from delta_text for subsequent processing - # NOTE: This preprocessing happens BEFORE the regex-based tool call - # parsing (from PR #24847) to ensure markers are removed cleanly - # before pattern matching. No double-stripping occurs because - # section markers and tool call markers are distinct. - delta_text, _, _ = self._check_and_strip_markers(delta_text) - - # Error recovery: If in tool section for too long, force exit - if self.in_tool_section: - self.section_char_count += len(delta_text) - if self.section_char_count > self.max_section_chars: - logger.warning( - "Tool section exceeded max length (%d chars), forcing exit. " - "This may indicate malformed model output.", - self.max_section_chars, - ) - self._reset_section_state() - # Deferred exit already handled by forced exit above - # Return remaining content as reasoning (or empty delta if no content) - return DeltaMessage(content=delta_text if delta_text.strip() else "") - try: - # figure out where we are in the parsing by counting tool call - # start & end tags - prev_tool_start_count = previous_token_ids.count( - self.tool_call_start_token_id - ) - prev_tool_end_count = previous_token_ids.count(self.tool_call_end_token_id) - cur_tool_start_count = current_token_ids.count( - self.tool_call_start_token_id - ) - cur_tool_end_count = current_token_ids.count(self.tool_call_end_token_id) - tool_call_portion = None - text_portion = None + # Extract any content before tool calls. + content = self._extract_content(current_text) + tool_calls = self._extract_tool_calls(current_text) + tool_call_deltas: list[DeltaToolCall] = [] - # case: if we're generating text, OR rounding out a tool call - if ( - cur_tool_start_count == cur_tool_end_count - and prev_tool_end_count == cur_tool_end_count - and self.tool_call_end_token not in delta_text - ): - # Suppress content between section begin and first tool begin - # (header noise). Don't suppress content between tools to avoid - # breaking potential delimiter characters. - if self.in_tool_section and cur_tool_start_count == 0: - logger.debug( - "In tool section before first tool, suppressing: %s", - delta_text, - ) - # Return empty delta to maintain iterator contract - return DeltaMessage(content="") - logger.debug("Generating text content! skipping tool parsing.") - return DeltaMessage(content=delta_text) + for i, tool_call in enumerate(tool_calls): + # First time seeing tool call at index i. + if i >= len(self.prev_tool_call_arr): + # Initialize streaming state. + self.prev_tool_call_arr.append({}) + self.streamed_args_for_tool.append("") - if self.tool_call_end_token in delta_text: - logger.debug("tool_call_end_token in delta_text") - full_text = current_text + delta_text - tool_call_portion = ( - full_text.split(self.tool_call_start_token)[-1] - .split(self.tool_call_end_token)[0] - .rstrip() - ) - delta_text = delta_text.split(self.tool_call_end_token)[0].rstrip() - text_portion = delta_text.split(self.tool_call_end_token)[-1].lstrip() + header, tool_args = self._split_tool_call(tool_call) - # case -- we're starting a new tool call - if ( - cur_tool_start_count > cur_tool_end_count - and cur_tool_start_count > prev_tool_start_count - ): - if len(delta_token_ids) > 1: - tool_call_portion = current_text.split(self.tool_call_start_token)[ - -1 - ] - else: - tool_call_portion = None - delta = None - - text_portion = None - - # set cursors and state appropriately - self.current_tool_id += 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append("") - logger.debug("Starting on a new tool %s", self.current_tool_id) - - # case -- we're updating an existing tool call - elif ( - cur_tool_start_count > cur_tool_end_count - and cur_tool_start_count == prev_tool_start_count - ): - # get the portion of the text that's the tool call - tool_call_portion = current_text.split(self.tool_call_start_token)[-1] - text_portion = None - - # case -- the current tool call is being closed. - elif ( - cur_tool_start_count == cur_tool_end_count - and cur_tool_end_count >= prev_tool_end_count - ): - if self.prev_tool_call_arr is None or len(self.prev_tool_call_arr) == 0: - logger.debug("attempting to close tool call, but no tool call") - # Handle deferred section exit before returning - if deferred_section_exit and self.in_tool_section: - self._reset_section_state() - return None - diff = self.prev_tool_call_arr[self.current_tool_id].get("arguments") - if diff: - diff = ( - diff.encode("utf-8").decode("unicode_escape") - if diff is str - else diff - ) - if '"}' not in delta_text: - # Handle deferred section exit before returning - if deferred_section_exit and self.in_tool_section: - self._reset_section_state() - return None - end_loc = delta_text.rindex('"}') - diff = delta_text[:end_loc] + '"}' - logger.debug( - "Finishing tool and found diff that had not " - "been streamed yet: %s", - diff, - ) - self.streamed_args_for_tool[self.current_tool_id] += diff - # Handle deferred section exit before returning - if deferred_section_exit and self.in_tool_section: - logger.debug("Completing deferred section exit") - self._reset_section_state() - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall(arguments=diff).model_dump( - exclude_none=True - ), - ) - ] - ) - - # case -- otherwise we're just generating text - else: - # Check if we're in tool section - if so, suppress - if self.in_tool_section: - logger.debug("In tool section, suppressing text generation") - # Handle deferred section exit before returning - if deferred_section_exit: - self._reset_section_state() - return DeltaMessage(content="") - text = delta_text.replace(self.tool_call_start_token, "") - text = text.replace(self.tool_call_end_token, "") - delta = DeltaMessage(tool_calls=[], content=text) - # Handle deferred section exit before returning - if deferred_section_exit and self.in_tool_section: - self._reset_section_state() - return delta - - current_tool_call = dict() - if tool_call_portion: - current_tool_call_matches = self.stream_tool_call_portion_regex.match( - tool_call_portion - ) - if current_tool_call_matches: - tool_id, tool_args = current_tool_call_matches.groups() - tool_name = tool_id.split(":")[0].split(".")[-1] - current_tool_call["id"] = tool_id.strip() - current_tool_call["name"] = tool_name - current_tool_call["arguments"] = tool_args - else: - current_tool_call_name_matches = ( - self.stream_tool_call_name_regex.match(tool_call_portion) - ) - if current_tool_call_name_matches: - (tool_id_str,) = current_tool_call_name_matches.groups() - tool_name = tool_id_str.split(":")[0].split(".")[-1] - current_tool_call["id"] = tool_id_str.strip() - current_tool_call["name"] = tool_name - current_tool_call["arguments"] = "" - else: - logger.debug("Not enough token") - return None - - # case - we haven't sent the tool name yet. If it's available, send - # it. otherwise, wait until it's available. - if not self.current_tool_name_sent: - if current_tool_call is None: - return None - function_name: str | None = current_tool_call.get("name") - tool_id = current_tool_call.get("id") - if function_name: - self.current_tool_name_sent = True - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - type="function", - id=tool_id, - function=DeltaFunctionCall( - name=function_name - ).model_dump(exclude_none=True), - ) - ] - ) - else: - return None - - # case -- otherwise, send the tool call delta - - # if the tool call portion is None, send the delta as text - if tool_call_portion is None: - # if there's text but not tool calls, send that - - # otherwise None to skip chunk - # CRITICAL: Never return content if we're in a tool section - if self.in_tool_section: - return None - delta = ( - DeltaMessage(content=delta_text) - if text_portion is not None - else None - ) - return delta - - # now, the nitty-gritty of tool calls - # now we have the portion to parse as tool call. - - logger.debug( - "Trying to parse current tool call with ID %s", self.current_tool_id - ) - - # if we're starting a new tool call, push an empty object in as - # a placeholder for the arguments - if len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - - # main logic for tool parsing here - compare prev. partially-parsed - # JSON to the current partially-parsed JSON - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get( - "arguments" - ) - cur_arguments = current_tool_call.get("arguments") - - logger.debug("diffing old arguments: %s", prev_arguments) - logger.debug("against new ones: %s", cur_arguments) - - # case -- no arguments have been created yet. skip sending a delta. - if not cur_arguments and not prev_arguments: - logger.debug("Skipping text %s - no arguments", delta_text) - delta = None - - # case -- prev arguments are defined, but non are now. - # probably impossible, but not a fatal error - just keep going - elif not cur_arguments and prev_arguments: - logger.error( - "should be impossible to have arguments reset " - "mid-call. skipping streaming anything." - ) - delta = None - - # case -- we now have the first info about arguments available from - # autocompleting the JSON - elif cur_arguments and not prev_arguments: - delta = DeltaMessage( - tool_calls=[ + # Stream back tool name. + if "name" not in self.prev_tool_call_arr[i]: + tool_id, tool_name = self._extract_tool_id_and_name(header) + if not tool_name: + # Can't skip to tool i+1 if i isn't ready + break + self.prev_tool_call_arr[i]["name"] = tool_name + self.prev_tool_call_arr[i]["id"] = tool_id + tool_call_deltas.append( DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=cur_arguments - ).model_dump(exclude_none=True), + index=i, + type="function", + id=tool_id, + function=DeltaFunctionCall(name=tool_name).model_dump( + exclude_none=True + ), ) - ] - ) - self.streamed_args_for_tool[self.current_tool_id] = cur_arguments - - # last case -- we have an update to existing arguments. - elif cur_arguments and prev_arguments: - if ( - isinstance(delta_text, str) - and cur_arguments != prev_arguments - and len(cur_arguments) > len(prev_arguments) - and cur_arguments.startswith(prev_arguments) - ): - delta_arguments = cur_arguments[len(prev_arguments) :] - logger.debug("got diff %s", delta_text) - - delta = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_id, - function=DeltaFunctionCall( - arguments=delta_arguments - ).model_dump(exclude_none=True), - ) - ] ) - self.streamed_args_for_tool[self.current_tool_id] = cur_arguments - else: - delta = None - # handle saving the state for the current tool into - # the "prev" list for use in diffing for the next iteration - if self.current_tool_id == len(self.prev_tool_call_arr) - 1: - self.prev_tool_call_arr[self.current_tool_id] = current_tool_call - else: - self.prev_tool_call_arr.append(current_tool_call) + # Stream back new tool args by diffing against what was sent. + args_diff = self._compute_args_diff(i, tool_args) + if args_diff: + tool_call_deltas.append( + DeltaToolCall( + index=i, + function=DeltaFunctionCall(arguments=args_diff).model_dump( + exclude_none=True + ), + ) + ) - # Handle deferred section exit after tool parsing completes - if deferred_section_exit and self.in_tool_section: - logger.debug("Completing deferred section exit") - self._reset_section_state() - - return delta + if content or tool_call_deltas: + return DeltaMessage( + content=content, + tool_calls=tool_call_deltas, + ) + return None except Exception: logger.exception("Error trying to handle streaming tool call.") - return None # do not stream a delta. skip this token ID. + return None