diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 128e511e69..4817d3b900 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -30,6 +30,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ) from vllm.parser.engine.registered_adapters import ( Gemma4Parser, + Glm47MoeParser, MinimaxM2Parser, NemotronV3Parser, Qwen3Parser, @@ -571,6 +572,80 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample: ) +# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ────────────── + +_GLM47_MOE_VOCAB: dict[str, int] = { + "": 50, + "": 51, + "": 60, + "": 61, + "": 62, + "": 63, + "": 64, + "": 65, +} + + +def _glm47_moe_arg_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False) + + +def _glm47_moe_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [ + ("", True), + (tc.name, False), + ] + for key, value in tc.arguments.items(): + segs.extend( + [ + ("", True), + (key, False), + ("", True), + ("", True), + (_glm47_moe_arg_value(value), False), + ("", True), + ] + ) + segs.append(("", True)) + return segs + + +def _glm47_moe_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + if scenario.reasoning is not None: + segs.append((scenario.reasoning, False)) + if scenario.content is not None or scenario.tool_calls: + segs.append(("", True)) + if scenario.content is not None: + segs.append((scenario.content, False)) + if scenario.tool_calls: + for tc in scenario.tool_calls: + segs.extend(_glm47_moe_tool_segments(tc)) + return segs + + +def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample: + sample = _make_sample( + sample_id=f"glm47_moe-{scenario.id}", + description=scenario.description, + vocab=_GLM47_MOE_VOCAB, + segments=_glm47_moe_segments(scenario), + expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "", + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, Glm47MoeParser) + return sample + + # ── Registry and public API ────────────────────────────────────────── _BUILDERS: dict[str, Any] = { @@ -578,6 +653,7 @@ _BUILDERS: dict[str, Any] = { "gemma4": _build_gemma4, "minimax_m2": _build_minimax_m2, "nemotron_v3": _build_nemotron_v3, + "glm47_moe": _build_glm47_moe, } diff --git a/tests/reasoning/test_glm4_moe_reasoning_parser.py b/tests/reasoning/test_glm4_moe_reasoning_parser.py index 6f7827e5b8..3d6f21b5e1 100644 --- a/tests/reasoning/test_glm4_moe_reasoning_parser.py +++ b/tests/reasoning/test_glm4_moe_reasoning_parser.py @@ -11,7 +11,7 @@ parser_name = "glm45" start_token = "" end_token = "" -REASONING_MODEL_NAME = "zai-org/GLM-4.5" +REASONING_MODEL_NAME = "zai-org/GLM-4.7" @pytest.fixture(scope="module") @@ -35,18 +35,32 @@ WITH_THINK_STREAM = { WITHOUT_THINK = { "output": "This is the rest", - "reasoning": None, - "content": "This is the rest", + "reasoning": "This is the rest", + "content": None, "is_reasoning_end": False, } WITHOUT_THINK_STREAM = { "output": "This is the rest", - "reasoning": None, - "content": "This is the rest", + "reasoning": "This is the rest", + "content": None, "is_reasoning_end": False, } +WITHOUT_OPEN_THINK = { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", + "is_reasoning_end": True, +} + +WITHOUT_OPEN_THINK_STREAM = { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", + "is_reasoning_end": True, +} + COMPLETE_REASONING = { "output": "This is a reasoning section", "reasoning": "This is a reasoning section", @@ -61,8 +75,8 @@ MULTILINE_REASONING = { } ONLY_OPEN_TAG = { "output": "This is a reasoning section", - "reasoning": None, - "content": "This is a reasoning section", + "reasoning": "This is a reasoning section", + "content": None, "is_reasoning_end": False, } @@ -94,6 +108,16 @@ TEST_CASES = [ WITHOUT_THINK_STREAM, id="without_think_stream", ), + pytest.param( + False, + WITHOUT_OPEN_THINK, + id="without_open_think", + ), + pytest.param( + True, + WITHOUT_OPEN_THINK_STREAM, + id="without_open_think_stream", + ), pytest.param( False, COMPLETE_REASONING, diff --git a/tests/tool_parsers/test_glm47_moe_tool_parser.py b/tests/tool_parsers/test_glm47_moe_tool_parser.py index 51696c9547..c9767f6f62 100644 --- a/tests/tool_parsers/test_glm47_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm47_moe_tool_parser.py @@ -16,7 +16,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.tokenizers import get_tokenizer from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser -MODEL = "zai-org/GLM-4.5" +MODEL = "zai-org/GLM-4.7" @pytest.fixture(scope="module") @@ -136,9 +136,10 @@ class TestGlm47Streaming: _reset(glm47_tool_parser) chunks = ["", "get_current_date", ""] current_text = "" + deltas = [] for chunk in chunks: current_text += chunk - glm47_tool_parser.extract_tool_calls_streaming( + delta = glm47_tool_parser.extract_tool_calls_streaming( previous_text="", current_text=current_text, delta_text=chunk, @@ -147,7 +148,23 @@ class TestGlm47Streaming: delta_token_ids=[], request=mock_request, ) - assert len(glm47_tool_parser.prev_tool_call_arr) >= 1 + if delta: + deltas.append(delta) + tool_calls = [ + tool_call for delta in deltas for tool_call in (delta.tool_calls or []) + ] + names = [ + tool_call.function.name + for tool_call in tool_calls + if tool_call.function and tool_call.function.name + ] + arguments = [ + tool_call.function.arguments + for tool_call in tool_calls + if tool_call.function and tool_call.function.arguments + ] + assert names == ["get_current_date"] + assert "".join(arguments) == "{}" def test_with_args(self, glm47_tool_parser, mock_request): _reset(glm47_tool_parser) @@ -161,9 +178,10 @@ class TestGlm47Streaming: "", ] current_text = "" + deltas = [] for chunk in chunks: current_text += chunk - glm47_tool_parser.extract_tool_calls_streaming( + delta = glm47_tool_parser.extract_tool_calls_streaming( previous_text="", current_text=current_text, delta_text=chunk, @@ -172,5 +190,13 @@ class TestGlm47Streaming: delta_token_ids=[], request=mock_request, ) - args = json.loads(glm47_tool_parser.prev_tool_call_arr[0]["arguments"]) + if delta: + deltas.append(delta) + arguments = [ + tool_call.function.arguments + for delta in deltas + for tool_call in (delta.tool_calls or []) + if tool_call.function and tool_call.function.arguments + ] + args = json.loads("".join(arguments)) assert args["city"] == "Beijing" diff --git a/tests/tool_parsers/test_glm4_moe_tool_parser.py b/tests/tool_parsers/test_glm4_moe_tool_parser.py index b0300297dd..ca110adac0 100644 --- a/tests/tool_parsers/test_glm4_moe_tool_parser.py +++ b/tests/tool_parsers/test_glm4_moe_tool_parser.py @@ -1,1067 +1,57 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Compatibility tests for GLM-4.5 using the shared GLM XML parser.""" import json -from unittest.mock import Mock - -import pytest -from openai.types.responses import FunctionTool +from typing import Any, TypedDict +from tests.parser.engine.replay_harness import MockTokenizer from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionToolsParam, FunctionDefinition, ) -from vllm.entrypoints.openai.engine.protocol import FunctionCall, ToolCall -from vllm.tokenizers import get_tokenizer -from vllm.tool_parsers.glm4_moe_tool_parser import ( - Glm4MoeModelToolParser, -) +from vllm.tool_parsers import ToolParserManager +from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser -# Use a common model that is likely to be available MODEL = "zai-org/GLM-4.5" - -@pytest.fixture(scope="module") -def glm4_moe_tokenizer(): - return get_tokenizer(tokenizer_name=MODEL) +_GLM_VOCAB = { + "": 50, + "": 51, + "": 60, + "": 61, + "": 62, + "": 63, + "": 64, + "": 65, +} -@pytest.fixture -def sample_tools(): +class _CollectedToolDelta(TypedDict): + name: str | None + args_fragments: list[str] + + +def _mock_tokenizer() -> MockTokenizer: + return MockTokenizer(vocab=_GLM_VOCAB, tokens=[]) + + +def _tools() -> list[ChatCompletionToolsParam]: return [ ChatCompletionToolsParam( function=FunctionDefinition( - name="get_weather", - parameters={"city": {"type": "string"}}, - ), - ), - ] - - -@pytest.fixture -def glm4_moe_tool_parser(glm4_moe_tokenizer, sample_tools): - return Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=sample_tools) - - -@pytest.fixture -def mock_request(sample_tools) -> ChatCompletionRequest: - request = Mock(spec=ChatCompletionRequest) - request.tools = sample_tools - return request - - -def assert_tool_calls( - actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall] -): - assert len(actual_tool_calls) == len(expected_tool_calls) - - for actual_tool_call, expected_tool_call in zip( - actual_tool_calls, expected_tool_calls - ): - assert isinstance(actual_tool_call.id, str) - assert len(actual_tool_call.id) > 0 - - assert actual_tool_call.type == "function" - assert actual_tool_call.function.name == expected_tool_call.function.name - # Compare arguments as JSON objects to handle formatting differences - actual_args = json.loads(actual_tool_call.function.arguments) - expected_args = json.loads(expected_tool_call.function.arguments) - assert actual_args == expected_args - - -def test_extract_tool_calls_no_tools(glm4_moe_tool_parser, mock_request): - model_output = "This is a test" - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - assert not extracted_tool_calls.tools_called - assert extracted_tool_calls.tool_calls == [] - assert extracted_tool_calls.content == model_output - - -@pytest.mark.parametrize( - ids=[ - "single_tool_call", - "multiple_tool_calls", - "tool_call_with_content_before", - "tool_call_with_mixed_args", - "tool_call_with_chinese_content", - ], - argnames=["model_output", "expected_tool_calls", "expected_content"], - argvalues=[ - ( - """get_current_weather - city - Dallas - state - TX - unit - fahrenheit - """, - [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps( - { - "city": "Dallas", - "state": "TX", - "unit": "fahrenheit", - } - ), - ) - ) - ], - None, - ), - ( - """get_current_weather - city - Dallas - state - TX - unit - fahrenheit - - get_current_weather - city - Orlando - state - FL - unit - fahrenheit - """, - [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps( - { - "city": "Dallas", - "state": "TX", - "unit": "fahrenheit", - } - ), - ) - ), - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps( - { - "city": "Orlando", - "state": "FL", - "unit": "fahrenheit", - } - ), - ) - ), - ], - None, - ), - ( - """I'll help you check the weather. get_current_weather - city - Seattle - state - WA - unit - celsius - """, - [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps( - { - "city": "Seattle", - "state": "WA", - "unit": "celsius", - } - ), - ) - ) - ], - "I'll help you check the weather. ", - ), - ( - """get_current_weather - city - New York - state - NY - unit - celsius - """, - [ - ToolCall( - function=FunctionCall( - name="get_current_weather", - arguments=json.dumps( - { - "city": "New York", - "state": "NY", - "unit": "celsius", - } - ), - ) - ) - ], - None, - ), - ( - """I will help you get the weather.get_weather - city - Beijing - date - 2025-08-01 - """, - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "city": "Beijing", - "date": "2025-08-01", - } - ), - ) - ) - ], - "I will help you get the weather.", - ), - ], -) -def test_extract_tool_calls( - glm4_moe_tool_parser, - mock_request, - model_output, - expected_tool_calls, - expected_content, -): - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - assert extracted_tool_calls.tools_called - assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) - - assert extracted_tool_calls.content == expected_content - - -def test_extract_tool_calls_with_thinking_tags(glm4_moe_tool_parser, mock_request): - """Test tool extraction when thinking tags are present.""" - model_output = """I want to get the weather. - -I will help you get the weather. -get_weather -city -Beijing -date -2025-08-01 -""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - assert len(extracted_tool_calls.tool_calls) == 1 - assert extracted_tool_calls.tool_calls[0].function.name == "get_weather" - - expected_content = """I want to get the weather. - -I will help you get the weather. -""" - assert extracted_tool_calls.content == expected_content - - -def test_extract_tool_calls_malformed_xml(glm4_moe_tool_parser, mock_request): - """Test that malformed XML is handled gracefully.""" - model_output = """get_weather -city -Seattle -incomplete_arg -value -""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - # Should handle malformed XML gracefully - # The parser should either extract what it can or return no tool calls - # depending on how robust we want the parsing to be - assert isinstance(extracted_tool_calls.tools_called, bool) - assert isinstance(extracted_tool_calls.tool_calls, list) - - -def test_extract_tool_calls_empty_arguments(glm4_moe_tool_parser, mock_request): - """Test tool calls with no arguments.""" - model_output = """get_current_time -""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - assert len(extracted_tool_calls.tool_calls) == 1 - assert extracted_tool_calls.tool_calls[0].function.name == "get_current_time" - # Empty arguments should result in empty JSON object - assert extracted_tool_calls.tool_calls[0].function.arguments == "{}" - - -def test_extract_tool_calls_mixed_content(glm4_moe_tool_parser, mock_request): - """Test extraction with mixed content and multiple tool calls.""" - model_output = """I will help you get the weather info. - -get_weather -city -Beijing -date -2025-08-01 - - -meaningwhile, I will also check the weather in Shanghai. - -get_weather -city -Shanghai -date -2025-08-01 -""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - assert len(extracted_tool_calls.tool_calls) == 2 - - # Check first tool call - assert extracted_tool_calls.tool_calls[0].function.name == "get_weather" - args1 = json.loads(extracted_tool_calls.tool_calls[0].function.arguments) - assert args1["city"] == "Beijing" - assert args1["date"] == "2025-08-01" - - # Check second tool call - assert extracted_tool_calls.tool_calls[1].function.name == "get_weather" - args2 = json.loads(extracted_tool_calls.tool_calls[1].function.arguments) - assert args2["city"] == "Shanghai" - assert args2["date"] == "2025-08-01" - - # Content should be everything before the first tool call - assert extracted_tool_calls.content == "I will help you get the weather info.\n\n" - - -def test_streaming_basic_functionality(glm4_moe_tool_parser, mock_request): - """Test basic streaming functionality.""" - _reset_streaming_state(glm4_moe_tool_parser) - - current_text = """get_weather -city -Beijing -""" - - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=current_text, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should return tool call with name and arguments in one shot - assert result is not None - assert result.tool_calls is not None - assert len(result.tool_calls) >= 1 - - -def test_streaming_no_tool_calls(glm4_moe_tool_parser, mock_request): - """Test streaming when there are no tool calls.""" - _reset_streaming_state(glm4_moe_tool_parser) - - current_text = "This is just regular text without any tool calls." - - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=current_text, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should return content - assert result is not None - assert result.content == current_text - - -def test_streaming_with_content_before_tool_calls(glm4_moe_tool_parser, mock_request): - """Test streaming when there's content before tool calls.""" - _reset_streaming_state(glm4_moe_tool_parser) - - current_text = "I will help you get the weather." - - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=current_text, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should return content before the tag - assert result is not None - assert result.content == "I will help you get the weather." - - -def test_extract_tool_calls_special_characters(glm4_moe_tool_parser, mock_request): - """Test tool calls with special characters and unicode.""" - model_output = """send_message -recipient -Amy -message -It is a nice day -priority -high -""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - assert len(extracted_tool_calls.tool_calls) == 1 - assert extracted_tool_calls.tool_calls[0].function.name == "send_message" - - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments) - assert args["recipient"] == "Amy" - assert args["message"] == "It is a nice day" - assert args["priority"] == "high" - - -def test_extract_tool_calls_incomplete_tool_call(glm4_moe_tool_parser, mock_request): - """Test incomplete tool calls (missing closing tag).""" - model_output = """get_weather -city -Beijing -date -2025-08-01""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - # Incomplete tool calls should not be extracted - assert not extracted_tool_calls.tools_called - assert extracted_tool_calls.tool_calls == [] - assert extracted_tool_calls.content == model_output - - -def _reset_streaming_state(parser): - """Helper to reset parser streaming state.""" - parser.current_tool_name_sent = False - parser.prev_tool_call_arr = [] - parser.current_tool_id = -1 - parser.streamed_args_for_tool = [] - parser._tool_call_ids = [] - parser._sent_content_idx = 0 - - -def test_streaming_incremental_string_value(glm4_moe_tool_parser, mock_request): - """Test incremental streaming of string argument values.""" - _reset_streaming_state(glm4_moe_tool_parser) - - # Simulate streaming a tool call chunk by chunk - chunks = [ - "", - "get_weather\n", - "city", - "", - "Bei", - "jing", - "", - "", - ] - - collected_fragments = [] - current_text = "" - for chunk in chunks: - current_text += chunk - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - if result is not None and result.tool_calls: - for tc in result.tool_calls: - func = tc.function - if isinstance(func, dict): - if func.get("arguments"): - collected_fragments.append(func["arguments"]) - if func.get("name"): - collected_fragments.append(f"name:{func['name']}") - else: - if func.arguments: - collected_fragments.append(func.arguments) - if func.name: - collected_fragments.append(f"name:{func.name}") - - # Verify we got incremental streaming of the argument value - assert len(collected_fragments) > 0 - # The fragments should include the tool name and argument pieces - combined = "".join(collected_fragments) - assert "get_weather" in combined or "name:get_weather" in combined - - -def test_streaming_empty_tool_call(glm4_moe_tool_parser, mock_request): - """Test that empty tool calls don't cause infinite loops.""" - _reset_streaming_state(glm4_moe_tool_parser) - - current_text = "" - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=current_text, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should not hang and should return something (None or content) - # The key is that this completes without hanging - assert result is None or hasattr(result, "content") or hasattr(result, "tool_calls") - - -def test_streaming_prev_tool_call_arr_updates(glm4_moe_tool_parser, mock_request): - """Test that prev_tool_call_arr is populated incrementally.""" - _reset_streaming_state(glm4_moe_tool_parser) - - chunks = [ - "get_weather\n", - "city", - "Beijing", - "", - ] - - current_text = "" - for chunk in chunks: - current_text += chunk - glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # After the tool call completes, prev_tool_call_arr should be populated - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1 - tool_entry = glm4_moe_tool_parser.prev_tool_call_arr[0] - assert tool_entry.get("name") == "get_weather" - - # arguments is a JSON string in the re-parse approach - args_str = tool_entry.get("arguments") - assert isinstance(args_str, str), f"Expected str, got {type(args_str)}" - parsed = json.loads(args_str) - assert parsed["city"] == "Beijing" - - # streamed_args_for_tool should match prev_tool_call_arr arguments - streamed = glm4_moe_tool_parser.streamed_args_for_tool[0] - assert streamed == args_str - - -def test_streaming_multiple_tool_calls_sequential(glm4_moe_tool_parser, mock_request): - """Test streaming multiple sequential tool calls.""" - _reset_streaming_state(glm4_moe_tool_parser) - - chunks = [ - "get_weather\n", - "city", - "Beijing", - "", - "get_weather\n", - "city", - "Shanghai", - "", - ] - - current_text = "" - for chunk in chunks: - current_text += chunk - glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should have two tool calls in prev_tool_call_arr - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2 - args0 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"]) - args1 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"]) - assert args0["city"] == "Beijing" - assert args1["city"] == "Shanghai" - - -def test_streaming_json_escape_in_string(glm4_moe_tool_parser, mock_request): - """Test that special characters in string values are properly escaped.""" - _reset_streaming_state(glm4_moe_tool_parser) - - chunks = [ - "send_message\n", - "message", - 'Hello "world"\nNew line', - "", - ] - - current_text = "" - for chunk in chunks: - current_text += chunk - glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # The streamed_args_for_tool should contain valid JSON - assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1 - args_json = glm4_moe_tool_parser.streamed_args_for_tool[0] - parsed = json.loads(args_json) - assert "message" in parsed - assert '"' in parsed["message"] or "world" in parsed["message"] - - -def test_streaming_long_content_incremental(glm4_moe_tokenizer): - """Test incremental streaming of long content (Issue #32829). - - This is the core fix: for long string values like code (4000+ chars), - the parser should stream incrementally rather than buffering until - complete. This test verifies we get many fragments, not just 1-3. - """ - - # Bubble sort example from Issue #32829 - realistic long content - bubble_sort_code = '''#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -Bubble Sort Implementation -""" - -def bubble_sort(arr): - n = len(arr) - for i in range(n): - swapped = False - for j in range(0, n - i - 1): - if arr[j] > arr[j + 1]: - arr[j], arr[j + 1] = arr[j + 1], arr[j] - swapped = True - if not swapped: - break - return arr - -if __name__ == "__main__": - test_arr = [64, 34, 25, 12, 22, 11, 90] - print(f"Original: {test_arr}") - sorted_arr = bubble_sort(test_arr.copy()) - print(f"Sorted: {sorted_arr}")''' - - # Create tools with schema to enable string type detection - # This is required for incremental streaming of string values - tools = [ - ChatCompletionToolsParam( - function=FunctionDefinition( - name="write_to_file", + name="get_current_weather", parameters={ "type": "object", "properties": { - "file_path": {"type": "string"}, - "content": {"type": "string"}, + "city": {"type": "string"}, + "state": {"type": "string"}, + "unit": {"type": "string"}, }, }, ), ), - ] - glm4_moe_tool_parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest( - model=MODEL, - messages=[], - tools=tools, - ) - - # Simulate token-based streaming (special tags as single tokens) - chunks = [ - "", - "write_to_file\n", - "file_path", - "/tmp/bubble_sort.py", - "content", - "", - ] - # Add content line by line (realistic token streaming) - for line in bubble_sort_code.split("\n"): - chunks.append(line + "\n") - chunks.append("") - chunks.append("") - - # Count argument fragments - fragment_count = 0 - current_text = "" - for chunk in chunks: - current_text += chunk - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=request, - ) - if result is not None and result.tool_calls: - for tc in result.tool_calls: - func = tc.function - if isinstance(func, dict): - args = func.get("arguments") - else: - args = getattr(func, "arguments", None) - if args: - fragment_count += 1 - - # For true incremental streaming, we expect many fragments (10+) - # Old buffered implementation would give only 1-3 fragments - assert fragment_count >= 10, ( - f"Expected >=10 fragments for incremental streaming, got {fragment_count}" - ) - - # Verify final result is valid JSON - assert len(glm4_moe_tool_parser.streamed_args_for_tool) == 1 - args_json = glm4_moe_tool_parser.streamed_args_for_tool[0] - parsed = json.loads(args_json) - assert parsed["file_path"] == "/tmp/bubble_sort.py" - assert "def bubble_sort" in parsed["content"] - - -def test_extract_tool_calls_numeric_deserialization(glm4_moe_tool_parser, mock_request): - """Test that numeric arguments are deserialized as numbers, not strings.""" - model_output = """calculate -operation -add -a -42 -b -3.14 -enabled -true -""" - - extracted_tool_calls = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted_tool_calls.tools_called - assert len(extracted_tool_calls.tool_calls) == 1 - - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments) - - # String should remain string - assert args["operation"] == "add" - assert isinstance(args["operation"], str) - - # Integer should be deserialized as int - assert args["a"] == 42 - assert isinstance(args["a"], int) - - # Float should be deserialized as float - assert args["b"] == 3.14 - assert isinstance(args["b"], float) - - # Boolean should be deserialized as bool - assert args["enabled"] is True - assert isinstance(args["enabled"], bool) - - -def test_whitespace_preserved_in_arg_values(glm4_moe_tokenizer): - """Test that string arguments preserve leading and trailing whitespace.""" - tools = [ - ChatCompletionToolsParam( - function=FunctionDefinition( - name="apply_diff", - parameters={ - "type": "object", - "properties": { - "s": {"type": "string"}, - }, - "required": ["s"], - }, - ), - ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - - model_output = """apply_diff -s - indented code -""" - - extracted_tool_calls = parser.extract_tool_calls(model_output, request=request) - args = json.loads(extracted_tool_calls.tool_calls[0].function.arguments) - - assert args["s"] == " indented code " - - -def test_zero_argument_tool_call(glm4_moe_tool_parser, mock_request): - """Regression: zero-argument tool call crash (PR #32321).""" - model_output = """get_time -""" - - extracted = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted.tools_called - assert len(extracted.tool_calls) == 1 - assert extracted.tool_calls[0].function.name == "get_time" - args = json.loads(extracted.tool_calls[0].function.arguments) - assert args == {} - - -def test_malformed_tool_call_no_regex_match(glm4_moe_tool_parser, mock_request): - """Regression: malformed tool_call with no regex match (PR #32321).""" - model_output = " " - - extracted = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted.tools_called is False - assert extracted.tool_calls == [] - - -def test_delimiter_preserved_transformers_5x(glm4_moe_tool_parser): - """Regression: adjust_request sets skip_special_tokens=False (PR #31622).""" - # Tools enabled - request_with_tools = ChatCompletionRequest( - model=MODEL, - messages=[], - tools=[ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - }, - } - ], - ) # type: ignore - adjusted = glm4_moe_tool_parser.adjust_request(request_with_tools) - assert adjusted.skip_special_tokens is False - - # tool_choice="none" - request_no_choice = ChatCompletionRequest( - model=MODEL, - messages=[], - tools=[ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - }, - } - ], - tool_choice="none", - ) # type: ignore - adjusted_none = glm4_moe_tool_parser.adjust_request(request_no_choice) - assert adjusted_none.skip_special_tokens is True - - # No tools at all - request_no_tools = ChatCompletionRequest( - model=MODEL, - messages=[], - ) # type: ignore - adjusted_empty = glm4_moe_tool_parser.adjust_request(request_no_tools) - assert adjusted_empty.skip_special_tokens is True - - -def test_unicode_characters_preserved(glm4_moe_tool_parser, mock_request): - """Regression: Unicode chars must not be escaped to \\uXXXX (PR #30920).""" - model_output = """send_message -greeting -你好世界 -emoji -🎉 -""" - - extracted = glm4_moe_tool_parser.extract_tool_calls( - model_output, request=mock_request - ) # type: ignore[arg-type] - - assert extracted.tools_called - assert len(extracted.tool_calls) == 1 - - raw_args = extracted.tool_calls[0].function.arguments - assert "你好世界" in raw_args - assert "🎉" in raw_args - assert "\\u4f60" not in raw_args - parsed_args = json.loads(raw_args) - assert parsed_args["greeting"] == "你好世界" - assert parsed_args["emoji"] == "🎉" - - -def test_streaming_multi_token_chunks(glm4_moe_tool_parser, mock_request): - """Test that multi-token chunks (stream_interval > 1) are handled correctly. - - With stream_interval > 1 or MTP, multiple XML tags arrive in one delta. - The old buffer-based parser could only return one delta per call, losing - data on the final output. The re-parse approach handles this correctly. - """ - _reset_streaming_state(glm4_moe_tool_parser) - - # Simulate stream_interval=3: chunks contain multiple XML tags - chunks = [ - "get_weather\ncityBei", - "jing", - "", - ] - - current_text = "" - for chunk in chunks: - current_text += chunk - glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # All data should be captured despite multi-token chunks - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1 - args = json.loads(glm4_moe_tool_parser.streamed_args_for_tool[0]) - assert args["city"] == "Beijing" - - -def test_streaming_entire_tool_call_at_once(glm4_moe_tool_parser, mock_request): - """Test that a complete tool call arriving in one delta works. - - This simulates the extreme MTP case where all tokens arrive at once. - """ - _reset_streaming_state(glm4_moe_tool_parser) - - full_text = ( - "get_weather\n" - "city" - "Beijing" - "" - ) - - result = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=full_text, - delta_text=full_text, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should emit tool call with complete arguments in one shot - assert result is not None - assert result.tool_calls is not None - - # Verify final state - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 1 - args = json.loads(glm4_moe_tool_parser.streamed_args_for_tool[0]) - assert args["city"] == "Beijing" - - -def test_streaming_content_between_tool_calls_multi_token( - glm4_moe_tool_parser, mock_request -): - """Test content between tool calls with multi-token chunks.""" - _reset_streaming_state(glm4_moe_tool_parser) - - # Deliver everything at once — worst case for the old buffer parser - full_text = ( - "I will check.\n" - "get_weather\n" - "city" - "Beijing" - "" - "\nAlso Shanghai.\n" - "get_weather\n" - "city" - "Shanghai" - "" - ) - - # First call with partial text (content only) - partial = "I will check.\n" - result1 = glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=partial, - delta_text=partial, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - assert result1 is not None - assert result1.content == "I will check.\n" - - # Second call with everything - glm4_moe_tool_parser.extract_tool_calls_streaming( - previous_text="", - current_text=full_text, - delta_text=full_text[len(partial) :], - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request, - ) - - # Should have both tool calls - assert len(glm4_moe_tool_parser.prev_tool_call_arr) == 2 - args0 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[0]["arguments"]) - args1 = json.loads(glm4_moe_tool_parser.prev_tool_call_arr[1]["arguments"]) - assert args0["city"] == "Beijing" - assert args1["city"] == "Shanghai" - - -def test_streaming_multi_token_with_multiple_args(glm4_moe_tokenizer): - """Test multi-token streaming with multiple arguments of mixed types.""" - tools = [ ChatCompletionToolsParam( function=FunctionDefinition( name="calculate", @@ -1071,415 +61,168 @@ def test_streaming_multi_token_with_multiple_args(glm4_moe_tokenizer): "operation": {"type": "string"}, "a": {"type": "number"}, "b": {"type": "number"}, + "enabled": {"type": "boolean"}, }, }, ), ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest( - model=MODEL, - messages=[], - tools=tools, - ) - - # All arguments arrive in two big chunks (simulates stream_interval=5) - chunks = [ - "calculate\noperationadda", - "42b3.14", - ] - - current_text = "" - for chunk in chunks: - current_text += chunk - parser.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=request, - ) - - args = json.loads(parser.streamed_args_for_tool[0]) - assert args["operation"] == "add" - assert args["a"] == 42 - assert args["b"] == 3.14 - - -def _simulate_streaming(tokenizer, parser, request, text, stream_interval=1): - """Simulate streaming with a given stream_interval. - - Tokens are batched into chunks of ``stream_interval`` tokens, - mimicking how the output processor delivers them. - Returns a list of non-None DeltaMessages. - """ - tokens = tokenizer.encode(text) - previous_text = "" - deltas = [] - for i in range(0, len(tokens), stream_interval): - chunk_ids = tokens[i : i + stream_interval] - delta_text = tokenizer.decode(chunk_ids) - current_text = previous_text + delta_text - delta = parser.extract_tool_calls_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_text, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=chunk_ids, - request=request, - ) - previous_text = current_text - if delta is not None: - deltas.append(delta) - return deltas - - -def _collect_from_deltas(deltas): - """Reconstruct tool call names/args and content from a delta stream.""" - tools: dict[int, dict] = {} - content_parts: list[str] = [] - for d in deltas: - if d.content: - content_parts.append(d.content) - if d.tool_calls: - for tc in d.tool_calls: - func = tc.function - if isinstance(func, dict): - name = func.get("name") - args = func.get("arguments") - else: - name = getattr(func, "name", None) - args = getattr(func, "arguments", None) - idx = tc.index - if idx not in tools: - tools[idx] = {"name": None, "args_fragments": []} - if name: - tools[idx]["name"] = name - if args: - tools[idx]["args_fragments"].append(args) - return content_parts, tools - - -@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) -def test_stream_interval_single_tool_call(glm4_moe_tokenizer, stream_interval): - """Tool call streaming produces correct name + args at any interval.""" - tools = [ ChatCompletionToolsParam( - function=FunctionDefinition( - name="get_weather", - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - ), - ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - - text = ( - "get_weather\n" - "city" - "Beijing" - "" - ) - - deltas = _simulate_streaming( - glm4_moe_tokenizer, parser, request, text, stream_interval - ) - _, tools_found = _collect_from_deltas(deltas) - - assert 0 in tools_found - assert tools_found[0]["name"] == "get_weather" - args_json = "".join(tools_found[0]["args_fragments"]) - parsed = json.loads(args_json) - assert parsed == {"city": "Beijing"} - - -@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) -def test_stream_interval_multiple_tool_calls(glm4_moe_tokenizer, stream_interval): - """Multiple sequential tool calls with correct indices at any interval.""" - tools = [ - ChatCompletionToolsParam( - function=FunctionDefinition( - name="get_weather", - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - ), - ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - - text = ( - "get_weather\n" - "city" - "Beijing" - "" - "get_weather\n" - "city" - "Shanghai" - "" - ) - - deltas = _simulate_streaming( - glm4_moe_tokenizer, parser, request, text, stream_interval - ) - _, tools_found = _collect_from_deltas(deltas) - - assert 0 in tools_found and 1 in tools_found - args0 = json.loads("".join(tools_found[0]["args_fragments"])) - args1 = json.loads("".join(tools_found[1]["args_fragments"])) - assert args0 == {"city": "Beijing"} - assert args1 == {"city": "Shanghai"} - - -@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8]) -def test_stream_interval_content_then_tool_call(glm4_moe_tokenizer, stream_interval): - """Content before a tool call is fully emitted before tool deltas.""" - tools = [ - ChatCompletionToolsParam( - function=FunctionDefinition( - name="get_weather", - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - ), - ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - - text = ( - "I will check the weather for you.\n" - "get_weather\n" - "city" - "Beijing" - "" - ) - - deltas = _simulate_streaming( - glm4_moe_tokenizer, parser, request, text, stream_interval - ) - content_parts, tools_found = _collect_from_deltas(deltas) - - # Content must be present and precede tool calls - full_content = "".join(content_parts) - assert "I will check the weather" in full_content - - # Tool call must be correct - assert 0 in tools_found - assert tools_found[0]["name"] == "get_weather" - args = json.loads("".join(tools_found[0]["args_fragments"])) - assert args == {"city": "Beijing"} - - -def test_stream_interval_extreme_single_chunk(glm4_moe_tokenizer): - """Extreme MTP: entire output arrives in one chunk (interval=9999).""" - tools = [ - ChatCompletionToolsParam( - function=FunctionDefinition( - name="get_weather", - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - ), - ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - - text = ( - "Here is the weather.\n" - "get_weather\n" - "city" - "Beijing" - "" - ) - - deltas = _simulate_streaming( - glm4_moe_tokenizer, parser, request, text, stream_interval=9999 - ) - content_parts, tools_found = _collect_from_deltas(deltas) - - assert "Here is the weather" in "".join(content_parts) - assert 0 in tools_found - assert tools_found[0]["name"] == "get_weather" - args = json.loads("".join(tools_found[0]["args_fragments"])) - assert args == {"city": "Beijing"} - - -@pytest.mark.parametrize("stream_interval", [1, 2, 5]) -def test_stream_interval_content_between_tool_calls( - glm4_moe_tokenizer, stream_interval -): - """Content between tool calls must be emitted, not silently dropped.""" - tools = [ - ChatCompletionToolsParam( - function=FunctionDefinition( - name="get_weather", - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - }, - ), - ), - ] - parser = Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=tools) - request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) - - text = ( - "Checking Beijing.\n" - "get_weather\n" - "city" - "Beijing" - "" - "\nAlso Shanghai.\n" - "get_weather\n" - "city" - "Shanghai" - "" - ) - - deltas = _simulate_streaming( - glm4_moe_tokenizer, parser, request, text, stream_interval - ) - content_parts, tools_found = _collect_from_deltas(deltas) - - full_content = "".join(content_parts) - # Both prefix and inter-tool-call content must appear - assert "Checking Beijing" in full_content - assert "Also Shanghai" in full_content - - # Both tool calls must be correct - assert 0 in tools_found and 1 in tools_found - args0 = json.loads("".join(tools_found[0]["args_fragments"])) - args1 = json.loads("".join(tools_found[1]["args_fragments"])) - assert args0 == {"city": "Beijing"} - assert args1 == {"city": "Shanghai"} - - -# ── FunctionTool (Responses API) tests ────────────────────────────── - - -@pytest.fixture -def function_tools(): - return [ - FunctionTool( - type="function", - name="get_weather", - parameters={ - "type": "object", - "properties": { - "city": {"type": "string"}, - "unit": {"type": "string"}, - }, - }, - ), - FunctionTool( - type="function", - name="calculate", - parameters={ - "type": "object", - "properties": { - "operation": {"type": "string"}, - "a": {"type": "number"}, - "b": {"type": "number"}, - }, - }, + function=FunctionDefinition(name="get_time", parameters={}), ), ] -@pytest.fixture -def glm4_moe_parser_function_tools(glm4_moe_tokenizer, function_tools): - return Glm4MoeModelToolParser(glm4_moe_tokenizer, tools=function_tools) +def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest: + return ChatCompletionRequest(model=MODEL, messages=[], tools=tools) -@pytest.fixture -def mock_request_function_tools(function_tools) -> ChatCompletionRequest: - request = Mock(spec=ChatCompletionRequest) - request.tools = function_tools - return request +def _parser(tools: list[ChatCompletionToolsParam] | None = None): + return Glm47MoeModelToolParser(_mock_tokenizer(), tools=tools) -def test_extract_tool_calls_with_function_tool( - glm4_moe_parser_function_tools, mock_request_function_tools -): - model_output = """get_weather +def _collect_tool_deltas(deltas: Any) -> dict[int, _CollectedToolDelta]: + calls: dict[int, _CollectedToolDelta] = {} + for delta in deltas: + if delta is None or not delta.tool_calls: + continue + for tool_call in delta.tool_calls: + entry = calls.setdefault( + tool_call.index, + {"name": None, "args_fragments": []}, + ) + function = tool_call.function + if function is None: + continue + if isinstance(function, dict): + name = function.get("name") + arguments = function.get("arguments") + else: + name = function.name + arguments = function.arguments + if isinstance(name, str) and name: + entry["name"] = name + if isinstance(arguments, str) and arguments: + entry["args_fragments"].append(arguments) + return calls + + +def test_glm45_uses_shared_glm47_parser(): + assert ToolParserManager.get_tool_parser("glm45") is Glm47MoeModelToolParser + assert ToolParserManager.get_tool_parser("glm47") is Glm47MoeModelToolParser + + +def test_extract_tool_calls_with_glm45_newline_format(): + tools = _tools() + parser = _parser(tools) + model_output = """I'll check it. get_current_weather city Dallas +state +TX unit fahrenheit """ - extracted = glm4_moe_parser_function_tools.extract_tool_calls( - model_output, request=mock_request_function_tools - ) + extracted = parser.extract_tool_calls(model_output, request=_request(tools)) + assert extracted.tools_called + assert extracted.content == "I'll check it." assert len(extracted.tool_calls) == 1 - assert extracted.tool_calls[0].function.name == "get_weather" - args = json.loads(extracted.tool_calls[0].function.arguments) - assert args["city"] == "Dallas" - assert args["unit"] == "fahrenheit" + tool_call = extracted.tool_calls[0] + assert tool_call.function.name == "get_current_weather" + assert json.loads(tool_call.function.arguments) == { + "city": "Dallas", + "state": "TX", + "unit": "fahrenheit", + } -def test_extract_tool_calls_with_function_tool_mixed_types( - glm4_moe_parser_function_tools, mock_request_function_tools -): - model_output = """calculate -operation -add -a -42 -b -3.14 +def test_extract_multiple_tool_calls_with_glm45_newline_format(): + tools = _tools() + parser = _parser(tools) + model_output = """get_current_weather +cityDallas + +get_current_weather +cityOrlando """ - extracted = glm4_moe_parser_function_tools.extract_tool_calls( - model_output, request=mock_request_function_tools - ) + extracted = parser.extract_tool_calls(model_output, request=_request(tools)) + assert extracted.tools_called - args = json.loads(extracted.tool_calls[0].function.arguments) - assert args["operation"] == "add" - assert isinstance(args["a"], (int, float)) - assert isinstance(args["b"], float) + assert [tc.function.name for tc in extracted.tool_calls] == [ + "get_current_weather", + "get_current_weather", + ] + assert [ + json.loads(tc.function.arguments)["city"] for tc in extracted.tool_calls + ] == ["Dallas", "Orlando"] -def test_streaming_with_function_tool( - glm4_moe_parser_function_tools, mock_request_function_tools -): - _reset_streaming_state(glm4_moe_parser_function_tools) +def test_extract_tool_calls_coerces_schema_types(): + tools = _tools() + parser = _parser(tools) + model_output = """calculate +operationadd +a42 +b3.14 +enabledtrue +""" + extracted = parser.extract_tool_calls(model_output, request=_request(tools)) + + assert extracted.tools_called + assert json.loads(extracted.tool_calls[0].function.arguments) == { + "operation": "add", + "a": 42, + "b": 3.14, + "enabled": True, + } + + +def test_extract_zero_argument_tool_call_with_glm45_newline_format(): + tools = _tools() + parser = _parser(tools) + + extracted = parser.extract_tool_calls( + "get_time\n", + request=_request(tools), + ) + + assert extracted.tools_called + assert extracted.tool_calls[0].function.name == "get_time" + assert json.loads(extracted.tool_calls[0].function.arguments) == {} + + +def test_streaming_tool_call_with_glm45_newline_format(): + tools = _tools() + parser = _parser(tools) + request = _request(tools) chunks = [ - "get_weather\n", + "", + "get_current_weather\n", "city", "Bei", - "jing", - "", + "jing", "", ] - + deltas = [] current_text = "" + for chunk in chunks: current_text += chunk - glm4_moe_parser_function_tools.extract_tool_calls_streaming( - previous_text="", - current_text=current_text, - delta_text=chunk, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=mock_request_function_tools, + deltas.append( + parser.extract_tool_calls_streaming( + previous_text="", + current_text=current_text, + delta_text=chunk, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=request, + ) ) - assert len(glm4_moe_parser_function_tools.prev_tool_call_arr) == 1 - args = json.loads(glm4_moe_parser_function_tools.prev_tool_call_arr[0]["arguments"]) - assert args["city"] == "Beijing" + calls = _collect_tool_deltas(deltas) + assert calls[0]["name"] == "get_current_weather" + assert json.loads("".join(calls[0]["args_fragments"])) == {"city": "Beijing"} diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index d45a82879f..9d670f3056 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and from vllm.parser.engine.adapters import make_adapters from vllm.parser.gemma4 import Gemma4Parser +from vllm.parser.glm47_moe import Glm47MoeParser from vllm.parser.minimax_m2 import MinimaxM2Parser from vllm.parser.nemotron_v3 import NemotronV3Parser from vllm.parser.qwen3 import Qwen3Parser @@ -32,3 +33,8 @@ from vllm.parser.qwen3 import Qwen3Parser Qwen3ParserReasoningAdapter, Qwen3ParserToolAdapter, ) = make_adapters(Qwen3Parser) + +( + Glm47MoeParserReasoningAdapter, + Glm47MoeParserToolAdapter, +) = make_adapters(Glm47MoeParser) diff --git a/vllm/parser/glm47_moe.py b/vllm/parser/glm47_moe.py new file mode 100644 index 0000000000..8aa4feef25 --- /dev/null +++ b/vllm/parser/glm47_moe.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GLM-4.7 parser for reasoning and tool calls. + +GLM-4.7 uses XML-like tool calls:: + + func_namekeyvalue + +The function name can be followed directly by the first ```` tag, +and tool calls may have no arguments. +""" + +from __future__ import annotations + +import functools +import json +from typing import TYPE_CHECKING + +import regex as re + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +THINK_START = "" +THINK_END = "" +TOOL_CALL_START = "" +TOOL_CALL_END = "" +ARG_KEY_START = "" +ARG_KEY_END = "" +ARG_VALUE_START = "" +ARG_VALUE_END = "" + +_ARG_RE = re.compile( + r"(?P.*?)\s*" + r"(?P.*?)", + re.DOTALL, +) +_PARTIAL_ARG_RE = re.compile( + r"(?P.*?)\s*" + r"(?P.*)$", + re.DOTALL, +) + + +def _glm47_arg_converter(raw_args: str, partial: bool) -> str: + params: dict[str, object] = {} + + for match in _ARG_RE.finditer(raw_args): + params[match.group("key").strip()] = match.group("value") + + if partial: + remaining = _ARG_RE.sub("", raw_args) + match = _PARTIAL_ARG_RE.search(remaining) + if match: + key = match.group("key").strip() + if key: + params[key] = match.group("value") + + return json.dumps(params, ensure_ascii=False) + + +@functools.cache +def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig: + arg_tag_transitions = { + (ParserState.TOOL_ARGS, terminal): Transition( + ParserState.TOOL_ARGS, + (EventType.ARG_VALUE_CHUNK,), + ) + for terminal in ( + "ARG_KEY_START", + "ARG_KEY_END", + "ARG_VALUE_START", + "ARG_VALUE_END", + ) + } + + reasoning_terminals = ( + { + "THINK_START": THINK_START, + "THINK_END": THINK_END, + } + if thinking + else {} + ) + reasoning_token_id_terminals = ( + { + "THINK_START": THINK_START, + "THINK_END": THINK_END, + } + if thinking + else {} + ) + reasoning_transitions = ( + { + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + (ParserState.CONTENT, "THINK_END"): Transition( + ParserState.CONTENT, + (), + ), + } + if thinking + else {} + ) + + return ParserEngineConfig( + name="glm47_moe", + initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + terminals={ + **reasoning_terminals, + "TOOL_START": TOOL_CALL_START, + "TOOL_END": TOOL_CALL_END, + "ARG_KEY_START": ARG_KEY_START, + "ARG_KEY_END": ARG_KEY_END, + "ARG_VALUE_START": ARG_VALUE_START, + "ARG_VALUE_END": ARG_VALUE_END, + }, + token_id_terminals={ + **reasoning_token_id_terminals, + "TOOL_START": TOOL_CALL_START, + "TOOL_END": TOOL_CALL_END, + }, + transitions={ + **reasoning_transitions, + (ParserState.REASONING, "THINK_START"): Transition( + ParserState.REASONING, + (), + ), + (ParserState.REASONING, "TOOL_START"): Transition( + ParserState.TOOL_NAME, + (EventType.REASONING_END, EventType.TOOL_CALL_START), + ), + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_NAME, "ARG_KEY_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.ARG_VALUE_CHUNK,), + ), + (ParserState.TOOL_NAME, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + **arg_tag_transitions, + }, + arg_converter=_glm47_arg_converter, + stream_arg_deltas=True, + tool_args_json=False, + validate_tool_names=True, + ) + + +class Glm47MoeParser(ParserEngine): + """GLM-4.7 parser backed by the declarative parser engine.""" + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + thinking = chat_kwargs.get("thinking", None) + enable_thinking = chat_kwargs.get("enable_thinking", None) + self.thinking_enabled = ( + True + if thinking is None and enable_thinking is None + else bool(thinking) or bool(enable_thinking) + ) + kwargs.setdefault( + "parser_engine_config", + glm47_moe_config(thinking=self.thinking_enabled), + ) + super().__init__(tokenizer, tools, **kwargs) + + def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None: + if name is not None: + name = name.strip() + super()._emit_name_delta(idx, deltas, name) + + def _handle_tool_end(self, event, deltas) -> None: + idx = event.tool_index + if 0 <= idx < len(self._tool_slots): + self._tool_slots[idx].name = self._tool_slots[idx].name.strip() + super()._handle_tool_end(event, deltas) + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + if not self.thinking_enabled: + return True + return super().is_reasoning_end(input_ids) + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + if not self.thinking_enabled: + return input_ids + return super().extract_content_ids(input_ids) + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + if not self.thinking_enabled: + return None, model_output + return super().extract_reasoning(model_output, request) diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 7d46faa6de..cbb1fa350f 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -53,8 +53,12 @@ _REASONING_PARSERS_TO_REGISTER = { "Gemma4ParserReasoningAdapter", ), "glm45": ( - "deepseek_v3_reasoning_parser", - "DeepSeekV3ReasoningWithThinkingParser", + "glm47_moe_reasoning_parser", + "Glm47MoeParserReasoningAdapter", + ), + "glm47": ( + "glm47_moe_reasoning_parser", + "Glm47MoeParserReasoningAdapter", ), "openai_gptoss": ( "gptoss_reasoning_parser", diff --git a/vllm/reasoning/glm47_moe_reasoning_parser.py b/vllm/reasoning/glm47_moe_reasoning_parser.py new file mode 100644 index 0000000000..8e963f88b0 --- /dev/null +++ b/vllm/reasoning/glm47_moe_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import Glm47MoeParserReasoningAdapter + +__all__ = ["Glm47MoeParserReasoningAdapter"] diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 407e57ca2f..bbc4d2edb1 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -51,8 +51,8 @@ _TOOL_PARSERS_TO_REGISTER = { "Ernie45ToolParser", ), "glm45": ( - "glm4_moe_tool_parser", - "Glm4MoeModelToolParser", + "glm47_moe_tool_parser", + "Glm47MoeModelToolParser", ), "glm47": ( "glm47_moe_tool_parser", diff --git a/vllm/tool_parsers/glm47_moe_tool_parser.py b/vllm/tool_parsers/glm47_moe_tool_parser.py index 80068264b7..70275a6ac0 100644 --- a/vllm/tool_parsers/glm47_moe_tool_parser.py +++ b/vllm/tool_parsers/glm47_moe_tool_parser.py @@ -1,41 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GLM-4.7 Tool Call Parser. -GLM-4.7 uses a slightly different tool call format compared to GLM-4.5: - - The function name may appear on the same line as ```` without - a newline separator before the first ````. - - Tool calls may have zero arguments - (e.g. ``func``). +from __future__ import annotations -This parser overrides the parent regex patterns to handle both formats. -""" - -import regex as re - -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import Tool -from vllm.tool_parsers.glm4_moe_tool_parser import Glm4MoeModelToolParser - -logger = init_logger(__name__) +from vllm.parser.engine.registered_adapters import Glm47MoeParserToolAdapter -class Glm47MoeModelToolParser(Glm4MoeModelToolParser): +class Glm47MoeModelToolParser(Glm47MoeParserToolAdapter): # type: ignore[valid-type, misc] supports_required_and_named = False structural_tag_model = "glm_4_7" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - # GLM-4.7 format: func_name[...]* - # The function name can be followed by a newline, whitespace, or - # directly by tags (no separator). The arg section is - # optional so that zero-argument calls are supported. - self.func_detail_regex = re.compile( - r"\s*(\S+?)\s*(.*)?", re.DOTALL - ) - self.func_arg_regex = re.compile( - r"(.*?)\s*(.*?)", - re.DOTALL, - ) diff --git a/vllm/tool_parsers/glm4_moe_tool_parser.py b/vllm/tool_parsers/glm4_moe_tool_parser.py deleted file mode 100644 index 213a774535..0000000000 --- a/vllm/tool_parsers/glm4_moe_tool_parser.py +++ /dev/null @@ -1,495 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -""" -GLM-4 Tool Call Parser with incremental string streaming support. - -This parser fixes the streaming issue reported in Issue #32829 where long string -parameters (e.g., file content with 4000+ characters of code) are buffered until -complete, causing multi-second delays before the user sees any content. - -The fix streams string values incrementally as they arrive, providing a true -streaming experience for long content. -""" - -import json -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.chat_utils import make_tool_call_id -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionNamedToolChoiceParam, - ChatCompletionRequest, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - extract_types_from_schema, - find_tool_properties, - partial_tag_overlap, - safe_literal_eval, -) - -logger = init_logger(__name__) - - -class Glm4MoeModelToolParser(ToolParser): - """Tool parser for GLM-4 models with incremental string streaming. - - On every streaming call the parser re-parses ``current_text`` to find - ```` regions, builds the JSON arguments string for each tool - call, and diffs against what was previously sent to emit only new content. - """ - - supports_required_and_named = False - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - # Stateful streaming fields - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict[str, Any]] = [] - self.current_tool_id: int = -1 - self.streamed_args_for_tool: list[str] = [] - - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - self.arg_key_start: str = "" - self.arg_key_end: str = "" - self.arg_val_start: str = "" - self.arg_val_end: str = "" - - self.tool_calls_start_token = self.tool_call_start_token - - self.func_call_regex = re.compile(r".*?", re.DOTALL) - self.func_detail_regex = re.compile( - r"([^\n]*)\n(.*)", re.DOTALL - ) - self.func_arg_regex = re.compile( - r"(.*?)\s*(.*?)", re.DOTALL - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - - # Pre-compiled pattern for finding the last ... - # before a partial (used in _build_args_json_so_far). - self._arg_key_pattern = re.compile( - re.escape(self.arg_key_start) + r"(.*?)" + re.escape(self.arg_key_end), - re.DOTALL, - ) - - # Streaming state for re-parse-and-diff approach - self._sent_content_idx: int = 0 - self._tool_call_ids: list[str] = [] - - @staticmethod - def _deserialize(value: str) -> Any: - try: - return json.loads(value) - except json.JSONDecodeError: - pass - - try: - return safe_literal_eval(value) - except (ValueError, SyntaxError): - pass - - return value - - @staticmethod - def _json_escape_string_content(s: str) -> str: - """JSON-escape string content for incremental streaming. - - This escapes the content that goes INSIDE a JSON string (between quotes), - not including the surrounding quotes themselves. - """ - if not s: - return "" - return json.dumps(s, ensure_ascii=False)[1:-1] - - def _is_string_type(self, tool_name: str, arg_name: str) -> bool: - tool_properties = find_tool_properties(self.tools, tool_name) - param_schema = tool_properties.get(arg_name) - if param_schema is None: - return False - param_types = extract_types_from_schema(param_schema) - return set(param_types) - {"null"} == {"string"} - - @staticmethod - def _tools_enabled(request: ChatCompletionRequest) -> bool: - """Return whether tool parsing should be applied for this request.""" - try: - tools = getattr(request, "tools", None) - tool_choice = getattr(request, "tool_choice", None) - return bool(tools) and tool_choice != "none" - except Exception: - logger.exception("Failed to determine if tools are enabled.") - return False - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - """Adjust request parameters for tool call token handling. - - For required/named tool_choice, skip setting structured_outputs - because GLM models output tool calls in XML format (per chat - template). Guided decoding would force JSON output, conflicting - with the XML format and causing parsing failures. - """ - if request.tools: - tc = request.tool_choice - if tc == "required" or isinstance(tc, ChatCompletionNamedToolChoiceParam): - # Do NOT call super().adjust_request() for required/named, - # because it would set structured_outputs and force JSON - # output via guided decoding. GLM models use XML tool-call - # syntax (defined in the chat template), so guided decoding - # must be skipped to let the model output XML freely. - # The tool_parser handles extraction from XML output. - if request.tool_choice != "none": - request.skip_special_tokens = False - return request - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # Ensure tool call tokens (, ) are not skipped - # during decoding. Even though they are not marked as special tokens, - # setting skip_special_tokens=False ensures proper handling in - # transformers 5.x where decoding behavior may have changed. - request.skip_special_tokens = False - return request - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - matched_tool_calls = self.func_call_regex.findall(model_output) - logger.debug("model_output: %s", model_output) - try: - tool_calls: list[ToolCall] = [] - for match in matched_tool_calls: - tc_detail = self.func_detail_regex.search(match) - if not tc_detail: - logger.warning( - "Failed to parse tool call details from: %s", - match, - ) - continue - tc_name = tc_detail.group(1).strip() - tc_args = tc_detail.group(2) - pairs = self.func_arg_regex.findall(tc_args) if tc_args else [] - arg_dct: dict[str, Any] = {} - for key, value in pairs: - arg_key = key.strip() - if self._is_string_type(tc_name, arg_key): - arg_val = value - else: - arg_val = self._deserialize(value.strip()) - logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val) - arg_dct[arg_key] = arg_val - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=tc_name, - arguments=json.dumps(arg_dct, ensure_ascii=False), - ), - ) - ) - except Exception: - logger.exception("Failed to extract tool call spec") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - else: - if len(tool_calls) > 0: - content: str | None = model_output[ - : model_output.find(self.tool_calls_start_token) - ] - # Normalize empty/whitespace-only content to None - if not content or not content.strip(): - content = None - return ExtractedToolCallInformation( - tools_called=True, tool_calls=tool_calls, content=content - ) - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def _extract_content(self, current_text: str) -> str | None: - """Return unsent non-tool-call text, or None. - - Collects all text outside ``...`` regions, - including text between consecutive tool calls. Holds back any - suffix that could be a partial ```` tag. - """ - # Build the "sendable index" — the furthest point we can send - # content up to. We scan through the text collecting segments - # that are outside tool-call regions. - content_segments: list[str] = [] - pos = self._sent_content_idx - - while pos < len(current_text): - start = current_text.find(self.tool_call_start_token, pos) - if start == -1: - # No more tool calls — send up to (len - partial-tag overlap) - tail = current_text[pos:] - overlap = partial_tag_overlap(tail, self.tool_call_start_token) - sendable = tail[: len(tail) - overlap] if overlap else tail - if sendable: - content_segments.append(sendable) - pos = len(current_text) - overlap - break - - # Text before this - if start > pos: - content_segments.append(current_text[pos:start]) - - # Skip past the (or to end if incomplete) - end = current_text.find(self.tool_call_end_token, start) - if end != -1: - pos = end + len(self.tool_call_end_token) - else: - # Incomplete tool call — nothing more to send - pos = start - break - - if content_segments: - self._sent_content_idx = pos - return "".join(content_segments) - # Even if no content, advance past completed tool-call regions - if pos > self._sent_content_idx: - self._sent_content_idx = pos - return None - - def _extract_tool_call_regions(self, text: str) -> list[tuple[str, bool]]: - """Extract ``(inner_text, is_complete)`` for each ```` region.""" - results: list[tuple[str, bool]] = [] - pos = 0 - while True: - start = text.find(self.tool_call_start_token, pos) - if start == -1: - break - inner_start = start + len(self.tool_call_start_token) - end = text.find(self.tool_call_end_token, inner_start) - if end != -1: - results.append((text[inner_start:end], True)) - pos = end + len(self.tool_call_end_token) - else: - # Incomplete tool call — strip partial suffix - raw = text[inner_start:] - overlap = partial_tag_overlap(raw, self.tool_call_end_token) - if overlap: - raw = raw[:-overlap] - results.append((raw, False)) - break - return results - - def _extract_tool_name_from_region(self, inner_text: str) -> str | None: - """Extract the tool name from the beginning of a tool-call region. - - The name is everything before the first ``\\n`` or ````. - Returns ``None`` if the name hasn't fully arrived yet. - """ - nl = inner_text.find("\n") - ak = inner_text.find(self.arg_key_start) - candidates = [i for i in [nl, ak] if i != -1] - if not candidates: - return None - cut = min(candidates) - name = inner_text[:cut].strip() - return name if name else None - - def _build_args_json_so_far( - self, - tool_name: str, - inner_text: str, - is_complete: bool, - ) -> str: - """Build the JSON arguments string from the XML pairs seen so far. - - For complete ``/`` pairs the value is fully - formatted. For the last argument whose ```` has been - opened but not closed, the partial string content is included - (JSON-escaped, with an opening ``"`` but no closing ``"``). - - The closing ``}`` is only appended when ``is_complete`` is True - (i.e. the ```` tag has arrived). - """ - # Find all complete arg pairs - pairs = self.func_arg_regex.findall(inner_text) - - parts: list[str] = [] - for key, value in pairs: - key = key.strip() - key_json = json.dumps(key, ensure_ascii=False) - if self._is_string_type(tool_name, key): - # Don't strip string values — whitespace is significant - # and must match the partial-value path for diffing. - val_json = json.dumps(value, ensure_ascii=False) - else: - val_json = json.dumps( - self._deserialize(value.strip()), ensure_ascii=False - ) - parts.append(f"{key_json}: {val_json}") - - # Check for a partial (incomplete) arg value - # Find the last that isn't closed - last_val_start = inner_text.rfind(self.arg_val_start) - last_val_end = inner_text.rfind(self.arg_val_end) - has_partial_value = last_val_start != -1 and ( - last_val_end == -1 or last_val_end < last_val_start - ) - - if has_partial_value: - # Find the key for this partial value - # Look for the last ... before this - last_key_match = None - for m in self._arg_key_pattern.finditer(inner_text[:last_val_start]): - last_key_match = m - - if last_key_match: - partial_key = last_key_match.group(1).strip() - partial_content_start = last_val_start + len(self.arg_val_start) - partial_content = inner_text[partial_content_start:] - - # Hold back any partial suffix - overlap = partial_tag_overlap(partial_content, self.arg_val_end) - if overlap: - partial_content = partial_content[:-overlap] - - key_json = json.dumps(partial_key, ensure_ascii=False) - if is_complete: - # Tool call finished but is missing - # (malformed output). Treat partial as complete value - # so the diff naturally closes any open quotes. - if self._is_string_type(tool_name, partial_key): - val_json = json.dumps(partial_content, ensure_ascii=False) - else: - val_json = json.dumps( - self._deserialize(partial_content.strip()), - ensure_ascii=False, - ) - parts.append(f"{key_json}: {val_json}") - elif self._is_string_type(tool_name, partial_key): - escaped = self._json_escape_string_content(partial_content) - # Open quote but no close — more content may arrive - parts.append(f'{key_json}: "{escaped}') - else: - # Non-string partial: include raw content, no wrapping - parts.append(f"{key_json}: {partial_content}") - - if not parts: - return "{}" if is_complete else "" - - joined = "{" + ", ".join(parts) - if is_complete: - joined += "}" - return joined - - def _compute_args_diff(self, index: int, args_so_far: str) -> str | None: - """Return new argument text not yet sent for tool *index*, or None.""" - if not args_so_far or len(args_so_far) <= len( - self.streamed_args_for_tool[index] - ): - return None - diff = args_so_far[len(self.streamed_args_for_tool[index]) :] - self.streamed_args_for_tool[index] = args_so_far - self.prev_tool_call_arr[index]["arguments"] = args_so_far - return diff - - def _ensure_tool_state_for(self, index: int) -> None: - """Grow state arrays so that *index* is valid.""" - while len(self._tool_call_ids) <= index: - self._tool_call_ids.append( - make_tool_call_id(id_type="random", func_name=None, idx=None) - ) - while len(self.streamed_args_for_tool) <= index: - self.streamed_args_for_tool.append("") - while len(self.prev_tool_call_arr) <= index: - self.prev_tool_call_arr.append({}) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - if not self._tools_enabled(request): - return DeltaMessage(content=delta_text) if delta_text else None - - content = self._extract_content(current_text) - regions = self._extract_tool_call_regions(current_text) - tool_call_deltas: list[DeltaToolCall] = [] - - for i, (inner_text, is_complete) in enumerate(regions): - self._ensure_tool_state_for(i) - - # Extract tool name - tool_name = self._extract_tool_name_from_region(inner_text) - if not tool_name: - break - - # Emit tool name (once per tool call) - if "name" not in self.prev_tool_call_arr[i]: - self.prev_tool_call_arr[i]["name"] = tool_name - tool_call_deltas.append( - DeltaToolCall( - index=i, - id=self._tool_call_ids[i], - type="function", - function=DeltaFunctionCall( - name=tool_name, - arguments="", - ).model_dump(exclude_none=True), - ) - ) - - # Build args JSON so far, diff, emit - args_so_far = self._build_args_json_so_far( - tool_name, inner_text, is_complete - ) - diff = self._compute_args_diff(i, args_so_far) - if diff: - tool_call_deltas.append( - DeltaToolCall( - index=i, - function=DeltaFunctionCall(arguments=diff).model_dump( - exclude_none=True - ), - ) - ) - - # Update current_tool_id for serving layer compatibility - if regions: - self.current_tool_id = len(regions) - 1 - - if content or tool_call_deltas: - return DeltaMessage( - content=content, - tool_calls=tool_call_deltas, - ) - return None