[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
-380
View File
@@ -1,380 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from dataclasses import dataclass, field
from types import NoneType
from typing import Any
import pytest
from tests.tool_parsers.utils import run_tool_extraction
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParserManager
@dataclass
class ToolParserTestConfig:
"""Configuration for a tool parser's common tests.
This dataclass contains all the test data and expected results needed
to run the common test suite for a parser. Each parser test file
creates one instance of this config with parser-specific values.
Attributes:
parser_name: Name used with ToolParserManager (e.g., "mistral")
Test data (model outputs):
no_tool_calls_output: Plain text without any tool syntax
single_tool_call_output: One tool call with simple arguments
parallel_tool_calls_output: Multiple tool calls in one response
various_data_types_output: Tool with various data types
empty_arguments_output: Tool call with no parameters
surrounding_text_output: Tool call mixed with regular text
escaped_strings_output: Tool call with escaped chars
malformed_input_outputs: List of invalid inputs
Expected results:
single_tool_call_expected_name: Expected function name
single_tool_call_expected_args: Expected arguments dict
parallel_tool_calls_count: Number of tools in parallel test
parallel_tool_calls_names: Function names in order
single_tool_call_expected_content: Content field when tool called
parallel_tool_calls_expected_content: Content for parallel test
xfail markers:
xfail_streaming: Mapping test name to xfail reason (streaming only)
xfail_nonstreaming: Mapping test name to xfail reason (non-streaming)
Special flags:
allow_empty_or_json_empty_args: True if "" or "{}" both valid for empty args
supports_typed_arguments: True if the parser supports typed function arguments
"""
# Parser identification
parser_name: str
# Test data - model outputs for each common test
no_tool_calls_output: str
single_tool_call_output: str
parallel_tool_calls_output: str
various_data_types_output: str
empty_arguments_output: str
surrounding_text_output: str
escaped_strings_output: str
malformed_input_outputs: list[str]
# Expected results for specific tests (optional overrides)
single_tool_call_expected_name: str = "get_weather"
single_tool_call_expected_args: dict[str, Any] = field(
default_factory=lambda: {"city": "Tokyo"}
)
parallel_tool_calls_count: int = 2
parallel_tool_calls_names: list[str] = field(
default_factory=lambda: ["get_weather", "get_time"]
)
# xfail configuration - maps test name to xfail reason
xfail_streaming: dict[str, str] = field(default_factory=dict)
xfail_nonstreaming: dict[str, str] = field(default_factory=dict)
# Content expectations (some parsers strip content, others don't)
single_tool_call_expected_content: str | None = None
parallel_tool_calls_expected_content: str | None = None
# Special assertions for edge cases
allow_empty_or_json_empty_args: bool = True # "{}" or "" for empty args
supports_typed_arguments: bool = True
class ToolParserTests:
"""Mixin class providing common test suite for tool parsers.
To use this mixin in a parser test file:
1. Create a test_config fixture that returns a ToolParserTestConfig instance
2. Inherit from this class
3. Add parser-specific tests as additional methods
Example:
class TestMistralToolParser(ToolParserTests):
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="mistral",
no_tool_calls_output="Plain text...",
# ... other config ...
)
# Parser-specific tests
def test_mistral_specific_feature(self, tool_parser):
# Custom test logic
pass
"""
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
"""Override this to provide parser-specific configuration."""
raise NotImplementedError(
"Subclass must provide test_config fixture returning ToolParserTestConfig"
)
@pytest.fixture
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
"""Override this to provide parser-specific tokenizer."""
return default_tokenizer
@pytest.fixture
def tool_parser(self, test_config: ToolParserTestConfig, tokenizer: TokenizerLike):
return ToolParserManager.get_tool_parser(test_config.parser_name)(tokenizer)
@pytest.fixture(params=[True, False])
def streaming(self, request: pytest.FixtureRequest) -> bool:
return request.param
def test_no_tool_calls(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles plain text without tool syntax."""
# Apply xfail markers if configured
test_name = "test_no_tool_calls"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.no_tool_calls_output, streaming=streaming
)
assert content == test_config.no_tool_calls_output, (
f"Expected content to match input, got {content}"
)
assert len(tool_calls) == 0, f"Expected no tool calls, got {len(tool_calls)}"
def test_single_tool_call_simple_args(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser extracts one tool with simple arguments."""
# Apply xfail markers if configured
test_name = "test_single_tool_call_simple_args"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.single_tool_call_output, streaming=streaming
)
# Content check (some parsers strip it)
if test_config.single_tool_call_expected_content is not None:
assert content == test_config.single_tool_call_expected_content
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
assert tool_calls[0].type == "function"
assert tool_calls[0].function.name == test_config.single_tool_call_expected_name
args = json.loads(tool_calls[0].function.arguments)
for key, value in test_config.single_tool_call_expected_args.items():
assert args.get(key) == value, (
f"Expected {key}={value}, got {args.get(key)}"
)
def test_parallel_tool_calls(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles multiple tools in one response."""
# Apply xfail markers if configured
test_name = "test_parallel_tool_calls"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser,
test_config.parallel_tool_calls_output,
streaming=streaming,
)
assert len(tool_calls) == test_config.parallel_tool_calls_count, (
f"Expected {test_config.parallel_tool_calls_count} "
f"tool calls, got {len(tool_calls)}"
)
# Verify tool names match expected
for i, expected_name in enumerate(test_config.parallel_tool_calls_names):
assert tool_calls[i].type == "function"
assert tool_calls[i].function.name == expected_name
# Verify unique IDs
ids = [tc.id for tc in tool_calls]
assert len(ids) == len(set(ids)), "Tool call IDs should be unique"
def test_various_data_types(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles all JSON types in arguments."""
# Apply xfail markers if configured
test_name = "test_various_data_types"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser,
test_config.various_data_types_output,
streaming=streaming,
)
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
args = json.loads(tool_calls[0].function.arguments)
# Verify all expected fields present
required_fields_types = {
"string_field": str,
"int_field": int,
"float_field": float,
"bool_field": bool,
"null_field": NoneType,
"array_field": list,
"object_field": dict,
}
for required_field, expected_type in required_fields_types.items():
assert required_field in args, (
f"Expected field '{required_field}' in arguments"
)
if test_config.supports_typed_arguments:
found_type = type(args[required_field])
assert found_type is expected_type, (
f"Expected field '{required_field}' to have type {expected_type}, "
f"got {found_type}"
)
def test_empty_arguments(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles parameterless tool calls."""
# Apply xfail markers if configured
test_name = "test_empty_arguments"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.empty_arguments_output, streaming=streaming
)
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
args = tool_calls[0].function.arguments
if test_config.allow_empty_or_json_empty_args:
assert args in ["{}", ""], f"Expected empty args, got {args}"
else:
assert args == "{}", f"Expected {{}}, got {args}"
def test_surrounding_text(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser extracts tools from mixed content."""
# Apply xfail markers if configured
test_name = "test_surrounding_text"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.surrounding_text_output, streaming=streaming
)
assert len(tool_calls) >= 1, (
f"Expected at least 1 tool call, got {len(tool_calls)}"
)
def test_escaped_strings(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser handles escaped characters in arguments."""
# Apply xfail markers if configured
test_name = "test_escaped_strings"
self.apply_xfail_mark(request, test_config, test_name, streaming)
content, tool_calls = run_tool_extraction(
tool_parser, test_config.escaped_strings_output, streaming=streaming
)
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
args = json.loads(tool_calls[0].function.arguments)
# At minimum, verify we can parse and have expected fields
# Exact escaping behavior varies by parser
assert len(args) > 0, "Expected some arguments with escaped strings"
def test_malformed_input(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
streaming: bool,
):
"""Verify parser gracefully handles invalid syntax."""
# Apply xfail markers if configured
test_name = "test_malformed_input"
self.apply_xfail_mark(request, test_config, test_name, streaming)
for malformed_input in test_config.malformed_input_outputs:
# Should not raise exception
content, tool_calls = run_tool_extraction(
tool_parser, malformed_input, streaming=streaming
)
# Parser should handle gracefully (exact behavior varies)
def test_streaming_reconstruction(
self,
request: pytest.FixtureRequest,
tool_parser: Any,
test_config: ToolParserTestConfig,
):
"""Verify streaming produces same result as non-streaming."""
test_name = "test_streaming_reconstruction"
self.apply_xfail_mark(request, test_config, test_name, True)
test_output = test_config.single_tool_call_output
# Non-streaming result
content_non, tools_non = run_tool_extraction(
tool_parser, test_output, streaming=False
)
# Streaming result
content_stream, tools_stream = run_tool_extraction(
tool_parser, test_output, streaming=True
)
# Compare results
assert content_non == content_stream, "Content should match between modes"
assert len(tools_non) == len(tools_stream), "Tool count should match"
if len(tools_non) > 0:
assert tools_non[0].function.name == tools_stream[0].function.name
assert tools_non[0].function.arguments == tools_stream[0].function.arguments
def apply_xfail_mark(self, request, test_config, test_name, streaming):
reason = None
if streaming and test_name in test_config.xfail_streaming:
reason = test_config.xfail_streaming[test_name]
elif not streaming and test_name in test_config.xfail_nonstreaming:
reason = test_config.xfail_nonstreaming[test_name]
if reason is not None:
mark = pytest.mark.xfail(reason=reason, strict=True)
request.node.add_marker(mark)
@@ -1,449 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tool_parsers.apertus_tool_parser import (
TOOL_CALLS_PREFIX,
TOOL_CALLS_SUFFIX,
ApertusToolParser,
)
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def mock_tokenizer():
tokenizer = MagicMock()
tokenizer.encode.return_value = [1, 2, 3]
# Include the tool call tokens in the vocab for the parser
tokenizer.get_vocab.return_value = {TOOL_CALLS_PREFIX: 100, TOOL_CALLS_SUFFIX: 101}
return tokenizer
@pytest.fixture
def parser(mock_tokenizer):
return ApertusToolParser(mock_tokenizer)
@pytest.fixture
def mock_request():
request = MagicMock(spec=ChatCompletionRequest)
request.tools = []
request.tool_choice = "auto"
return request
# ---------------------------------------------------------------------------
# Non-streaming extraction tests
# ---------------------------------------------------------------------------
class TestExtractToolCalls:
def test_no_tool_calls(self, parser, mock_request):
model_output = "Hello, how can I help you today?"
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is False
assert result.tool_calls == []
assert result.content == model_output
def test_single_tool_call(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "London"}
def test_multiple_arguments(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": '
'{"location": "San Francisco", '
'"unit": "celsius"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "San Francisco", "unit": "celsius"}
def test_text_before_tool_call(self, parser, mock_request):
model_output = (
"Let me check the weather for you. "
'<|tools_prefix|>[{"get_weather": {"location": "Paris"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert result.content == "Let me check the weather for you."
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
def test_multiple_tool_calls(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": '
'{"location": "London"}}, '
'{"get_time": {"location": "London"}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "get_weather"
assert result.tool_calls[1].function.name == "get_time"
def test_nested_arguments(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"complex_function": '
'{"nested": {"inner": "value"}, '
'"list": ["a", "b"]}}]<|tools_suffix|>'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "complex_function"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]}
def test_incomplete_tool_call(self, parser, mock_request):
model_output = '<|tools_prefix|>[{"get_weather": {"location": "London"}'
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "London"}
def test_missing_tool_suffix(self, parser, mock_request):
model_output = (
'<|tools_prefix|>[{"get_weather": '
'{"location": "San Francisco", "unit": "celsius"}}]'
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "San Francisco", "unit": "celsius"}
# ---------------------------------------------------------------------------
# Streaming extraction tests
# ---------------------------------------------------------------------------
class TestStreamingExtraction:
def _simulate_streaming(
self, parser: ApertusToolParser, mock_request: Any, chunks: list[str]
) -> list[tuple[Any, str]]:
results: list[tuple[Any, str]] = []
previous_text: str = ""
previous_token_ids: list[int] = []
for chunk in chunks:
current_text = previous_text + chunk
# Simulate a token ID sequence matching the chunk progression
delta_token_ids: list[int] = [0] * max(1, len(chunk) // 4)
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=tuple(previous_token_ids),
current_token_ids=tuple(current_token_ids),
delta_token_ids=tuple(delta_token_ids),
request=mock_request,
)
results.append((delta, current_text))
previous_text = current_text
previous_token_ids = list(current_token_ids)
return results
def _collect_tool_calls(self, results) -> dict[int, dict[str, Any]]:
"""Properly tracks and concatenates streamed tool arguments by their Index."""
tool_calls = {}
for delta, _ in results:
if not delta or not getattr(delta, "tool_calls", None):
continue
for tc in delta.tool_calls:
idx = (
tc.get("index", 0)
if isinstance(tc, dict)
else getattr(tc, "index", 0)
)
func = (
tc.get("function", {})
if isinstance(tc, dict)
else getattr(tc, "function", None)
)
if not func:
continue
name = (
func.get("name")
if isinstance(func, dict)
else getattr(func, "name", None)
)
args = (
func.get("arguments")
if isinstance(func, dict)
else getattr(func, "arguments", None)
)
if idx not in tool_calls:
tool_calls[idx] = {"name": "", "arguments": ""}
if name:
tool_calls[idx]["name"] += name
if args:
tool_calls[idx]["arguments"] += args
return tool_calls
def _collect_content(self, results) -> str:
"""Collects generated normal text outside of the tool calls."""
return "".join(
delta.content
for delta, _ in results
if delta and getattr(delta, "content", None)
)
def test_basic_streaming_single_tool(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
'[{"get_weather": ',
'{"location": "Paris, ',
'France"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
@pytest.mark.parametrize(
"arguments",
['{"value": -2e-3}', '{"value": 1.5e3}', '{"values": [1.5e3, -2e-3]}'],
)
def test_streaming_scientific_notation(self, parser, mock_request, arguments):
model_output = (
f'{TOOL_CALLS_PREFIX}[{{"calculate": {arguments}}}]{TOOL_CALLS_SUFFIX}'
)
results = self._simulate_streaming(parser, mock_request, list(model_output))
tool_calls = self._collect_tool_calls(results)
assert len(tool_calls) == 1
assert tool_calls[0]["name"] == "calculate"
assert json.loads(tool_calls[0]["arguments"]) == json.loads(arguments)
def test_streaming_missing_tool_suffix(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
'[{"get_weather": ',
'{"location": "Paris, ',
'France"}}]',
]
results = self._simulate_streaming(parser, mock_request, chunks)
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
def test_streaming_partial_tag_buffering_missing_tool_suffix(
self, parser, mock_request
):
chunks = ["Content", "<|tools_", "prefix|>", '[{"f": ', '{"a": 1}}]']
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Content" in content
assert "<|tools_prefix|>" not in content
assert "<|tools_suffix|>" not in content
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "f"
assert json.loads(tcs[0]["arguments"]) == {"a": 1}
def test_streaming_multi_tool(self, parser, mock_request):
chunks = [
"<|tools_prefix|>",
'[{"get_weather": {"location": "Tokyo"}}',
', {"get_time": {"location": "Tokyo"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
tcs = self._collect_tool_calls(results)
assert len(tcs) == 2
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "Tokyo"}
assert tcs[1]["name"] == "get_time"
assert json.loads(tcs[1]["arguments"]) == {"location": "Tokyo"}
def test_streaming_text_before_tool_call(self, parser, mock_request):
chunks = [
"Let me check ",
"the weather. ",
"<|tools_prefix|>",
'[{"get_weather": {"location": "London"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert content.strip() == "Let me check the weather."
tcs = self._collect_tool_calls(results)
assert len(tcs) == 1
assert tcs[0]["name"] == "get_weather"
assert json.loads(tcs[0]["arguments"]) == {"location": "London"}
def test_streaming_partial_tag_buffering(self, parser, mock_request):
chunks = [
"Content",
"<|tools_",
"prefix|>",
'[{"f": {"a": 1}}]',
"<|tools_suf",
"fix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Content" in content
assert "<|tools_prefix|>" not in content
assert "<|tools_suffix|>" not in content
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "f"
assert json.loads(tc[0]["arguments"]) == {"a": 1}
# ---------------------------------------------------------------------------
# Edge Cases: Multi-Token Prediction (MTP) & vLLM Chunking Anomalies
# ---------------------------------------------------------------------------
def test_mtp_streaming_massive_chunk(self, parser, mock_request):
"""Simulates MTP predicting text, tool calls,
and trailing text all in a single chunk."""
chunks = [
"Sure! "
'<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Sure! " in content
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
def test_mtp_streaming_multiple_tools_burst(self, parser, mock_request):
"""Simulates MTP predicting an array of multiple tools in one single chunk."""
chunks = [
'<|tools_prefix|>[{"get_weather": '
'{"location": "London"}}, '
'{"get_time": {"location": "Paris"}}]<|tools_suffix|>'
]
results = self._simulate_streaming(parser, mock_request, chunks)
tc = self._collect_tool_calls(results)
assert len(tc) == 2
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
assert tc[1]["name"] == "get_time"
assert json.loads(tc[1]["arguments"]) == {"location": "Paris"}
def test_mtp_streaming_skip_and_catch_up(self, parser, mock_request):
"""Simulates MTP chunks that jump over entire tools
(e.g., from middle of tool 1 to middle of tool 3)."""
chunks = [
'<|tools_prefix|>[{"t1": {"a": 1}',
'}, {"t2": {"b": 2}}, {"t3": {"c": 3',
"}}]<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
tc = self._collect_tool_calls(results)
assert len(tc) == 3
assert tc[0]["name"] == "t1"
assert json.loads(tc[0]["arguments"]) == {"a": 1}
assert tc[1]["name"] == "t2"
assert json.loads(tc[1]["arguments"]) == {"b": 2}
assert tc[2]["name"] == "t3"
assert json.loads(tc[2]["arguments"]) == {"c": 3}
def test_vllm_streaming_character_by_character(self, parser, mock_request):
"""Simulates worst-case vLLM fragmentation where
chunks arrive character-by-character."""
text = (
'Hi <|tools_prefix|>[{"get_weather": '
'{"location": "London"}}]<|tools_suffix|> '
)
chunks = list(text)
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert "Hi" in content
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
def test_vllm_streaming_empty_deltas(self, parser, mock_request):
"""Simulates vLLM stream producing empty string chunks
(e.g., hidden tokens or artifacts)."""
chunks = [
"Wait",
"",
"<|tools_prefix|>",
"",
'[{"get_weather": ',
"",
'{"location": "London"}}]',
"<|tools_suffix|>",
]
results = self._simulate_streaming(parser, mock_request, chunks)
content = self._collect_content(results)
assert content == "Wait"
tc = self._collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.deepseekv31_tool_parser import (
DeepSeekV31ToolParser,
)
MODEL = "deepseek-ai/DeepSeek-V3.1"
@pytest.fixture(scope="module")
def deepseekv31_tokenizer():
return get_tokenizer(tokenizer_name=MODEL)
@pytest.fixture
def parser(deepseekv31_tokenizer):
return DeepSeekV31ToolParser(deepseekv31_tokenizer)
def test_extract_tool_calls_with_tool(parser):
model_output = (
"normal text"
"<|tool▁calls▁begin|>"
'<|tool▁call▁begin|>foo<|tool▁sep|>{"x":1}<|tool▁call▁end|>'
"<|tool▁calls▁end|>"
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "foo"
assert result.tool_calls[0].function.arguments == '{"x":1}'
assert result.content == "normal text"
def test_extract_tool_calls_with_multiple_tools(parser):
model_output = (
"some prefix text"
"<|tool▁calls▁begin|>"
'<|tool▁call▁begin|>foo<|tool▁sep|>{"x":1}<|tool▁call▁end|>'
'<|tool▁call▁begin|>bar<|tool▁sep|>{"y":2}<|tool▁call▁end|>'
"<|tool▁calls▁end|>"
" some suffix text"
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "foo"
assert result.tool_calls[0].function.arguments == '{"x":1}'
assert result.tool_calls[1].function.name == "bar"
assert result.tool_calls[1].function.arguments == '{"y":2}'
# prefix is content
assert result.content == "some prefix text"
File diff suppressed because it is too large Load Diff
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from vllm.tokenizers import TokenizerLike, get_tokenizer
class TestDeepSeekV3ToolParser(ToolParserTests):
@pytest.fixture(scope="class")
def tokenizer(self) -> TokenizerLike:
return get_tokenizer("deepseek-ai/DeepSeek-V3")
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="deepseek_v3",
# Test data
no_tool_calls_output=(
"How can I help you today? I can check weather for you."
),
single_tool_call_output="""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
```json
{"city": "Tokyo", "unit": "celsius"}
```<|tool▁call▁end|><|tool▁calls▁end|>""",
parallel_tool_calls_output="""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
```json
{"city": "Tokyo", "unit": "celsius"}
```<|tool▁call▁end|><|tool▁call▁begin|>function<|tool▁sep|>search_hotels
```json
{"location": "Tokyo", "check_in": "2025-01-15"}
```<|tool▁call▁end|><|tool▁calls▁end|>""",
various_data_types_output=(
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>test_function
```json
"""
"""{"string_field": "hello", "int_field": 42, "float_field": 3.14, """
""""bool_field": true, "null_field": null, """
""""array_field": ["a", "b", "c"], """
""""object_field": {"nested": "value"}, """
""""empty_array": [], "empty_object": {}}
```<|tool▁call▁end|><|tool▁calls▁end|>"""
),
empty_arguments_output="""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_current_time
```json
{}
```<|tool▁call▁end|><|tool▁calls▁end|>""",
surrounding_text_output=(
"""Let me check the weather for you."""
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
```json
{"city": "Paris"}
```<|tool▁call▁end|><|tool▁calls▁end|>"""
),
escaped_strings_output=(
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>send_message
```json
"""
"""{"text": "He said \\"hello\\"", "path": "C:\\\\Users\\\\file", """
""""newline": "line1\\nline2"}
```<|tool▁call▁end|><|tool▁calls▁end|>"""
),
malformed_input_outputs=[
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
```json
{"city": "Tokyo"
```<|tool▁call▁end|><|tool▁calls▁end|>""",
"""<|tool▁calls▁begin|>function<|tool▁sep|>get_weather
```json
{"city": "Tokyo"}
```<|tool▁calls▁end|>""",
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo", "unit": "celsius"},
single_tool_call_expected_content=None,
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "search_hotels"],
# xfail markers
xfail_streaming={},
xfail_nonstreaming={
"test_malformed_input": (
"Parser sets tools_called=True even when tool_calls is "
"empty (detects start token but fails to parse)"
),
},
)
@@ -1,447 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for DeepSeekV4EngineToolParser."""
import json
from unittest.mock import MagicMock
import pytest
from xgrammar import StructuralTag
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedFunction,
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.deepseekv4_engine_tool_parser import (
DeepSeekV4EngineToolParser,
)
pytestmark = pytest.mark.skip_global_cleanup
MOCK_TOKENIZER = MagicMock()
MOCK_TOKENIZER.get_vocab.return_value = {}
TC_START = "<|DSML|tool_calls>"
TC_END = "</|DSML|tool_calls>"
INV_START = '<|DSML|invoke name="'
INV_END = "</|DSML|invoke>"
PARAM_START = '<|DSML|parameter name="'
PARAM_END = "</|DSML|parameter>"
@pytest.fixture
def sample_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_current_weather",
"description": "Get the current weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string", "description": "The city name"},
"state": {"type": "string", "description": "The state code"},
"unit": {"type": "string", "enum": ["fahrenheit", "celsius"]},
},
"required": ["city", "state"],
},
},
),
ChatCompletionToolsParam(
type="function",
function={
"name": "calculate_area",
"description": "Calculate area of a shape",
"parameters": {
"type": "object",
"properties": {
"shape": {"type": "string"},
"dimensions": {"type": "object"},
"precision": {"type": "integer"},
},
},
},
),
]
def make_parser(tools=None) -> DeepSeekV4EngineToolParser:
return DeepSeekV4EngineToolParser(MOCK_TOKENIZER, tools=tools)
def make_request(tools=None) -> MagicMock:
req = MagicMock()
req.tools = tools
return req
def build_tool_call(func_name: str, params: dict[str, str]) -> str:
param_strs = "".join(
f'{PARAM_START}{k}" string="true">{v}{PARAM_END}\n' for k, v in params.items()
)
return f'{TC_START}\n{INV_START}{func_name}">\n{param_strs}{INV_END}\n{TC_END}'
def stream(parser: DeepSeekV4EngineToolParser, full_text: str, chunk_size: int = 7):
deltas = []
previous_text = ""
for start in range(0, len(full_text), chunk_size):
delta_text = full_text[start : start + chunk_size]
current_text = previous_text + delta_text
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[1],
request=make_request(),
)
previous_text = current_text
if delta is not None:
deltas.append(delta)
return deltas
def reconstruct_args(deltas, tool_index: int = 0) -> str:
fragments = []
for delta in deltas:
if delta.tool_calls:
for tool_call in delta.tool_calls:
if (
tool_call.index == tool_index
and tool_call.function
and tool_call.function.arguments
):
fragments.append(tool_call.function.arguments)
return "".join(fragments)
def test_registered():
assert (
ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4EngineToolParser
)
def test_extract_tool_calls():
parser = make_parser()
model_output = "Let me check. " + build_tool_call(
"get_weather", {"location": "Beijing", "unit": "celsius"}
)
result = parser.extract_tool_calls(model_output, make_request())
assert result.tools_called
assert result.content == "Let me check. "
assert len(result.tool_calls) == 1
tool_call = result.tool_calls[0]
assert tool_call.function.name == "get_weather"
assert json.loads(tool_call.function.arguments) == {
"location": "Beijing",
"unit": "celsius",
}
def test_function_calls_wrapper_is_not_recognized():
# The V3.2 wrapper is not a V4 terminal, so it passes through as content,
# but the invoke inside it is still parsed (tool calls anchor on the invoke).
parser = make_parser()
model_output = build_tool_call("search", {"query": "vllm"}).replace(
"tool_calls", "function_calls"
)
result = parser.extract_tool_calls(model_output, make_request())
assert result.tools_called
assert result.tool_calls[0].function.name == "search"
assert result.content == "<|DSML|function_calls>\n"
def test_missing_tool_calls_wrapper_is_recovered():
# Regression for #48931: at long context the model omits the
# <|DSML|tool_calls> START token but still emits a complete invoke.
parser = make_parser()
model_output = build_tool_call("search", {"query": "vllm"}).replace(
TC_START + "\n", ""
)
assert TC_START not in model_output
result = parser.extract_tool_calls(model_output, make_request())
assert result.tools_called
assert result.tool_calls[0].function.name == "search"
assert json.loads(result.tool_calls[0].function.arguments) == {"query": "vllm"}
assert result.content is None
deltas = stream(make_parser(), model_output, chunk_size=3)
names = [
tc.function.name
for d in deltas
if d.tool_calls
for tc in d.tool_calls
if tc.function.name
]
assert names == ["search"]
assert json.loads(reconstruct_args(deltas)) == {"query": "vllm"}
assert not any(d.content and "DSML" in d.content for d in deltas)
def test_streaming_extracts_complete_invokes():
parser = make_parser()
full_text = build_tool_call("search", {"query": "deepseek v4"})
deltas = stream(parser, full_text, chunk_size=5)
names = [
tool_call.function.name
for delta in deltas
if delta.tool_calls
for tool_call in delta.tool_calls
if tool_call.function.name
]
assert names == ["search"]
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
def test_streaming_emits_incremental_argument_chunks():
tool = ChatCompletionToolsParam(
function=FunctionDefinition(
name="plan_trip",
parameters={
"type": "object",
"properties": {
"days": {"type": "integer"},
"flexible": {"type": "boolean"},
"cities": {"type": "array", "items": {"type": "string"}},
"notes": {"type": "string"},
},
},
),
)
parser = make_parser(tools=[tool])
full_text = (
f"{TC_START}\n"
f'{INV_START}plan_trip">\n'
f'{PARAM_START}days" string="false">3{PARAM_END}\n'
f'{PARAM_START}flexible" string="false">false{PARAM_END}\n'
f'{PARAM_START}cities" string="false">'
f'["Beijing","Shanghai","Tokyo","New York"]{PARAM_END}\n'
f'{PARAM_START}notes" string="true">靠窗座位{PARAM_END}\n'
f"{INV_END}\n"
f"{TC_END}"
)
deltas = stream(parser, full_text, chunk_size=4)
arg_chunks = [
tool_call.function.arguments
for delta in deltas
for tool_call in delta.tool_calls or []
if tool_call.function and tool_call.function.arguments is not None
]
assert len([chunk for chunk in arg_chunks if chunk]) > 2
assert json.loads("".join(arg_chunks)) == {
"days": 3,
"flexible": False,
"cities": ["Beijing", "Shanghai", "Tokyo", "New York"],
"notes": "靠窗座位",
}
def _with_strict(
tools: list[ChatCompletionToolsParam],
) -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
type=t.type,
function=FunctionDefinition(
name=t.function.name,
description=t.function.description,
parameters=t.function.parameters,
strict=True,
),
)
for t in tools
]
def test_get_vllm_registry_structural_tag_returns_structural_tag(
sample_tools: list[ChatCompletionToolsParam],
) -> None:
parser = make_parser()
strict_tools = _with_strict(sample_tools)
req = ChatCompletionRequest(
messages=[],
model="m",
tools=strict_tools,
tool_choice="auto",
)
tag = parser.get_structural_tag(req)
assert isinstance(tag, StructuralTag)
req = ChatCompletionRequest(
messages=[],
model="m",
tools=sample_tools,
tool_choice="required",
)
tag = parser.get_structural_tag(req)
assert isinstance(tag, StructuralTag)
if sample_tools:
tool = sample_tools[0]
req = ChatCompletionRequest(
messages=[],
model="m",
tools=sample_tools,
)
req.tool_choice = ChatCompletionNamedToolChoiceParam(
function=ChatCompletionNamedFunction(name=tool.function.name)
)
tag = parser.get_structural_tag(req)
assert isinstance(tag, StructuralTag)
def test_extract_tool_calls_arguments_wrapper():
mock_tokenizer = MagicMock()
mock_tokenizer.get_vocab.return_value = {}
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"location": {"type": "string"}},
},
},
)
parser = DeepSeekV4EngineToolParser(mock_tokenizer, tools=[tool])
request = MagicMock()
request.tools = [tool]
model_output = (
f"{TC_START}"
f'{INV_START}get_weather">'
f'{PARAM_START}arguments" string="false">{{"location":"Beijing"}}{PARAM_END}'
f"{INV_END}"
f"{TC_END}"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"location": "Beijing"}
_ANGLE_BRACKET_TOOL = ChatCompletionToolsParam(
function=FunctionDefinition(
name="run_command",
parameters={
"type": "object",
"properties": {
"command": {"type": "string"},
},
},
),
)
@pytest.mark.parametrize(
"tools",
[[_ANGLE_BRACKET_TOOL], None],
ids=["with_tools", "without_tools"],
)
def test_no_dsml_closing_tag_leak_in_streamed_args(tools):
"""Streaming must not leak </|DSML|parameter> into argument values.
When a parameter value contains '>' (e.g. shell redirects like
'2>&1'), certain chunk boundaries cause the incremental lexer to
emit the closing delimiter text as part of the content token. The
partial regex then captures it as part of the value, violating the
prefix invariant and corrupting the streamed JSON.
"""
full_text = build_tool_call("run_command", {"command": "git --version 2>&1"})
expected = {"command": "git --version 2>&1"}
for chunk_size in range(1, len(full_text) + 1):
parser = make_parser(tools=tools)
deltas = stream(parser, full_text, chunk_size=chunk_size)
args_str = reconstruct_args(deltas)
assert args_str, f"No args emitted at chunk_size={chunk_size}"
assert "DSML" not in args_str, (
f"DSML marker leaked into args at chunk_size={chunk_size}: {args_str!r}"
)
parsed = json.loads(args_str)
assert parsed == expected, (
f"Args mismatch at chunk_size={chunk_size}: "
f"got {parsed!r}, expected {expected!r}"
)
def test_non_streaming_extract_with_angle_brackets():
"""Non-streaming extraction must correctly handle '>' in values."""
parser = make_parser()
full_text = build_tool_call("run_command", {"command": "git --version 2>&1"})
result = parser.extract_tool_calls(full_text, make_request())
assert result.tools_called
assert len(result.tool_calls) == 1
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"command": "git --version 2>&1"}
assert "DSML" not in result.tool_calls[0].function.arguments
def test_composed_schema_converts_object_and_array_params():
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "set_timer",
"parameters": {
"type": "object",
"properties": {
"wait": {
"anyOf": [
{"type": "object"},
{"type": "null"},
],
},
"patches": {
"allOf": [
{"type": "array", "items": {"type": "object"}},
],
},
},
},
},
)
parser = make_parser(tools=[tool])
request = make_request(tools=[tool])
model_output = (
f"{TC_START}\n"
f'{INV_START}set_timer">\n'
f'{PARAM_START}wait" string="false">'
f'{{"type":"for","minutes":2880}}'
f"{PARAM_END}\n"
f'{PARAM_START}patches" string="false">'
f'[{{"op":"replace","path":"/schedule","value":"quiet"}}]'
f"{PARAM_END}\n"
f"{INV_END}\n"
f"{TC_END}"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {
"wait": {"type": "for", "minutes": 2880},
"patches": [{"op": "replace", "path": "/schedule", "value": "quiet"}],
}
-275
View File
@@ -1,275 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Iterable
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.dots_tool_parser import DotsToolParser
def _tool(name: str, properties: dict) -> ChatCompletionToolsParam:
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"description": "test tool",
"parameters": {"type": "object", "properties": properties},
},
)
def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=tools,
tool_choice="auto",
)
def _stream(
parser: DotsToolParser,
chunks: Iterable[str],
request: ChatCompletionRequest,
) -> list[DeltaMessage]:
previous_text = ""
messages: list[DeltaMessage] = []
for chunk in chunks:
current_text = previous_text + chunk
delta = parser.extract_tool_calls_streaming(
previous_text,
current_text,
chunk,
[],
[],
[],
request,
)
if delta is not None:
messages.append(delta)
previous_text = current_text
return messages
@pytest.fixture
def parser() -> DotsToolParser:
return DotsToolParser(MagicMock())
def test_registered_in_tool_parser_manager() -> None:
assert ToolParserManager.get_tool_parser("dots") is DotsToolParser
def test_non_stream_xml_converts_schema_types_and_resolves_ref(
parser: DotsToolParser,
) -> None:
tool = ChatCompletionToolsParam(
type="function",
function={
"name": "set_location",
"description": "Set location",
"parameters": {
"type": "object",
"properties": {
"location": {"$ref": "#/$defs/Location"},
"days": {"type": "integer"},
"include_weather": {"type": "boolean"},
},
"$defs": {
"Location": {
"type": "object",
"properties": {"city": {"type": "string"}},
}
},
},
},
)
request = _request([tool])
text = (
"ok<dots_function_call>"
'<invoke name="set_location">'
'<parameter name="location">{"city": "Shanghai"}</parameter>'
'<parameter name="days">3</parameter>'
'<parameter name="include_weather">true</parameter>'
"</invoke>"
"</dots_function_call>"
)
result = parser.extract_tool_calls(text, request)
assert result.content == "ok"
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "set_location"
assert json.loads(result.tool_calls[0].function.arguments) == {
"location": {"city": "Shanghai"},
"days": 3,
"include_weather": True,
}
def test_non_stream_supports_multiple_invokes_and_json_fallback(
parser: DotsToolParser,
) -> None:
tools = [
_tool("search", {"query": {"type": "string"}}),
_tool("open", {"id": {"type": "integer"}}),
]
request = _request(tools)
text = (
"<dots_function_call>"
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
'<invoke name="open"><parameter name="id">7</parameter></invoke>'
"</dots_function_call>"
'<dots_function_call>{"name":"search","arguments":{"query":"tables"}}'
"</dots_function_call>"
)
result = parser.extract_tool_calls(text, request)
assert [call.function.name for call in result.tool_calls] == [
"search",
"open",
"search",
]
assert [json.loads(call.function.arguments) for call in result.tool_calls] == [
{"query": "chairs"},
{"id": 7},
{"query": "tables"},
]
def test_non_stream_unknown_tool_is_left_as_content(parser: DotsToolParser) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
text = (
"<dots_function_call>"
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
"</dots_function_call>"
)
result = parser.extract_tool_calls(text, request)
assert not result.tools_called
assert result.tool_calls == []
assert result.content == text
def test_streaming_buffers_partial_marker_and_emits_all_complete_calls(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
chunks = [
"visible<dots_func",
(
"tion_call>"
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
"</dots_function_call>"
"<dots_function_call>"
'<invoke name="search"><parameter name="query">tables</parameter></invoke>'
"</dots_function_call>"
),
]
messages = _stream(parser, chunks, request)
assert "".join(message.content or "" for message in messages) == "visible"
calls = [call for message in messages for call in message.tool_calls]
assert [call.index for call in calls] == [0, 1]
assert [json.loads(call.function.arguments or "") for call in calls] == [
{"query": "chairs"},
{"query": "tables"},
]
def test_streaming_filters_unknown_tools_and_surfaces_content(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
text = (
"<dots_function_call>"
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
"</dots_function_call>"
)
messages = _stream(parser, [text], request)
assert len(messages) == 1
assert messages[0].tool_calls == []
assert "ghost" in (messages[0].content or "")
assert parser._buffer == ""
def test_streaming_malformed_block_does_not_block_later_valid_call(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(
parser,
[
"<dots_function_call>garbage</dots_function_call>",
(
"<dots_function_call>"
'<invoke name="search">'
'<parameter name="query">chairs</parameter>'
"</invoke></dots_function_call>"
),
],
request,
)
assert messages[0].content == "garbage"
assert messages[0].tool_calls == []
assert messages[1].tool_calls[0].function.name == "search"
def test_streaming_strips_stray_end_marker(parser: DotsToolParser) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(parser, ["some text </dots_function_call>"], request)
assert len(messages) == 1
assert messages[0].content == "some text "
assert messages[0].tool_calls == []
def test_streaming_flushes_partial_opening_marker_at_eof(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(parser, ["answer <dots_func"], request)
assert messages[0].content == "answer "
assert parser.flush_pending_normal_text() == "<dots_func"
assert parser.flush_pending_normal_text() == ""
def test_streaming_emits_complete_json_before_end_marker_without_duplication(
parser: DotsToolParser,
) -> None:
request = _request([_tool("search", {"query": {"type": "string"}})])
messages = _stream(
parser,
[
"<dots_function_call>",
'{"name":"search","arguments":{"query":"chairs"}}',
"</dots_function_call>",
],
request,
)
calls = [call for message in messages for call in message.tool_calls]
assert [call.function.name for call in calls] == ["search", None]
assert "".join(call.function.arguments or "" for call in calls) == (
'{"query": "chairs"}'
)
assert messages[-1].tool_calls == calls
@@ -1,359 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import json
from collections.abc import Generator
import pytest
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tool_parsers.ernie45_tool_parser import Ernie45ToolParser
# Use a common model that is likely to be available
MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking"
@pytest.fixture(scope="module")
def ernie45_tokenizer():
return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True)
@pytest.fixture
def ernie45_tool_parser(ernie45_tokenizer):
return Ernie45ToolParser(ernie45_tokenizer)
def assert_tool_calls(
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
):
assert len(actual_tool_calls) == len(expected_tool_calls)
for actual_tool_call, expected_tool_call in zip(
actual_tool_calls, expected_tool_calls
):
assert isinstance(actual_tool_call.id, str)
assert len(actual_tool_call.id) > 0
assert actual_tool_call.type == "function"
assert actual_tool_call.function.name == expected_tool_call.function.name
# Compare arguments as JSON objects to handle formatting differences
actual_args = json.loads(actual_tool_call.function.arguments)
expected_args = json.loads(expected_tool_call.function.arguments)
assert actual_args == expected_args
def test_extract_tool_calls_no_tools(ernie45_tool_parser):
model_output = "This is a test"
extracted_tool_calls = ernie45_tool_parser.extract_tool_calls(
model_output, request=None
)
assert not extracted_tool_calls.tools_called
assert extracted_tool_calls.tool_calls == []
assert extracted_tool_calls.content == model_output
@pytest.mark.parametrize(
ids=[
"single_tool_call",
"multiple_tool_calls",
"tool_call_with_content_before",
],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
(
"""<tool_call>
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
</tool_call>
""",
[
ToolCall(
function=FunctionCall(
name="get_current_temperature",
arguments=json.dumps(
{
"location": "Beijing",
}
),
)
)
],
None,
),
(
"""<tool_call>
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
</tool_call>
<tool_call>
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
</tool_call>
""",
[
ToolCall(
function=FunctionCall(
name="get_current_temperature",
arguments=json.dumps(
{
"location": "Beijing",
}
),
)
),
ToolCall(
function=FunctionCall(
name="get_temperature_unit",
arguments=json.dumps(
{
"location": "Guangzhou",
"unit": "c",
}
),
)
),
],
None,
),
(
"""I need to call two tools to handle these two issues separately.
</think>
<tool_call>
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
</tool_call>
<tool_call>
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
</tool_call>
""",
[
ToolCall(
function=FunctionCall(
name="get_current_temperature",
arguments=json.dumps(
{
"location": "Beijing",
}
),
)
),
ToolCall(
function=FunctionCall(
name="get_temperature_unit",
arguments=json.dumps(
{
"location": "Guangzhou",
"unit": "c",
}
),
)
),
],
"I need to call two tools to handle these two issues separately.\n</think>",
),
],
)
def test_extract_tool_calls(
ernie45_tool_parser, model_output, expected_tool_calls, expected_content
):
extracted_tool_calls = ernie45_tool_parser.extract_tool_calls(
model_output, request=None
)
assert extracted_tool_calls.tools_called
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
assert extracted_tool_calls.content == expected_content
def stream_delta_message_generator(
ernie45_tool_parser: Ernie45ToolParser,
ernie45_tokenizer: TokenizerLike,
model_output: str,
request: ChatCompletionRequest | None = None,
) -> Generator[DeltaMessage, None, None]:
all_token_ids = ernie45_tokenizer.encode(model_output, add_special_tokens=False)
previous_text = ""
previous_tokens = None
prefix_offset = 0
read_offset = 0
for i, delta_token in enumerate(all_token_ids):
delta_token_ids = [delta_token]
previous_token_ids = all_token_ids[:i]
current_token_ids = all_token_ids[: i + 1]
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
detokenize_incrementally(
tokenizer=ernie45_tokenizer,
all_input_ids=current_token_ids,
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=False,
spaces_between_special_tokens=True,
)
)
current_text = previous_text + delta_text
delta_message = ernie45_tool_parser.extract_tool_calls_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
request=request,
)
if delta_message:
yield delta_message
previous_text = current_text
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
prefix_offset = new_prefix_offset
read_offset = new_read_offset
@pytest.mark.parametrize(
ids=[
"single_tool_call",
"multiple_tool_calls",
"tool_call_with_content_before",
],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
(
"""<tool_call>
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
</tool_call>
""",
[
ToolCall(
function=FunctionCall(
name="get_current_temperature",
arguments=json.dumps(
{
"location": "Beijing",
}
),
)
)
],
None,
),
(
"""<tool_call>
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
</tool_call>
<tool_call>
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
</tool_call>
""",
[
ToolCall(
function=FunctionCall(
name="get_current_temperature",
arguments=json.dumps(
{
"location": "Beijing",
}
),
)
),
ToolCall(
function=FunctionCall(
name="get_temperature_unit",
arguments=json.dumps(
{
"location": "Guangzhou",
"unit": "c",
}
),
)
),
],
None,
),
(
"""I need to call two tools to handle these two issues separately.
</think>
<tool_call>
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
</tool_call>
<tool_call>
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
</tool_call>
""",
[
ToolCall(
function=FunctionCall(
name="get_current_temperature",
arguments=json.dumps(
{
"location": "Beijing",
}
),
)
),
ToolCall(
function=FunctionCall(
name="get_temperature_unit",
arguments=json.dumps(
{
"location": "Guangzhou",
"unit": "c",
}
),
)
),
],
"I need to call two tools to handle these two issues separately.\n</think>",
),
],
)
def test_extract_tool_calls_streaming_incremental(
ernie45_tool_parser,
ernie45_tokenizer,
model_output,
expected_tool_calls,
expected_content,
):
"""Verify the Ernie45 Parser streaming behavior by verifying each chunk is as expected.""" # noqa: E501
request = ChatCompletionRequest(model=MODEL, messages=[])
tool_calls_dict = {}
for delta_message in stream_delta_message_generator(
ernie45_tool_parser, ernie45_tokenizer, model_output, request
):
if (
delta_message.role is None
and delta_message.content is None
and delta_message.reasoning is None
and len(delta_message.tool_calls) == 0
):
continue
tool_calls = delta_message.tool_calls
for tool_call_chunk in tool_calls:
index = tool_call_chunk.index
if index not in tool_calls_dict:
if tool_call_chunk.function.arguments is None:
tool_call_chunk.function.arguments = ""
tool_calls_dict[index] = tool_call_chunk
else:
tool_calls_dict[
index
].function.arguments += tool_call_chunk.function.arguments
actual_tool_calls = list(tool_calls_dict.values())
assert len(actual_tool_calls) > 0
# check tool call format
assert_tool_calls(actual_tool_calls, expected_tool_calls)
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tool_parsers.functiongemma_tool_parser import FunctionGemmaToolParser
@pytest.fixture
def mock_tokenizer():
tokenizer = MagicMock()
tokenizer.encode.return_value = [1, 2, 3]
tokenizer.get_vocab.return_value = {}
return tokenizer
@pytest.fixture
def parser(mock_tokenizer):
return FunctionGemmaToolParser(mock_tokenizer)
@pytest.fixture
def mock_request():
request = MagicMock(spec=ChatCompletionRequest)
request.tools = []
request.tool_choice = "auto"
request.parallel_tool_calls = True
return request
class TestExtractToolCalls:
def test_no_tool_calls(self, parser, mock_request):
model_output = "Hello, how can I help you today?"
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is False
assert result.tool_calls == []
assert result.content == model_output
def test_single_tool_call(self, parser, mock_request):
model_output = (
"<start_function_call>call:get_weather{location:<escape>London<escape>}"
"<end_function_call>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
assert '"location": "London"' in result.tool_calls[0].function.arguments
def test_multiple_arguments(self, parser, mock_request):
model_output = (
"<start_function_call>call:get_weather{"
"location:<escape>San Francisco<escape>,"
"unit:<escape>celsius<escape>}"
"<end_function_call>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = result.tool_calls[0].function.arguments
assert "San Francisco" in args
assert "celsius" in args
def test_text_before_tool_call(self, parser, mock_request):
model_output = (
"Let me check the weather for you. "
"<start_function_call>call:get_weather{location:<escape>Paris<escape>}"
"<end_function_call>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert result.content == "Let me check the weather for you."
def test_multiple_tool_calls(self, parser, mock_request):
model_output = (
"<start_function_call>call:get_weather{location:<escape>London<escape>}"
"<end_function_call>"
"<start_function_call>call:get_time{timezone:<escape>UTC<escape>}"
"<end_function_call>"
)
result = parser.extract_tool_calls(model_output, mock_request)
assert result.tools_called is True
assert len(result.tool_calls) == 2
assert result.tool_calls[0].function.name == "get_weather"
assert result.tool_calls[1].function.name == "get_time"
class TestParseArguments:
def test_empty_arguments(self, parser):
result = parser._parse_arguments("")
assert result == {}
def test_single_string_argument(self, parser):
result = parser._parse_arguments("city:<escape>Tokyo<escape>")
assert result == {"city": "Tokyo"}
def test_multiple_arguments(self, parser):
args_str = "city:<escape>Tokyo<escape>,country:<escape>Japan<escape>"
result = parser._parse_arguments(args_str)
assert result == {"city": "Tokyo", "country": "Japan"}
def test_numeric_argument(self, parser):
result = parser._parse_arguments("count:<escape>42<escape>")
assert result == {"count": 42}
def test_boolean_argument(self, parser):
result = parser._parse_arguments("enabled:<escape>true<escape>")
assert result == {"enabled": True}
def test_argument_with_spaces(self, parser):
result = parser._parse_arguments("message:<escape>Hello World<escape>")
assert result == {"message": "Hello World"}
class TestAdjustRequest:
def test_skip_special_tokens_disabled(self, parser, mock_request):
mock_request.tools = [{"type": "function", "function": {"name": "test"}}]
mock_request.tool_choice = "auto"
mock_request.skip_special_tokens = True
result = parser.adjust_request(mock_request)
assert result.skip_special_tokens is False
def test_skip_special_tokens_when_tool_choice_none(self, parser, mock_request):
mock_request.tools = [{"type": "function", "function": {"name": "test"}}]
mock_request.tool_choice = "none"
mock_request.skip_special_tokens = True
result = parser.adjust_request(mock_request)
assert result.skip_special_tokens is True
class TestBufferDeltaText:
def test_regular_text_not_buffered(self, parser):
result = parser._buffer_delta_text("hello")
assert result == "hello"
assert parser.buffered_delta_text == ""
def test_complete_tag_flushed(self, parser):
parser.buffered_delta_text = "<start_function_"
result = parser._buffer_delta_text("call>")
assert "<start_function_call>" in result
if __name__ == "__main__":
pytest.main([__file__, "-v"])
File diff suppressed because it is too large Load Diff
@@ -1,350 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from transformers import AutoTokenizer
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser, ToolParserManager
@pytest.fixture(scope="function")
def default_tokenizer() -> TokenizerLike:
"""Override module-scoped default_tokenizer because gigachat tests
mutate the tokenizer via ``add_tokens``."""
return AutoTokenizer.from_pretrained("openai-community/gpt2")
MSG_SEP_TOKEN = "<|message_sep|>\n\n"
ROLE_SEP_TOKEN = "<|role_sep|>\n"
EOS_TOKEN = "</s>"
TOOL_HEADER_GIGACHAT3 = f"function call{ROLE_SEP_TOKEN}"
TOOL_HEADER_GIGACHAT31 = "<|function_call|>"
SIMPLE_ARGS_DICT = {
"action": "create",
"id": "preferences",
}
SIMPLE_FUNCTION_JSON = json.dumps(
{
"name": "manage_user_memory",
"arguments": SIMPLE_ARGS_DICT,
},
ensure_ascii=False,
)
SIMPLE_FUNCTION_OUTPUT_GIGACHAT3 = (
f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{SIMPLE_FUNCTION_JSON}"
)
SIMPLE_FUNCTION_OUTPUT_GIGACHAT31 = f"{TOOL_HEADER_GIGACHAT31}{SIMPLE_FUNCTION_JSON}"
SIMPLE_FUNCTION_CALL = FunctionCall(
name="manage_user_memory",
arguments=json.dumps(SIMPLE_ARGS_DICT, ensure_ascii=False),
)
PARAMETERLESS_FUNCTION_JSON = json.dumps(
{
"name": "manage_user_memory",
"arguments": {},
},
ensure_ascii=False,
)
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3 = (
f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{PARAMETERLESS_FUNCTION_JSON}"
)
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31 = (
f"{TOOL_HEADER_GIGACHAT31}{PARAMETERLESS_FUNCTION_JSON}"
)
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
name="manage_user_memory",
arguments=json.dumps({}, ensure_ascii=False),
)
COMPLEX_ARGS_DICT = {
"action": "create",
"id": "preferences",
"content": {
"short_answers": True,
"hate_emojis": True,
"english_ui": False,
"russian_math_explanations": True,
},
}
COMPLEX_FUNCTION_JSON = json.dumps(
{
"name": "manage_user_memory",
"arguments": COMPLEX_ARGS_DICT,
},
ensure_ascii=False,
)
COMPLEX_FUNCTION_OUTPUT_GIGACHAT3 = (
f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{COMPLEX_FUNCTION_JSON}"
)
COMPLEX_FUNCTION_OUTPUT_GIGACHAT31 = f"{TOOL_HEADER_GIGACHAT31}{COMPLEX_FUNCTION_JSON}"
COMPLEX_FUNCTION_CALL = FunctionCall(
name="manage_user_memory",
arguments=json.dumps(COMPLEX_ARGS_DICT, ensure_ascii=False),
)
CONTENT_TEXT = "I'll check that for you."
MIXED_OUTPUT_GIGACHAT3 = f"{CONTENT_TEXT}{SIMPLE_FUNCTION_OUTPUT_GIGACHAT3}"
MIXED_OUTPUT_GIGACHAT31 = f"{CONTENT_TEXT}{SIMPLE_FUNCTION_OUTPUT_GIGACHAT31}"
@pytest.fixture(name="gigachat_tokenizer")
def fixture_gigachat_tokenizer(default_tokenizer: TokenizerLike):
default_tokenizer.add_tokens(
[
MSG_SEP_TOKEN,
ROLE_SEP_TOKEN,
TOOL_HEADER_GIGACHAT31,
EOS_TOKEN,
]
)
return default_tokenizer
@pytest.mark.parametrize("streaming", [True, False])
def test_no_tool_call(streaming: bool, gigachat_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
gigachat_tokenizer
)
model_output = "How can I help you today?"
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content == model_output
assert len(tool_calls) == 0
TEST_CASES = [
pytest.param(
True,
SIMPLE_FUNCTION_OUTPUT_GIGACHAT3,
[SIMPLE_FUNCTION_CALL],
None,
id="simple_streaming_gigachat3",
),
pytest.param(
False,
SIMPLE_FUNCTION_OUTPUT_GIGACHAT3,
[SIMPLE_FUNCTION_CALL],
None,
id="simple_nonstreaming_gigachat3",
),
pytest.param(
True,
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3,
[PARAMETERLESS_FUNCTION_CALL],
None,
id="parameterless_streaming_gigachat3",
),
pytest.param(
False,
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3,
[PARAMETERLESS_FUNCTION_CALL],
None,
id="parameterless_nonstreaming_gigachat3",
),
pytest.param(
True,
COMPLEX_FUNCTION_OUTPUT_GIGACHAT3,
[COMPLEX_FUNCTION_CALL],
None,
id="complex_streaming_gigachat3",
),
pytest.param(
False,
COMPLEX_FUNCTION_OUTPUT_GIGACHAT3,
[COMPLEX_FUNCTION_CALL],
None,
id="complex_nonstreaming_gigachat3",
),
pytest.param(
True,
MIXED_OUTPUT_GIGACHAT3,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_streaming_gigachat3",
),
pytest.param(
False,
MIXED_OUTPUT_GIGACHAT3,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_nonstreaming_gigachat3",
),
pytest.param(
True,
MIXED_OUTPUT_GIGACHAT3 + EOS_TOKEN,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_streaming_with_eos_gigachat3",
),
pytest.param(
False,
MIXED_OUTPUT_GIGACHAT3 + EOS_TOKEN,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_nonstreaming_with_eos_gigachat3",
),
pytest.param(
True,
SIMPLE_FUNCTION_OUTPUT_GIGACHAT31,
[SIMPLE_FUNCTION_CALL],
None,
id="simple_streaming_gigachat31",
),
pytest.param(
False,
SIMPLE_FUNCTION_OUTPUT_GIGACHAT31,
[SIMPLE_FUNCTION_CALL],
None,
id="simple_nonstreaming_gigachat31",
),
pytest.param(
True,
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31,
[PARAMETERLESS_FUNCTION_CALL],
None,
id="parameterless_streaming_gigachat31",
),
pytest.param(
False,
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31,
[PARAMETERLESS_FUNCTION_CALL],
None,
id="parameterless_nonstreaming_gigachat31",
),
pytest.param(
True,
COMPLEX_FUNCTION_OUTPUT_GIGACHAT31,
[COMPLEX_FUNCTION_CALL],
None,
id="complex_streaming_gigachat31",
),
pytest.param(
False,
COMPLEX_FUNCTION_OUTPUT_GIGACHAT31,
[COMPLEX_FUNCTION_CALL],
None,
id="complex_nonstreaming_gigachat31",
),
pytest.param(
True,
MIXED_OUTPUT_GIGACHAT31,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_streaming_gigachat31",
),
pytest.param(
False,
MIXED_OUTPUT_GIGACHAT31,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_nonstreaming_gigachat31",
),
pytest.param(
True,
MIXED_OUTPUT_GIGACHAT31 + EOS_TOKEN,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_streaming_with_eos_gigachat31",
),
pytest.param(
False,
MIXED_OUTPUT_GIGACHAT31 + EOS_TOKEN,
[SIMPLE_FUNCTION_CALL],
CONTENT_TEXT,
id="mixed_content_nonstreaming_with_eos_gigachat31",
),
]
@pytest.mark.parametrize(
"streaming, model_output, expected_tool_calls, expected_content", TEST_CASES
)
def test_tool_call(
streaming: bool,
model_output: str,
expected_tool_calls: list[FunctionCall],
expected_content: str | None,
gigachat_tokenizer: TokenizerLike,
):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
gigachat_tokenizer
)
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
if content == "":
content = None
assert content == expected_content
assert len(tool_calls) == len(expected_tool_calls)
for actual, expected in zip(tool_calls, expected_tool_calls):
assert actual.type == "function"
assert actual.function.name == expected.name
actual_args = json.loads(actual.function.arguments)
expected_args = json.loads(expected.arguments)
assert actual_args == expected_args
@pytest.mark.parametrize(
"model_output_deltas",
[
pytest.param(
[
CONTENT_TEXT[:3],
CONTENT_TEXT[3:5],
CONTENT_TEXT[5:],
MSG_SEP_TOKEN,
TOOL_HEADER_GIGACHAT3,
COMPLEX_FUNCTION_JSON[:40],
COMPLEX_FUNCTION_JSON[40:-1],
COMPLEX_FUNCTION_JSON[-1],
],
id="gigachat3",
),
pytest.param(
[
CONTENT_TEXT[:3],
CONTENT_TEXT[3:5],
CONTENT_TEXT[5:],
TOOL_HEADER_GIGACHAT31,
COMPLEX_FUNCTION_JSON[:40],
COMPLEX_FUNCTION_JSON[40:-1],
COMPLEX_FUNCTION_JSON[-1],
],
id="gigachat31",
),
],
)
def test_streaming_tool_call_with_large_steps(
model_output_deltas: list[str],
gigachat_tokenizer: TokenizerLike,
):
"""Test that the closing braces are streamed correctly."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
gigachat_tokenizer
)
reconstructor = run_tool_extraction_streaming(
tool_parser,
model_output_deltas,
assert_one_tool_per_delta=False,
)
assert len(reconstructor.tool_calls) == 1
call = reconstructor.tool_calls[0]
assert call.type == "function"
assert call.function.name == "manage_user_memory"
args_dict = json.loads(call.function.arguments)
assert args_dict == COMPLEX_ARGS_DICT
@@ -1,268 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""Tests for the GLM-4.7 tool call parser."""
import json
from unittest.mock import Mock
import pytest
from openai.types.responses import ResponseFunctionToolCall
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.entrypoints.openai.responses.utils import build_response_output_items
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
MODEL = "zai-org/GLM-4.7"
@pytest.fixture(scope="module")
def glm47_tokenizer():
return get_tokenizer(tokenizer_name=MODEL)
@pytest.fixture
def sample_tools():
return [
ChatCompletionToolsParam(
function=FunctionDefinition(name="get_current_date", parameters={}),
),
ChatCompletionToolsParam(
function=FunctionDefinition(
name="get_weather",
parameters={
"type": "object",
"properties": {
"city": {"type": "string"},
"date": {"type": "string"},
},
},
),
),
]
@pytest.fixture
def glm47_tool_parser(glm47_tokenizer, sample_tools):
return Glm47MoeModelToolParser(glm47_tokenizer, tools=sample_tools)
@pytest.fixture
def mock_request(sample_tools) -> ChatCompletionRequest:
request = Mock(spec=ChatCompletionRequest)
request.tools = sample_tools
request.tool_choice = "auto"
return request
@pytest.fixture
def namespace_tool_request() -> ResponsesRequest:
return ResponsesRequest.model_validate(
{
"input": "hi",
"tools": [
{
"type": "namespace",
"name": "mcp__computer_use",
"description": "Computer use tools.",
"tools": [
{
"type": "function",
"name": "get_app_state",
"description": "Get app state.",
"parameters": {
"type": "object",
"properties": {
"app": {"type": "string"},
},
},
}
],
}
],
}
)
class TestGlm47ExtractToolCalls:
def test_namespace_tool_call_round_trip_to_responses_output(
self, glm47_tokenizer, namespace_tool_request
):
parser = Glm47MoeModelToolParser(
glm47_tokenizer, tools=namespace_tool_request.tools
)
out = (
"<tool_call>mcp__computer_use__get_app_state"
"<arg_key>app</arg_key>"
"<arg_value>Google Chrome</arg_value>"
"</tool_call>"
)
result = parser.extract_tool_calls(out, request=namespace_tool_request)
assert result.tools_called
tool_call = result.tool_calls[0].function
assert tool_call == FunctionCall(
name="mcp__computer_use__get_app_state",
arguments='{"app": "Google Chrome"}',
)
output_items = build_response_output_items(
reasoning=None,
content=None,
tool_calls=[tool_call],
tools=namespace_tool_request.tools,
)
output_tool_call = output_items[0]
assert isinstance(output_tool_call, ResponseFunctionToolCall)
assert output_tool_call.name == "get_app_state"
assert output_tool_call.namespace == "mcp__computer_use"
def test_no_tool_call(self, glm47_tool_parser, mock_request):
out = "This is a plain response."
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert not r.tools_called
assert r.content == out
def test_zero_arg_inline(self, glm47_tool_parser, mock_request):
out = "<tool_call>get_current_date</tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert r.tool_calls[0].function.name == "get_current_date"
assert json.loads(r.tool_calls[0].function.arguments) == {}
assert r.content is None
def test_zero_arg_newline(self, glm47_tool_parser, mock_request):
out = "<tool_call>get_current_date\n</tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert r.tool_calls[0].function.name == "get_current_date"
def test_args_same_line(self, glm47_tool_parser, mock_request):
out = "<tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value></tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert json.loads(r.tool_calls[0].function.arguments) == {"city": "Beijing"}
def test_args_with_newlines(self, glm47_tool_parser, mock_request):
out = "<tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>Beijing</arg_value>\n</tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert json.loads(r.tool_calls[0].function.arguments) == {"city": "Beijing"}
def test_whitespace_preserved_in_arg_values(self, glm47_tool_parser, mock_request):
out = "<tool_call>get_weather<arg_key>city</arg_key><arg_value> Beijing </arg_value></tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert json.loads(r.tool_calls[0].function.arguments) == {"city": " Beijing "}
def test_content_before(self, glm47_tool_parser, mock_request):
out = "Checking.<tool_call>get_current_date</tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert r.content == "Checking."
def test_multiple(self, glm47_tool_parser, mock_request):
out = (
"<tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value></tool_call>"
"<tool_call>get_weather<arg_key>city</arg_key><arg_value>Shanghai</arg_value></tool_call>"
)
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert len(r.tool_calls) == 2
def test_empty_content_none(self, glm47_tool_parser, mock_request):
out = "<tool_call>get_current_date</tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.content is None
def test_whitespace_content_none(self, glm47_tool_parser, mock_request):
out = " \n <tool_call>get_current_date</tool_call>"
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.content is None
def _reset(parser):
parser.current_tool_name_sent = False
parser.prev_tool_call_arr = []
parser.current_tool_id = -1
parser.streamed_args_for_tool = []
parser._tool_call_ids = []
parser._sent_content_idx = 0
class TestGlm47Streaming:
def test_no_args(self, glm47_tool_parser, mock_request):
_reset(glm47_tool_parser)
chunks = ["<tool_call>", "get_current_date", "</tool_call>"]
current_text = ""
deltas = []
for chunk in chunks:
current_text += chunk
delta = glm47_tool_parser.extract_tool_calls_streaming(
previous_text="",
current_text=current_text,
delta_text=chunk,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=mock_request,
)
if delta:
deltas.append(delta)
tool_calls = [
tool_call for delta in deltas for tool_call in (delta.tool_calls or [])
]
names = [
tool_call.function.name
for tool_call in tool_calls
if tool_call.function and tool_call.function.name
]
arguments = [
tool_call.function.arguments
for tool_call in tool_calls
if tool_call.function and tool_call.function.arguments
]
assert names == ["get_current_date"]
assert "".join(arguments) == "{}"
def test_with_args(self, glm47_tool_parser, mock_request):
_reset(glm47_tool_parser)
chunks = [
"<tool_call>",
"get_weather\n",
"<arg_key>city</arg_key>",
"<arg_value>",
"Beijing",
"</arg_value>",
"</tool_call>",
]
current_text = ""
deltas = []
for chunk in chunks:
current_text += chunk
delta = glm47_tool_parser.extract_tool_calls_streaming(
previous_text="",
current_text=current_text,
delta_text=chunk,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=mock_request,
)
if delta:
deltas.append(delta)
arguments = [
tool_call.function.arguments
for delta in deltas
for tool_call in (delta.tool_calls or [])
if tool_call.function and tool_call.function.arguments
]
args = json.loads("".join(arguments))
assert args["city"] == "Beijing"
@@ -1,228 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compatibility tests for GLM-4.5 using the shared GLM XML parser."""
import json
from typing import Any, TypedDict
from tests.parser.engine.replay_harness import MockTokenizer
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
MODEL = "zai-org/GLM-4.5"
_GLM_VOCAB = {
"<think>": 50,
"</think>": 51,
"<tool_call>": 60,
"</tool_call>": 61,
"<arg_key>": 62,
"</arg_key>": 63,
"<arg_value>": 64,
"</arg_value>": 65,
}
class _CollectedToolDelta(TypedDict):
name: str | None
args_fragments: list[str]
def _mock_tokenizer() -> MockTokenizer:
return MockTokenizer(vocab=_GLM_VOCAB, tokens=[])
def _tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="get_current_weather",
parameters={
"type": "object",
"properties": {
"city": {"type": "string"},
"state": {"type": "string"},
"unit": {"type": "string"},
},
},
),
),
ChatCompletionToolsParam(
function=FunctionDefinition(
name="calculate",
parameters={
"type": "object",
"properties": {
"operation": {"type": "string"},
"a": {"type": "number"},
"b": {"type": "number"},
"enabled": {"type": "boolean"},
},
},
),
),
ChatCompletionToolsParam(
function=FunctionDefinition(name="get_time", parameters={}),
),
]
def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
return ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
def _parser(tools: list[ChatCompletionToolsParam] | None = None):
return Glm47MoeModelToolParser(_mock_tokenizer(), tools=tools)
def _collect_tool_deltas(deltas: Any) -> dict[int, _CollectedToolDelta]:
calls: dict[int, _CollectedToolDelta] = {}
for delta in deltas:
if delta is None or not delta.tool_calls:
continue
for tool_call in delta.tool_calls:
entry = calls.setdefault(
tool_call.index,
{"name": None, "args_fragments": []},
)
function = tool_call.function
if function is None:
continue
if isinstance(function, dict):
name = function.get("name")
arguments = function.get("arguments")
else:
name = function.name
arguments = function.arguments
if isinstance(name, str) and name:
entry["name"] = name
if isinstance(arguments, str) and arguments:
entry["args_fragments"].append(arguments)
return calls
def test_glm45_uses_shared_glm47_parser():
assert ToolParserManager.get_tool_parser("glm45") is Glm47MoeModelToolParser
assert ToolParserManager.get_tool_parser("glm47") is Glm47MoeModelToolParser
def test_extract_tool_calls_with_glm45_newline_format():
tools = _tools()
parser = _parser(tools)
model_output = """I'll check it. <tool_call>get_current_weather
<arg_key>city</arg_key>
<arg_value>Dallas</arg_value>
<arg_key>state</arg_key>
<arg_value>TX</arg_value>
<arg_key>unit</arg_key>
<arg_value>fahrenheit</arg_value>
</tool_call>"""
extracted = parser.extract_tool_calls(model_output, request=_request(tools))
assert extracted.tools_called
assert extracted.content == "I'll check it."
assert len(extracted.tool_calls) == 1
tool_call = extracted.tool_calls[0]
assert tool_call.function.name == "get_current_weather"
assert json.loads(tool_call.function.arguments) == {
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
def test_extract_multiple_tool_calls_with_glm45_newline_format():
tools = _tools()
parser = _parser(tools)
model_output = """<tool_call>get_current_weather
<arg_key>city</arg_key><arg_value>Dallas</arg_value>
</tool_call>
<tool_call>get_current_weather
<arg_key>city</arg_key><arg_value>Orlando</arg_value>
</tool_call>"""
extracted = parser.extract_tool_calls(model_output, request=_request(tools))
assert extracted.tools_called
assert [tc.function.name for tc in extracted.tool_calls] == [
"get_current_weather",
"get_current_weather",
]
assert [
json.loads(tc.function.arguments)["city"] for tc in extracted.tool_calls
] == ["Dallas", "Orlando"]
def test_extract_tool_calls_coerces_schema_types():
tools = _tools()
parser = _parser(tools)
model_output = """<tool_call>calculate
<arg_key>operation</arg_key><arg_value>add</arg_value>
<arg_key>a</arg_key><arg_value>42</arg_value>
<arg_key>b</arg_key><arg_value>3.14</arg_value>
<arg_key>enabled</arg_key><arg_value>true</arg_value>
</tool_call>"""
extracted = parser.extract_tool_calls(model_output, request=_request(tools))
assert extracted.tools_called
assert json.loads(extracted.tool_calls[0].function.arguments) == {
"operation": "add",
"a": 42,
"b": 3.14,
"enabled": True,
}
def test_extract_zero_argument_tool_call_with_glm45_newline_format():
tools = _tools()
parser = _parser(tools)
extracted = parser.extract_tool_calls(
"<tool_call>get_time\n</tool_call>",
request=_request(tools),
)
assert extracted.tools_called
assert extracted.tool_calls[0].function.name == "get_time"
assert json.loads(extracted.tool_calls[0].function.arguments) == {}
def test_streaming_tool_call_with_glm45_newline_format():
tools = _tools()
parser = _parser(tools)
request = _request(tools)
chunks = [
"<tool_call>",
"get_current_weather\n",
"<arg_key>city</arg_key>",
"<arg_value>Bei",
"jing</arg_value>",
"</tool_call>",
]
deltas = []
current_text = ""
for chunk in chunks:
current_text += chunk
deltas.append(
parser.extract_tool_calls_streaming(
previous_text="",
current_text=current_text,
delta_text=chunk,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=request,
)
)
calls = _collect_tool_deltas(deltas)
assert calls[0]["name"] == "get_current_weather"
assert json.loads("".join(calls[0]["args_fragments"])) == {"city": "Beijing"}
@@ -1,147 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import random
from typing import Any
import pytest
from transformers import AutoTokenizer
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
MODEL = "ibm-granite/granite-4.0-h-tiny"
def create_complex_input(create_string_args: bool):
coord_arg: dict | str = {
"coordinates": [[23.54, 43.1], [-12.2, 54.3], [4, 5]],
"coordinate_type": "latlong",
}
if create_string_args:
# test granite behavior
coord_arg = json.dumps(coord_arg)
return [
{"name": "find_bbox", "arguments": coord_arg},
{
"name": "get_stock_price",
"arguments": {
"symbol": "AAPL",
"start_date": "2021-01-01",
"end_date": "2021-12-31",
},
},
{"name": "find_bbox", "arguments": coord_arg},
]
def random_chunks(s: str, min_len: int, max_len: int):
chunks = []
i = 0
n = len(s)
while i < n:
size = random.randint(min_len, max_len)
chunks.append(s[i : i + size])
i += size
return chunks
@pytest.fixture(scope="module")
def tokenizer():
return AutoTokenizer.from_pretrained(MODEL)
# create a variety of input chunk sizes
@pytest.mark.parametrize(
"min_chunk, max_chunk",
[
(1, 1),
(1, 2),
(5, 7),
(6, 20),
],
)
def test_tool_call_parser_complex(min_chunk: int, max_chunk: int, tokenizer):
input_dicts = create_complex_input(True)
formatted_tcs = [
"<tool_call> " + json.dumps(call) + " </tool_call>" for call in input_dicts
]
text_messages = [
"Here goes the bbox call: \n",
" Now the stock price call: \n ",
" Now another bbox call: \n ",
" See? I'm a helpful assistant.",
]
test_input = (
text_messages[0]
+ formatted_tcs[0]
+ text_messages[1]
+ formatted_tcs[1]
+ text_messages[2]
+ formatted_tcs[2]
+ text_messages[3]
)
any_chat_request = ChatCompletionRequest(
seed=42,
model=MODEL,
messages=[],
)
parser = Granite4ToolParser(tokenizer=tokenizer)
delta_messages = list[DeltaMessage]()
for text in random_chunks(test_input, min_chunk, max_chunk):
delta = parser.extract_tool_calls_streaming(
previous_text="",
current_text="",
delta_text=text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=any_chat_request,
)
if delta is not None:
delta_messages.append(delta)
content = ""
tool_calls = list[dict[str, Any]]()
current_name = "__start__"
current_args = ""
for msg in delta_messages:
if msg.content:
content += msg.content
for tool_call in msg.tool_calls:
if delta_func := tool_call.function:
if delta_func.name is not None:
if current_name == "__start__":
current_name = delta_func.name
if delta_func.name != current_name:
tool_calls.append(
{
"name": current_name,
"arguments": json.loads(current_args),
}
)
current_name = delta_func.name
current_args = ""
if delta_func.arguments:
current_args += delta_func.arguments
if current_name != "__start__":
tool_calls.append({"name": current_name, "arguments": json.loads(current_args)})
assert content == "".join(text_messages)
assert tool_calls == create_complex_input(False)
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
split_string_into_token_deltas,
)
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.granite_engine_tool_parser import GraniteEngineToolParser
class TestGraniteToolParser(ToolParserTests):
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="granite",
# Test data
no_tool_calls_output="This is a regular response without any tool calls.",
single_tool_call_output=(
'<|tool_call|> [{"name": "get_weather", '
'"arguments": {"city": "Tokyo"}}]'
),
parallel_tool_calls_output="""<|tool_call|> [
{"name": "get_weather", "arguments": {"city": "Tokyo"}},
{"name": "get_time", "arguments": {"timezone": "Asia/Tokyo"}}
]""",
various_data_types_output="""<tool_call> [{
"name": "test_function",
"arguments": {
"string_field": "hello",
"int_field": 42,
"float_field": 3.14,
"bool_field": true,
"null_field": null,
"array_field": ["a", "b", "c"],
"object_field": {"nested": "value"},
"empty_array": [],
"empty_object": {}
}
}]""",
empty_arguments_output=(
'<|tool_call|> [{"name": "refresh", "arguments": {}}]'
),
surrounding_text_output="""Let me check the weather for you.
<|tool_call|> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]
I'll get that information.""",
escaped_strings_output="""<tool_call> [{
"name": "test_function",
"arguments": {
"quoted": "He said \\"hello\\"",
"path": "C:\\\\Users\\\\file.txt",
"newline": "line1\\nline2",
"unicode": "emoji: 🎉"
}
}]""",
malformed_input_outputs=[
'<|tool_call|> [{"name": "func", "arguments": {',
'<|tool_call|> {"name": "func", "arguments": {}}', # Not an array
'[{"name": "func", "arguments": "not a dict"}]',
'Some text [{"name": "func"}]', # JSON but not tool call format
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo"},
# Granite strips content when tool calls present
single_tool_call_expected_content=None,
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "get_time"],
)
# Granite-Specific Tests
@pytest.mark.parametrize("streaming", [True, False])
def test_granite_token_prefix_format(self, tool_parser, streaming):
"""Verify parser handles Granite 3.0 <|tool_call|> token format."""
single_tool_call_token = (
'<|tool_call|> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
)
content, tool_calls = run_tool_extraction(
tool_parser, single_tool_call_token, streaming=streaming
)
assert len(tool_calls) == 1, (
f"Expected 1 tool call from token format, got {len(tool_calls)}"
)
assert tool_calls[0].function.name == "get_weather"
@pytest.mark.parametrize("streaming", [True, False])
def test_granite_string_prefix_format(self, tool_parser, streaming):
"""Verify parser handles Granite 3.1 <tool_call> string format."""
single_tool_call_string = (
'<tool_call> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
)
content, tool_calls = run_tool_extraction(
tool_parser, single_tool_call_string, streaming=streaming
)
assert len(tool_calls) == 1, (
f"Expected 1 tool call from string format, got {len(tool_calls)}"
)
assert tool_calls[0].function.name == "get_weather"
# granite emits arguments before name and its own tokenizer (not gpt2) is used
# here so the token boundaries match production; get_tokenizer only fetches the
# small tokenizer files, not the model weights.
@pytest.fixture(scope="module")
def granite_tokenizer():
return get_tokenizer(tokenizer_name="ibm-granite/granite-3.1-8b-instruct")
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size):
"""A batched delta (multiple tokens) spanning the boundary between two
parallel calls must not drop the first call's name. granite streams
arguments before name, so the name only completes as the next call appears.
"""
parser = GraniteEngineToolParser(granite_tokenizer)
model_output = (
'<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, '
'{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]'
)
token_deltas = split_string_into_token_deltas(granite_tokenizer, model_output)
batched = [
"".join(token_deltas[i : i + chunk_size])
for i in range(0, len(token_deltas), chunk_size)
]
reconstructor = run_tool_extraction_streaming(
parser, batched, assert_one_tool_per_delta=False
)
names = [tc.function.name for tc in reconstructor.tool_calls]
assert names == ["get_weather", "get_time"]
# trailing args of the final call are flushed by the serving layer
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"city": "Tokyo"
}
@@ -1,414 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
CONFIGS = {
"llama": {
"tool_parser": Hermes2ProToolParser,
},
"granite4": {
"tool_parser": Granite4ToolParser,
},
}
@pytest.fixture
def qwen_tokenizer() -> TokenizerLike:
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")
@pytest.fixture(params=CONFIGS.keys())
def hermes_parser(request, qwen_tokenizer: TokenizerLike) -> ToolParser:
config = CONFIGS[request.param]
return config["tool_parser"](qwen_tokenizer)
@pytest.fixture
def any_chat_request() -> ChatCompletionRequest:
return ChatCompletionRequest(
seed=42,
model="Qwen/Qwen3-32B",
messages=[],
)
def test_hermes_parser_streaming_just_forward_text(
qwen_tokenizer: TokenizerLike,
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
text = """This is some prior text that has nothing to do with tool calling."""
tokens = qwen_tokenizer.encode(text)
previous_text = ""
delta_messages = []
for token in tokens:
delta_text = qwen_tokenizer.decode([token])
current_text = previous_text + delta_text
delta = hermes_parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=any_chat_request,
)
previous_text = current_text
delta_messages.append(delta)
for delta in delta_messages:
assert delta is not None
assert not delta.tool_calls
print(delta_messages)
assert "".join([delta.content for delta in delta_messages]) == text
def test_hermes_parser_streaming_failure_case_bug_19056(
qwen_tokenizer: TokenizerLike,
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
text = """<tool_call>
{"name": "final_answer", "arguments": {"trigger": true}}
</tool_call>"""
tokens = qwen_tokenizer.encode(text)
previous_text = ""
delta_messages = []
for token in tokens:
text = qwen_tokenizer.decode([token])
current_text = previous_text + text
delta = hermes_parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=any_chat_request,
)
previous_text = current_text
if delta is not None:
delta_messages.append(delta)
assert delta_messages[0].tool_calls[0].function.name == "final_answer"
tool_call_args = "".join(
delta.tool_calls[0].function.arguments or "" for delta in delta_messages
)
assert tool_call_args == '{"trigger": true}'
def test_hermes_parser_streaming(
qwen_tokenizer: TokenizerLike,
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
text = '<tool_call>\
{"name": "get_current_temperature",\
"arguments": {"location":\
"San Francisco, California, United States", "unit": "celsius"}}\
</tool_call>'
tokens = qwen_tokenizer.encode(text)
previous_text = ""
delta_messages = []
for token in tokens:
text = qwen_tokenizer.decode([token])
current_text = previous_text + text
delta = hermes_parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=any_chat_request,
)
previous_text = current_text
if delta is not None:
delta_messages.append(delta)
print(delta_messages)
assert delta_messages[0].tool_calls[0].function.name == "get_current_temperature"
# load to normalize whitespace
tool_call_args = json.loads(
"".join(
delta.tool_calls[0].function.arguments or "" for delta in delta_messages
)
)
assert tool_call_args == {
"location": "San Francisco, California, United States",
"unit": "celsius",
}
def _simulate_streaming(
tokenizer: TokenizerLike,
parser: ToolParser,
request: ChatCompletionRequest,
text: str,
stream_interval: int = 1,
) -> list:
"""Simulate streaming with a given stream_interval.
Tokens are batched into chunks of `stream_interval` tokens,
mimicking how the output processor delivers them.
Returns a list of non-None DeltaMessages.
"""
tokens = tokenizer.encode(text)
previous_text = ""
delta_messages = []
for i in range(0, len(tokens), stream_interval):
chunk_ids = tokens[i : i + stream_interval]
delta_text = tokenizer.decode(chunk_ids)
current_text = previous_text + delta_text
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=chunk_ids,
request=request,
)
previous_text = current_text
if delta is not None:
delta_messages.append(delta)
return delta_messages
@pytest.mark.parametrize("stream_interval", [2, 3, 5, 8])
def test_hermes_streaming_tool_call_with_stream_interval(
qwen_tokenizer: TokenizerLike,
any_chat_request: ChatCompletionRequest,
stream_interval: int,
) -> None:
"""Tool call streaming must produce correct name + args at any interval."""
text = (
'<tool_call>{"name": "get_current_temperature", '
'"arguments": {"location": "San Francisco", "unit": "celsius"}}'
"</tool_call>"
)
parser = Hermes2ProToolParser(qwen_tokenizer)
deltas = _simulate_streaming(
qwen_tokenizer, parser, any_chat_request, text, stream_interval
)
# Flatten all DeltaToolCalls across all deltas.
tool_deltas = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
assert tool_deltas, "Expected at least one tool call delta"
assert tool_deltas[0].function.name == "get_current_temperature"
# Concatenated arguments must be valid JSON matching the original.
args_str = "".join(tc.function.arguments or "" for tc in tool_deltas)
assert json.loads(args_str) == {
"location": "San Francisco",
"unit": "celsius",
}
@pytest.mark.parametrize("stream_interval", [2, 3, 5, 8])
def test_hermes_streaming_content_then_tool_call_with_stream_interval(
qwen_tokenizer: TokenizerLike,
any_chat_request: ChatCompletionRequest,
stream_interval: int,
) -> None:
"""Content before a tool call must be fully streamed, then tool call."""
text = (
"Sure, let me check the weather."
'<tool_call>{"name": "get_weather", '
'"arguments": {"city": "NYC"}}</tool_call>'
)
parser = Hermes2ProToolParser(qwen_tokenizer)
deltas = _simulate_streaming(
qwen_tokenizer, parser, any_chat_request, text, stream_interval
)
content_deltas = [d for d in deltas if d.content]
tool_deltas = [d for d in deltas if d.tool_calls]
# Content must reconstruct the prefix.
content_str = "".join(d.content for d in content_deltas)
assert content_str == "Sure, let me check the weather."
# Tool call must be correct.
tool_calls = [tc for d in tool_deltas for tc in d.tool_calls]
assert tool_calls[0].function.name == "get_weather"
args_str = "".join(tc.function.arguments or "" for tc in tool_calls)
assert json.loads(args_str) == {"city": "NYC"}
@pytest.mark.parametrize("stream_interval", [1, 2, 4])
def test_hermes_streaming_multiple_tool_calls_with_stream_interval(
qwen_tokenizer: TokenizerLike,
any_chat_request: ChatCompletionRequest,
stream_interval: int,
) -> None:
"""Multiple sequential tool calls must each be streamed correctly."""
text = (
'<tool_call>{"name": "search", "arguments": {"q": "cats"}}</tool_call>'
'<tool_call>{"name": "search", "arguments": {"q": "dogs"}}</tool_call>'
)
parser = Hermes2ProToolParser(qwen_tokenizer)
deltas = _simulate_streaming(
qwen_tokenizer, parser, any_chat_request, text, stream_interval
)
# Flatten all DeltaToolCalls across all deltas.
all_tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
# Separate by tool index.
tool0 = [tc for tc in all_tool_calls if tc.index == 0]
tool1 = [tc for tc in all_tool_calls if tc.index == 1]
assert tool0[0].function.name == "search"
args0 = "".join(tc.function.arguments or "" for tc in tool0)
assert json.loads(args0) == {"q": "cats"}
assert tool1[0].function.name == "search"
args1 = "".join(tc.function.arguments or "" for tc in tool1)
assert json.loads(args1) == {"q": "dogs"}
@pytest.mark.parametrize("stream_interval", [2, 5])
def test_hermes_streaming_boolean_args_with_stream_interval(
qwen_tokenizer: TokenizerLike,
any_chat_request: ChatCompletionRequest,
stream_interval: int,
) -> None:
"""Regression test for bug #19056 with stream_interval > 1."""
text = (
"<tool_call>\n"
'{"name": "final_answer", "arguments": {"trigger": true}}\n'
"</tool_call>"
)
parser = Hermes2ProToolParser(qwen_tokenizer)
deltas = _simulate_streaming(
qwen_tokenizer, parser, any_chat_request, text, stream_interval
)
tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
assert tool_calls[0].function.name == "final_answer"
args_str = "".join(tc.function.arguments or "" for tc in tool_calls)
assert json.loads(args_str) == {"trigger": True}
@pytest.mark.parametrize("stream_interval", [2, 3, 5])
def test_hermes_streaming_just_forward_text_with_stream_interval(
qwen_tokenizer: TokenizerLike,
any_chat_request: ChatCompletionRequest,
stream_interval: int,
) -> None:
"""Plain text with no tool calls must be fully forwarded."""
text = "This is plain text with no tool calling involved."
parser = Hermes2ProToolParser(qwen_tokenizer)
deltas = _simulate_streaming(
qwen_tokenizer, parser, any_chat_request, text, stream_interval
)
for d in deltas:
assert not d.tool_calls
assert "".join(d.content for d in deltas) == text
def test_hermes_parser_non_streaming_no_tool_call(
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
text = """This is not a tool call."""
tool_call = hermes_parser.extract_tool_calls(
model_output=text,
request=any_chat_request,
)
assert tool_call is not None
assert not tool_call.tools_called
def test_hermes_parser_non_streaming_tool_call_between_tags(
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
text = """<tool_call>
{"name": "final_answer", "arguments": {"trigger": true}}
</tool_call>"""
tool_call = hermes_parser.extract_tool_calls(
model_output=text,
request=any_chat_request,
)
assert tool_call is not None
assert tool_call.tools_called
assert tool_call.tool_calls[0].function.name == "final_answer"
assert tool_call.tool_calls[0].function.arguments == '{"trigger": true}'
def test_hermes_parser_non_streaming_tool_call_until_eos(
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
if isinstance(hermes_parser, Granite4ToolParser):
pytest.skip(reason="The Granite4 tool parser enforces a complete response")
text = """<tool_call>
{"name": "final_answer", "arguments": {"trigger": true}}"""
tool_call = hermes_parser.extract_tool_calls(
model_output=text,
request=any_chat_request,
)
assert tool_call is not None
assert tool_call.tools_called
assert tool_call.tool_calls[0].function.name == "final_answer"
assert tool_call.tool_calls[0].function.arguments == '{"trigger": true}'
def test_hermes_parser_non_streaming_tool_call_invalid_json(
hermes_parser: ToolParser,
any_chat_request: ChatCompletionRequest,
) -> None:
# Missing closing brace to trigger exception
text = """<tool_call>
{"name": "final_answer", "arguments": {"trigger": true}"""
tool_call = hermes_parser.extract_tool_calls(
model_output=text,
request=any_chat_request,
)
assert tool_call is not None
assert not tool_call.tools_called
def test_hermes_streaming_content_and_tool_call_in_single_chunk(
qwen_tokenizer: TokenizerLike,
any_chat_request: ChatCompletionRequest,
) -> None:
"""Content + complete tool call in one chunk must both be emitted."""
text = 'Hi!<tool_call>{"name": "f", "arguments": {"x": 1}}</tool_call>'
# Use a stream_interval large enough to guarantee a single chunk.
parser = Hermes2ProToolParser(qwen_tokenizer)
deltas = _simulate_streaming(
qwen_tokenizer,
parser,
any_chat_request,
text,
stream_interval=9999,
)
content_parts = [d.content for d in deltas if d.content]
tool_parts = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
assert "".join(content_parts) == "Hi!"
assert tool_parts[0].function.name == "f"
args_str = "".join(tc.function.arguments or "" for tc in tool_parts)
assert json.loads(args_str) == {"x": 1}
@@ -1,191 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import json
from unittest.mock import MagicMock
import pytest
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall, ToolCall
from vllm.tool_parsers import ToolParser, ToolParserManager
def make_tool_call(name, arguments):
return ToolCall(
type="function",
function=FunctionCall(name=name, arguments=json.dumps(arguments)),
)
# TODO: add reason prefix and suffix.
@pytest.mark.parametrize(
"model_output,expected_tool_calls,expected_content",
[
# No tool call
("How can I help you today?", [], "How can I help you today?"),
# Single tool call, no content
(
'<tool_calls>[{"name": "get_weather", "arguments": {"city": "San Francisco", "metric": "celsius"}}]</tool_calls>', # noqa: E501
[
make_tool_call(
"get_weather", {"city": "San Francisco", "metric": "celsius"}
)
],
None,
),
# Multiple tool calls
(
'<tool_calls>[{"name": "get_weather", "arguments": {"city": "San Francisco", "metric": "celsius"}}, {"name": "register_user", "arguments": {"name": "John Doe", "age": 37, "address": {"city": "San Francisco", "state": "CA"}, "role": null, "passed_test": true, "aliases": ["John", "Johnny"]}}]</tool_calls>', # noqa: E501
[
make_tool_call(
"get_weather", {"city": "San Francisco", "metric": "celsius"}
),
make_tool_call(
"register_user",
{
"name": "John Doe",
"age": 37,
"address": {"city": "San Francisco", "state": "CA"},
"role": None,
"passed_test": True,
"aliases": ["John", "Johnny"],
},
),
],
None,
),
# Content before tool call
(
'I will call the tool now. <tool_calls>[{"name": "get_weather", "arguments": {"city": "Boston"}}]</tool_calls>', # noqa: E501
[make_tool_call("get_weather", {"city": "Boston"})],
"I will call the tool now. ",
),
# Content after tool call (should be stripped)
(
'<tool_calls>[{"name": "get_weather", "arguments": {"city": "Seattle"}}]</tool_calls>\nThank you!', # noqa: E501
[make_tool_call("get_weather", {"city": "Seattle"})],
None,
),
(
'<tool_calls>[{"name": "complex_tool", "arguments": {"level1": {"level2": {"level3": {"value": 123}}}}}]</tool_calls>',
[
make_tool_call(
"complex_tool", {"level1": {"level2": {"level3": {"value": 123}}}}
)
],
None,
),
],
)
def test_hunyuan_a13b_tool_parser_extract(
model_output, expected_tool_calls, expected_content
):
mock_tokenizer = MagicMock()
tool_parser: ToolParser = ToolParserManager.get_tool_parser("hunyuan_a13b")(
mock_tokenizer
)
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=False
)
# align the random id.
for idx in range(len(tool_calls)):
tool_calls[idx].id = expected_tool_calls[idx].id
assert tool_calls == expected_tool_calls
assert content == expected_content
# Streaming test: simulate incremental output
@pytest.mark.parametrize(
"model_deltas,expected_tool_calls",
[
(
[
'<tool_calls>[{"name": "get_weather", ',
'"arguments": {"city": "San Francisco", ',
'"metric": "celsius"}}]',
"</tool_calls>",
],
[
make_tool_call(
"get_weather", {"city": "San Francisco", "metric": "celsius"}
)
],
),
(
[
'<tool_calls>[{"name":',
' "get_weather",',
' "arguments":',
' {"city": "Boston"}',
"}]",
"</tool_calls>",
],
[make_tool_call("get_weather", {"city": "Boston"})],
),
(
[
"",
'<tool_calls>[{"name":',
' "get_weather",',
' "arguments":',
' {"city": "Boston"}',
"}]",
"</tool_calls>",
"\n</answer>",
],
[make_tool_call("get_weather", {"city": "Boston"})],
),
pytest.param(
[
'<tool_calls>[{"name": "complex_tool",',
' "arguments": ',
' {"level1": {"level2": ',
'{"level3": {"value": 123}}}}}',
"]</tool_calls>",
],
[
make_tool_call(
"complex_tool", {"level1": {"level2": {"level3": {"value": 123}}}}
)
],
marks=pytest.mark.xfail(
reason="stream parsing not support nested json yet."
),
),
],
)
def test_hunyuan_a13b_tool_parser_streaming(model_deltas, expected_tool_calls):
mock_tokenizer = MagicMock()
tool_parser: ToolParser = ToolParserManager.get_tool_parser("hunyuan_a13b")(
mock_tokenizer
)
reconstructor = run_tool_extraction_streaming(
tool_parser, model_deltas, assert_one_tool_per_delta=False
)
# align the random id.
for idx in range(len(reconstructor.tool_calls)):
reconstructor.tool_calls[idx].id = expected_tool_calls[idx].id
assert reconstructor.tool_calls == expected_tool_calls
def test_hunyuan_a13b_tool_parser_non_ascii():
mock_tokenizer = MagicMock()
tool_parser: ToolParser = ToolParserManager.get_tool_parser("hunyuan_a13b")(
mock_tokenizer
)
model_output = '<tool_calls>[{"name": "get_weather", "arguments": {"city": "北京"}}]</tool_calls>'
_, tool_calls = run_tool_extraction(tool_parser, model_output, streaming=False)
args = tool_calls[0].function.arguments
assert "北京" in args
assert "\\u" not in args
@@ -1,274 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""Tests for the HYV3 tool call parser."""
import json
from unittest.mock import Mock
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.hy_v3_tool_parser import HYV3ToolParser
parser_name = "hy_v3"
MODEL = "tencent/Hy3-preview"
@pytest.fixture(scope="module")
def hy_v3_tokenizer():
return get_tokenizer(tokenizer_name=MODEL)
@pytest.fixture
def hy_v3_tool_parser(hy_v3_tokenizer):
return HYV3ToolParser(hy_v3_tokenizer)
@pytest.fixture
def mock_request() -> ChatCompletionRequest:
request = Mock(spec=ChatCompletionRequest)
request.tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(name="get_current_date", parameters={}),
),
ChatCompletionToolsParam(
function=FunctionDefinition(
name="get_weather",
parameters={
"type": "object",
"properties": {
"city": {"type": "string"},
"date": {"type": "string"},
},
},
),
),
]
request.tool_choice = "auto"
return request
class TestHYV3ExtractToolCalls:
def test_no_tool_call(self, hy_v3_tool_parser, mock_request):
out = "This is a plain response."
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert not r.tools_called
assert r.content == out
def test_zero_arg_inline(self, hy_v3_tool_parser, mock_request):
out = (
"<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>"
)
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert r.tool_calls[0].function.name == "get_current_date"
assert json.loads(r.tool_calls[0].function.arguments) == {}
assert r.content is None
def test_zero_arg_newline(self, hy_v3_tool_parser, mock_request):
out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert r.tool_calls[0].function.name == "get_current_date"
def test_args_same_line(self, hy_v3_tool_parser, mock_request):
out = (
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing"
"</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>"
)
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert json.loads(r.tool_calls[0].function.arguments) == {
"city": "Beijing",
"date": "2026-03-30",
}
def test_args_with_newlines(self, hy_v3_tool_parser, mock_request):
out = (
"<tool_calls>\n<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Beijing"
"</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n</tool_calls>"
)
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert json.loads(r.tool_calls[0].function.arguments) == {
"city": "Beijing",
"date": "2026-03-30",
}
def test_content_before(self, hy_v3_tool_parser, mock_request):
out = "Checking.<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.tools_called
assert r.content == "Checking."
def test_multiple(self, hy_v3_tool_parser, mock_request):
out = (
"<tool_calls>\n<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Beijing"
"</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n"
"<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Hangzhou</arg_value>\n"
"<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n</tool_calls>"
)
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert len(r.tool_calls) == 2
def test_empty_content_none(self, hy_v3_tool_parser, mock_request):
out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
assert r.content is None
def _simulate_streaming(
parser: HYV3ToolParser,
deltas: list[str],
request: ChatCompletionRequest,
) -> list[DeltaMessage | None]:
results: list[DeltaMessage | None] = []
previous_text = ""
previous_token_ids: list[int] = []
vocab = parser.vocab
for delta_text in deltas:
current_text = previous_text + delta_text
delta_token_ids = [tid for tok, tid in vocab.items() if tok in delta_text]
current_token_ids = previous_token_ids + delta_token_ids
result = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=previous_token_ids,
current_token_ids=current_token_ids,
delta_token_ids=delta_token_ids,
request=request,
)
results.append(result)
previous_text = current_text
previous_token_ids = current_token_ids
return results
def _collect_streaming_tool_calls(results: list[DeltaMessage | None]) -> list[dict]:
tool_calls: dict[int, dict] = {}
for result in results:
if result is None or not result.tool_calls:
continue
for tc in result.tool_calls:
idx = tc.index
if idx not in tool_calls:
tool_calls[idx] = {
"name": tc.function.name or "",
"arguments": tc.function.arguments or "",
}
else:
if tc.function.name:
tool_calls[idx]["name"] += tc.function.name
if tc.function.arguments:
tool_calls[idx]["arguments"] += tc.function.arguments
return [tool_calls[i] for i in sorted(tool_calls.keys())]
def _collect_streaming_content(results: list[DeltaMessage | None]) -> str:
parts = []
for result in results:
if result is not None and result.content:
parts.append(result.content)
return "".join(parts)
class TestHYV3ExtractToolCallsStreaming:
def test_no_tool_call_streaming(self, hy_v3_tool_parser, mock_request):
deltas = ["This is ", "a plain ", "response."]
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
content = _collect_streaming_content(results)
assert content == "This is a plain response."
assert len(_collect_streaming_tool_calls(results)) == 0
def test_zero_arg_streaming(self, hy_v3_tool_parser, mock_request):
deltas = [
"<tool_calls>",
"\n<tool_call>",
"get_current_date",
"<tool_sep>",
"\n</tool_call>",
"\n</tool_calls>",
]
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
tc = _collect_streaming_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_current_date"
assert json.loads(tc[0]["arguments"]) == {}
def test_args_streaming(self, hy_v3_tool_parser, mock_request):
deltas = [
"<tool_calls>",
"\n<tool_call>",
"get_weather",
"<tool_sep>",
"\n<arg_key>city</arg_key>",
"\n<arg_value>Beijing</arg_value>",
"\n<arg_key>date</arg_key>",
"\n<arg_value>2026-03-30</arg_value>",
"\n</tool_call>",
"\n</tool_calls>",
]
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
tc = _collect_streaming_tool_calls(results)
assert len(tc) == 1 and tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {
"city": "Beijing",
"date": "2026-03-30",
}
def test_content_before_streaming(self, hy_v3_tool_parser, mock_request):
deltas = [
"Checking.",
"<tool_calls>",
"\n<tool_call>",
"get_current_date",
"<tool_sep>",
"\n</tool_call>",
"\n</tool_calls>",
]
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
assert "Checking." in _collect_streaming_content(results)
tc = _collect_streaming_tool_calls(results)
assert len(tc) == 1 and tc[0]["name"] == "get_current_date"
def test_multiple_streaming(self, hy_v3_tool_parser, mock_request):
deltas = [
"<tool_calls>",
"\n<tool_call>",
"get_weather",
"<tool_sep>",
"\n<arg_key>city</arg_key>",
"\n<arg_value>Beijing</arg_value>",
"\n<arg_key>date</arg_key>",
"\n<arg_value>2026-03-30</arg_value>",
"\n</tool_call>",
"\n<tool_call>",
"get_weather",
"<tool_sep>",
"\n<arg_key>city</arg_key>",
"\n<arg_value>Hangzhou</arg_value>",
"\n<arg_key>date</arg_key>",
"\n<arg_value>2026-03-30</arg_value>",
"\n</tool_call>",
"\n</tool_calls>",
]
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
tc = _collect_streaming_tool_calls(results)
assert len(tc) == 2
assert json.loads(tc[0]["arguments"])["city"] == "Beijing"
assert json.loads(tc[1]["arguments"])["city"] == "Hangzhou"
def test_all_in_one_delta_streaming(self, hy_v3_tool_parser, mock_request):
out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
results = _simulate_streaming(hy_v3_tool_parser, [out], mock_request)
tc = _collect_streaming_tool_calls(results)
assert len(tc) == 1 and tc[0]["name"] == "get_current_date"
assert json.loads(tc[0]["arguments"]) == {}
@@ -1,192 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import MagicMock
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from tests.tool_parsers.utils import run_tool_extraction_streaming
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.internlm2_tool_parser import Internlm2ToolParser
class TestInternLM2ToolParser(ToolParserTests):
@pytest.fixture
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
"""Add some internlm2 specific tokens to the default vocab."""
tokenizer_vocab = default_tokenizer.get_vocab()
default_tokenizer.get_vocab = MagicMock()
tokenizer_vocab.update(
{
"<|action_start|>": 92540,
"<|plugin|>": 92541,
"<|action_end|>": 92542,
}
)
default_tokenizer.get_vocab.return_value = tokenizer_vocab
return default_tokenizer
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="internlm",
# Test data
no_tool_calls_output="This is a regular response without any tool calls.",
single_tool_call_output=(
'<|action_start|><|plugin|>{"name": "get_weather", '
'"parameters": {"city": "Tokyo"}}<|action_end|>'
),
# InternLM2 doesn't support parallel calls
parallel_tool_calls_output=(
'<|action_start|><|plugin|>{"name": "get_weather", '
'"parameters": {"city": "Tokyo"}}<|action_end|>'
),
various_data_types_output="""<|action_start|><|plugin|>{
"name": "test_function",
"parameters": {
"string_field": "hello",
"int_field": 42,
"float_field": 3.14,
"bool_field": true,
"null_field": null,
"array_field": ["a", "b", "c"],
"object_field": {"nested": "value"},
"empty_array": [],
"empty_object": {}
}
}<|action_end|>""",
empty_arguments_output=(
'<|action_start|><|plugin|>{"name": "refresh", '
'"parameters": {}}<|action_end|>'
),
surrounding_text_output=(
"Let me check the weather for you. "
'<|action_start|><|plugin|>{"name": "get_weather", '
'"parameters": {"city": "Tokyo"}}<|action_end|>'
),
escaped_strings_output="""<|action_start|><|plugin|>{
"name": "test_function",
"parameters": {
"quoted": "He said \\"hello\\"",
"path": "C:\\\\Users\\\\file.txt",
"newline": "line1\\nline2",
"unicode": "emoji: 🎉"
}
}<|action_end|>""",
malformed_input_outputs=[
'<|action_start|><|plugin|>{"name": "func", "parameters": {',
(
'<|action_start|><|plugin|>{"name": "func", '
'"parameters": "not a dict"}<|action_end|>'
),
"<|action_start|><|plugin|>not json<|action_end|>",
"<|action_start|><|plugin|>",
'<|action_start|>{"name": "func"}',
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo"},
single_tool_call_expected_content=None,
parallel_tool_calls_count=1, # InternLM2 only supports single tool calls
parallel_tool_calls_names=["get_weather"],
# Parser-specific settings
allow_empty_or_json_empty_args=True,
# xfail markers
xfail_streaming={
"test_single_tool_call_simple_args": (
"InternLM2 streaming not fully implemented"
),
"test_parallel_tool_calls": (
"InternLM2 streaming not fully implemented"
),
"test_various_data_types": (
"InternLM2 streaming not fully implemented"
),
"test_empty_arguments": ("InternLM2 streaming not fully implemented"),
"test_surrounding_text": ("InternLM2 streaming not fully implemented"),
"test_escaped_strings": ("InternLM2 streaming not fully implemented"),
"test_streaming_reconstruction": (
"InternLM2 streaming parser returns '<|action_start|' as "
"content instead of None - streaming/non-streaming inconsistency"
),
},
xfail_nonstreaming={
"test_malformed_input": (
"InternLM2 parser raises JSONDecodeError on malformed JSON "
"instead of gracefully handling it"
),
},
)
def test_streaming_arguments_in_single_delta(default_tokenizer: TokenizerLike) -> None:
"""Arguments arriving whole in one delta must not be dropped."""
tokenizer_vocab = default_tokenizer.get_vocab()
default_tokenizer.get_vocab = MagicMock()
tokenizer_vocab.update(
{
"<|action_start|>": 92540,
"<|plugin|>": 92541,
"<|action_end|>": 92542,
}
)
default_tokenizer.get_vocab.return_value = tokenizer_vocab
parser = Internlm2ToolParser(default_tokenizer)
deltas = [
'<|action_start|><|plugin|>{"name": "get_weather"',
', "parameters": {"city": "Dallas", "state": "TX"}}<|action_end|>',
]
streamed = ""
current_text = ""
for delta_text in deltas:
previous_text = current_text
current_text += delta_text
delta_message = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=None,
)
if delta_message and delta_message.tool_calls:
arguments = delta_message.tool_calls[0].function.arguments
if arguments:
streamed += arguments
assert json.loads(streamed) == {"city": "Dallas", "state": "TX"}
@pytest.mark.parametrize(
"arguments_deltas",
[
['{"location": "Beijing', '", "days": 3}}<|action_end|>'],
[*'{"location": "Beijing", "days": 3}}', "<|action_end|>"],
],
ids=["split_inside_string", "char_by_char"],
)
def test_streaming_arguments_match_final_arguments(
default_tokenizer: TokenizerLike, arguments_deltas: list[str]
) -> None:
"""Streamed argument deltas must add up to the final arguments, wherever
the chunk boundaries fall."""
parser = Internlm2ToolParser(default_tokenizer)
deltas = [
'<|action_start|><|plugin|>{"name": "get_weather", "parameters": ',
*arguments_deltas,
]
reconstructor = run_tool_extraction_streaming(parser, deltas)
assert len(reconstructor.tool_calls) == 1
streamed = reconstructor.tool_calls[0].function.arguments
streamed += parser.get_remaining_unstreamed_args()
assert streamed == '{"location": "Beijing", "days": 3}'
@@ -1,339 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Generator
import partial_json_parser
import pytest
from partial_json_parser.core.options import Allow
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall, ToolCall
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
MODEL = "ai21labs/Jamba-tiny-dev"
@pytest.fixture(scope="module")
def jamba_tokenizer():
return get_tokenizer(tokenizer_name=MODEL)
@pytest.fixture
def jamba_tool_parser(jamba_tokenizer):
return JambaToolParser(jamba_tokenizer)
def assert_tool_calls(
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
):
assert len(actual_tool_calls) == len(expected_tool_calls)
for actual_tool_call, expected_tool_call in zip(
actual_tool_calls, expected_tool_calls
):
assert isinstance(actual_tool_call.id, str)
assert len(actual_tool_call.id) > 16
assert actual_tool_call.type == "function"
assert actual_tool_call.function == expected_tool_call.function
def stream_delta_message_generator(
jamba_tool_parser: JambaToolParser,
jamba_tokenizer: TokenizerLike,
model_output: str,
) -> Generator[DeltaMessage, None, None]:
all_token_ids = jamba_tokenizer.encode(model_output, add_special_tokens=False)
previous_text = ""
previous_tokens = None
prefix_offset = 0
read_offset = 0
for i, delta_token in enumerate(all_token_ids):
delta_token_ids = [delta_token]
previous_token_ids = all_token_ids[:i]
current_token_ids = all_token_ids[: i + 1]
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
detokenize_incrementally(
tokenizer=jamba_tokenizer,
all_input_ids=current_token_ids,
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=False,
spaces_between_special_tokens=True,
)
)
current_text = previous_text + delta_text
delta_message = jamba_tool_parser.extract_tool_calls_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
request=None,
)
if delta_message:
yield delta_message
previous_text = current_text
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
prefix_offset = new_prefix_offset
read_offset = new_read_offset
def test_extract_tool_calls_no_tools(jamba_tool_parser):
model_output = "This is a test"
extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
model_output, request=None
)
assert not extracted_tool_calls.tools_called
assert extracted_tool_calls.tool_calls == []
assert extracted_tool_calls.content == model_output
@pytest.mark.parametrize(
ids=[
"single_tool",
"single_tool_with_content",
"parallel_tools",
],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
(
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
),
)
)
],
None,
),
(
""" Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
),
)
)
],
" Sure! let me call the tool for you.",
),
(
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
),
)
),
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Orlando", "state": "FL", "unit": "fahrenheit"}
),
)
),
],
None,
),
],
)
def test_extract_tool_calls(
jamba_tool_parser, model_output, expected_tool_calls, expected_content
):
extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
model_output, request=None
)
assert extracted_tool_calls.tools_called
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
assert extracted_tool_calls.content == expected_content
@pytest.mark.parametrize(
ids=[
"no_tools",
"single_tool",
"single_tool_with_content",
"parallel_tools",
],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
("""This is a test""", [], """This is a test"""),
(
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
),
)
)
],
" ",
),
(
""" Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
),
)
)
],
" Sure! let me call the tool for you.",
),
(
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
),
)
),
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{"city": "Orlando", "state": "FL", "unit": "fahrenheit"}
),
)
),
],
" ",
),
],
)
def test_extract_tool_calls_streaming(
jamba_tool_parser,
jamba_tokenizer,
model_output,
expected_tool_calls,
expected_content,
):
other_content: str = ""
function_names: list[str] = []
function_args_strs: list[str] = []
tool_call_idx: int = -1
tool_call_ids: list[str | None] = []
for delta_message in stream_delta_message_generator(
jamba_tool_parser, jamba_tokenizer, model_output
):
# role should never be streamed from tool parser
assert not delta_message.role
if delta_message.content:
other_content += delta_message.content
streamed_tool_calls = delta_message.tool_calls
if streamed_tool_calls and len(streamed_tool_calls) > 0:
# make sure only one diff is present - correct even for parallel
assert len(streamed_tool_calls) == 1
tool_call = streamed_tool_calls[0]
# if a new tool is being called, set up empty arguments
if tool_call.index != tool_call_idx:
tool_call_idx = tool_call.index
function_args_strs.append("")
tool_call_ids.append(None)
# if a tool call ID is streamed, make sure one hasn't been already
if tool_call.id and not tool_call_ids[tool_call.index]:
tool_call_ids[tool_call.index] = tool_call.id
# if parts of the function start being streamed
if tool_call.function:
# if the function name is defined, set it. it should be streamed
# IN ENTIRETY, exactly one time.
if tool_call.function.name:
assert isinstance(tool_call.function.name, str)
function_names.append(tool_call.function.name)
if tool_call.function.arguments:
# make sure they're a string and then add them to the list
assert isinstance(tool_call.function.arguments, str)
function_args_strs[tool_call.index] += tool_call.function.arguments
assert other_content == expected_content
actual_tool_calls = [
ToolCall(
id=tool_call_id,
function=FunctionCall(
name=function_name,
arguments=partial_json_parser.ensure_json(
function_args_str, Allow.OBJ | Allow.STR
),
),
)
for tool_call_id, function_name, function_args_str in zip(
tool_call_ids, function_names, function_args_strs
)
]
assert_tool_calls(actual_tool_calls, expected_tool_calls)
def test_extract_tool_calls_streaming_arguments_in_single_delta(jamba_tool_parser):
"""Arguments delivered whole in one coarse delta must not be dropped."""
deltas = [
'<tool_calls>[{"name": "get_current_weather"',
",",
' "arguments": {"city": "Dallas", "state": "TX"}}]',
"</tool_calls>",
]
streamed_arguments = ""
current_text = ""
for delta_text in deltas:
previous_text = current_text
current_text += delta_text
delta_message = jamba_tool_parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=None,
)
if delta_message and delta_message.tool_calls:
arguments = delta_message.tool_calls[0].function.arguments
if arguments:
streamed_arguments += arguments
assert json.loads(streamed_arguments) == {"city": "Dallas", "state": "TX"}
@@ -1,330 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.parser_manager import ParserManager
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.k2_horizon_tool_parser import K2HorizonToolParser
pytestmark = pytest.mark.skip_global_cleanup
TOOLS = [
{
"type": "function",
"function": {
"name": "lookup",
"parameters": {
"type": "object",
"properties": {
"query": {"type": "string"},
"limit": {"type": "integer"},
"enabled": {"type": "boolean"},
},
},
},
},
{
"type": "function",
"function": {
"name": "ping",
"parameters": {"type": "object", "properties": {}},
},
},
]
@pytest.fixture
def tokenizer():
tokenizer = MagicMock()
tokenizer.get_vocab.return_value = {}
return tokenizer
def _request(
tool_format: str | None = None,
tool_choice="auto",
) -> ChatCompletionRequest:
chat_template_kwargs = (
{"tool_call_format": tool_format} if tool_format is not None else None
)
return ChatCompletionRequest(
model="k2-horizon",
messages=[{"role": "user", "content": "test"}],
tools=TOOLS,
tool_choice=tool_choice,
chat_template_kwargs=chat_template_kwargs,
)
def _group(*calls: str, prefix: str = "", suffix: str = "") -> str:
return prefix + "<ifm|tool_calls>" + "\n".join(calls) + "</ifm|tool_calls>" + suffix
def _json_call(name: str, arguments: dict) -> str:
payload = json.dumps({"name": name, "arguments": arguments})
return f"<ifm|tool_call>{payload}</ifm|tool_call>"
def _xml_call(
name: str,
arguments: list[tuple[str, str]],
*,
typed: bool = False,
) -> str:
parts = [f"<ifm|tool_call>{name}"]
for key, value in arguments:
parts.append(f"<ifm|arg_key>{key}</ifm|arg_key>")
if typed:
arg_type = "integer" if key == "limit" else "string"
parts.append(f"<ifm|arg_type>{arg_type}</ifm|arg_type>")
parts.append(f"<ifm|arg_value>{value}</ifm|arg_value>")
parts.append("</ifm|tool_call>")
return "".join(parts)
def _collect_stream(
parser: K2HorizonToolParser,
request: ChatCompletionRequest,
output: str,
) -> tuple[str, list[tuple[str, dict]]]:
messages: list[DeltaMessage | None] = []
for char in output:
messages.append(
parser.extract_tool_calls_streaming(
previous_text="",
current_text="",
delta_text=char,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=request,
)
)
messages.append(parser.finish_streaming())
content: list[str] = []
calls: list[tuple[str, dict]] = []
for message in messages:
if message is None:
continue
if message.content is not None:
content.append(message.content)
for tool_call in message.tool_calls:
assert tool_call.function is not None
assert tool_call.function.name is not None
assert tool_call.function.arguments is not None
calls.append(
(
tool_call.function.name,
json.loads(tool_call.function.arguments),
)
)
return "".join(content), calls
def test_xml_default_parses_multiple_calls_and_coerces_schema(tokenizer):
parser = K2HorizonToolParser(tokenizer, _request().tools)
output = _group(
_xml_call("lookup", [("query", "123"), ("limit", "3")]),
_xml_call("ping", []),
prefix="Before ",
suffix=" after.",
)
result = parser.extract_tool_calls(output, _request())
assert parser.tool_format == "xml"
assert result.tools_called
assert result.content == "Before after."
assert [call.function.name for call in result.tool_calls] == ["lookup", "ping"]
assert json.loads(result.tool_calls[0].function.arguments) == {
"query": "123",
"limit": 3,
}
@pytest.mark.parametrize(
("tool_format", "call", "expected"),
[
("json", _json_call("lookup", {"limit": "2"}), {"limit": 2}),
(
"xml_typed",
_xml_call("lookup", [("query", "weather"), ("limit", "2")], typed=True),
{"query": "weather", "limit": 2},
),
],
)
def test_request_level_format_overrides(tokenizer, tool_format, call, expected):
request = _request(tool_format)
parser = K2HorizonToolParser(tokenizer, request.tools)
result = parser.extract_tool_calls(_group(call), request)
assert result.tools_called
assert json.loads(result.tool_calls[0].function.arguments) == expected
@pytest.mark.parametrize(
"tool_format",
["yaml", "python", "xml_untyped", "", None, 1],
)
def test_invalid_request_format_rejected_before_generation(tokenizer, tool_format):
parser = K2HorizonToolParser(tokenizer, _request().tools)
request = _request().model_copy(
update={"chat_template_kwargs": {"tool_call_format": tool_format}}
)
with pytest.raises(ValueError, match="Unsupported tool_call_format"):
parser.adjust_request(request)
@pytest.mark.parametrize(
("tool_request", "call"),
[
(_request(), _json_call("ping", {})),
(_request("json"), _xml_call("ping", [])),
(_request("xml"), _xml_call("lookup", [("query", "x")], typed=True)),
(_request("xml_typed"), _xml_call("lookup", [("query", "x")])),
],
)
def test_mismatched_formats_fail_closed(tokenizer, tool_request, call):
parser = K2HorizonToolParser(tokenizer, tool_request.tools)
output = _group(call)
result = parser.extract_tool_calls(output, tool_request)
assert not result.tools_called
assert result.content == output
@pytest.mark.parametrize(
"tool_choice",
[
"auto",
"required",
{"type": "function", "function": {"name": "lookup"}},
],
)
def test_auto_required_and_named_use_ifm_parser(tokenizer, tool_choice):
request = _request("json", tool_choice)
parser_cls = ParserManager.get_parser(
tool_parser_name="k2_horizon",
enable_auto_tools=True,
)
assert parser_cls is not None
parser = parser_cls(tokenizer, request.tools)
assert parser.adjust_request(request) is request
assert request.structured_outputs is None
_, content, tool_calls = parser.parse(
_group(_json_call("lookup", {"enabled": "true"})),
request,
enable_auto_tools=True,
)
assert content is None
assert tool_calls is not None
assert json.loads(tool_calls[0].arguments) == {"enabled": True}
def test_unknown_or_wrong_named_tool_fails_closed(tokenizer):
named_request = _request(
"json", {"type": "function", "function": {"name": "lookup"}}
)
parser = K2HorizonToolParser(tokenizer, named_request.tools)
unknown = parser.extract_tool_calls(
_group(_json_call("missing", {})), _request("json")
)
wrong_named = parser.extract_tool_calls(
_group(_json_call("ping", {})), named_request
)
assert not unknown.tools_called
assert not wrong_named.tools_called
@pytest.mark.parametrize(
"output",
[
"<ifm|tool_calls><ifm|tool_call>ping",
"<ifm|tool_calls><ifm|tool_call>ping</ifm|tool_call>",
_group("junk"),
],
)
def test_malformed_and_incomplete_input_is_content(tokenizer, output):
request = _request()
parser = K2HorizonToolParser(tokenizer, request.tools)
result = parser.extract_tool_calls(output, request)
assert not result.tools_called
assert result.content == output
@pytest.mark.parametrize(
("tool_request", "call", "expected"),
[
(_request(), _xml_call("lookup", [("limit", "2")]), {"limit": 2}),
(_request("json"), _json_call("lookup", {"limit": "2"}), {"limit": 2}),
(
_request("xml_typed"),
_xml_call("lookup", [("limit", "2")], typed=True),
{"limit": 2},
),
],
)
def test_character_split_streaming(tokenizer, tool_request, call, expected):
parser = K2HorizonToolParser(tokenizer, tool_request.tools)
output = _group(call, prefix="Before", suffix="after")
content, calls = _collect_stream(parser, tool_request, output)
assert content == "Beforeafter"
assert calls == [("lookup", expected)]
def test_streaming_flushes_incomplete_markup_as_content(tokenizer):
request = _request()
parser = K2HorizonToolParser(tokenizer, request.tools)
output = "Before<ifm|tool_calls><ifm|tool_call>ping"
content, calls = _collect_stream(parser, request, output)
assert content == output
assert calls == []
def test_whitespace_only_surrounding_content_is_not_preserved(tokenizer):
request = _request()
parser = K2HorizonToolParser(tokenizer, request.tools)
result = parser.extract_tool_calls(
_group(_xml_call("ping", []), prefix=" \n", suffix="\t "), request
)
assert result.tools_called
assert result.content is None
def test_tool_parser_registered():
assert ToolParserManager.get_tool_parser("k2_horizon") is K2HorizonToolParser
assert K2HorizonToolParser.supports_required_and_named is False
def test_named_tool_resolves_custom_choice():
# Custom tools ride the function shim; a ToolChoiceCustom must resolve to
# its name like ToolChoiceFunction does.
request = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
assert K2HorizonToolParser._named_tool(request) == "emit_command"
@@ -1,583 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import json
from unittest.mock import MagicMock
import pytest
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
MODEL = "moonshotai/Kimi-K2-Instruct"
@pytest.fixture(scope="module")
def kimi_k2_tokenizer():
return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True)
@pytest.fixture
def parser(kimi_k2_tokenizer):
return KimiK2ToolParser(kimi_k2_tokenizer)
SECTION_BEGIN = "<|tool_calls_section_begin|>"
SECTION_END = "<|tool_calls_section_end|>"
TOOL_BEGIN = "<|tool_call_begin|>"
TOOL_END = "<|tool_call_end|>"
ARG_BEGIN = "<|tool_call_argument_begin|>"
def _tool(tool_id: str, args: str) -> str:
return f"{TOOL_BEGIN}{tool_id} {ARG_BEGIN}{args}{TOOL_END}"
def _wrap(*tool_strs: str) -> str:
return SECTION_BEGIN + "".join(tool_strs) + SECTION_END
class TestExtractToolCalls:
def test_no_tools(self, parser):
content, tool_calls = run_tool_extraction(
parser, "This is a test", streaming=False
)
assert content == "This is a test"
assert tool_calls == []
@pytest.mark.parametrize(
"model_output, expected_names, expected_args_list, expected_content",
[
pytest.param(
"I'll check. "
+ _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')),
["get_weather"],
[{"city": "Beijing"}],
"I'll check. ",
id="single_tool_call",
),
pytest.param(
"Compare weather. "
+ _wrap(
_tool("functions.get_weather:0", '{"city": "Beijing"}'),
_tool("functions.get_weather:1", '{"city": "Shanghai"}'),
),
["get_weather", "get_weather"],
[{"city": "Beijing"}, {"city": "Shanghai"}],
"Compare weather. ",
id="parallel_tool_calls",
),
pytest.param(
"Multiple tasks. "
+ _wrap(
_tool("functions.get_weather:0", '{"city": "New York"}'),
_tool("functions.get_news:1", '{"topic": "technology"}'),
_tool(
"functions.send_email:2",
'{"to": "user@example.com", "subject": "Daily Update"}',
),
),
["get_weather", "get_news", "send_email"],
[
{"city": "New York"},
{"topic": "technology"},
{"to": "user@example.com", "subject": "Daily Update"},
],
"Multiple tasks. ",
id="three_tool_calls",
),
pytest.param(
"Process HTML. "
+ _wrap(
_tool("functions.process_html:0", '{"html": "<div>content</div>"}')
),
["process_html"],
[{"html": "<div>content</div>"}],
"Process HTML. ",
id="angle_brackets_in_json",
),
pytest.param(
"Formatted. "
+ _wrap(
_tool(
"functions.process_data:0",
'{\n "name": "test",\n "value": 123\n}',
)
),
["process_data"],
[{"name": "test", "value": 123}],
"Formatted. ",
id="multiline_json",
),
pytest.param(
"No prefix. " + _wrap(_tool("get_weather:0", '{"city": "Tokyo"}')),
["get_weather"],
[{"city": "Tokyo"}],
"No prefix. ",
id="no_functions_prefix",
),
pytest.param(
"Empty args. " + _wrap(_tool("functions.test:0", "{}")),
["test"],
[{}],
"Empty args. ",
id="empty_arguments",
),
],
)
def test_extract_tool_calls(
self, parser, model_output, expected_names, expected_args_list, expected_content
):
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
assert content == expected_content
assert len(tool_calls) == len(expected_names)
for tc, name, expected_args in zip(
tool_calls, expected_names, expected_args_list
):
assert tc.type == "function"
assert tc.function.name == name
assert json.loads(tc.function.arguments) == expected_args
# id format: "something:digit"
assert tc.id.split(":")[-1].isdigit()
def test_invalid_json_still_extracted(self, parser):
"""Tool calls with invalid JSON are still returned (arguments as-is)."""
model_output = (
"Help. "
+ SECTION_BEGIN
+ _tool("functions.bad:0", '{"city": "Beijing"')
+ _tool("functions.good:1", '{"city": "Shanghai"}')
+ SECTION_END
)
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
assert len(tool_calls) == 2
assert tool_calls[0].function.name == "bad"
assert tool_calls[1].function.name == "good"
def test_invalid_funcall_id_skipped(self, parser):
"""Tool calls with malformed id (no colon+digit) are skipped."""
model_output = (
"Help. "
+ SECTION_BEGIN
+ _tool("functions.invalid.0", '{"city": "Beijing"}')
+ _tool("functions.valid:1", '{"city": "Shanghai"}')
+ SECTION_END
)
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
assert len(tool_calls) == 1
assert tool_calls[0].function.name == "valid"
def test_native_id_extracted(self, parser):
"""Regression: parser extracts native ID onto ToolCall (PR #32768)."""
model_output = "Checking weather. " + _wrap(
_tool("functions.get_weather:0", '{"city": "Tokyo"}')
)
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
assert len(tool_calls) == 1
assert tool_calls[0].id == "functions.get_weather:0"
assert tool_calls[0].function.name == "get_weather"
assert json.loads(tool_calls[0].function.arguments) == {"city": "Tokyo"}
def test_multi_turn_native_id_continuity(self, kimi_k2_tokenizer):
"""Regression: native IDs from turn 1 preserved across turns (PR #32768)."""
turn1_parser = KimiK2ToolParser(kimi_k2_tokenizer)
turn1_output = "Let me check. " + _wrap(
_tool("functions.get_weather:0", '{"city": "Beijing"}')
)
_, turn1_tools = run_tool_extraction(
turn1_parser, turn1_output, streaming=False
)
assert len(turn1_tools) == 1
assert turn1_tools[0].id == "functions.get_weather:0"
# Fresh parser for turn 2
turn2_parser = KimiK2ToolParser(kimi_k2_tokenizer)
turn2_output = "Now let me get news. " + _wrap(
_tool("functions.get_news:0", '{"topic": "weather in Beijing"}')
)
_, turn2_tools = run_tool_extraction(
turn2_parser, turn2_output, streaming=False
)
assert len(turn2_tools) == 1
assert turn2_tools[0].id == "functions.get_news:0"
def _split_tool_output_to_deltas(
content: str, tool_strs: list[tuple[str, str]]
) -> list[str]:
"""Build a list of string deltas with special tokens as separate chunks.
Args:
content: text before tool section
tool_strs: list of (tool_id, args_json)
"""
deltas = [content, SECTION_BEGIN]
for tool_id, args_json in tool_strs:
deltas.extend(
[
TOOL_BEGIN,
f"{tool_id} ",
ARG_BEGIN,
f"{args_json} ",
TOOL_END,
]
)
deltas.append(SECTION_END)
return deltas
class TestStreamingHappyPath:
def test_single_tool_call(self, parser):
"""Verify DeltaToolCall output: name, id, arguments for one tool."""
deltas = _split_tool_output_to_deltas(
"I'll help. ",
[("functions.get_weather:0", '{"city": "Beijing"}')],
)
rec = run_tool_extraction_streaming(parser, deltas)
assert len(rec.tool_calls) == 1
tc = rec.tool_calls[0]
assert tc.function.name == "get_weather"
assert tc.id == "functions.get_weather:0"
assert json.loads(tc.function.arguments) == {"city": "Beijing"}
def test_multiple_tool_calls(self, parser):
"""Two tool calls emitted with correct indices, names, arguments."""
deltas = _split_tool_output_to_deltas(
"Compare weather. ",
[
("functions.get_weather:0", '{"city": "Tokyo"}'),
("functions.get_weather:1", '{"city": "NYC"}'),
],
)
rec = run_tool_extraction_streaming(parser, deltas)
assert len(rec.tool_calls) == 2
assert rec.tool_calls[0].function.name == "get_weather"
assert rec.tool_calls[0].id == "functions.get_weather:0"
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"}
assert rec.tool_calls[1].function.name == "get_weather"
assert rec.tool_calls[1].id == "functions.get_weather:1"
assert json.loads(rec.tool_calls[1].function.arguments) == {"city": "NYC"}
def test_content_before_tools(self, parser):
"""Content before section is streamed; markers/args don't leak."""
deltas = _split_tool_output_to_deltas(
"I'll check the weather. ",
[("functions.get_weather:0", '{"city": "Tokyo"}')],
)
rec = run_tool_extraction_streaming(parser, deltas)
assert "check the weather" in rec.other_content
# No markers or tool content leaked
for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]:
assert marker not in rec.other_content
assert "get_weather" not in rec.other_content
assert "Tokyo" not in rec.other_content
def test_no_tool_calls(self, parser):
"""Plain text streaming returns content only."""
deltas = ["This is just ", "regular text ", "without tools."]
rec = run_tool_extraction_streaming(parser, deltas)
assert rec.other_content == "This is just regular text without tools."
assert rec.tool_calls == []
def test_incremental_arguments(self, parser):
"""Arguments split across small chunks accumulate correctly."""
deltas = [
"Help. ",
SECTION_BEGIN,
TOOL_BEGIN,
"functions.get_weather:0 ",
ARG_BEGIN,
'{"ci',
'ty": "Be',
'ijing"}',
" ",
TOOL_END,
SECTION_END,
]
rec = run_tool_extraction_streaming(parser, deltas)
assert len(rec.tool_calls) == 1
assert rec.tool_calls[0].function.name == "get_weather"
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"}
@pytest.mark.parametrize(
"model_output",
[
pytest.param(
"Single. "
+ _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')),
id="single_tool",
),
pytest.param(
"Multi. "
+ _wrap(
_tool("functions.get_weather:0", '{"city": "Tokyo"}'),
_tool("functions.get_news:1", '{"topic": "tech"}'),
),
id="parallel_tools",
),
pytest.param(
"No prefix id. " + _wrap(_tool("get_weather:0", '{"city": "NYC"}')),
id="no_functions_prefix",
),
],
)
def test_streaming_matches_nonstreaming(self, parser, model_output):
"""Streaming reconstruction matches non-streaming extraction."""
content_non, tools_non = run_tool_extraction(
parser, model_output, streaming=False
)
content_stream, tools_stream = run_tool_extraction(
parser, model_output, streaming=True
)
assert len(tools_non) == len(tools_stream)
for tc_non, tc_stream in zip(tools_non, tools_stream):
assert tc_non.function.name == tc_stream.function.name
assert json.loads(tc_non.function.arguments) == json.loads(
tc_stream.function.arguments
)
class TestStreamingEdgeCases:
def test_marker_suppression(self, parser):
"""No special-token markers appear in reconstructed content."""
deltas = _split_tool_output_to_deltas(
"I'll check. ",
[("functions.get_weather:0", '{"city": "Tokyo"}')],
)
rec = run_tool_extraction_streaming(parser, deltas)
forbidden = [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]
for marker in forbidden:
assert marker not in rec.other_content, (
f"Marker leaked: {marker!r} in {rec.other_content!r}"
)
def test_noise_between_markers_suppressed(self, parser):
"""Text between section_begin and tool_call_begin doesn't leak."""
deltas = [
"Reasoning. ",
SECTION_BEGIN,
" spurious noise ",
TOOL_BEGIN,
"functions.test:0 ",
ARG_BEGIN,
'{"k": "v"} ',
TOOL_END,
SECTION_END,
]
rec = run_tool_extraction_streaming(parser, deltas)
assert "spurious" not in rec.other_content
assert "noise" not in rec.other_content
def test_empty_tool_section(self, parser):
"""Empty section (begin immediately followed by end) doesn't crash."""
deltas = ["Reasoning. ", SECTION_BEGIN, SECTION_END]
rec = run_tool_extraction_streaming(parser, deltas)
assert rec.tool_calls == []
def test_three_different_tools(self, parser):
"""Three tool calls with different functions stream correctly."""
deltas = _split_tool_output_to_deltas(
"Multiple tasks. ",
[
("functions.get_weather:0", '{"city": "NYC"}'),
("functions.get_news:1", '{"topic": "tech"}'),
("functions.send_email:2", '{"to": "a@b.com"}'),
],
)
rec = run_tool_extraction_streaming(parser, deltas)
assert len(rec.tool_calls) == 3
names = [tc.function.name for tc in rec.tool_calls]
assert names == ["get_weather", "get_news", "send_email"]
ids = [tc.id for tc in rec.tool_calls]
assert len(set(ids)) == 3 # unique ids
def test_truncated_tool_call_no_end_marker(self, parser):
"""Stream ending mid-tool-call (max_tokens) doesn't crash."""
deltas = [
"I'll check. ",
SECTION_BEGIN,
TOOL_BEGIN,
"functions.get_weather:0 ",
ARG_BEGIN,
'{"city": "Bei',
# Stream ends here — no TOOL_END, no SECTION_END
]
rec = run_tool_extraction_streaming(parser, deltas)
# Should not crash; tool name and partial args extracted
assert len(rec.tool_calls) == 1
assert rec.tool_calls[0].function.name == "get_weather"
assert rec.tool_calls[0].id == "functions.get_weather:0"
assert rec.tool_calls[0].function.arguments == '{"city": "Bei'
# No markers leaked into content
for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]:
assert marker not in rec.other_content
def test_content_after_tool_section(self, parser):
"""Trailing text after section_end doesn't crash or leak markers."""
deltas = [
"Before. ",
SECTION_BEGIN,
TOOL_BEGIN,
"functions.get_weather:0 ",
ARG_BEGIN,
'{"city": "Tokyo"} ',
TOOL_END,
SECTION_END,
" After tools.",
]
rec = run_tool_extraction_streaming(parser, deltas)
# Tool call extracted correctly
assert len(rec.tool_calls) == 1
assert rec.tool_calls[0].function.name == "get_weather"
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"}
# Trailing content after tool section is dropped
assert "After tools." not in rec.other_content
# No markers leaked into content
for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]:
assert marker not in rec.other_content
class TestAdjustRequest:
def test_sets_skip_special_tokens_false(self, parser):
request = MagicMock(spec=ChatCompletionRequest)
request.tools = [{"type": "function", "function": {"name": "test"}}]
request.tool_choice = "auto"
request.skip_special_tokens = True
result = parser.adjust_request(request)
assert result.skip_special_tokens is False
def test_no_change_when_tool_choice_none(self, parser):
request = MagicMock(spec=ChatCompletionRequest)
request.tools = [{"type": "function", "function": {"name": "test"}}]
request.tool_choice = "none"
request.skip_special_tokens = True
result = parser.adjust_request(request)
assert result.skip_special_tokens is True
def test_no_change_when_no_tools(self, parser):
request = MagicMock(spec=ChatCompletionRequest)
request.tools = None
request.tool_choice = "auto"
request.skip_special_tokens = True
result = parser.adjust_request(request)
assert result.skip_special_tokens is True
def _chunk_tokenized_deltas(tokenizer, text: str, stream_interval: int) -> list[str]:
"""Encode text, group tokens into chunks of stream_interval, decode each."""
token_ids = tokenizer.encode(text, add_special_tokens=False)
deltas = []
prev = ""
for i in range(0, len(token_ids), stream_interval):
decoded = tokenizer.decode(
token_ids[: i + stream_interval], skip_special_tokens=False
)
deltas.append(decoded[len(prev) :])
prev = decoded
return deltas
class TestStreamingIntervals:
"""Test streaming at various token-chunk sizes to catch boundary bugs."""
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
def test_single_tool_call_at_interval(self, kimi_k2_tokenizer, stream_interval):
text = "Help. " + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}'))
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
parser = KimiK2ToolParser(kimi_k2_tokenizer)
rec = run_tool_extraction_streaming(
parser, deltas, assert_one_tool_per_delta=False
)
assert len(rec.tool_calls) == 1
assert rec.tool_calls[0].function.name == "get_weather"
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"}
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
def test_content_then_tool_call_at_interval(
self, kimi_k2_tokenizer, stream_interval
):
text = "Sure, let me check. " + _wrap(
_tool("functions.get_weather:0", '{"city": "Tokyo"}')
)
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
parser = KimiK2ToolParser(kimi_k2_tokenizer)
rec = run_tool_extraction_streaming(
parser, deltas, assert_one_tool_per_delta=False
)
assert "let me check" in rec.other_content
assert "get_weather" not in rec.other_content
assert len(rec.tool_calls) == 1
assert rec.tool_calls[0].function.name == "get_weather"
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"}
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
def test_multiple_tool_calls_at_interval(self, kimi_k2_tokenizer, stream_interval):
text = "Compare. " + _wrap(
_tool("functions.search:0", '{"q": "cats"}'),
_tool("functions.search:1", '{"q": "dogs"}'),
)
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
parser = KimiK2ToolParser(kimi_k2_tokenizer)
rec = run_tool_extraction_streaming(
parser, deltas, assert_one_tool_per_delta=False
)
assert len(rec.tool_calls) == 2
assert rec.tool_calls[0].function.name == "search"
assert json.loads(rec.tool_calls[0].function.arguments) == {"q": "cats"}
assert rec.tool_calls[1].function.name == "search"
assert json.loads(rec.tool_calls[1].function.arguments) == {"q": "dogs"}
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
def test_plain_text_at_interval(self, kimi_k2_tokenizer, stream_interval):
text = "This is plain text with no tool calling involved."
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
parser = KimiK2ToolParser(kimi_k2_tokenizer)
rec = run_tool_extraction_streaming(
parser, deltas, assert_one_tool_per_delta=False
)
assert rec.other_content == text
assert rec.tool_calls == []
def test_content_and_tool_call_in_single_chunk(self, kimi_k2_tokenizer):
"""Content + complete tool call in one chunk must both be emitted."""
text = "Hi! " + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}'))
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval=9999)
parser = KimiK2ToolParser(kimi_k2_tokenizer)
rec = run_tool_extraction_streaming(
parser, deltas, assert_one_tool_per_delta=False
)
assert "Hi!" in rec.other_content
assert "get_weather" not in rec.other_content
assert len(rec.tool_calls) == 1
assert rec.tool_calls[0].function.name == "get_weather"
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"}
@@ -1,86 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Named tool choice for Kimi K3: allowed when the XTML structural tag is
attached (strict tool calling), rejected otherwise."""
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import StructuredOutputsParams
class _DummyTokenizer:
def get_vocab(self):
return {}
def encode(self, text, add_special_tokens=False):
return [ord(c) for c in text]
def _request(with_tag: bool) -> ChatCompletionRequest:
req = ChatCompletionRequest(
model="k3",
messages=[{"role": "user", "content": "hi"}],
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
],
tool_choice={"type": "function", "function": {"name": "get_weather"}},
)
if with_tag:
req.structured_outputs = StructuredOutputsParams(
structural_tag='{"type": "structural_tag", "format": {}}'
)
return req
def _parser():
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
return KimiK3ToolParser(_DummyTokenizer())
def test_named_choice_allowed_with_structural_tag():
req = _parser().adjust_request(_request(with_tag=True))
assert req.skip_special_tokens is False
def test_named_choice_rejected_without_structural_tag():
with pytest.raises(VLLMValidationError):
_parser().adjust_request(_request(with_tag=False))
def _custom_request(with_tag: bool) -> ResponsesRequest:
req = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
if with_tag:
req.structured_outputs = StructuredOutputsParams(
structural_tag='{"type": "structural_tag", "format": {}}'
)
return req
def test_named_custom_choice_allowed_with_structural_tag():
req = _parser().adjust_request(_custom_request(with_tag=True))
assert req.skip_special_tokens is False
def test_named_custom_choice_rejected_without_structural_tag():
with pytest.raises(VLLMValidationError):
_parser().adjust_request(_custom_request(with_tag=False))
-666
View File
@@ -1,666 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import pytest
from transformers import AutoTokenizer
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser, ToolParserManager
TOOL_CALL_START = "<|tool_call_start|>"
TOOL_CALL_END = "<|tool_call_end|>"
SIMPLE_FUNCTION_OUTPUT = "get_candidate_status(candidate_id='12345')"
SIMPLE_FUNCTION_CALL = FunctionCall(
name="get_candidate_status",
arguments='{"candidate_id": "12345"}',
)
MORE_TYPES_FUNCTION_OUTPUT = (
"register_user(name='John Doe', "
"age=37, "
"address={'city': 'San Francisco', 'state': 'CA'}, "
"role=None, "
"passed_test=True, "
"aliases=['John', 'Johnny'])"
)
MORE_TYPES_FUNCTION_CALL = FunctionCall(
name="register_user",
arguments='{"name": "John Doe", '
'"age": 37, '
'"address": {"city": "San Francisco", "state": "CA"}, '
'"role": null, '
'"passed_test": true, '
'"aliases": ["John", "Johnny"]}',
)
PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments="{}",
)
EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"additional_data": {}}',
)
EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"steps": []}',
)
SET_ARG_FUNCTION_OUTPUT = "label(tags={'urgent', 'bug'})"
SET_ARG_FUNCTION_CALL = FunctionCall(
name="label",
arguments='{"tags": ["urgent", "bug"]}',
)
LEADING_ZERO_FUNCTION_OUTPUT = "set_date(month=07, day=05)"
LEADING_ZERO_FUNCTION_CALL = FunctionCall(
name="set_date",
arguments='{"month": 7, "day": 5}',
)
NESTED_QUOTE_FUNCTION_OUTPUT = (
"bash(command='sed -n '360,450p' /testbed/sympy/matrices/common.py')"
)
NESTED_QUOTE_FUNCTION_CALL = FunctionCall(
name="bash",
arguments=('{"command": "sed -n \'360,450p\' /testbed/sympy/matrices/common.py"}'),
)
NESTED_PYTHON_C_FUNCTION_OUTPUT = (
"bash(command='cd /testbed && python3 -c \"from sympy import latex\n"
"print(latex(3*x, mul_symbol='\\,'))\"')"
)
NESTED_PYTHON_C_FUNCTION_CALL = FunctionCall(
name="bash",
arguments=(
'{"command": "cd /testbed && python3 -c \\"from sympy import latex\\n'
"print(latex(3*x, mul_symbol='\\\\,'))\\\"\"}"
),
)
ESCAPED_STRING_FUNCTION_OUTPUT = (
r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
)
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
)
DOTTED_NAME_FUNCTION_OUTPUT = (
"grocery.orderIngredients("
"ingredientList=[{'name': 'Lasagna noodles', 'amount': 250, 'unit': 'g'}], "
"deliveryAddress='845 Willow Lane, Springfield, IL 62704')"
)
DOTTED_NAME_FUNCTION_CALL = FunctionCall(
name="grocery.orderIngredients",
arguments=(
'{"ingredientList": ['
'{"name": "Lasagna noodles", "amount": 250, "unit": "g"}], '
'"deliveryAddress": "845 Willow Lane, Springfield, IL 62704"}'
),
)
# Agentic shell commands: a bracket inside a string argument must not corrupt
# the streaming bracket tracker.
BRACKET_IN_STRING_FUNCTION_OUTPUT = "exec(command='grep -F \"]\" log.txt')"
BRACKET_IN_STRING_FUNCTION_CALL = FunctionCall(
name="exec",
arguments='{"command": "grep -F \\"]\\" log.txt"}',
)
# A raw newline inside a string argument (multi-line shell command) is invalid
# Python; the parser must recover the full value instead of dropping the call.
MULTILINE_FUNCTION_OUTPUT = (
"exec(command='cat > f.py << EOF\nimport csv\nprint(1)\nEOF')"
)
MULTILINE_FUNCTION_CALL = FunctionCall(
name="exec",
arguments='{"command": "cat > f.py << EOF\\nimport csv\\nprint(1)\\nEOF"}',
)
# A NUL byte inside a string argument makes ast.parse raise ValueError (not
# SyntaxError) for the whole source; the escape path must recover the call.
NUL_BYTE_FUNCTION_OUTPUT = "exec(command='printf a\x00b')"
NUL_BYTE_FUNCTION_CALL = FunctionCall(
name="exec",
arguments='{"command": "printf a\\u0000b"}',
)
@pytest.fixture(scope="module")
def lfm2_tokenizer() -> TokenizerLike:
return AutoTokenizer.from_pretrained("LiquidAI/LFM2.5-1.2B-Instruct")
def _wrap(tool_text: str, content_after: str = "") -> str:
"""Wrap pythonic tool call in LFM2.5 sentinel tokens."""
result = f"{TOOL_CALL_START}[{tool_text}]{TOOL_CALL_END}"
if content_after:
result += f"\n{content_after}"
return result
@pytest.mark.parametrize("streaming", [True, False])
def test_no_tool_call(streaming: bool, lfm2_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
model_output = "How can I help you today?"
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content == model_output
assert len(tool_calls) == 0
TEST_CASES = [
pytest.param(
True,
_wrap(SIMPLE_FUNCTION_OUTPUT),
[SIMPLE_FUNCTION_CALL],
None,
id="simple_streaming",
),
pytest.param(
False,
_wrap(SIMPLE_FUNCTION_OUTPUT),
[SIMPLE_FUNCTION_CALL],
None,
id="simple_nonstreaming",
),
pytest.param(
True,
_wrap(MORE_TYPES_FUNCTION_OUTPUT),
[MORE_TYPES_FUNCTION_CALL],
None,
id="more_types_streaming",
),
pytest.param(
False,
_wrap(MORE_TYPES_FUNCTION_OUTPUT),
[MORE_TYPES_FUNCTION_CALL],
None,
id="more_types_nonstreaming",
),
pytest.param(
True,
_wrap(PARAMETERLESS_FUNCTION_OUTPUT),
[PARAMETERLESS_FUNCTION_CALL],
None,
id="parameterless_streaming",
),
pytest.param(
False,
_wrap(PARAMETERLESS_FUNCTION_OUTPUT),
[PARAMETERLESS_FUNCTION_CALL],
None,
id="parameterless_nonstreaming",
),
pytest.param(
True,
_wrap(EMPTY_DICT_FUNCTION_OUTPUT),
[EMPTY_DICT_FUNCTION_CALL],
None,
id="empty_dict_streaming",
),
pytest.param(
False,
_wrap(EMPTY_DICT_FUNCTION_OUTPUT),
[EMPTY_DICT_FUNCTION_CALL],
None,
id="empty_dict_nonstreaming",
),
pytest.param(
True,
_wrap(EMPTY_LIST_FUNCTION_OUTPUT),
[EMPTY_LIST_FUNCTION_CALL],
None,
id="empty_list_streaming",
),
pytest.param(
False,
_wrap(EMPTY_LIST_FUNCTION_OUTPUT),
[EMPTY_LIST_FUNCTION_CALL],
None,
id="empty_list_nonstreaming",
),
pytest.param(
True,
_wrap(ESCAPED_STRING_FUNCTION_OUTPUT),
[ESCAPED_STRING_FUNCTION_CALL],
None,
id="escaped_string_streaming",
),
pytest.param(
False,
_wrap(ESCAPED_STRING_FUNCTION_OUTPUT),
[ESCAPED_STRING_FUNCTION_CALL],
None,
id="escaped_string_nonstreaming",
),
pytest.param(
True,
_wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"),
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
None,
id="parallel_calls_streaming",
),
pytest.param(
False,
_wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"),
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
None,
id="parallel_calls_nonstreaming",
),
# LFM2.5 specific: content AFTER tool call
pytest.param(
False,
_wrap(
SIMPLE_FUNCTION_OUTPUT,
content_after="Checking the current status of candidate ID 12345.",
),
[SIMPLE_FUNCTION_CALL],
"Checking the current status of candidate ID 12345.",
id="content_after_tool_call_nonstreaming",
),
# Dotted / class-method function names: grocery.orderIngredients(...)
pytest.param(
True,
_wrap(DOTTED_NAME_FUNCTION_OUTPUT),
[DOTTED_NAME_FUNCTION_CALL],
None,
id="dotted_name_streaming",
),
pytest.param(
False,
_wrap(DOTTED_NAME_FUNCTION_OUTPUT),
[DOTTED_NAME_FUNCTION_CALL],
None,
id="dotted_name_nonstreaming",
),
# Messy agentic shell commands: literal bracket inside a string argument
pytest.param(
True,
_wrap(BRACKET_IN_STRING_FUNCTION_OUTPUT),
[BRACKET_IN_STRING_FUNCTION_CALL],
None,
id="bracket_in_string_streaming",
),
pytest.param(
False,
_wrap(BRACKET_IN_STRING_FUNCTION_OUTPUT),
[BRACKET_IN_STRING_FUNCTION_CALL],
None,
id="bracket_in_string_nonstreaming",
),
# Multi-line string argument (raw newlines in a shell command)
pytest.param(
True,
_wrap(MULTILINE_FUNCTION_OUTPUT),
[MULTILINE_FUNCTION_CALL],
None,
id="multiline_string_arg_streaming",
),
pytest.param(
False,
_wrap(MULTILINE_FUNCTION_OUTPUT),
[MULTILINE_FUNCTION_CALL],
None,
id="multiline_string_arg_nonstreaming",
),
# Set argument decoded as a list (JSON has no set type)
pytest.param(
True,
_wrap(SET_ARG_FUNCTION_OUTPUT),
[SET_ARG_FUNCTION_CALL],
None,
id="set_arg_streaming",
),
pytest.param(
False,
_wrap(SET_ARG_FUNCTION_OUTPUT),
[SET_ARG_FUNCTION_CALL],
None,
id="set_arg_nonstreaming",
),
# Unescaped same-style quotes nested in a shell command argument
pytest.param(
True,
_wrap(NESTED_QUOTE_FUNCTION_OUTPUT),
[NESTED_QUOTE_FUNCTION_CALL],
None,
id="nested_quote_streaming",
),
pytest.param(
False,
_wrap(NESTED_QUOTE_FUNCTION_OUTPUT),
[NESTED_QUOTE_FUNCTION_CALL],
None,
id="nested_quote_nonstreaming",
),
# Doubly nested: multi-line python -c payload with its own inner quotes
pytest.param(
True,
_wrap(NESTED_PYTHON_C_FUNCTION_OUTPUT),
[NESTED_PYTHON_C_FUNCTION_CALL],
None,
id="nested_python_c_streaming",
),
pytest.param(
False,
_wrap(NESTED_PYTHON_C_FUNCTION_OUTPUT),
[NESTED_PYTHON_C_FUNCTION_CALL],
None,
id="nested_python_c_nonstreaming",
),
# Zero-padded integer arguments (SyntaxError: leading zeros)
pytest.param(
True,
_wrap(LEADING_ZERO_FUNCTION_OUTPUT),
[LEADING_ZERO_FUNCTION_CALL],
None,
id="leading_zero_int_streaming",
),
pytest.param(
False,
_wrap(LEADING_ZERO_FUNCTION_OUTPUT),
[LEADING_ZERO_FUNCTION_CALL],
None,
id="leading_zero_int_nonstreaming",
),
# NUL byte in a string argument (ValueError from ast.parse, not
# SyntaxError)
pytest.param(
True,
_wrap(NUL_BYTE_FUNCTION_OUTPUT),
[NUL_BYTE_FUNCTION_CALL],
None,
id="nul_byte_string_arg_streaming",
),
pytest.param(
False,
_wrap(NUL_BYTE_FUNCTION_OUTPUT),
[NUL_BYTE_FUNCTION_CALL],
None,
id="nul_byte_string_arg_nonstreaming",
),
]
@pytest.mark.parametrize(
"streaming, model_output, expected_tool_calls, expected_content",
TEST_CASES,
)
def test_tool_call(
streaming: bool,
model_output: str,
expected_tool_calls: list[FunctionCall],
expected_content: str | None,
lfm2_tokenizer: TokenizerLike,
):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
if expected_content and not streaming:
assert content == expected_content
assert len(tool_calls) == len(expected_tool_calls)
for actual, expected in zip(tool_calls, expected_tool_calls):
assert actual.type == "function"
assert actual.function == expected
@pytest.mark.parametrize("body", ["", " "])
def test_empty_tool_call_block(body: str, lfm2_tokenizer: TokenizerLike):
"""An empty block ([] or [ ]) means the model called no tools; it must
not report tools_called=True with zero tool calls (the invariant
run_tool_extraction asserts)."""
cls = ToolParserManager.get_tool_parser("lfm2")
model_output = f"{TOOL_CALL_START}[{body}]{TOOL_CALL_END}"
content, tool_calls = run_tool_extraction(
cls(lfm2_tokenizer), model_output, streaming=False
)
assert len(tool_calls) == 0
reconstructor = run_tool_extraction_streaming(
cls(lfm2_tokenizer), [model_output], assert_one_tool_per_delta=False
)
assert len(reconstructor.tool_calls) == 0
def test_whitespace_after_start_token(lfm2_tokenizer: TokenizerLike):
"""Whitespace between <|tool_call_start|> and the opening bracket must
not break parsing. The streaming path used to feed the indented text to
ast.parse verbatim, so every completion candidate raised
IndentationError and the call was silently dropped (non-streaming
stripped it and succeeded)."""
cls = ToolParserManager.get_tool_parser("lfm2")
model_output = f"{TOOL_CALL_START} [{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
content, tool_calls = run_tool_extraction(
cls(lfm2_tokenizer), model_output, streaming=False
)
assert len(tool_calls) == 1
assert tool_calls[0].function == SIMPLE_FUNCTION_CALL
reconstructor = run_tool_extraction_streaming(
cls(lfm2_tokenizer), [model_output], assert_one_tool_per_delta=False
)
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
def test_streaming_tool_call_with_large_steps(lfm2_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
model_output_deltas = [
f"{TOOL_CALL_START}[get_candidate_status(candidate_id='12345'), "
f"{PARAMETERLESS_FUNCTION_OUTPUT}, "
f"{EMPTY_LIST_FUNCTION_OUTPUT}]{TOOL_CALL_END}",
]
reconstructor = run_tool_extraction_streaming(
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
)
assert len(reconstructor.tool_calls) == 3
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
def test_streaming_full_block_and_trailing_in_single_delta(
lfm2_tokenizer: TokenizerLike,
):
"""The entire <|tool_call_start|>[...]<|tool_call_end|> block plus
trailing assistant text arrive in one delta. Trailing content must
still be emitted — not silently dropped."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
full_text = f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone."
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert "Done." in reconstructor.other_content
def test_streaming_leading_content_and_full_block_in_single_delta(
lfm2_tokenizer: TokenizerLike,
):
"""Leading assistant text plus the entire tool block arrive in one
delta. Leading content must be emitted — not silently dropped."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
full_text = (
f"Let me check. {TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
)
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert "Let me check." in reconstructor.other_content
def test_streaming_leading_block_and_trailing_in_single_delta(
lfm2_tokenizer: TokenizerLike,
):
"""Leading text + complete tool block + trailing text in one delta.
Both leading and trailing content must be preserved."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
full_text = (
"Let me check. "
f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone."
)
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert "Let me check." in reconstructor.other_content
assert "Done." in reconstructor.other_content
def test_echoed_tool_call_body_not_leaked_to_content(
lfm2_tokenizer: TokenizerLike,
):
"""LFM2 sometimes emits the tool call body again after the first
<|tool_call_end|>, capped with a second <|tool_call_end|>. The
echoed body must not surface as assistant content — neither in
streaming nor non-streaming paths."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
body = (
"[grocery.orderIngredients("
"ingredientList=[{'name': 'apple', 'quantity': '2'}], "
"deliveryAddress='123 Main St')]"
)
model_output = f"{TOOL_CALL_START}{body}{TOOL_CALL_END}{body}{TOOL_CALL_END}"
# Non-streaming
content_ns, tool_calls_ns = run_tool_extraction(
tool_parser, model_output, streaming=False
)
assert len(tool_calls_ns) == 1
assert tool_calls_ns[0].function.name == "grocery.orderIngredients"
assert content_ns in (None, "")
# Streaming: re-fetch a fresh parser since state was mutated above.
tool_parser2: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
content_s, tool_calls_s = run_tool_extraction(
tool_parser2, model_output, streaming=True
)
assert len(tool_calls_s) == 1
assert tool_calls_s[0].function.name == "grocery.orderIngredients"
# Echoed body must not leak as content.
assert content_s in (None, "")
assert "grocery.orderIngredients" not in (content_s or "")
assert TOOL_CALL_END not in (content_s or "")
def test_streaming_char_by_char_multi_dict_list(lfm2_tokenizer: TokenizerLike):
"""Stream a tool call containing a list of multiple dicts one
character at a time. Every prefix lands in some partial-parse state
(mid-key, mid-value, open quote inside dict, empty dict, etc.). The
parser must not raise — incomplete prefixes should silently wait for
more text instead of logging exceptions."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
full_text = (
f"{TOOL_CALL_START}[grocery.orderIngredients("
"ingredientList=["
'{"name": "apple", "quantity": "2"}, '
'{"name": "bread", "quantity": "1"}'
f"])]{TOOL_CALL_END}"
)
deltas = [c for c in full_text]
reconstructor = run_tool_extraction_streaming(
tool_parser, deltas, assert_one_tool_per_delta=False
)
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function.name == "grocery.orderIngredients"
import json
args = json.loads(reconstructor.tool_calls[0].function.arguments)
assert args == {
"ingredientList": [
{"name": "apple", "quantity": "2"},
{"name": "bread", "quantity": "1"},
]
}
def test_streaming_dotted_name_in_single_delta(lfm2_tokenizer: TokenizerLike):
"""A pythonic call with a dotted/attribute function name (e.g.
``domain.method(arg=...)``) must be parsed correctly in streaming mode
just as in non-streaming mode."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
full_text = f"{TOOL_CALL_START}[{DOTTED_NAME_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function == DOTTED_NAME_FUNCTION_CALL
def test_adjust_request_disables_skip_special_tokens(
lfm2_tokenizer: TokenizerLike,
):
"""When tools are present, the parser must force
``skip_special_tokens=False`` so the engine does not strip the
<|tool_call_start|>/<|tool_call_end|> sentinels before they reach the
parser."""
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
request_with_tools = ChatCompletionRequest(
messages=[],
model="test-model",
tools=[
{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
},
},
}
],
)
assert request_with_tools.skip_special_tokens is True
adjusted = tool_parser.adjust_request(request_with_tools)
assert adjusted.skip_special_tokens is False
# No tools → no override; default behaviour preserved.
request_no_tools = ChatCompletionRequest(messages=[], model="test-model")
assert request_no_tools.skip_special_tokens is True
adjusted_no_tools = tool_parser.adjust_request(request_no_tools)
assert adjusted_no_tools.skip_special_tokens is True
@pytest.mark.parametrize("streaming", [False])
def test_regex_timeout_handling(streaming: bool, lfm2_tokenizer: TokenizerLike):
"""Test regex timeout is handled gracefully."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
fake_input = f"{TOOL_CALL_START}[A(A=" + "\t)A(A=,\t" * 2
fake_input += f"]{TOOL_CALL_END}"
mock_regex = MagicMock()
mock_regex.match.side_effect = TimeoutError("Regex timeout")
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
content, tool_calls = run_tool_extraction(
tool_parser, fake_input, streaming=streaming
)
assert content == fake_input
assert len(tool_calls) == 0
mock_regex.match.assert_called_once()
@@ -1,269 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import pytest
from transformers import AutoTokenizer
from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
@pytest.fixture(scope="module")
def llama_tokenizer():
return AutoTokenizer.from_pretrained(LLAMA_MODEL)
@pytest.fixture
def parser(llama_tokenizer):
return Llama3JsonToolParser(llama_tokenizer)
def test_extract_tool_calls_simple(parser):
# Test with a simple tool call
model_output = (
'Here is the result: {"name": "getOpenIncidentsTool", '
'"parameters": {}} Would you like to know more?'
)
result = parser.extract_tool_calls(model_output, None)
assert isinstance(result, ExtractedToolCallInformation)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].type == "function"
assert result.tool_calls[0].function.name == "getOpenIncidentsTool"
assert result.tool_calls[0].function.arguments == "{}"
assert result.content is None
def test_extract_tool_calls_with_arguments(parser):
# Test with a tool call that has arguments
model_output = (
'{"name": "searchTool", "parameters": {"query": "test query", "limit": 10}}'
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "searchTool"
assert '"query": "test query"' in result.tool_calls[0].function.arguments
assert '"limit": 10' in result.tool_calls[0].function.arguments
def test_extract_tool_calls_no_json(parser):
# Test with text that doesn't contain a JSON object
model_output = "This is just some text without any tool calls"
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is False
assert len(result.tool_calls) == 0
assert result.content == model_output
def test_extract_tool_calls_invalid_json(parser):
# Test with invalid JSON
model_output = '{"name": "invalidTool", "parameters": {invalid json}'
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is False
assert len(result.tool_calls) == 0
assert result.content == model_output
def test_extract_tool_calls_with_arguments_key(parser):
# Test with a tool call that uses "arguments" instead of "parameters"
model_output = '{"name": "searchTool", "arguments": {"query": "test"}}'
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "searchTool"
assert '"query": "test"' in result.tool_calls[0].function.arguments
def test_extract_tool_calls_multiple_json(parser):
# Test with multiple JSONs separated by semicolons
model_output = (
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
'{"name": "searchTool", "parameters": {"query": "test2"}}'
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 3
# Check first tool call
assert result.tool_calls[0].function.name == "searchTool"
assert '"query": "test1"' in result.tool_calls[0].function.arguments
# Check second tool call
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
assert result.tool_calls[1].function.arguments == "{}"
# Check third tool call
assert result.tool_calls[2].function.name == "searchTool"
assert '"query": "test2"' in result.tool_calls[2].function.arguments
def test_extract_tool_calls_multiple_json_with_whitespace(parser):
# Test with multiple JSONs separated by semicolons and extra whitespace
model_output = (
'{"name": "searchTool", "parameters": {"query": "test1"}} ; '
'{"name": "getOpenIncidentsTool", "parameters": {}} ; '
'{"name": "searchTool", "parameters": {"query": "test2"}}'
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 3
assert result.tool_calls[0].function.name == "searchTool"
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
assert result.tool_calls[2].function.name == "searchTool"
def test_extract_tool_calls_multiple_json_with_surrounding_text(parser):
# Test with multiple JSONs and surrounding text
model_output = (
"Here are the results: "
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
'{"name": "searchTool", "parameters": {"query": "test2"}} '
"Would you like to know more?"
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 3
assert result.tool_calls[0].function.name == "searchTool"
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
assert result.tool_calls[2].function.name == "searchTool"
def test_extract_tool_calls_deeply_nested_json(parser):
# Test with deeply nested JSON parameters (5 levels)
model_output = (
'{"name": "complexTool", '
'"parameters": {'
'"level1": {'
'"level2": {'
'"level3": {'
'"level4": {'
'"value": "deep"'
"}}}}}}"
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "complexTool"
# Verify the nested structure is preserved in the arguments
import json
args = json.loads(result.tool_calls[0].function.arguments)
assert args["level1"]["level2"]["level3"]["level4"]["value"] == "deep"
def test_extract_tool_calls_multiple_with_deep_nesting(parser):
# Test with multiple tool calls where some have deeply nested parameters
model_output = (
'{"name": "simpleTool", "parameters": {"value": "test"}}; '
'{"name": "complexTool", "parameters": '
'{"config": {"database": {"connection": {"pool": {"size": 10}}}}}}'
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 2
# Check first tool call
assert result.tool_calls[0].function.name == "simpleTool"
import json
args0 = json.loads(result.tool_calls[0].function.arguments)
assert args0["value"] == "test"
# Check second tool call with deep nesting
assert result.tool_calls[1].function.name == "complexTool"
args1 = json.loads(result.tool_calls[1].function.arguments)
assert args1["config"]["database"]["connection"]["pool"]["size"] == 10
def test_extract_tool_calls_with_quotes_and_brackets_in_string(parser):
# Test with quotes and brackets inside quoted string values
model_output = (
'{"name": "searchTool", '
'"parameters": {'
'"query": "test {value} [complex]",'
'"nested": {"inner": "more {brackets}"}'
"}}"
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "searchTool"
# Verify the string values are preserved including brackets and quotes
import json
args = json.loads(result.tool_calls[0].function.arguments)
assert args["query"] == "test {value} [complex]"
assert args["nested"]["inner"] == "more {brackets}"
def test_extract_tool_calls_with_escaped_quotes_in_nested_json(parser):
# Test with escaped quotes in deeply nested JSON
model_output = (
'{"name": "parserTool", "parameters": {"text": "He said \\"Hello {world}\\""}}'
)
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is True
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "parserTool"
# Verify escaped quotes are preserved
import json
args = json.loads(result.tool_calls[0].function.arguments)
assert args["text"] == 'He said "Hello {world}"'
def test_extract_tool_calls_missing_name_key(parser):
# Test that missing "name" key returns content
model_output = '{"parameters": {}}'
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is False
assert len(result.tool_calls) == 0
assert result.content == model_output
def test_extract_tool_calls_missing_parameters_and_arguments_key(parser):
# Test that missing both "parameters" and "arguments" keys returns content
model_output = '{"name": "toolWithoutParams"}'
result = parser.extract_tool_calls(model_output, None)
assert result.tools_called is False
assert len(result.tool_calls) == 0
assert result.content == model_output
def test_regex_timeout_handling(parser):
"""Test regex timeout is handled gracefully."""
fake_problematic_input = "{hello world[A(A=" + "\t)A(A=,\t" * 2
# create a mock regex that raises TimeoutError
mock_regex = MagicMock()
mock_regex.finditer.side_effect = TimeoutError("Regex timeout")
with patch.object(parser, "tool_call_start_regex", mock_regex):
result = parser.extract_tool_calls(fake_problematic_input, None)
# should treat as regular text when regex times out
assert result.content == fake_problematic_input
assert result.tools_called is False
assert len(result.tool_calls) == 0
mock_regex.finditer.assert_called_once()
@@ -1,269 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import pytest
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser, ToolParserManager
# Test cases similar to pythonic parser but with Llama4 specific format
SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]"
SIMPLE_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "LA", "metric": "C"}',
)
MORE_TYPES_FUNCTION_OUTPUT = (
"[register_user(name='Doe', "
"age=9, "
"address={'city': 'LA', 'state': 'CA'}, "
"role=None, "
"passed_test=True, "
"aliases=['John', 'Johnny'])]"
)
MORE_TYPES_FUNCTION_CALL = FunctionCall(
name="register_user",
arguments='{"name": "Doe", '
'"age": 9, '
'"address": {"city": "LA", "state": "CA"}, '
'"role": null, '
'"passed_test": true, '
'"aliases": ["John", "Johnny"]}',
)
PARAMETERLESS_FUNCTION_OUTPUT = "[get_weather()]"
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments="{}",
)
EMPTY_DICT_FUNCTION_OUTPUT = "[do_something_cool(additional_data={})]"
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"additional_data": {}}',
)
EMPTY_LIST_FUNCTION_OUTPUT = "[do_something_cool(steps=[])]"
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"steps": []}',
)
ESCAPED_STRING_FUNCTION_OUTPUT = (
r"[get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')]"
)
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
)
PYTHON_TAG_FUNCTION_OUTPUT = (
"<|python_start|>[get_weather(city='LA', metric='C')]<|python_end|>"
)
@pytest.mark.parametrize("streaming", [True, False])
def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
default_tokenizer
)
model_output = "How can I help you today?"
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content == model_output
assert len(tool_calls) == 0
test_str = "<|python_start|>"
test_str += "[get_weather(city='LA', metric='C'),"
test_str += "register_user(name='Doe', age=9)]"
TEST_CASES = [
pytest.param(
True,
ESCAPED_STRING_FUNCTION_OUTPUT,
[ESCAPED_STRING_FUNCTION_CALL],
id="simple_streaming",
),
pytest.param(
False, SIMPLE_FUNCTION_OUTPUT, [SIMPLE_FUNCTION_CALL], id="simple_nonstreaming"
),
pytest.param(
True,
MORE_TYPES_FUNCTION_OUTPUT,
[MORE_TYPES_FUNCTION_CALL],
id="more_types_streaming",
),
pytest.param(
False,
MORE_TYPES_FUNCTION_OUTPUT,
[MORE_TYPES_FUNCTION_CALL],
id="more_types_nonstreaming",
),
pytest.param(
True,
PARAMETERLESS_FUNCTION_OUTPUT,
[PARAMETERLESS_FUNCTION_CALL],
id="parameterless_streaming",
),
pytest.param(
False,
PARAMETERLESS_FUNCTION_OUTPUT,
[PARAMETERLESS_FUNCTION_CALL],
id="parameterless_nonstreaming",
),
pytest.param(
True,
EMPTY_DICT_FUNCTION_OUTPUT,
[EMPTY_DICT_FUNCTION_CALL],
id="empty_dict_streaming",
),
pytest.param(
False,
EMPTY_DICT_FUNCTION_OUTPUT,
[EMPTY_DICT_FUNCTION_CALL],
id="empty_dict_nonstreaming",
),
pytest.param(
True,
EMPTY_LIST_FUNCTION_OUTPUT,
[EMPTY_LIST_FUNCTION_CALL],
id="empty_list_streaming",
),
pytest.param(
False,
EMPTY_LIST_FUNCTION_OUTPUT,
[EMPTY_LIST_FUNCTION_CALL],
id="empty_list_nonstreaming",
),
pytest.param(
True,
ESCAPED_STRING_FUNCTION_OUTPUT,
[ESCAPED_STRING_FUNCTION_CALL],
id="escaped_string_streaming",
),
pytest.param(
False,
ESCAPED_STRING_FUNCTION_OUTPUT,
[ESCAPED_STRING_FUNCTION_CALL],
id="escaped_string_nonstreaming",
),
pytest.param(
True,
"[get_weather(city='LA',metric='C'),register_user(name='Doe',age=9)]",
[
SIMPLE_FUNCTION_CALL,
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
],
id="parallel_calls_streaming",
),
pytest.param(
False,
"[get_weather(city='LA',metric='C'),register_user(name='Doe',age=9)]",
[
SIMPLE_FUNCTION_CALL,
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
],
id="parallel_calls_nonstreaming",
),
pytest.param(
True,
PYTHON_TAG_FUNCTION_OUTPUT,
[SIMPLE_FUNCTION_CALL],
id="python_tag_streaming",
),
pytest.param(
False,
PYTHON_TAG_FUNCTION_OUTPUT,
[SIMPLE_FUNCTION_CALL],
id="python_tag_nonstreaming",
),
pytest.param(
True,
test_str,
[
SIMPLE_FUNCTION_CALL,
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
],
id="parallel_calls_streaming",
),
pytest.param(
False,
"<|python_start|>[get_weather(city='LA', metric='C'), "
"register_user(name='Doe', age=9)]",
[
SIMPLE_FUNCTION_CALL,
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
],
id="parallel_calls_nonstreaming",
),
]
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
def test_tool_call(
streaming: bool,
model_output: str,
expected_tool_calls: list[FunctionCall],
default_tokenizer: TokenizerLike,
):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
default_tokenizer
)
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert len(tool_calls) == len(expected_tool_calls)
for actual, expected in zip(tool_calls, expected_tool_calls):
assert actual.type == "function"
assert actual.function == expected
def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
default_tokenizer
)
model_output_deltas = [
"<|python_start|>[get_weather(city='LA', metric='C'), "
"get_weather(), "
"do_something_cool(steps=[])]<|python_end|>",
]
reconstructor = run_tool_extraction_streaming(
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
)
assert reconstructor.other_content == ""
assert len(reconstructor.tool_calls) == 3
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
@pytest.mark.parametrize("streaming", [False])
def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):
"""Test regex timeout is handled gracefully."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
default_tokenizer
)
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
# create a mock regex that raises TimeoutError
mock_regex = MagicMock()
mock_regex.match.side_effect = TimeoutError("Regex timeout")
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
content, tool_calls = run_tool_extraction(
tool_parser, fake_problematic_input, streaming=streaming
)
# should treat as regular text when regex times out
assert content == fake_problematic_input
assert len(tool_calls) == 0
mock_regex.match.assert_called_once()
@@ -1,101 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from vllm.tokenizers import TokenizerLike
class TestLongCatToolParser(ToolParserTests):
@pytest.fixture
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
"""Add some longcat specific tokens to the default vocab."""
tokenizer = default_tokenizer
tokenizer_vocab = tokenizer.get_vocab()
tokenizer.get_vocab = MagicMock()
tokenizer_vocab.update(
{
"<longcat_tool_call>": 32000,
"</longcat_tool_call>": 32001,
}
)
tokenizer.get_vocab.return_value = tokenizer_vocab
return tokenizer
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="longcat",
# Test data
no_tool_calls_output="This is a regular response without any tool calls.",
single_tool_call_output=(
'<longcat_tool_call>{"name": "get_weather", '
'"arguments": {"city": "Tokyo"}}</longcat_tool_call>'
),
parallel_tool_calls_output=(
'<longcat_tool_call>{"name": "get_weather", '
'"arguments": {"city": "Tokyo"}}</longcat_tool_call>\n'
'<longcat_tool_call>{"name": "get_time", '
'"arguments": {"timezone": "Asia/Tokyo"}}</longcat_tool_call>'
),
various_data_types_output="""<longcat_tool_call>{
"name": "test_function",
"arguments": {
"string_field": "hello",
"int_field": 42,
"float_field": 3.14,
"bool_field": true,
"null_field": null,
"array_field": ["a", "b", "c"],
"object_field": {"nested": "value"},
"empty_array": [],
"empty_object": {}
}
}</longcat_tool_call>""",
empty_arguments_output=(
'<longcat_tool_call>{"name": "refresh", "arguments": {}}'
"</longcat_tool_call>"
),
surrounding_text_output=(
"Let me check the weather for you.\n"
'<longcat_tool_call>{"name": "get_weather", '
'"arguments": {"city": "Tokyo"}}</longcat_tool_call>\n'
"Here is the result."
),
escaped_strings_output="""<longcat_tool_call>{
"name": "test_function",
"arguments": {
"quoted": "He said \\"hello\\"",
"path": "C:\\\\Users\\\\file.txt",
"newline": "line1\\nline2",
"unicode": "emoji: 🎉"
}
}</longcat_tool_call>""",
malformed_input_outputs=[
'<longcat_tool_call>{"name": "func", "arguments": {',
(
'<longcat_tool_call>{"name": "func", '
'"arguments": "not a dict"}</longcat_tool_call>'
),
"Some text with <longcat_tool_call>invalid json",
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo"},
single_tool_call_expected_content=None,
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "get_time"],
# xfail markers
xfail_streaming={
"test_malformed_input": "Streaming has complex buffering behavior",
},
xfail_nonstreaming={},
# Configuration
allow_empty_or_json_empty_args=True,
)
@@ -1,787 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
import json
import random
from unittest.mock import MagicMock
import pytest
from tests.tool_parsers.utils import run_tool_extraction_streaming
from vllm.entrypoints.generate.base.protocol import FunctionCall, ToolCall
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.tool_parsers import ToolParser, ToolParserManager
from vllm.tool_parsers.minicpm5xml_tool_parser import MiniCPM5XMLToolParser
def _tool(name: str, parameters: dict) -> ChatCompletionToolsParam:
return ChatCompletionToolsParam(
type="function",
function={
"name": name,
"parameters": parameters,
},
)
def make_tools_weather() -> list[ChatCompletionToolsParam]:
return [
_tool(
"get_weather",
{
"type": "object",
"properties": {
"city": {"type": "string"},
"date": {"type": "string"},
},
"required": ["city"],
},
)
]
def make_tools_sum() -> list[ChatCompletionToolsParam]:
return [
_tool(
"sum_values",
{
"type": "object",
"properties": {
"nums": {"type": "array"},
"exact": {"type": "boolean"},
},
"required": ["nums"],
},
)
]
def make_tools_no_required() -> list[ChatCompletionToolsParam]:
return [
_tool(
"noop",
{
"type": "object",
"properties": {"note": {"type": "string"}},
"required": [],
},
)
]
def make_request(
tools: list[ChatCompletionToolsParam],
tool_choice: str = "auto",
) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
tools=tools,
tool_choice=tool_choice,
)
def make_tool_call(name: str, arguments: dict) -> ToolCall:
return ToolCall(
type="function",
function=FunctionCall(
name=name,
arguments=json.dumps(arguments, ensure_ascii=False),
),
)
def assert_tool_calls(
actual: list[ToolCall],
expected: list[ToolCall],
) -> None:
assert len(actual) == len(expected)
for act, exp in zip(actual, expected):
assert act.type == "function"
assert act.function.name == exp.function.name
assert json.loads(act.function.arguments) == json.loads(exp.function.arguments)
@pytest.fixture
def parser() -> ToolParser:
mock_tokenizer = MagicMock()
return MiniCPM5XMLToolParser(mock_tokenizer)
def test_registered_in_tool_parser_manager() -> None:
cls = ToolParserManager.get_tool_parser("minicpm5")
assert cls is MiniCPM5XMLToolParser
def test_adjust_request_skip_special_tokens(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
assert request.skip_special_tokens is True
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is False
def test_adjust_request_tool_choice_none(parser: ToolParser) -> None:
request = make_request(make_tools_weather(), tool_choice="none")
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is True
def test_no_tool_call(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
out = parser.extract_tool_calls("How can I help you?", request)
assert not out.tools_called
assert out.tool_calls == []
assert out.content == "How can I help you?"
def test_single_call_with_surrounding_text(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
"Intro before.\n"
'<function name="get_weather">'
'<param name="city">上海</param>'
'<param name="date">2024-06-27</param>'
"</function>\n"
"Outro after.\n"
)
out = parser.extract_tool_calls(text, request)
assert out.tools_called
assert_tool_calls(
out.tool_calls,
[
make_tool_call(
"get_weather",
{
"city": "上海",
"date": "2024-06-27",
},
)
],
)
assert out.content is None
def test_cdata_multiline(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather">'
'<param name="city"><![CDATA[北\n京]]></param>'
'<param name="date">2024-06-27</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args["city"] == "北\n京"
assert args["date"] == "2024-06-27"
def test_tokenizer_space_marker(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function\u0120name="get_weather">'
'<param\u0120name="city">上海</param>'
'<param\u0120name="date">2024-06-27</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert out.tools_called
assert_tool_calls(
out.tool_calls,
[
make_tool_call(
"get_weather",
{
"city": "上海",
"date": "2024-06-27",
},
)
],
)
def test_collapsed_function_and_param_tags(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<functionname="get_weather">'
'<paramname="city">上海</param>'
'<paramname="date">2024-06-27</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert out.tools_called
assert_tool_calls(
out.tool_calls,
[
make_tool_call(
"get_weather",
{
"city": "上海",
"date": "2024-06-27",
},
)
],
)
def test_collapsed_param_tags_with_tokenizer_space_function(
parser: ToolParser,
) -> None:
request = make_request(make_tools_weather())
text = (
'<function\u0120name="get_weather">'
'<paramname="city">上海</param>'
'<paramname="date">2024-06-27</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert out.tools_called
assert_tool_calls(
out.tool_calls,
[
make_tool_call(
"get_weather",
{
"city": "上海",
"date": "2024-06-27",
},
)
],
)
def test_extract_tool_calls_streaming_partial_chunks(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
chunks = [
'<function name="get_weather">',
'<param name="city">',
'上海</param><param name="date">2024-06-27</param></function>\n',
]
reconstructor = run_tool_extraction_streaming(
parser,
chunks,
request,
assert_one_tool_per_delta=False,
)
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function.name == "get_weather"
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"city": "上海",
"date": "2024-06-27",
}
def test_extract_tool_calls_streaming_tokenizer_space_marker(
parser: ToolParser,
) -> None:
request = make_request(make_tools_weather())
chunks = [
'<function\u0120name="get_weather">',
'<param\u0120name="city">',
'上海</param><param\u0120name="date">2024-06-27</param></function>\n',
]
reconstructor = run_tool_extraction_streaming(
parser,
chunks,
request,
assert_one_tool_per_delta=False,
)
assert len(reconstructor.tool_calls) == 1
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"city": "上海",
"date": "2024-06-27",
}
def test_extract_tool_calls_streaming_collapsed_tags_weather(
parser: ToolParser,
) -> None:
request = make_request(make_tools_weather())
text = (
'<functionname="get_weather">'
'<paramname="city">上海</param>'
'<paramname="date">2024-06-27</param>'
"</function>\n"
)
random.seed(2)
reconstructor = run_tool_extraction_streaming(
parser,
_random_chunks(text, 1, 4),
request,
)
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function.name == "get_weather"
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"city": "上海",
"date": "2024-06-27",
}
def test_collapsed_tags_current_weather(parser: ToolParser) -> None:
request = make_request(
[
_tool(
"get_current_weather",
{
"type": "object",
"properties": {
"city": {"type": "string"},
"state": {"type": "string"},
"unit": {"type": "string"},
},
"required": ["city", "state", "unit"],
},
)
]
)
text = (
'<functionname="get_current_weather">'
'<paramname="city">Dallas</param>'
'<paramname="state">TX</param>'
'<paramname="unit">fahrenheit</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert out.tools_called
assert json.loads(out.tool_calls[0].function.arguments) == {
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
def test_extract_tool_calls_streaming_incremental_arguments(
parser: ToolParser,
) -> None:
request = make_request(
[
_tool(
"get_current_weather",
{
"type": "object",
"properties": {
"city": {"type": "string"},
"state": {"type": "string"},
"unit": {"type": "string"},
},
"required": ["city", "state", "unit"],
},
)
]
)
text = (
'<function name="get_current_weather">'
'<param name="city">Dallas</param>'
'<param name="state">TX</param>'
'<param name="unit">fahrenheit</param>'
"</function>"
)
prev = ""
arguments = ""
for chunk in [text[:37], text[37:70], text[70:]]:
current = prev + chunk
delta = parser.extract_tool_calls_streaming(
prev,
current,
chunk,
[],
[],
[],
request,
)
if delta and delta.tool_calls:
arg_delta = delta.tool_calls[0].function.arguments
if arg_delta:
arguments += arg_delta
prev = current
assert json.loads(arguments) == {
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
def test_unknown_tool_block_preserved(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = '<function name="unknown"><param name="x">1</param></function>\n'
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
assert "unknown" in (out.content or "")
def test_non_string_types(parser: ToolParser) -> None:
request = make_request(make_tools_sum())
text = (
'<function name="sum_values">'
'<param name="nums">[1, 2, 3]</param>'
'<param name="exact">true</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args["nums"] == [1, 2, 3]
assert args["exact"] is True
def test_multiple_calls_interleaved_text(parser: ToolParser) -> None:
tools = make_tools_weather() + make_tools_sum()
request = make_request(tools)
text = (
"Head\n"
'<function name="get_weather"><param name="city">北京</param></function>\n'
"TXT\n"
'<function name="sum_values"><param name="nums">[7,8,9]</param>'
'<param name="exact">false</param></function>\n'
"Tail\n"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 2
args0 = json.loads(out.tool_calls[0].function.arguments)
assert args0["city"] == "北京"
args1 = json.loads(out.tool_calls[1].function.arguments)
assert args1["nums"] == [7, 8, 9]
assert args1["exact"] is False
assert out.content is None
def test_incomplete_missing_function_end(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = '<function name="get_weather"><param name="city">北京</param>'
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
assert "get_weather" in (out.content or "")
def test_param_missing_name_invalid(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather">'
"<param>北京</param>"
'<param name="date">2024-06-27</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
assert "<param>北京</param>" in (out.content or "")
def test_duplicate_param_names_invalid(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather">'
'<param name="city">北京</param>'
'<param name="city">上海</param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
def test_case_sensitive_param_name_invalid(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = '<function name="get_weather"><param name="City">北京</param></function>\n'
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
def test_no_required_and_zero_param_valid(parser: ToolParser) -> None:
request = make_request(make_tools_no_required())
text = '<function name="noop"></function>\n'
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {}
def test_thinking_only_sentencepiece_normalized_in_content(
parser: ToolParser,
) -> None:
request = make_request(make_tools_weather())
text = "\u010aFirst,\u0120I\u0120need\u0120to\u0120check\u0120the\u0120weather."
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
assert out.content is not None
assert "\u0120" not in out.content
assert "\u010a" not in out.content
assert "First, I need to check the weather." in out.content
def test_properties_wrapped_arguments(parser: ToolParser) -> None:
tools = [
_tool(
"get_customer_by_phone",
{
"type": "object",
"properties": {"phone_number": {"type": "string"}},
"required": ["phone_number"],
},
)
]
request = make_request(tools)
text = (
'<function name="get_customer_by_phone">'
"<param name=\"properties\">{'phone_number': '555-123-2002'}</param>"
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "get_customer_by_phone"
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"phone_number": "555-123-2002"}
def test_arguments_wrapped_arguments(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather">'
'<param name="arguments">{"city": "上海", "date": "2024-06-27"}</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"city": "上海", "date": "2024-06-27"}
def test_wrapped_arguments_still_validate_schema(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather">'
'<param name="properties">{"unknown": "x"}</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
def test_extra_arguments_ignored_when_required_present(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather">'
'<param name="city">上海</param>'
'<param name="unknown">ignored</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"city": "上海"}
def test_extra_arguments_do_not_satisfy_required(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
'<function name="get_weather"><param name="unknown">ignored</param></function>'
)
out = parser.extract_tool_calls(text, request)
assert not out.tools_called
def test_zero_arg_tool_ignores_extra_arguments(parser: ToolParser) -> None:
request = make_request(make_tools_no_required())
text = (
'<function name="noop">'
'<param name="note">ignored</param>'
'<param name="extra">ignored</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"note": "ignored"}
def test_alias_get_details_by_phone(parser: ToolParser) -> None:
tools = [
_tool(
"get_customer_by_phone",
{
"type": "object",
"properties": {"phone_number": {"type": "string"}},
"required": ["phone_number"],
},
)
]
request = make_request(tools)
text = (
'<function name="get_details_by_phone">'
'<param name="phone_number">555-123-2002</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "get_customer_by_phone"
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"phone_number": "555-123-2002"}
def test_alias_get_line_details(parser: ToolParser) -> None:
tools = [
_tool(
"get_details_by_id",
{
"type": "object",
"properties": {"id": {"type": "string"}},
"required": ["id"],
},
)
]
request = make_request(tools)
text = (
'<function name="get_line_details">'
'<param name="customer_id">C1001</param>'
'<param name="line_id">L1001</param>'
"</function>"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "get_details_by_id"
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"id": "L1001"}
def test_alias_enable_roaming(parser: ToolParser) -> None:
tools = [
_tool(
"toggle_roaming",
{
"type": "object",
"properties": {
"line_id": {"type": "string"},
"enabled": {"type": "boolean"},
},
"required": ["line_id", "enabled"],
},
)
]
request = make_request(tools)
text = (
'<function name="enable_roaming"><param name="line_id">L1001</param></function>'
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
assert out.tool_calls[0].function.name == "toggle_roaming"
args = json.loads(out.tool_calls[0].function.arguments)
assert args == {"line_id": "L1001", "enabled": True}
def _random_chunks(text: str, min_len: int, max_len: int) -> list[str]:
chunks: list[str] = []
index = 0
while index < len(text):
size = random.randint(min_len, max_len)
chunks.append(text[index : index + size])
index += size
return chunks
def test_extract_tool_calls_streaming_single(parser: ToolParser) -> None:
request = make_request(make_tools_weather())
text = (
"Intro before.\n"
'<function name="get_weather">'
'<param name="city">上海</param>'
'<param name="date">2024-06-27</param>'
"</function>\n"
"Outro after.\n"
)
random.seed(0)
reconstructor = run_tool_extraction_streaming(
parser,
_random_chunks(text, 1, 4),
request,
)
assert "Intro before." in reconstructor.other_content
assert "Outro after." in reconstructor.other_content
assert len(reconstructor.tool_calls) == 1
assert reconstructor.tool_calls[0].function.name == "get_weather"
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"city": "上海",
"date": "2024-06-27",
}
def test_extract_tool_calls_streaming_multiple(parser: ToolParser) -> None:
tools = make_tools_weather() + make_tools_sum()
request = make_request(tools)
text = (
"Head\n"
'<function name="get_weather"><param name="city">北京</param></function>\n'
"TXT\n"
'<function name="sum_values"><param name="nums">[7,8,9]</param>'
'<param name="exact">false</param></function>\n'
"Tail\n"
)
random.seed(1)
reconstructor = run_tool_extraction_streaming(
parser,
_random_chunks(text, 1, 5),
request,
)
assert "Head" in reconstructor.other_content
assert "TXT" in reconstructor.other_content
assert "Tail" in reconstructor.other_content
assert len(reconstructor.tool_calls) == 2
assert json.loads(reconstructor.tool_calls[0].function.arguments)["city"] == "北京"
assert json.loads(reconstructor.tool_calls[1].function.arguments) == {
"nums": [7, 8, 9],
"exact": False,
}
def make_tools_write() -> list[ChatCompletionToolsParam]:
return [
_tool(
"write_file",
{
"type": "object",
"properties": {"content": {"type": "string"}},
"required": ["content"],
},
)
]
class TestParameterWhitespace:
"""CDATA is verbatim, so its whitespace must survive."""
def test_cdata_whitespace_preserved(self, parser: ToolParser) -> None:
request = make_request(make_tools_write())
text = (
'<function name="write_file">'
'<param name="content"><![CDATA[ def foo():\n pass\n]]></param>'
"</function>\n"
)
out = parser.extract_tool_calls(text, request)
assert len(out.tool_calls) == 1
args = json.loads(out.tool_calls[0].function.arguments)
assert args["content"] == " def foo():\n pass\n"
def test_cdata_whitespace_preserved_streaming(self, parser: ToolParser) -> None:
"""Value split across chunks, i.e. the partial path."""
request = make_request(make_tools_write())
chunks = [
'<function name="write_file">',
'<param name="content"><![CDATA[ def foo():\n',
" pass\n]]></param></function>\n",
]
reconstructor = run_tool_extraction_streaming(
parser,
chunks,
request,
assert_one_tool_per_delta=False,
)
assert len(reconstructor.tool_calls) == 1
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
"content": " def foo():\n pass\n"
}
@@ -1,603 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.tool_parsers.minimax_m2_tool_parser import (
MinimaxM2ToolParser,
)
pytestmark = pytest.mark.cpu_test
# Token IDs matching FakeTokenizer.vocab
TC_START_ID = 1
TC_END_ID = 2
class FakeTokenizer:
"""Minimal fake tokenizer for unit tests."""
def __init__(self):
self.model_tokenizer = True
self.vocab = {
"<minimax:tool_call>": TC_START_ID,
"</minimax:tool_call>": TC_END_ID,
}
def get_vocab(self):
return self.vocab
def decode(self, token_ids):
id_to_token = {v: k for k, v in self.vocab.items()}
return "".join(id_to_token.get(token_id, "") for token_id in token_ids)
@pytest.fixture
def parser():
return MinimaxM2ToolParser(FakeTokenizer())
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _feed(parser, chunks, request=None):
"""Feed chunks through the streaming parser and collect results.
Each element in *chunks* is either:
- a ``str``: used as delta_text (current_text accumulates automatically)
- a ``(delta_text, delta_token_ids)`` tuple for special-token scenarios
Returns a list of non-None DeltaMessage objects.
"""
previous = ""
results = []
for chunk in chunks:
if isinstance(chunk, tuple):
delta, delta_ids = chunk
else:
delta = chunk
delta_ids = []
current = previous + delta
result = parser.extract_tool_calls_streaming(
previous_text=previous,
current_text=current,
delta_text=delta,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=delta_ids,
request=request,
)
if result is not None:
results.append(result)
previous = current
return results
def _collect_content(results):
"""Join all content strings from a list of DeltaMessages."""
return "".join(r.content for r in results if r.content)
def _collect_tool_calls(results):
"""Aggregate tool calls by index from a list of DeltaMessages.
Returns a dict: index -> {"id": ..., "name": ..., "arguments": ...}
"""
tool_calls = {}
for r in results:
for tc in r.tool_calls or []:
if tc.index not in tool_calls:
tool_calls[tc.index] = {
"id": None,
"name": "",
"arguments": "",
}
if tc.id:
tool_calls[tc.index]["id"] = tc.id
if tc.function:
if tc.function.name:
tool_calls[tc.index]["name"] += tc.function.name
if tc.function.arguments:
tool_calls[tc.index]["arguments"] += tc.function.arguments
return tool_calls
# ---------------------------------------------------------------------------
# Phase 1: content before tool calls
# ---------------------------------------------------------------------------
class TestContentStreaming:
"""Tests for plain content (no tool calls)."""
def test_plain_content(self, parser):
"""No tool call tokens — all text is streamed as content."""
results = _feed(parser, ["Hello ", "world"])
assert _collect_content(results) == "Hello world"
def test_content_before_tool_call(self, parser):
"""Text before <minimax:tool_call> is streamed as content."""
results = _feed(
parser,
[
"Let me check. ",
'<minimax:tool_call><invoke name="get_weather">'
'<parameter name="city">Seattle</parameter>'
"</invoke></minimax:tool_call>",
],
)
assert _collect_content(results) == "Let me check. "
def test_empty_delta_no_crash(self, parser):
"""Empty delta_text with no token IDs returns None."""
results = _feed(parser, [("", [])])
assert results == []
# ---------------------------------------------------------------------------
# Phase 2: tool call parsing
# ---------------------------------------------------------------------------
class TestSingleInvoke:
"""Tests for a single <invoke> block."""
def test_incremental_chunks(self, parser):
"""Each XML element arrives in a separate chunk."""
results = _feed(
parser,
[
"<minimax:tool_call>",
'<invoke name="get_weather">',
'<parameter name="city">Seattle</parameter>',
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
assert tc[0]["name"] == "get_weather"
assert json.loads(tc[0]["arguments"]) == {"city": "Seattle"}
assert tc[0]["id"] is not None
def test_single_chunk_complete(self, parser):
"""Entire tool call arrives in one delta."""
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="get_weather">'
'<parameter name="city">Seattle</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
assert json.loads(tc[0]["arguments"]) == {"city": "Seattle"}
def test_multiple_params(self, parser):
"""Multiple parameters in one invoke."""
results = _feed(
parser,
[
"<minimax:tool_call>",
'<invoke name="get_weather">',
'<parameter name="city">Seattle</parameter>',
'<parameter name="days">5</parameter>',
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert json.loads(tc[0]["arguments"]) == {
"city": "Seattle",
"days": "5",
}
class TestMultipleInvokes:
"""Tests for multiple <invoke> blocks in one tool call."""
def test_two_invokes_incremental(self, parser):
"""Two invokes arriving one chunk at a time."""
results = _feed(
parser,
[
"<minimax:tool_call>",
'<invoke name="search_web">'
'<parameter name="query">OpenAI</parameter>'
"</invoke>",
'<invoke name="search_web">'
'<parameter name="query">Gemini</parameter>'
"</invoke>",
"</minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 2
assert tc[0]["name"] == "search_web"
assert tc[1]["name"] == "search_web"
assert json.loads(tc[0]["arguments"]) == {"query": "OpenAI"}
assert json.loads(tc[1]["arguments"]) == {"query": "Gemini"}
def test_two_invokes_in_single_delta(self, parser):
"""Both invokes close in the same delta — loop must emit both."""
results = _feed(
parser,
[
"<minimax:tool_call>",
'<invoke name="fn_a"><parameter name="x">1</parameter></invoke>'
'<invoke name="fn_b"><parameter name="y">2</parameter></invoke>',
"</minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 2
assert tc[0]["name"] == "fn_a"
assert tc[1]["name"] == "fn_b"
def test_different_functions(self, parser):
"""Parallel calls to different functions."""
results = _feed(
parser,
[
"<minimax:tool_call>",
'<invoke name="get_weather">'
'<parameter name="city">NYC</parameter>'
"</invoke>",
'<invoke name="get_stock">'
'<parameter name="ticker">AAPL</parameter>'
"</invoke>",
"</minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert tc[0]["name"] == "get_weather"
assert tc[1]["name"] == "get_stock"
# ---------------------------------------------------------------------------
# DeltaMessage structure
# ---------------------------------------------------------------------------
class TestDeltaMessageFormat:
"""Verify the shape of emitted DeltaMessage / DeltaToolCall."""
def test_tool_call_fields(self, parser):
"""Each emitted tool call has id, name, arguments, type, index."""
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="fn">'
'<parameter name="k">v</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc_deltas = [tc for r in results for tc in (r.tool_calls or [])]
assert len(tc_deltas) == 1
tc = tc_deltas[0]
assert tc.index == 0
assert tc.type == "function"
assert tc.id is not None
assert tc.function.name == "fn"
assert json.loads(tc.function.arguments) == {"k": "v"}
def test_multi_invoke_indices(self, parser):
"""Multiple invokes get sequential indices."""
results = _feed(
parser,
[
"<minimax:tool_call>",
'<invoke name="a"><parameter name="x">1</parameter></invoke>',
'<invoke name="b"><parameter name="x">2</parameter></invoke>',
"</minimax:tool_call>",
],
)
tc_deltas = [tc for r in results for tc in (r.tool_calls or [])]
indices = [tc.index for tc in tc_deltas]
assert indices == [0, 1]
# ---------------------------------------------------------------------------
# Large chunks (stream_interval > 1)
# ---------------------------------------------------------------------------
class TestLargeChunks:
"""Simulate stream_interval > 1 where many tokens arrive at once."""
def test_header_and_params_in_separate_chunks(self, parser):
"""Header in chunk 1, all params + close in chunk 2."""
chunk1 = '<minimax:tool_call><invoke name="get_weather">'
chunk2 = (
'<parameter name="city">Seattle</parameter>'
'<parameter name="days">5</parameter>'
"</invoke></minimax:tool_call>"
)
results = _feed(
parser,
[
chunk1,
chunk2,
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed == {"city": "Seattle", "days": "5"}
class TestAnyOfNullableParam:
"""Regression: anyOf nullable parameter parsing (PR #32342)."""
def test_anyof_nullable_param_non_null_value(self):
"""A valid non-null string should be preserved, not collapsed to None."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="update_profile",
parameters={
"type": "object",
"properties": {
"nickname": {
"anyOf": [{"type": "string"}, {"type": "null"}],
},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="update_profile">'
'<parameter name="nickname">Alice</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["nickname"] == "Alice"
def test_anyof_nullable_param_null_value(self):
"""An actual null-like value should be returned as None/null."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="update_profile",
parameters={
"type": "object",
"properties": {
"nickname": {
"anyOf": [{"type": "string"}, {"type": "null"}],
},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="update_profile">'
'<parameter name="nickname">null</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["nickname"] is None
def test_anyof_nullable_param_object_value(self):
"""A valid object value in anyOf with null should parse as dict."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="update_settings",
parameters={
"type": "object",
"properties": {
"config": {
"anyOf": [{"type": "object"}, {"type": "null"}],
},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="update_settings">'
'<parameter name="config">{"theme": "dark", "fontSize": 14}'
"</parameter>"
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["config"] == {"theme": "dark", "fontSize": 14}
assert isinstance(parsed["config"], dict)
class TestNoneStringPreservation:
"""Regression tests for #39567: 'none' as a string must not become None."""
def test_none_string_preserved_in_enum(self):
"""'none' in an enum must stay as the string 'none', not Python None."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="set_theme",
parameters={
"type": "object",
"properties": {
"theme": {
"type": "string",
"enum": ["dark", "light", "none"],
},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="set_theme">'
'<parameter name="theme">none</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["theme"] == "none"
assert parsed["theme"] is not None
def test_none_string_preserved_plain_string(self):
"""'none' as a plain string param must stay as 'none'."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="echo",
parameters={
"type": "object",
"properties": {
"message": {"type": "string"},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="echo">'
'<parameter name="message">none</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["message"] == "none"
def test_null_still_converts_to_none(self):
"""'null' in a nullable param must still become Python None."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="update_profile",
parameters={
"type": "object",
"properties": {
"nickname": {
"anyOf": [{"type": "string"}, {"type": "null"}],
},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="update_profile">'
'<parameter name="nickname">null</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["nickname"] is None
def test_nil_string_preserved(self):
"""'nil' must stay as the string 'nil', not become None."""
tools = [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="echo",
parameters={
"type": "object",
"properties": {
"value": {"type": "string"},
},
},
),
)
]
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="echo">'
'<parameter name="value">nil</parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
parsed = json.loads(tc[0]["arguments"])
assert parsed["value"] == "nil"
class TestParameterWhitespace:
"""Parameter values must preserve surrounding whitespace."""
def test_whitespace_preserved(self, parser):
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="echo">'
'<parameter name="msg"> hi </parameter>'
"</invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
assert json.loads(tc[0]["arguments"]) == {"msg": " hi "}
def test_whitespace_preserved_across_chunks(self, parser):
"""Value split before </parameter> arrives, i.e. the partial path."""
results = _feed(
parser,
[
'<minimax:tool_call><invoke name="write">'
'<parameter name="content"> def foo():\n',
" return 1\n",
"</parameter></invoke></minimax:tool_call>",
],
)
tc = _collect_tool_calls(results)
assert len(tc) == 1
assert json.loads(tc[0]["arguments"]) == {
"content": " def foo():\n return 1\n"
}
@@ -1,276 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from typing import Any
import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
FunctionDefinition,
)
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
NS = "]<]minimax[>["
EOS_ID = 99
class FakeTokenizer:
"""Minimal fake tokenizer for unit tests."""
def __init__(self):
self.model_tokenizer = True
self.vocab: dict[str, int] = {}
def get_vocab(self) -> dict[str, int]:
return self.vocab
def sample_tools() -> list[ChatCompletionToolsParam]:
return [
ChatCompletionToolsParam(
function=FunctionDefinition(
name="create_order",
parameters={
"type": "object",
"properties": {
"user_id": {"type": "integer"},
"urgent": {"type": "boolean"},
"note": {"type": "string"},
"shipping": {
"type": "object",
"properties": {
"city": {"type": "string"},
"zip": {"type": "integer"},
},
},
"items": {
"type": "array",
"items": {
"type": "object",
"properties": {
"sku": {"type": "string"},
"qty": {"type": "integer"},
},
},
},
"metadata": {
"type": "object",
"additionalProperties": {"type": "string"},
},
"duplicate_demo": {"type": "object"},
},
},
),
)
]
@pytest.fixture
def parser() -> MinimaxM3ToolParser:
return MinimaxM3ToolParser(FakeTokenizer(), tools=sample_tools())
def build_order_call() -> str:
return (
f"{NS}<tool_call>\n"
f'{NS}<invoke name="create_order">'
f"{NS}<user_id>42{NS}</user_id>"
f"{NS}<urgent>true{NS}</urgent>"
f"{NS}<note>Please leave at front desk.{NS}</note>"
f"{NS}<shipping>"
f"{NS}<city>Singapore{NS}</city>"
f"{NS}<zip>018956{NS}</zip>"
f"{NS}</shipping>"
f"{NS}<items>"
f"{NS}<item>{NS}<sku>book-001{NS}</sku>{NS}<qty>2{NS}</qty>{NS}</item>"
f"{NS}<item>{NS}<sku>pen-007{NS}</sku>{NS}<qty>5{NS}</qty>{NS}</item>"
f"{NS}</items>"
f"{NS}<metadata>"
f"{NS}<source>mobile{NS}</source>"
f"{NS}<campaign>may-launch{NS}</campaign>"
f"{NS}</metadata>"
f"{NS}<duplicate_demo>"
f"{NS}<tag>a{NS}</tag>"
f"{NS}<tag>b{NS}</tag>"
f"{NS}</duplicate_demo>"
f"{NS}</invoke>\n"
f"{NS}</tool_call>"
)
ORDER_ARGUMENTS = {
"user_id": 42,
"urgent": True,
"note": "Please leave at front desk.",
"shipping": {"city": "Singapore", "zip": 18956},
"items": [
{"sku": "book-001", "qty": 2},
{"sku": "pen-007", "qty": 5},
],
"metadata": {
"source": "mobile",
"campaign": "may-launch",
},
"duplicate_demo": {"tag": ["a", "b"]},
}
def build_order_invocation(user_id: int) -> str:
return (
f'{NS}<invoke name="create_order">'
f"{NS}<user_id>{user_id}{NS}</user_id>"
f"{NS}</invoke>"
)
def build_multiple_order_call() -> str:
return (
f"{NS}<tool_call>\n"
f"{build_order_invocation(1)}\n"
f"{build_order_invocation(2)}\n"
f"{NS}</tool_call>"
)
def _feed(
parser: MinimaxM3ToolParser, chunks: list[str | tuple[str, list[int]]]
) -> list[DeltaMessage]:
previous = ""
results: list[DeltaMessage] = []
for chunk in chunks:
if isinstance(chunk, tuple):
delta, delta_ids = chunk
else:
delta = chunk
delta_ids = []
current = previous + delta
result = parser.extract_tool_calls_streaming(
previous_text=previous,
current_text=current,
delta_text=delta,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=delta_ids,
request=None,
)
if result is not None:
results.append(result)
previous = current
return results
def _collect_content(results: list[DeltaMessage]) -> str:
return "".join(result.content for result in results if result.content)
def _collect_tool_calls(results: list[DeltaMessage]) -> dict[int, dict[str, Any]]:
tool_calls: dict[int, dict[str, Any]] = {}
for result in results:
for tool_call in result.tool_calls or []:
tool_calls.setdefault(
tool_call.index,
{"id": None, "name": "", "arguments": ""},
)
if tool_call.id:
tool_calls[tool_call.index]["id"] = tool_call.id
if tool_call.function:
if tool_call.function.name:
tool_calls[tool_call.index]["name"] += tool_call.function.name
if tool_call.function.arguments:
tool_calls[tool_call.index]["arguments"] += (
tool_call.function.arguments
)
return tool_calls
def test_minimax_m3_parser_registered():
assert ToolParserManager.get_tool_parser("minimax_m3") is MinimaxM3ToolParser
def test_required_tool_choice_keeps_native_syntax(parser):
request = ChatCompletionRequest(
model="minimax-m3",
messages=[{"role": "user", "content": "order"}],
tools=sample_tools(),
tool_choice="required",
)
adjusted = parser.adjust_request(request)
assert adjusted.structured_outputs is None
assert adjusted.skip_special_tokens
def test_non_streaming_nested_tool_call(parser):
result = parser.extract_tool_calls(
"I will create it.\n" + build_order_call(),
request=None,
)
assert result.tools_called
assert result.content == "I will create it.\n"
assert len(result.tool_calls) == 1
tool_call = result.tool_calls[0]
assert tool_call.function.name == "create_order"
assert json.loads(tool_call.function.arguments) == ORDER_ARGUMENTS
def test_non_streaming_without_tool_call_keeps_content(parser):
result = parser.extract_tool_calls("plain response", request=None)
assert not result.tools_called
assert result.tool_calls == []
assert result.content == "plain response"
def test_non_streaming_multiple_tool_calls(parser):
result = parser.extract_tool_calls(build_multiple_order_call(), request=None)
assert result.tools_called
assert result.content is None
assert [tool_call.function.name for tool_call in result.tool_calls] == [
"create_order",
"create_order",
]
assert [
json.loads(tool_call.function.arguments)["user_id"]
for tool_call in result.tool_calls
] == [1, 2]
def test_streaming_without_tool_call_emits_text(parser):
results = _feed(parser, ["plain ", "response"])
assert _collect_content(results) == "plain response"
assert _collect_tool_calls(results) == {}
def test_streaming_nested_tool_call(parser):
tool_call_text = build_order_call()
results = _feed(
parser,
[
"I will create it.\n",
tool_call_text[:5],
tool_call_text[5:17],
tool_call_text[17:120],
tool_call_text[120:],
("", [EOS_ID]),
],
)
assert _collect_content(results) == "I will create it.\n"
tool_calls = _collect_tool_calls(results)
assert len(tool_calls) == 1
assert tool_calls[0]["name"] == "create_order"
assert tool_calls[0]["id"] is not None
assert json.loads(tool_calls[0]["arguments"]) == ORDER_ARGUMENTS
assert results[-1].content is None
@@ -1,251 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import pytest
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser, ToolParserManager
# https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1
SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"
SIMPLE_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "San Francisco", "metric": "celsius"}',
)
MORE_TYPES_FUNCTION_OUTPUT = (
"register_user(name='John Doe', "
"age=37, "
"address={'city': 'San Francisco', 'state': 'CA'}, "
"role=None, "
"passed_test=True, "
"aliases=['John', 'Johnny'])"
)
MORE_TYPES_FUNCTION_OUTPUT_JSON_LITERALS = (
"register_user(name='John Doe', "
"age=37, "
"address={'city': 'San Francisco', 'state': 'CA'}, "
"role=null, "
"passed_test=true, "
"aliases=['John', 'Johnny'])"
)
MORE_TYPES_FUNCTION_CALL = FunctionCall(
name="register_user",
arguments='{"name": "John Doe", '
'"age": 37, '
'"address": {"city": "San Francisco", "state": "CA"}, '
'"role": null, '
'"passed_test": true, '
'"aliases": ["John", "Johnny"]}',
)
PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments="{}",
)
EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"additional_data": {}}',
)
EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"steps": []}',
)
ESCAPED_STRING_FUNCTION_OUTPUT = (
r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
)
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
)
@pytest.mark.parametrize("streaming", [True, False])
def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
default_tokenizer
)
model_output = "How can I help you today?"
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content == model_output
assert len(tool_calls) == 0
TEST_CASES = [
pytest.param(
True,
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}</function_calls>",
[SIMPLE_FUNCTION_CALL],
id="simple_streaming",
),
pytest.param(
False,
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}</function_calls>",
[SIMPLE_FUNCTION_CALL],
id="simple_nonstreaming",
),
pytest.param(
True,
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
[MORE_TYPES_FUNCTION_CALL],
id="more_types_streaming",
),
pytest.param(
False,
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
[MORE_TYPES_FUNCTION_CALL],
id="more_types_nonstreaming",
),
pytest.param(
True,
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT_JSON_LITERALS}</function_calls>",
[MORE_TYPES_FUNCTION_CALL],
id="more_types_streaming_json_literals",
),
pytest.param(
False,
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT_JSON_LITERALS}</function_calls>",
[MORE_TYPES_FUNCTION_CALL],
id="more_types_nonstreaming_json_literals",
),
pytest.param(
True,
f"<function_calls>{PARAMETERLESS_FUNCTION_OUTPUT}</function_calls>",
[PARAMETERLESS_FUNCTION_CALL],
id="parameterless_streaming",
),
pytest.param(
False,
f"<function_calls>{PARAMETERLESS_FUNCTION_OUTPUT}</function_calls>",
[PARAMETERLESS_FUNCTION_CALL],
id="parameterless_nonstreaming",
),
pytest.param(
True,
f"<function_calls>{EMPTY_DICT_FUNCTION_OUTPUT}</function_calls>",
[EMPTY_DICT_FUNCTION_CALL],
id="empty_dict_streaming",
),
pytest.param(
False,
f"<function_calls>{EMPTY_DICT_FUNCTION_OUTPUT}</function_calls>",
[EMPTY_DICT_FUNCTION_CALL],
id="empty_dict_nonstreaming",
),
pytest.param(
True,
f"<function_calls>{EMPTY_LIST_FUNCTION_OUTPUT}</function_calls>",
[EMPTY_LIST_FUNCTION_CALL],
id="empty_list_streaming",
),
pytest.param(
False,
f"<function_calls>{EMPTY_LIST_FUNCTION_OUTPUT}</function_calls>",
[EMPTY_LIST_FUNCTION_CALL],
id="empty_list_nonstreaming",
),
pytest.param(
True,
f"<function_calls>{ESCAPED_STRING_FUNCTION_OUTPUT}</function_calls>",
[ESCAPED_STRING_FUNCTION_CALL],
id="escaped_string_streaming",
),
pytest.param(
False,
f"<function_calls>{ESCAPED_STRING_FUNCTION_OUTPUT}</function_calls>",
[ESCAPED_STRING_FUNCTION_CALL],
id="escaped_string_nonstreaming",
),
pytest.param(
True,
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}\n{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
id="parallel_calls_streaming",
),
pytest.param(
False,
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}\n{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
id="parallel_calls_nonstreaming",
),
]
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
def test_tool_call(
streaming: bool,
model_output: str,
expected_tool_calls: list[FunctionCall],
default_tokenizer: TokenizerLike,
):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
default_tokenizer
)
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content is None
assert len(tool_calls) == len(expected_tool_calls)
for actual, expected in zip(tool_calls, expected_tool_calls):
assert actual.type == "function"
assert actual.function == expected
def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
default_tokenizer
)
model_output_deltas = [
"<function_calls>get_weather(city='San",
" Francisco', metric='celsius')\n"
f"{PARAMETERLESS_FUNCTION_OUTPUT}\n"
f"{EMPTY_LIST_FUNCTION_OUTPUT}</function_calls>",
]
reconstructor = run_tool_extraction_streaming(
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
)
assert reconstructor.other_content == ""
assert len(reconstructor.tool_calls) == 3
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
@pytest.mark.parametrize("streaming", [False])
def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):
"""Test regex timeout is handled gracefully."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
default_tokenizer
)
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
# create a mock regex that raises TimeoutError
mock_regex = MagicMock()
mock_regex.match.side_effect = TimeoutError("Regex timeout")
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
content, tool_calls = run_tool_extraction(
tool_parser, fake_problematic_input, streaming=streaming
)
# should treat as regular text when regex times out
assert content == fake_problematic_input
assert len(tool_calls) == 0
mock_regex.match.assert_called_once()
@@ -1,109 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from vllm.tokenizers import TokenizerLike
class TestPhi4MiniToolParser(ToolParserTests):
@pytest.fixture
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
"""Add some phi4mini specific tokens to the default vocab."""
tokenizer = default_tokenizer
tokenizer_vocab = tokenizer.get_vocab()
tokenizer.get_vocab = MagicMock()
tokenizer_vocab.update(
{
"functools": 32000,
}
)
tokenizer.get_vocab.return_value = tokenizer_vocab
return tokenizer
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="phi4_mini_json",
# Test data
no_tool_calls_output="This is a regular response without any tool calls.",
single_tool_call_output=(
'functools[{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
),
parallel_tool_calls_output="""functools[
{"name": "get_weather", "arguments": {"city": "Tokyo"}},
{"name": "get_time", "arguments": {"timezone": "Asia/Tokyo"}}
]""",
various_data_types_output="""functools[{
"name": "test_function",
"arguments": {
"string_field": "hello",
"int_field": 42,
"float_field": 3.14,
"bool_field": true,
"null_field": null,
"array_field": ["a", "b", "c"],
"object_field": {"nested": "value"},
"empty_array": [],
"empty_object": {}
}
}]""",
empty_arguments_output='functools[{"name": "refresh", "arguments": {}}]',
surrounding_text_output="""Let me check the weather for you.
functools[{"name": "get_weather", "arguments": {"city": "Tokyo"}}]
Would you like to know more?""",
escaped_strings_output="""functools[{
"name": "test_function",
"arguments": {
"quoted": "He said \\"hello\\"",
"path": "C:\\\\Users\\\\file.txt",
"newline": "line1\\nline2",
"unicode": "emoji: 🎉"
}
}]""",
malformed_input_outputs=[
'functools[{"name": "func", "arguments": {',
'functools[{"name": "func", "arguments": "not a dict"}]',
'functools{"name": "func"}', # Missing brackets
'functools[{"name": "func"}]', # Missing arguments/parameters
"functools[] This is just text", # Empty functools
"functools[ This is just text ]", # functools with invalid JSON
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo"},
# Phi-4 Mini strips content when tool calls present
single_tool_call_expected_content=None,
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "get_time"],
parallel_tool_calls_expected_content=None,
# xfail markers
xfail_streaming={
"test_no_tool_calls": "Phi4 Mini streaming not implemented",
"test_single_tool_call_simple_args": (
"Phi4 Mini streaming not implemented"
),
"test_parallel_tool_calls": "Phi4 Mini streaming not implemented",
"test_various_data_types": "Phi4 Mini streaming not implemented",
"test_empty_arguments": "Phi4 Mini streaming not implemented",
"test_surrounding_text": "Phi4 Mini streaming not implemented",
"test_escaped_strings": "Phi4 Mini streaming not implemented",
"test_streaming_reconstruction": "Phi4 Mini streaming not implemented",
},
xfail_nonstreaming={
"test_various_data_types": (
"Phi4MiniJsonToolParser regex has nesting limitations "
"with nested objects"
),
"test_malformed_input": (
"Phi4MiniJsonToolParser incorrectly sets "
"tools_called=True on empty array"
),
},
)
@@ -1,366 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression tests for ``PoolsideV1ToolParser``.
Covers two bugs:
1. ``adjust_request`` did not skip the forced ``structured_outputs`` JSON
for ``required``/named tool choice. These models emit XML tool calls
(``<tool_call>...<arg_value>...</arg_value></tool_call>``) per the chat
template, so guided JSON decoding conflicts with the format: the call
leaks as content with empty ``tool_calls``. ``adjust_request`` now skips
the constraint for both ChatCompletion (``ChatCompletionNamedToolChoice``)
and Responses (``ToolChoiceFunction``) named choices.
2. ``extract_tool_calls`` stripped string-typed argument values, corrupting
content whose whitespace is significant (e.g. code/file bodies losing
leading indent and trailing newline). String values are now kept verbatim;
only non-string types are stripped/deserialized.
"""
from __future__ import annotations
import json
from typing import Any
from openai.types.responses.tool_param import FunctionToolParam
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.tool_parsers.poolside_v1_tool_parser import PoolsideV1ToolParser
def _write_file_tool() -> dict[str, Any]:
"""Tool with a string arg (``content``) and a non-string arg (``mode``)."""
return {
"type": "function",
"function": {
"name": "write_file",
"description": "Write content to a file",
"parameters": {
"type": "object",
"properties": {
"content": {"type": "string"},
"mode": {"type": "integer"},
},
"required": ["content"],
},
},
}
def _responses_write_file_tool() -> FunctionToolParam:
return FunctionToolParam(
type="function",
name="write_file",
description="Write content to a file",
parameters={
"type": "object",
"properties": {
"content": {"type": "string"},
"mode": {"type": "integer"},
},
"required": ["content"],
},
strict=True,
)
def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionRequest:
return ChatCompletionRequest.model_validate(
{
"model": "poolside-test",
"messages": [{"role": "user", "content": "write the file"}],
"tools": [_write_file_tool()],
"tool_choice": tool_choice,
}
)
def _build_responses_request(
*,
tool_choice: str | dict[str, Any],
include: list[str] | None = None,
tools: list[dict[str, Any]] | None = None,
) -> ResponsesRequest:
return ResponsesRequest(
model="poolside-test",
input=[{"role": "user", "content": "write the file"}],
tools=tools if tools is not None else [_responses_write_file_tool()],
tool_choice=tool_choice,
stream=True,
max_output_tokens=200,
include=include,
)
class _StubTokenizer:
"""Minimal tokenizer stub to satisfy ``PoolsideV1ToolParser.__init__``."""
def get_vocab(self) -> dict[str, int]:
return {"<tool_call>": 151_657, "</tool_call>": 151_658}
def _make_parser(request: ChatCompletionRequest) -> PoolsideV1ToolParser:
return PoolsideV1ToolParser(_StubTokenizer(), tools=request.tools)
# ---------------------------------------------------------------------------
# Bug 1: required/named must skip forced structured_outputs (#39870 pattern)
# ---------------------------------------------------------------------------
def test_required_skips_structured_outputs_chatcompletion() -> None:
request = _build_chat_request(tool_choice="required")
_make_parser(request).adjust_request(request)
assert request.structured_outputs is None
assert request.skip_special_tokens is False
def test_named_skips_structured_outputs_chatcompletion() -> None:
request = _build_chat_request(
tool_choice={"type": "function", "function": {"name": "write_file"}}
)
_make_parser(request).adjust_request(request)
assert request.structured_outputs is None
assert request.skip_special_tokens is False
def test_required_skips_structured_outputs_responses() -> None:
request = _build_responses_request(tool_choice="required")
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_named_skips_structured_outputs_responses() -> None:
# Responses-API named choice parses to ToolChoiceFunction, a different
# type than the ChatCompletion named choice; both must be handled.
request = _build_responses_request(
tool_choice={"type": "function", "name": "write_file"}
)
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_named_custom_tool_choice_skips_structured_outputs_responses() -> None:
# A named custom choice parses to ToolChoiceCustom; custom tools ride the
# function shim, so it must route like ToolChoiceFunction.
request = _build_responses_request(
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_auto_still_keeps_special_tokens() -> None:
request = _build_chat_request(tool_choice="auto")
_make_parser(request).adjust_request(request)
assert request.skip_special_tokens is False
# ---------------------------------------------------------------------------
# Bug 2: string arg whitespace must be preserved (#42026 pattern)
# ---------------------------------------------------------------------------
def test_string_arg_preserves_whitespace() -> None:
request = _build_chat_request(tool_choice="auto")
parser = _make_parser(request)
content = " def f():\n return 1\n"
model_output = (
"<tool_call>write_file\n"
"<arg_key>content</arg_key>\n"
f"<arg_value>{content}</arg_value>\n"
"</tool_call>"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
args = json.loads(result.tool_calls[0].function.arguments)
# Leading indent and trailing newline must survive verbatim.
assert args["content"] == content
def test_non_string_arg_still_deserialized() -> None:
request = _build_chat_request(tool_choice="auto")
parser = _make_parser(request)
model_output = (
"<tool_call>write_file\n"
"<arg_key>content</arg_key>\n"
"<arg_value>hi</arg_value>\n"
"<arg_key>mode</arg_key>\n"
"<arg_value> 420 </arg_value>\n"
"</tool_call>"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
args = json.loads(result.tool_calls[0].function.arguments)
assert args["content"] == "hi"
# Non-string value is stripped and parsed to its native type.
assert args["mode"] == 420
def test_responses_extract_tool_calls_with_flat_tools() -> None:
# required/named Responses calls route into extract_tool_calls with flat
# FunctionTool (.name); _is_string_type must not raise.
request = _build_responses_request(tool_choice="required")
parser = PoolsideV1ToolParser(_StubTokenizer(), tools=request.tools)
content = " x = 1\n"
model_output = (
"<tool_call>write_file\n"
"<arg_key>content</arg_key>\n"
f"<arg_value>{content}</arg_value>\n"
"</tool_call>"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
args = json.loads(result.tool_calls[0].function.arguments)
assert args["content"] == content
def _weather_tool() -> dict[str, Any]:
return {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
},
},
}
def _build_weather_request(*, tool_choice: str) -> ChatCompletionRequest:
return ChatCompletionRequest.model_validate(
{
"model": "poolside-test",
"messages": [{"role": "user", "content": "weather in Paris?"}],
"tools": [_weather_tool()],
"tool_choice": tool_choice,
}
)
def test_no_newline_after_name_non_streaming() -> None:
request = _build_weather_request(tool_choice="auto")
parser = _make_parser(request)
model_output = (
"<tool_call>get_weather"
"<arg_key>city</arg_key><arg_value>Paris</arg_value>"
"</tool_call>"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"city": "Paris"}
def test_newline_after_name_still_parses_non_streaming() -> None:
request = _build_weather_request(tool_choice="auto")
parser = _make_parser(request)
model_output = (
"<tool_call>get_weather\n"
"<arg_key>city</arg_key>\n<arg_value>Paris</arg_value>\n"
"</tool_call>"
)
result = parser.extract_tool_calls(model_output, request)
assert result.tools_called
assert result.tool_calls[0].function.name == "get_weather"
args = json.loads(result.tool_calls[0].function.arguments)
assert args == {"city": "Paris"}
def test_no_newline_after_name_streaming() -> None:
request = _build_weather_request(tool_choice="auto")
parser = _make_parser(request)
model_output = (
"<tool_call>get_weather"
"<arg_key>city</arg_key><arg_value>Paris</arg_value>"
"</tool_call>"
)
name = ""
args = ""
prev = ""
for ch in model_output:
cur = prev + ch
delta = parser.extract_tool_calls_streaming(
previous_text=prev,
current_text=cur,
delta_text=ch,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=request,
)
prev = cur
if delta is None:
continue
for tc in delta.tool_calls or []:
if tc.function is None:
continue
if tc.function.name:
name += tc.function.name
if tc.function.arguments:
args += tc.function.arguments
assert name == "get_weather"
assert json.loads(args) == {"city": "Paris"}
def _stream_partial_start_token(request: ResponsesRequest):
parser = _make_parser(request)
delta = parser.tool_call_start_token[0]
return parser.extract_tool_calls_streaming(
previous_text="",
current_text=delta,
delta_text=delta,
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=request,
)
def test_streaming_responses_request_without_logprobs() -> None:
request = _build_responses_request(tool_choice="auto")
assert _stream_partial_start_token(request) is None
def test_streaming_responses_request_with_logprobs_emits_empty_delta() -> None:
request = _build_responses_request(
tool_choice="auto", include=["message.output_text.logprobs"]
)
result = _stream_partial_start_token(request)
assert result is not None
assert result.content == ""
@@ -1,231 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock, patch
import pytest
from tests.tool_parsers.utils import (
run_tool_extraction,
run_tool_extraction_streaming,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser, ToolParserManager
# https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1
SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"
SIMPLE_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "San Francisco", "metric": "celsius"}',
)
MORE_TYPES_FUNCTION_OUTPUT = (
"register_user(name='John Doe', "
"age=37, "
"address={'city': 'San Francisco', 'state': 'CA'}, "
"role=None, "
"passed_test=True, "
"aliases=['John', 'Johnny'])"
)
MORE_TYPES_FUNCTION_CALL = FunctionCall(
name="register_user",
arguments='{"name": "John Doe", '
'"age": 37, '
'"address": {"city": "San Francisco", "state": "CA"}, '
'"role": null, '
'"passed_test": true, '
'"aliases": ["John", "Johnny"]}',
)
PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments="{}",
)
EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"additional_data": {}}',
)
EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
name="do_something_cool",
arguments='{"steps": []}',
)
ESCAPED_STRING_FUNCTION_OUTPUT = (
r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
)
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
name="get_weather",
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
)
@pytest.mark.parametrize("streaming", [True, False])
def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
default_tokenizer
)
model_output = "How can I help you today?"
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content == model_output
assert len(tool_calls) == 0
TEST_CASES = [
pytest.param(
True,
f"[{SIMPLE_FUNCTION_OUTPUT}]",
[SIMPLE_FUNCTION_CALL],
id="simple_streaming",
),
pytest.param(
False,
f"[{SIMPLE_FUNCTION_OUTPUT}]",
[SIMPLE_FUNCTION_CALL],
id="simple_nonstreaming",
),
pytest.param(
True,
f"[{MORE_TYPES_FUNCTION_OUTPUT}]",
[MORE_TYPES_FUNCTION_CALL],
id="more_types_streaming",
),
pytest.param(
False,
f"[{MORE_TYPES_FUNCTION_OUTPUT}]",
[MORE_TYPES_FUNCTION_CALL],
id="more_types_nonstreaming",
),
pytest.param(
True,
f"[{PARAMETERLESS_FUNCTION_OUTPUT}]",
[PARAMETERLESS_FUNCTION_CALL],
id="parameterless_streaming",
),
pytest.param(
False,
f"[{PARAMETERLESS_FUNCTION_OUTPUT}]",
[PARAMETERLESS_FUNCTION_CALL],
id="parameterless_nonstreaming",
),
pytest.param(
True,
f"[{EMPTY_DICT_FUNCTION_OUTPUT}]",
[EMPTY_DICT_FUNCTION_CALL],
id="empty_dict_streaming",
),
pytest.param(
False,
f"[{EMPTY_DICT_FUNCTION_OUTPUT}]",
[EMPTY_DICT_FUNCTION_CALL],
id="empty_dict_nonstreaming",
),
pytest.param(
True,
f"[{EMPTY_LIST_FUNCTION_OUTPUT}]",
[EMPTY_LIST_FUNCTION_CALL],
id="empty_list_streaming",
),
pytest.param(
False,
f"[{EMPTY_LIST_FUNCTION_OUTPUT}]",
[EMPTY_LIST_FUNCTION_CALL],
id="empty_list_nonstreaming",
),
pytest.param(
True,
f"[{ESCAPED_STRING_FUNCTION_OUTPUT}]",
[ESCAPED_STRING_FUNCTION_CALL],
id="escaped_string_streaming",
),
pytest.param(
False,
f"[{ESCAPED_STRING_FUNCTION_OUTPUT}]",
[ESCAPED_STRING_FUNCTION_CALL],
id="escaped_string_nonstreaming",
),
pytest.param(
True,
f"[{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}]",
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
id="parallel_calls_streaming",
),
pytest.param(
False,
f"[{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}]",
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
id="parallel_calls_nonstreaming",
),
]
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
def test_tool_call(
streaming: bool,
model_output: str,
expected_tool_calls: list[FunctionCall],
default_tokenizer: TokenizerLike,
):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
default_tokenizer
)
content, tool_calls = run_tool_extraction(
tool_parser, model_output, streaming=streaming
)
assert content is None
assert len(tool_calls) == len(expected_tool_calls)
for actual, expected in zip(tool_calls, expected_tool_calls):
assert actual.type == "function"
assert actual.function == expected
def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
default_tokenizer
)
model_output_deltas = [
"[get_weather(city='San",
" Francisco', metric='celsius'), "
f"{PARAMETERLESS_FUNCTION_OUTPUT}, "
f"{EMPTY_LIST_FUNCTION_OUTPUT}]",
]
reconstructor = run_tool_extraction_streaming(
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
)
assert reconstructor.other_content == ""
assert len(reconstructor.tool_calls) == 3
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
@pytest.mark.parametrize("streaming", [False])
def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):
"""Test regex timeout is handled gracefully."""
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
default_tokenizer
)
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
# create a mock regex that raises TimeoutError
mock_regex = MagicMock()
mock_regex.match.side_effect = TimeoutError("Regex timeout")
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
content, tool_calls = run_tool_extraction(
tool_parser, fake_problematic_input, streaming=streaming
)
# should treat as regular text when regex times out
assert content == fake_problematic_input
assert len(tool_calls) == 0
mock_regex.match.assert_called_once()
File diff suppressed because it is too large Load Diff
@@ -1,112 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.tool_parsers.common_tests import (
ToolParserTestConfig,
ToolParserTests,
)
from vllm.tokenizers import TokenizerLike, get_tokenizer
class TestStep3ToolParser(ToolParserTests):
@pytest.fixture(scope="class")
def tokenizer(self) -> TokenizerLike:
return get_tokenizer("stepfun-ai/step3")
@pytest.fixture
def test_config(self) -> ToolParserTestConfig:
return ToolParserTestConfig(
parser_name="step3",
# Test data
no_tool_calls_output="This is a regular response without any tool calls.",
single_tool_call_output=(
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="get_weather">'
'<steptml:parameter name="city">Tokyo</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
),
parallel_tool_calls_output=(
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="get_weather">'
'<steptml:parameter name="city">Tokyo</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_sep|>"
'<|tool_call_begin|><steptml:invoke name="get_time">'
'<steptml:parameter name="timezone">Asia/Tokyo</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
),
various_data_types_output=(
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="test_function">'
'<steptml:parameter name="string_field">hello</steptml:parameter>'
'<steptml:parameter name="int_field">42</steptml:parameter>'
'<steptml:parameter name="float_field">3.14</steptml:parameter>'
'<steptml:parameter name="bool_field">true</steptml:parameter>'
'<steptml:parameter name="null_field">null</steptml:parameter>'
'<steptml:parameter name="array_field">'
'["a", "b", "c"]</steptml:parameter>'
'<steptml:parameter name="object_field">'
'{"nested": "value"}</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
),
empty_arguments_output=(
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="refresh"></steptml:invoke>'
"<|tool_call_end|><|tool_calls_end|>"
),
surrounding_text_output=(
"Let me check the weather for you.\n\n"
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="get_weather">'
'<steptml:parameter name="city">Tokyo</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>\n\n"
"I'll get that information."
),
escaped_strings_output=(
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="test_function">'
'<steptml:parameter name="quoted">He said "hello"</steptml:parameter>'
'<steptml:parameter name="path">C:\\Users\\file.txt</steptml:parameter>'
'<steptml:parameter name="newline">line1\nline2</steptml:parameter>'
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
),
malformed_input_outputs=[
(
"<|tool_calls_begin|><|tool_call_begin|>"
'<steptml:invoke name="func">'
),
(
'<|tool_call_begin|><steptml:invoke name="func">'
"</steptml:invoke><|tool_call_end|>"
),
],
# Expected results
single_tool_call_expected_name="get_weather",
single_tool_call_expected_args={"city": "Tokyo"},
parallel_tool_calls_count=2,
parallel_tool_calls_names=["get_weather", "get_time"],
# xfail markers
xfail_nonstreaming={
"test_single_tool_call_simple_args": (
"Step3 parser non-streaming has bugs"
),
"test_parallel_tool_calls": ("Step3 parser non-streaming has bugs"),
"test_various_data_types": "Step3 parser non-streaming has bugs",
"test_empty_arguments": "Step3 parser non-streaming has bugs",
"test_surrounding_text": "Step3 parser non-streaming has bugs",
"test_escaped_strings": "Step3 parser non-streaming has bugs",
},
xfail_streaming={
"test_parallel_tool_calls": (
"Step3 parser has significant bugs in both streaming "
"and non-streaming"
),
"test_streaming_reconstruction": (
"Step3 parser non-streaming has bugs, so streaming "
"doesn't match non-streaming"
),
},
supports_typed_arguments=False,
)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-557
View File
@@ -1,557 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Generator
import pytest
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
from vllm.tool_parsers.xlam_tool_parser import xLAMToolParser
# Use a common model that is likely to be available
MODEL = "Salesforce/Llama-xLAM-2-8B-fc-r"
@pytest.fixture(scope="module")
def xlam_tokenizer():
return get_tokenizer(tokenizer_name=MODEL)
@pytest.fixture
def xlam_tool_parser(xlam_tokenizer):
return xLAMToolParser(xlam_tokenizer)
def assert_tool_calls(
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
):
assert len(actual_tool_calls) == len(expected_tool_calls)
for actual_tool_call, expected_tool_call in zip(
actual_tool_calls, expected_tool_calls
):
assert isinstance(actual_tool_call.id, str)
assert len(actual_tool_call.id) > 16
assert actual_tool_call.type == "function"
assert actual_tool_call.function == expected_tool_call.function
def stream_delta_message_generator(
xlam_tool_parser: xLAMToolParser,
xlam_tokenizer: TokenizerLike,
model_output: str,
request: ChatCompletionRequest | None = None,
) -> Generator[DeltaMessage, None, None]:
all_token_ids = xlam_tokenizer.encode(model_output, add_special_tokens=False)
previous_text = ""
previous_tokens = None
prefix_offset = 0
read_offset = 0
for i, delta_token in enumerate(all_token_ids):
delta_token_ids = [delta_token]
previous_token_ids = all_token_ids[:i]
current_token_ids = all_token_ids[: i + 1]
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
detokenize_incrementally(
tokenizer=xlam_tokenizer,
all_input_ids=current_token_ids,
prev_tokens=previous_tokens,
prefix_offset=prefix_offset,
read_offset=read_offset,
skip_special_tokens=False,
spaces_between_special_tokens=True,
)
)
current_text = previous_text + delta_text
delta_message = xlam_tool_parser.extract_tool_calls_streaming(
previous_text,
current_text,
delta_text,
previous_token_ids,
current_token_ids,
delta_token_ids,
request=request,
)
if delta_message:
yield delta_message
previous_text = current_text
previous_tokens = (
previous_tokens + new_tokens if previous_tokens else new_tokens
)
prefix_offset = new_prefix_offset
read_offset = new_read_offset
def test_extract_tool_calls_no_tools(xlam_tool_parser):
model_output = "This is a test"
extracted_tool_calls = xlam_tool_parser.extract_tool_calls(
model_output, request=None
)
assert not extracted_tool_calls.tools_called
assert extracted_tool_calls.tool_calls == []
assert extracted_tool_calls.content == model_output
@pytest.mark.parametrize(
ids=[
"parallel_tool_calls",
"single_tool_with_think_tag",
"single_tool_with_json_code_block",
"single_tool_with_tool_calls_tag",
"single_tool_with_tool_call_xml_tags",
],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
(
"""[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}, {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
),
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Orlando",
"state": "FL",
"unit": "fahrenheit",
}
),
)
),
],
None,
),
(
"""<think>I'll help you with that.</think>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"<think>I'll help you with that.</think>",
),
(
"""I'll help you with that.\n```json\n[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\n```""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"I'll help you with that.",
),
(
"""I'll check the weather for you.[TOOL_CALLS][{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"I'll check the weather for you.",
),
(
"""I'll help you check the weather.<tool_call>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]</tool_call>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"I'll help you check the weather.",
),
],
)
def test_extract_tool_calls(
xlam_tool_parser, model_output, expected_tool_calls, expected_content
):
extracted_tool_calls = xlam_tool_parser.extract_tool_calls(
model_output, request=None
)
assert extracted_tool_calls.tools_called
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
assert extracted_tool_calls.content == expected_content
@pytest.mark.parametrize(
ids=["list_structured_tool_call"],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
(
"""[{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Seattle",
"state": "WA",
"unit": "celsius",
}
),
)
)
],
None,
),
],
)
def test_extract_tool_calls_list_structure(
xlam_tool_parser, model_output, expected_tool_calls, expected_content
):
"""Test extraction of tool calls when the model outputs a list-structured tool call.""" # noqa: E501
extracted_tool_calls = xlam_tool_parser.extract_tool_calls(
model_output, request=None
)
assert extracted_tool_calls.tools_called
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
assert extracted_tool_calls.content == expected_content
# Test for preprocess_model_output method
def test_preprocess_model_output(xlam_tool_parser):
# Test with list structure
model_output = (
"""[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]""" # noqa: E501
)
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
model_output
)
assert content is None
assert potential_tool_calls == model_output
# Test with thinking tag
model_output = """<think>I'll help you with that.</think>[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]""" # noqa: E501
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
model_output
)
assert content == "<think>I'll help you with that.</think>"
assert (
potential_tool_calls
== '[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]'
)
# Test with JSON code block
model_output = """I'll help you with that.
```json
[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]
```"""
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
model_output
)
assert content == "I'll help you with that."
assert "get_current_weather" in potential_tool_calls
# Test with no tool calls
model_output = """I'll help you with that."""
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
model_output
)
assert content == model_output
assert potential_tool_calls is None
# Simulate streaming to test extract_tool_calls_streaming
def test_streaming_with_list_structure(xlam_tool_parser):
# Reset streaming state
xlam_tool_parser.prev_tool_calls = []
xlam_tool_parser.current_tools_sent = []
xlam_tool_parser.streamed_args = []
xlam_tool_parser.current_tool_id = -1
# Simulate receiving a message with list structure
current_text = (
"""[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]""" # noqa: E501
)
# First call to set up the tool
xlam_tool_parser.extract_tool_calls_streaming(
previous_text="",
current_text=current_text,
delta_text="]",
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=None,
)
# Make sure the tool is set up correctly
assert xlam_tool_parser.current_tool_id >= 0, "Tool index should be initialized"
# Manually set up the state for sending the tool name
xlam_tool_parser.current_tools_sent = [False]
# Call to send the function name
result = xlam_tool_parser.extract_tool_calls_streaming(
previous_text=current_text,
current_text=current_text,
delta_text="",
previous_token_ids=[],
current_token_ids=[],
delta_token_ids=[],
request=None,
)
# Check that we get a result with the proper tool call
if result is not None:
assert hasattr(result, "tool_calls")
assert len(result.tool_calls) == 1
assert result.tool_calls[0].function.name == "get_current_weather"
@pytest.mark.parametrize(
ids=[
"parallel_tool_calls",
"single_tool_with_think_tag",
"single_tool_with_json_code_block",
"single_tool_with_tool_calls_tag",
"single_tool_with_tool_call_xml_tags",
],
argnames=["model_output", "expected_tool_calls", "expected_content"],
argvalues=[
(
"""[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}, {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
),
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Orlando",
"state": "FL",
"unit": "fahrenheit",
}
),
)
),
],
"",
),
(
"""<think>I'll help you with that.</think>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"<think>I'll help you with that.</think>",
),
(
"""```json\n[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\n```""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"",
),
(
"""[TOOL_CALLS][{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"",
),
(
"""I can help with that.<tool_call>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]</tool_call>""", # noqa: E501
[
ToolCall(
function=FunctionCall(
name="get_current_weather",
arguments=json.dumps(
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
}
),
)
)
],
"I can help with that.",
),
],
)
def test_extract_tool_calls_streaming_incremental(
xlam_tool_parser,
xlam_tokenizer,
model_output,
expected_tool_calls,
expected_content,
):
"""Verify the XLAM Parser streaming behavior by verifying each chunk is as expected.""" # noqa: E501
request = ChatCompletionRequest(model=MODEL, messages=[])
chunks = []
for delta_message in stream_delta_message_generator(
xlam_tool_parser, xlam_tokenizer, model_output, request
):
chunks.append(delta_message)
# Should have multiple chunks
assert len(chunks) >= 3
# Should have a chunk with tool header (id, name, type) for the first tool call # noqa: E501
header_found = False
expected_first_tool = expected_tool_calls[0]
for chunk in chunks:
if chunk.tool_calls and chunk.tool_calls[0].id:
header_found = True
assert (
chunk.tool_calls[0].function.name == expected_first_tool.function.name
)
assert chunk.tool_calls[0].type == "function"
# Arguments may be empty initially or None
if chunk.tool_calls[0].function.arguments is not None:
# If present, should be empty string initially
assert chunk.tool_calls[0].function.arguments == ""
break
assert header_found
# Should have chunks with incremental arguments
arg_chunks = []
for chunk in chunks:
if (
chunk.tool_calls
and chunk.tool_calls[0].function.arguments
and chunk.tool_calls[0].function.arguments != ""
and chunk.tool_calls[0].index
== 0 # Only collect arguments from the first tool call
):
arg_chunks.append(chunk.tool_calls[0].function.arguments)
# Arguments should be streamed incrementally
assert len(arg_chunks) > 1
# Concatenated arguments should form valid JSON for the first tool call
full_args = "".join(arg_chunks)
parsed_args = json.loads(full_args)
expected_args = json.loads(expected_first_tool.function.arguments)
assert parsed_args == expected_args
@pytest.mark.parametrize("streaming", [False, True])
def test_extract_tool_calls_non_ascii(xlam_tool_parser, xlam_tokenizer, streaming):
# Use parallel tool calls so the streaming path re-serializes arguments
# (the ensure_ascii fix only runs when tool_count > 1).
model_output = """[{"name": "get_current_weather", "arguments": {"city": "北京"}}, {"name": "get_current_weather", "arguments": {"city": "上海"}}]""" # noqa: E501
if streaming:
request = ChatCompletionRequest(model=MODEL, messages=[])
args = "".join(
delta.tool_calls[0].function.arguments
for delta in stream_delta_message_generator(
xlam_tool_parser, xlam_tokenizer, model_output, request
)
if delta.tool_calls and delta.tool_calls[0].function.arguments
)
else:
extracted = xlam_tool_parser.extract_tool_calls(model_output, request=None)
args = "".join(tc.function.arguments for tc in extracted.tool_calls)
assert "北京" in args
assert "\\u" not in args
-167
View File
@@ -1,167 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from vllm.entrypoints.generate.base.protocol import (
DeltaMessage,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers import ToolParser
class StreamingToolReconstructor:
def __init__(self, assert_one_tool_per_delta: bool = True):
self.tool_calls: list[ToolCall] = []
self.other_content: str = ""
self._assert_one_tool_per_delta = assert_one_tool_per_delta
def append_delta(self, delta: DeltaMessage):
if delta.content is not None:
self.other_content += delta.content
else:
assert delta.tool_calls, (
"Streaming results should have either content or tool calls (or both)"
)
if self._assert_one_tool_per_delta:
# Note: This isn't strictly required by the API and may not be
# possible to adhere to depending on the token space and number of
# tokens per streamed response from the model, but it is required
# by tool_use tests, so we enforce it here by default also.
assert len(delta.tool_calls) < 2, (
"Streaming should include only one tool call per update."
)
for call_delta in delta.tool_calls:
assert call_delta.type is None or call_delta.type == "function", (
"Streaming tool calls should only emit function calls. Got "
f"{call_delta.type}"
)
current_tool_call = (
self.tool_calls[call_delta.index]
if call_delta.index < len(self.tool_calls)
else None
)
if current_tool_call:
assert not call_delta.function.name, (
"Streaming tool calls should emit the full function name "
f"exactly once. Got {call_delta.function.name}"
)
assert not call_delta.id, (
"Streaming tool calls must emit function id only once. Got "
f"{call_delta.id}"
)
assert call_delta.index == len(self.tool_calls) - 1, (
f"Incorrect index for tool delta. Got {call_delta.index}, "
f"expected {len(self.tool_calls) - 1}"
)
current_tool_call.function.arguments += call_delta.function.arguments
else:
assert call_delta.id is not None, (
"Streaming tool calls must have an id on first appearance"
)
assert call_delta.function.name is not None, (
"Streaming tool calls must have a function name on first appearance"
)
assert call_delta.index == len(self.tool_calls), (
f"Incorrect index for tool delta. Got {call_delta.index}, "
f"expected {len(self.tool_calls)}"
)
self.tool_calls.append(
ToolCall(
id=call_delta.id,
function=FunctionCall(
name=call_delta.function.name,
arguments=call_delta.function.arguments or "",
),
)
)
def run_tool_extraction(
tool_parser: ToolParser,
model_output: str,
request: ChatCompletionRequest | None = None,
streaming: bool = False,
assert_one_tool_per_delta: bool = True,
) -> tuple[str | None, list[ToolCall]]:
if streaming:
reconstructor = run_tool_extraction_streaming(
tool_parser,
model_output,
request,
assert_one_tool_per_delta=assert_one_tool_per_delta,
)
return reconstructor.other_content or None, reconstructor.tool_calls
else:
extracted = run_tool_extraction_nonstreaming(tool_parser, model_output, request)
assert extracted.tools_called == bool(extracted.tool_calls)
return extracted.content, extracted.tool_calls
def run_tool_extraction_nonstreaming(
tool_parser: ToolParser,
model_output: str,
request: ChatCompletionRequest | None = None,
) -> ExtractedToolCallInformation:
request = request or ChatCompletionRequest(messages=[], model="test-model")
return tool_parser.extract_tool_calls(model_output, request)
def split_string_into_token_deltas(tokenizer: TokenizerLike, text: str) -> list[str]:
# Split a string into a series of deltas using the provided tokenizer. Each
# delta will be the string equivalent of a single token.
token_ids = tokenizer.encode(text, add_special_tokens=False)
previously_decoded_text = ""
deltas = []
for i in range(1, len(token_ids) + 1):
current_tokens = token_ids[:i]
current_text = tokenizer.decode(current_tokens)
new_text = current_text[len(previously_decoded_text) :]
previously_decoded_text = current_text
deltas.append(new_text)
return deltas
def run_tool_extraction_streaming(
tool_parser: ToolParser,
model_deltas: Iterable[str],
request: ChatCompletionRequest | None = None,
assert_one_tool_per_delta: bool = True,
) -> StreamingToolReconstructor:
if isinstance(model_deltas, str):
model_deltas = split_string_into_token_deltas(
tool_parser.model_tokenizer, model_deltas
)
request = request or ChatCompletionRequest(messages=[], model="test-model")
reconstructor = StreamingToolReconstructor(
assert_one_tool_per_delta=assert_one_tool_per_delta
)
previous_text = ""
previous_tokens: list[int] = []
for delta in model_deltas:
token_delta = [
tool_parser.vocab.get(token)
for token in tool_parser.model_tokenizer.tokenize(delta)
if token in tool_parser.vocab
]
current_text = previous_text + delta
current_tokens = previous_tokens + token_delta
delta_message = tool_parser.extract_tool_calls_streaming(
previous_text,
current_text,
delta,
previous_tokens,
current_tokens,
token_delta,
request,
)
if delta_message is not None:
reconstructor.append_delta(delta_message)
previous_text = current_text
previous_tokens = current_tokens
return reconstructor