[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,380 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from types import NoneType
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import run_tool_extraction
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolParserTestConfig:
|
||||
"""Configuration for a tool parser's common tests.
|
||||
|
||||
This dataclass contains all the test data and expected results needed
|
||||
to run the common test suite for a parser. Each parser test file
|
||||
creates one instance of this config with parser-specific values.
|
||||
|
||||
Attributes:
|
||||
parser_name: Name used with ToolParserManager (e.g., "mistral")
|
||||
|
||||
Test data (model outputs):
|
||||
no_tool_calls_output: Plain text without any tool syntax
|
||||
single_tool_call_output: One tool call with simple arguments
|
||||
parallel_tool_calls_output: Multiple tool calls in one response
|
||||
various_data_types_output: Tool with various data types
|
||||
empty_arguments_output: Tool call with no parameters
|
||||
surrounding_text_output: Tool call mixed with regular text
|
||||
escaped_strings_output: Tool call with escaped chars
|
||||
malformed_input_outputs: List of invalid inputs
|
||||
|
||||
Expected results:
|
||||
single_tool_call_expected_name: Expected function name
|
||||
single_tool_call_expected_args: Expected arguments dict
|
||||
parallel_tool_calls_count: Number of tools in parallel test
|
||||
parallel_tool_calls_names: Function names in order
|
||||
single_tool_call_expected_content: Content field when tool called
|
||||
parallel_tool_calls_expected_content: Content for parallel test
|
||||
|
||||
xfail markers:
|
||||
xfail_streaming: Mapping test name to xfail reason (streaming only)
|
||||
xfail_nonstreaming: Mapping test name to xfail reason (non-streaming)
|
||||
|
||||
Special flags:
|
||||
allow_empty_or_json_empty_args: True if "" or "{}" both valid for empty args
|
||||
supports_typed_arguments: True if the parser supports typed function arguments
|
||||
|
||||
"""
|
||||
|
||||
# Parser identification
|
||||
parser_name: str
|
||||
|
||||
# Test data - model outputs for each common test
|
||||
no_tool_calls_output: str
|
||||
single_tool_call_output: str
|
||||
parallel_tool_calls_output: str
|
||||
various_data_types_output: str
|
||||
empty_arguments_output: str
|
||||
surrounding_text_output: str
|
||||
escaped_strings_output: str
|
||||
malformed_input_outputs: list[str]
|
||||
|
||||
# Expected results for specific tests (optional overrides)
|
||||
single_tool_call_expected_name: str = "get_weather"
|
||||
single_tool_call_expected_args: dict[str, Any] = field(
|
||||
default_factory=lambda: {"city": "Tokyo"}
|
||||
)
|
||||
parallel_tool_calls_count: int = 2
|
||||
parallel_tool_calls_names: list[str] = field(
|
||||
default_factory=lambda: ["get_weather", "get_time"]
|
||||
)
|
||||
|
||||
# xfail configuration - maps test name to xfail reason
|
||||
xfail_streaming: dict[str, str] = field(default_factory=dict)
|
||||
xfail_nonstreaming: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
# Content expectations (some parsers strip content, others don't)
|
||||
single_tool_call_expected_content: str | None = None
|
||||
parallel_tool_calls_expected_content: str | None = None
|
||||
|
||||
# Special assertions for edge cases
|
||||
allow_empty_or_json_empty_args: bool = True # "{}" or "" for empty args
|
||||
supports_typed_arguments: bool = True
|
||||
|
||||
|
||||
class ToolParserTests:
|
||||
"""Mixin class providing common test suite for tool parsers.
|
||||
|
||||
To use this mixin in a parser test file:
|
||||
|
||||
1. Create a test_config fixture that returns a ToolParserTestConfig instance
|
||||
2. Inherit from this class
|
||||
3. Add parser-specific tests as additional methods
|
||||
|
||||
Example:
|
||||
class TestMistralToolParser(ToolParserTests):
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="mistral",
|
||||
no_tool_calls_output="Plain text...",
|
||||
# ... other config ...
|
||||
)
|
||||
|
||||
# Parser-specific tests
|
||||
def test_mistral_specific_feature(self, tool_parser):
|
||||
# Custom test logic
|
||||
pass
|
||||
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
"""Override this to provide parser-specific configuration."""
|
||||
raise NotImplementedError(
|
||||
"Subclass must provide test_config fixture returning ToolParserTestConfig"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""Override this to provide parser-specific tokenizer."""
|
||||
return default_tokenizer
|
||||
|
||||
@pytest.fixture
|
||||
def tool_parser(self, test_config: ToolParserTestConfig, tokenizer: TokenizerLike):
|
||||
return ToolParserManager.get_tool_parser(test_config.parser_name)(tokenizer)
|
||||
|
||||
@pytest.fixture(params=[True, False])
|
||||
def streaming(self, request: pytest.FixtureRequest) -> bool:
|
||||
return request.param
|
||||
|
||||
def test_no_tool_calls(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles plain text without tool syntax."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_no_tool_calls"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.no_tool_calls_output, streaming=streaming
|
||||
)
|
||||
assert content == test_config.no_tool_calls_output, (
|
||||
f"Expected content to match input, got {content}"
|
||||
)
|
||||
assert len(tool_calls) == 0, f"Expected no tool calls, got {len(tool_calls)}"
|
||||
|
||||
def test_single_tool_call_simple_args(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser extracts one tool with simple arguments."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_single_tool_call_simple_args"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.single_tool_call_output, streaming=streaming
|
||||
)
|
||||
|
||||
# Content check (some parsers strip it)
|
||||
if test_config.single_tool_call_expected_content is not None:
|
||||
assert content == test_config.single_tool_call_expected_content
|
||||
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
assert tool_calls[0].type == "function"
|
||||
assert tool_calls[0].function.name == test_config.single_tool_call_expected_name
|
||||
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
for key, value in test_config.single_tool_call_expected_args.items():
|
||||
assert args.get(key) == value, (
|
||||
f"Expected {key}={value}, got {args.get(key)}"
|
||||
)
|
||||
|
||||
def test_parallel_tool_calls(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles multiple tools in one response."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_parallel_tool_calls"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser,
|
||||
test_config.parallel_tool_calls_output,
|
||||
streaming=streaming,
|
||||
)
|
||||
|
||||
assert len(tool_calls) == test_config.parallel_tool_calls_count, (
|
||||
f"Expected {test_config.parallel_tool_calls_count} "
|
||||
f"tool calls, got {len(tool_calls)}"
|
||||
)
|
||||
|
||||
# Verify tool names match expected
|
||||
for i, expected_name in enumerate(test_config.parallel_tool_calls_names):
|
||||
assert tool_calls[i].type == "function"
|
||||
assert tool_calls[i].function.name == expected_name
|
||||
|
||||
# Verify unique IDs
|
||||
ids = [tc.id for tc in tool_calls]
|
||||
assert len(ids) == len(set(ids)), "Tool call IDs should be unique"
|
||||
|
||||
def test_various_data_types(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles all JSON types in arguments."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_various_data_types"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser,
|
||||
test_config.various_data_types_output,
|
||||
streaming=streaming,
|
||||
)
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
# Verify all expected fields present
|
||||
required_fields_types = {
|
||||
"string_field": str,
|
||||
"int_field": int,
|
||||
"float_field": float,
|
||||
"bool_field": bool,
|
||||
"null_field": NoneType,
|
||||
"array_field": list,
|
||||
"object_field": dict,
|
||||
}
|
||||
for required_field, expected_type in required_fields_types.items():
|
||||
assert required_field in args, (
|
||||
f"Expected field '{required_field}' in arguments"
|
||||
)
|
||||
if test_config.supports_typed_arguments:
|
||||
found_type = type(args[required_field])
|
||||
assert found_type is expected_type, (
|
||||
f"Expected field '{required_field}' to have type {expected_type}, "
|
||||
f"got {found_type}"
|
||||
)
|
||||
|
||||
def test_empty_arguments(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles parameterless tool calls."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_empty_arguments"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.empty_arguments_output, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
|
||||
args = tool_calls[0].function.arguments
|
||||
if test_config.allow_empty_or_json_empty_args:
|
||||
assert args in ["{}", ""], f"Expected empty args, got {args}"
|
||||
else:
|
||||
assert args == "{}", f"Expected {{}}, got {args}"
|
||||
|
||||
def test_surrounding_text(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser extracts tools from mixed content."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_surrounding_text"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.surrounding_text_output, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) >= 1, (
|
||||
f"Expected at least 1 tool call, got {len(tool_calls)}"
|
||||
)
|
||||
|
||||
def test_escaped_strings(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser handles escaped characters in arguments."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_escaped_strings"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, test_config.escaped_strings_output, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) == 1, f"Expected 1 tool call, got {len(tool_calls)}"
|
||||
|
||||
args = json.loads(tool_calls[0].function.arguments)
|
||||
# At minimum, verify we can parse and have expected fields
|
||||
# Exact escaping behavior varies by parser
|
||||
assert len(args) > 0, "Expected some arguments with escaped strings"
|
||||
|
||||
def test_malformed_input(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
streaming: bool,
|
||||
):
|
||||
"""Verify parser gracefully handles invalid syntax."""
|
||||
# Apply xfail markers if configured
|
||||
test_name = "test_malformed_input"
|
||||
self.apply_xfail_mark(request, test_config, test_name, streaming)
|
||||
|
||||
for malformed_input in test_config.malformed_input_outputs:
|
||||
# Should not raise exception
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, malformed_input, streaming=streaming
|
||||
)
|
||||
# Parser should handle gracefully (exact behavior varies)
|
||||
|
||||
def test_streaming_reconstruction(
|
||||
self,
|
||||
request: pytest.FixtureRequest,
|
||||
tool_parser: Any,
|
||||
test_config: ToolParserTestConfig,
|
||||
):
|
||||
"""Verify streaming produces same result as non-streaming."""
|
||||
test_name = "test_streaming_reconstruction"
|
||||
self.apply_xfail_mark(request, test_config, test_name, True)
|
||||
|
||||
test_output = test_config.single_tool_call_output
|
||||
|
||||
# Non-streaming result
|
||||
content_non, tools_non = run_tool_extraction(
|
||||
tool_parser, test_output, streaming=False
|
||||
)
|
||||
|
||||
# Streaming result
|
||||
content_stream, tools_stream = run_tool_extraction(
|
||||
tool_parser, test_output, streaming=True
|
||||
)
|
||||
|
||||
# Compare results
|
||||
assert content_non == content_stream, "Content should match between modes"
|
||||
assert len(tools_non) == len(tools_stream), "Tool count should match"
|
||||
if len(tools_non) > 0:
|
||||
assert tools_non[0].function.name == tools_stream[0].function.name
|
||||
assert tools_non[0].function.arguments == tools_stream[0].function.arguments
|
||||
|
||||
def apply_xfail_mark(self, request, test_config, test_name, streaming):
|
||||
reason = None
|
||||
if streaming and test_name in test_config.xfail_streaming:
|
||||
reason = test_config.xfail_streaming[test_name]
|
||||
elif not streaming and test_name in test_config.xfail_nonstreaming:
|
||||
reason = test_config.xfail_nonstreaming[test_name]
|
||||
if reason is not None:
|
||||
mark = pytest.mark.xfail(reason=reason, strict=True)
|
||||
request.node.add_marker(mark)
|
||||
@@ -1,449 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tool_parsers.apertus_tool_parser import (
|
||||
TOOL_CALLS_PREFIX,
|
||||
TOOL_CALLS_SUFFIX,
|
||||
ApertusToolParser,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.encode.return_value = [1, 2, 3]
|
||||
# Include the tool call tokens in the vocab for the parser
|
||||
tokenizer.get_vocab.return_value = {TOOL_CALLS_PREFIX: 100, TOOL_CALLS_SUFFIX: 101}
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return ApertusToolParser(mock_tokenizer)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request():
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = []
|
||||
request.tool_choice = "auto"
|
||||
return request
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming extraction tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractToolCalls:
|
||||
def test_no_tool_calls(self, parser, mock_request):
|
||||
model_output = "Hello, how can I help you today?"
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert result.tool_calls == []
|
||||
assert result.content == model_output
|
||||
|
||||
def test_single_tool_call(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "London"}
|
||||
|
||||
def test_multiple_arguments(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tools_prefix|>[{"get_weather": '
|
||||
'{"location": "San Francisco", '
|
||||
'"unit": "celsius"}}]<|tools_suffix|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "San Francisco", "unit": "celsius"}
|
||||
|
||||
def test_text_before_tool_call(self, parser, mock_request):
|
||||
model_output = (
|
||||
"Let me check the weather for you. "
|
||||
'<|tools_prefix|>[{"get_weather": {"location": "Paris"}}]<|tools_suffix|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.content == "Let me check the weather for you."
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
def test_multiple_tool_calls(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tools_prefix|>[{"get_weather": '
|
||||
'{"location": "London"}}, '
|
||||
'{"get_time": {"location": "London"}}]<|tools_suffix|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert result.tool_calls[1].function.name == "get_time"
|
||||
|
||||
def test_nested_arguments(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tools_prefix|>[{"complex_function": '
|
||||
'{"nested": {"inner": "value"}, '
|
||||
'"list": ["a", "b"]}}]<|tools_suffix|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "complex_function"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]}
|
||||
|
||||
def test_incomplete_tool_call(self, parser, mock_request):
|
||||
model_output = '<|tools_prefix|>[{"get_weather": {"location": "London"}'
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "London"}
|
||||
|
||||
def test_missing_tool_suffix(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tools_prefix|>[{"get_weather": '
|
||||
'{"location": "San Francisco", "unit": "celsius"}}]'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "San Francisco", "unit": "celsius"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming extraction tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStreamingExtraction:
|
||||
def _simulate_streaming(
|
||||
self, parser: ApertusToolParser, mock_request: Any, chunks: list[str]
|
||||
) -> list[tuple[Any, str]]:
|
||||
results: list[tuple[Any, str]] = []
|
||||
previous_text: str = ""
|
||||
previous_token_ids: list[int] = []
|
||||
|
||||
for chunk in chunks:
|
||||
current_text = previous_text + chunk
|
||||
# Simulate a token ID sequence matching the chunk progression
|
||||
delta_token_ids: list[int] = [0] * max(1, len(chunk) // 4)
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=tuple(previous_token_ids),
|
||||
current_token_ids=tuple(current_token_ids),
|
||||
delta_token_ids=tuple(delta_token_ids),
|
||||
request=mock_request,
|
||||
)
|
||||
results.append((delta, current_text))
|
||||
previous_text = current_text
|
||||
previous_token_ids = list(current_token_ids)
|
||||
|
||||
return results
|
||||
|
||||
def _collect_tool_calls(self, results) -> dict[int, dict[str, Any]]:
|
||||
"""Properly tracks and concatenates streamed tool arguments by their Index."""
|
||||
tool_calls = {}
|
||||
for delta, _ in results:
|
||||
if not delta or not getattr(delta, "tool_calls", None):
|
||||
continue
|
||||
|
||||
for tc in delta.tool_calls:
|
||||
idx = (
|
||||
tc.get("index", 0)
|
||||
if isinstance(tc, dict)
|
||||
else getattr(tc, "index", 0)
|
||||
)
|
||||
func = (
|
||||
tc.get("function", {})
|
||||
if isinstance(tc, dict)
|
||||
else getattr(tc, "function", None)
|
||||
)
|
||||
if not func:
|
||||
continue
|
||||
|
||||
name = (
|
||||
func.get("name")
|
||||
if isinstance(func, dict)
|
||||
else getattr(func, "name", None)
|
||||
)
|
||||
args = (
|
||||
func.get("arguments")
|
||||
if isinstance(func, dict)
|
||||
else getattr(func, "arguments", None)
|
||||
)
|
||||
|
||||
if idx not in tool_calls:
|
||||
tool_calls[idx] = {"name": "", "arguments": ""}
|
||||
|
||||
if name:
|
||||
tool_calls[idx]["name"] += name
|
||||
if args:
|
||||
tool_calls[idx]["arguments"] += args
|
||||
|
||||
return tool_calls
|
||||
|
||||
def _collect_content(self, results) -> str:
|
||||
"""Collects generated normal text outside of the tool calls."""
|
||||
return "".join(
|
||||
delta.content
|
||||
for delta, _ in results
|
||||
if delta and getattr(delta, "content", None)
|
||||
)
|
||||
|
||||
def test_basic_streaming_single_tool(self, parser, mock_request):
|
||||
chunks = [
|
||||
"<|tools_prefix|>",
|
||||
'[{"get_weather": ',
|
||||
'{"location": "Paris, ',
|
||||
'France"}}]',
|
||||
"<|tools_suffix|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
tcs = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tcs) == 1
|
||||
assert tcs[0]["name"] == "get_weather"
|
||||
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments",
|
||||
['{"value": -2e-3}', '{"value": 1.5e3}', '{"values": [1.5e3, -2e-3]}'],
|
||||
)
|
||||
def test_streaming_scientific_notation(self, parser, mock_request, arguments):
|
||||
model_output = (
|
||||
f'{TOOL_CALLS_PREFIX}[{{"calculate": {arguments}}}]{TOOL_CALLS_SUFFIX}'
|
||||
)
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, list(model_output))
|
||||
tool_calls = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["name"] == "calculate"
|
||||
assert json.loads(tool_calls[0]["arguments"]) == json.loads(arguments)
|
||||
|
||||
def test_streaming_missing_tool_suffix(self, parser, mock_request):
|
||||
chunks = [
|
||||
"<|tools_prefix|>",
|
||||
'[{"get_weather": ',
|
||||
'{"location": "Paris, ',
|
||||
'France"}}]',
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
tcs = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tcs) == 1
|
||||
assert tcs[0]["name"] == "get_weather"
|
||||
assert json.loads(tcs[0]["arguments"]) == {"location": "Paris, France"}
|
||||
|
||||
def test_streaming_partial_tag_buffering_missing_tool_suffix(
|
||||
self, parser, mock_request
|
||||
):
|
||||
chunks = ["Content", "<|tools_", "prefix|>", '[{"f": ', '{"a": 1}}]']
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
content = self._collect_content(results)
|
||||
|
||||
assert "Content" in content
|
||||
assert "<|tools_prefix|>" not in content
|
||||
assert "<|tools_suffix|>" not in content
|
||||
|
||||
tcs = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tcs) == 1
|
||||
assert tcs[0]["name"] == "f"
|
||||
assert json.loads(tcs[0]["arguments"]) == {"a": 1}
|
||||
|
||||
def test_streaming_multi_tool(self, parser, mock_request):
|
||||
chunks = [
|
||||
"<|tools_prefix|>",
|
||||
'[{"get_weather": {"location": "Tokyo"}}',
|
||||
', {"get_time": {"location": "Tokyo"}}]',
|
||||
"<|tools_suffix|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
tcs = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tcs) == 2
|
||||
assert tcs[0]["name"] == "get_weather"
|
||||
assert json.loads(tcs[0]["arguments"]) == {"location": "Tokyo"}
|
||||
assert tcs[1]["name"] == "get_time"
|
||||
assert json.loads(tcs[1]["arguments"]) == {"location": "Tokyo"}
|
||||
|
||||
def test_streaming_text_before_tool_call(self, parser, mock_request):
|
||||
chunks = [
|
||||
"Let me check ",
|
||||
"the weather. ",
|
||||
"<|tools_prefix|>",
|
||||
'[{"get_weather": {"location": "London"}}]',
|
||||
"<|tools_suffix|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
content = self._collect_content(results)
|
||||
|
||||
assert content.strip() == "Let me check the weather."
|
||||
tcs = self._collect_tool_calls(results)
|
||||
|
||||
assert len(tcs) == 1
|
||||
assert tcs[0]["name"] == "get_weather"
|
||||
assert json.loads(tcs[0]["arguments"]) == {"location": "London"}
|
||||
|
||||
def test_streaming_partial_tag_buffering(self, parser, mock_request):
|
||||
chunks = [
|
||||
"Content",
|
||||
"<|tools_",
|
||||
"prefix|>",
|
||||
'[{"f": {"a": 1}}]',
|
||||
"<|tools_suf",
|
||||
"fix|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
content = self._collect_content(results)
|
||||
|
||||
assert "Content" in content
|
||||
assert "<|tools_prefix|>" not in content
|
||||
assert "<|tools_suffix|>" not in content
|
||||
|
||||
tc = self._collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert tc[0]["name"] == "f"
|
||||
assert json.loads(tc[0]["arguments"]) == {"a": 1}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge Cases: Multi-Token Prediction (MTP) & vLLM Chunking Anomalies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def test_mtp_streaming_massive_chunk(self, parser, mock_request):
|
||||
"""Simulates MTP predicting text, tool calls,
|
||||
and trailing text all in a single chunk."""
|
||||
chunks = [
|
||||
"Sure! "
|
||||
'<|tools_prefix|>[{"get_weather": {"location": "London"}}]<|tools_suffix|>'
|
||||
]
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
content = self._collect_content(results)
|
||||
assert "Sure! " in content
|
||||
|
||||
tc = self._collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert tc[0]["name"] == "get_weather"
|
||||
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
|
||||
|
||||
def test_mtp_streaming_multiple_tools_burst(self, parser, mock_request):
|
||||
"""Simulates MTP predicting an array of multiple tools in one single chunk."""
|
||||
chunks = [
|
||||
'<|tools_prefix|>[{"get_weather": '
|
||||
'{"location": "London"}}, '
|
||||
'{"get_time": {"location": "Paris"}}]<|tools_suffix|>'
|
||||
]
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
tc = self._collect_tool_calls(results)
|
||||
assert len(tc) == 2
|
||||
assert tc[0]["name"] == "get_weather"
|
||||
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
|
||||
assert tc[1]["name"] == "get_time"
|
||||
assert json.loads(tc[1]["arguments"]) == {"location": "Paris"}
|
||||
|
||||
def test_mtp_streaming_skip_and_catch_up(self, parser, mock_request):
|
||||
"""Simulates MTP chunks that jump over entire tools
|
||||
(e.g., from middle of tool 1 to middle of tool 3)."""
|
||||
chunks = [
|
||||
'<|tools_prefix|>[{"t1": {"a": 1}',
|
||||
'}, {"t2": {"b": 2}}, {"t3": {"c": 3',
|
||||
"}}]<|tools_suffix|>",
|
||||
]
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
tc = self._collect_tool_calls(results)
|
||||
assert len(tc) == 3
|
||||
assert tc[0]["name"] == "t1"
|
||||
assert json.loads(tc[0]["arguments"]) == {"a": 1}
|
||||
assert tc[1]["name"] == "t2"
|
||||
assert json.loads(tc[1]["arguments"]) == {"b": 2}
|
||||
assert tc[2]["name"] == "t3"
|
||||
assert json.loads(tc[2]["arguments"]) == {"c": 3}
|
||||
|
||||
def test_vllm_streaming_character_by_character(self, parser, mock_request):
|
||||
"""Simulates worst-case vLLM fragmentation where
|
||||
chunks arrive character-by-character."""
|
||||
text = (
|
||||
'Hi <|tools_prefix|>[{"get_weather": '
|
||||
'{"location": "London"}}]<|tools_suffix|> '
|
||||
)
|
||||
chunks = list(text)
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
content = self._collect_content(results)
|
||||
assert "Hi" in content
|
||||
|
||||
tc = self._collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert tc[0]["name"] == "get_weather"
|
||||
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
|
||||
|
||||
def test_vllm_streaming_empty_deltas(self, parser, mock_request):
|
||||
"""Simulates vLLM stream producing empty string chunks
|
||||
(e.g., hidden tokens or artifacts)."""
|
||||
chunks = [
|
||||
"Wait",
|
||||
"",
|
||||
"<|tools_prefix|>",
|
||||
"",
|
||||
'[{"get_weather": ',
|
||||
"",
|
||||
'{"location": "London"}}]',
|
||||
"<|tools_suffix|>",
|
||||
]
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
content = self._collect_content(results)
|
||||
assert content == "Wait"
|
||||
|
||||
tc = self._collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert tc[0]["name"] == "get_weather"
|
||||
assert json.loads(tc[0]["arguments"]) == {"location": "London"}
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.deepseekv31_tool_parser import (
|
||||
DeepSeekV31ToolParser,
|
||||
)
|
||||
|
||||
MODEL = "deepseek-ai/DeepSeek-V3.1"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def deepseekv31_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(deepseekv31_tokenizer):
|
||||
return DeepSeekV31ToolParser(deepseekv31_tokenizer)
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_tool(parser):
|
||||
model_output = (
|
||||
"normal text"
|
||||
"<|tool▁calls▁begin|>"
|
||||
'<|tool▁call▁begin|>foo<|tool▁sep|>{"x":1}<|tool▁call▁end|>'
|
||||
"<|tool▁calls▁end|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "foo"
|
||||
assert result.tool_calls[0].function.arguments == '{"x":1}'
|
||||
assert result.content == "normal text"
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_multiple_tools(parser):
|
||||
model_output = (
|
||||
"some prefix text"
|
||||
"<|tool▁calls▁begin|>"
|
||||
'<|tool▁call▁begin|>foo<|tool▁sep|>{"x":1}<|tool▁call▁end|>'
|
||||
'<|tool▁call▁begin|>bar<|tool▁sep|>{"y":2}<|tool▁call▁end|>'
|
||||
"<|tool▁calls▁end|>"
|
||||
" some suffix text"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 2
|
||||
|
||||
assert result.tool_calls[0].function.name == "foo"
|
||||
assert result.tool_calls[0].function.arguments == '{"x":1}'
|
||||
|
||||
assert result.tool_calls[1].function.name == "bar"
|
||||
assert result.tool_calls[1].function.arguments == '{"y":2}'
|
||||
|
||||
# prefix is content
|
||||
assert result.content == "some prefix text"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
|
||||
|
||||
class TestDeepSeekV3ToolParser(ToolParserTests):
|
||||
@pytest.fixture(scope="class")
|
||||
def tokenizer(self) -> TokenizerLike:
|
||||
return get_tokenizer("deepseek-ai/DeepSeek-V3")
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="deepseek_v3",
|
||||
# Test data
|
||||
no_tool_calls_output=(
|
||||
"How can I help you today? I can check weather for you."
|
||||
),
|
||||
single_tool_call_output="""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
|
||||
```json
|
||||
{"city": "Tokyo", "unit": "celsius"}
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>""",
|
||||
parallel_tool_calls_output="""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
|
||||
```json
|
||||
{"city": "Tokyo", "unit": "celsius"}
|
||||
```<|tool▁call▁end|><|tool▁call▁begin|>function<|tool▁sep|>search_hotels
|
||||
```json
|
||||
{"location": "Tokyo", "check_in": "2025-01-15"}
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>""",
|
||||
various_data_types_output=(
|
||||
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>test_function
|
||||
```json
|
||||
"""
|
||||
"""{"string_field": "hello", "int_field": 42, "float_field": 3.14, """
|
||||
""""bool_field": true, "null_field": null, """
|
||||
""""array_field": ["a", "b", "c"], """
|
||||
""""object_field": {"nested": "value"}, """
|
||||
""""empty_array": [], "empty_object": {}}
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>"""
|
||||
),
|
||||
empty_arguments_output="""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_current_time
|
||||
```json
|
||||
{}
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>""",
|
||||
surrounding_text_output=(
|
||||
"""Let me check the weather for you."""
|
||||
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
|
||||
```json
|
||||
{"city": "Paris"}
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>"""
|
||||
),
|
||||
escaped_strings_output=(
|
||||
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>send_message
|
||||
```json
|
||||
"""
|
||||
"""{"text": "He said \\"hello\\"", "path": "C:\\\\Users\\\\file", """
|
||||
""""newline": "line1\\nline2"}
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>"""
|
||||
),
|
||||
malformed_input_outputs=[
|
||||
"""<|tool▁calls▁begin|><|tool▁call▁begin|>function<|tool▁sep|>get_weather
|
||||
```json
|
||||
{"city": "Tokyo"
|
||||
```<|tool▁call▁end|><|tool▁calls▁end|>""",
|
||||
"""<|tool▁calls▁begin|>function<|tool▁sep|>get_weather
|
||||
```json
|
||||
{"city": "Tokyo"}
|
||||
```<|tool▁calls▁end|>""",
|
||||
],
|
||||
# Expected results
|
||||
single_tool_call_expected_name="get_weather",
|
||||
single_tool_call_expected_args={"city": "Tokyo", "unit": "celsius"},
|
||||
single_tool_call_expected_content=None,
|
||||
parallel_tool_calls_count=2,
|
||||
parallel_tool_calls_names=["get_weather", "search_hotels"],
|
||||
# xfail markers
|
||||
xfail_streaming={},
|
||||
xfail_nonstreaming={
|
||||
"test_malformed_input": (
|
||||
"Parser sets tools_called=True even when tool_calls is "
|
||||
"empty (detects start token but fails to parse)"
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -1,447 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Unit tests for DeepSeekV4EngineToolParser."""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from xgrammar import StructuralTag
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedFunction,
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.deepseekv4_engine_tool_parser import (
|
||||
DeepSeekV4EngineToolParser,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
MOCK_TOKENIZER = MagicMock()
|
||||
MOCK_TOKENIZER.get_vocab.return_value = {}
|
||||
|
||||
TC_START = "<|DSML|tool_calls>"
|
||||
TC_END = "</|DSML|tool_calls>"
|
||||
INV_START = '<|DSML|invoke name="'
|
||||
INV_END = "</|DSML|invoke>"
|
||||
PARAM_START = '<|DSML|parameter name="'
|
||||
PARAM_END = "</|DSML|parameter>"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_tools() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string", "description": "The city name"},
|
||||
"state": {"type": "string", "description": "The state code"},
|
||||
"unit": {"type": "string", "enum": ["fahrenheit", "celsius"]},
|
||||
},
|
||||
"required": ["city", "state"],
|
||||
},
|
||||
},
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "calculate_area",
|
||||
"description": "Calculate area of a shape",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"shape": {"type": "string"},
|
||||
"dimensions": {"type": "object"},
|
||||
"precision": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def make_parser(tools=None) -> DeepSeekV4EngineToolParser:
|
||||
return DeepSeekV4EngineToolParser(MOCK_TOKENIZER, tools=tools)
|
||||
|
||||
|
||||
def make_request(tools=None) -> MagicMock:
|
||||
req = MagicMock()
|
||||
req.tools = tools
|
||||
return req
|
||||
|
||||
|
||||
def build_tool_call(func_name: str, params: dict[str, str]) -> str:
|
||||
param_strs = "".join(
|
||||
f'{PARAM_START}{k}" string="true">{v}{PARAM_END}\n' for k, v in params.items()
|
||||
)
|
||||
return f'{TC_START}\n{INV_START}{func_name}">\n{param_strs}{INV_END}\n{TC_END}'
|
||||
|
||||
|
||||
def stream(parser: DeepSeekV4EngineToolParser, full_text: str, chunk_size: int = 7):
|
||||
deltas = []
|
||||
previous_text = ""
|
||||
for start in range(0, len(full_text), chunk_size):
|
||||
delta_text = full_text[start : start + chunk_size]
|
||||
current_text = previous_text + delta_text
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[1],
|
||||
request=make_request(),
|
||||
)
|
||||
previous_text = current_text
|
||||
if delta is not None:
|
||||
deltas.append(delta)
|
||||
return deltas
|
||||
|
||||
|
||||
def reconstruct_args(deltas, tool_index: int = 0) -> str:
|
||||
fragments = []
|
||||
for delta in deltas:
|
||||
if delta.tool_calls:
|
||||
for tool_call in delta.tool_calls:
|
||||
if (
|
||||
tool_call.index == tool_index
|
||||
and tool_call.function
|
||||
and tool_call.function.arguments
|
||||
):
|
||||
fragments.append(tool_call.function.arguments)
|
||||
return "".join(fragments)
|
||||
|
||||
|
||||
def test_registered():
|
||||
assert (
|
||||
ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4EngineToolParser
|
||||
)
|
||||
|
||||
|
||||
def test_extract_tool_calls():
|
||||
parser = make_parser()
|
||||
model_output = "Let me check. " + build_tool_call(
|
||||
"get_weather", {"location": "Beijing", "unit": "celsius"}
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, make_request())
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content == "Let me check. "
|
||||
assert len(result.tool_calls) == 1
|
||||
tool_call = result.tool_calls[0]
|
||||
assert tool_call.function.name == "get_weather"
|
||||
assert json.loads(tool_call.function.arguments) == {
|
||||
"location": "Beijing",
|
||||
"unit": "celsius",
|
||||
}
|
||||
|
||||
|
||||
def test_function_calls_wrapper_is_not_recognized():
|
||||
# The V3.2 wrapper is not a V4 terminal, so it passes through as content,
|
||||
# but the invoke inside it is still parsed (tool calls anchor on the invoke).
|
||||
parser = make_parser()
|
||||
model_output = build_tool_call("search", {"query": "vllm"}).replace(
|
||||
"tool_calls", "function_calls"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, make_request())
|
||||
|
||||
assert result.tools_called
|
||||
assert result.tool_calls[0].function.name == "search"
|
||||
assert result.content == "<|DSML|function_calls>\n"
|
||||
|
||||
|
||||
def test_missing_tool_calls_wrapper_is_recovered():
|
||||
# Regression for #48931: at long context the model omits the
|
||||
# <|DSML|tool_calls> START token but still emits a complete invoke.
|
||||
parser = make_parser()
|
||||
model_output = build_tool_call("search", {"query": "vllm"}).replace(
|
||||
TC_START + "\n", ""
|
||||
)
|
||||
assert TC_START not in model_output
|
||||
|
||||
result = parser.extract_tool_calls(model_output, make_request())
|
||||
|
||||
assert result.tools_called
|
||||
assert result.tool_calls[0].function.name == "search"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {"query": "vllm"}
|
||||
assert result.content is None
|
||||
|
||||
deltas = stream(make_parser(), model_output, chunk_size=3)
|
||||
names = [
|
||||
tc.function.name
|
||||
for d in deltas
|
||||
if d.tool_calls
|
||||
for tc in d.tool_calls
|
||||
if tc.function.name
|
||||
]
|
||||
assert names == ["search"]
|
||||
assert json.loads(reconstruct_args(deltas)) == {"query": "vllm"}
|
||||
assert not any(d.content and "DSML" in d.content for d in deltas)
|
||||
|
||||
|
||||
def test_streaming_extracts_complete_invokes():
|
||||
parser = make_parser()
|
||||
full_text = build_tool_call("search", {"query": "deepseek v4"})
|
||||
|
||||
deltas = stream(parser, full_text, chunk_size=5)
|
||||
|
||||
names = [
|
||||
tool_call.function.name
|
||||
for delta in deltas
|
||||
if delta.tool_calls
|
||||
for tool_call in delta.tool_calls
|
||||
if tool_call.function.name
|
||||
]
|
||||
assert names == ["search"]
|
||||
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
|
||||
|
||||
|
||||
def test_streaming_emits_incremental_argument_chunks():
|
||||
tool = ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="plan_trip",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"days": {"type": "integer"},
|
||||
"flexible": {"type": "boolean"},
|
||||
"cities": {"type": "array", "items": {"type": "string"}},
|
||||
"notes": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
full_text = (
|
||||
f"{TC_START}\n"
|
||||
f'{INV_START}plan_trip">\n'
|
||||
f'{PARAM_START}days" string="false">3{PARAM_END}\n'
|
||||
f'{PARAM_START}flexible" string="false">false{PARAM_END}\n'
|
||||
f'{PARAM_START}cities" string="false">'
|
||||
f'["Beijing","Shanghai","Tokyo","New York"]{PARAM_END}\n'
|
||||
f'{PARAM_START}notes" string="true">靠窗座位{PARAM_END}\n'
|
||||
f"{INV_END}\n"
|
||||
f"{TC_END}"
|
||||
)
|
||||
|
||||
deltas = stream(parser, full_text, chunk_size=4)
|
||||
arg_chunks = [
|
||||
tool_call.function.arguments
|
||||
for delta in deltas
|
||||
for tool_call in delta.tool_calls or []
|
||||
if tool_call.function and tool_call.function.arguments is not None
|
||||
]
|
||||
|
||||
assert len([chunk for chunk in arg_chunks if chunk]) > 2
|
||||
assert json.loads("".join(arg_chunks)) == {
|
||||
"days": 3,
|
||||
"flexible": False,
|
||||
"cities": ["Beijing", "Shanghai", "Tokyo", "New York"],
|
||||
"notes": "靠窗座位",
|
||||
}
|
||||
|
||||
|
||||
def _with_strict(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
) -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
type=t.type,
|
||||
function=FunctionDefinition(
|
||||
name=t.function.name,
|
||||
description=t.function.description,
|
||||
parameters=t.function.parameters,
|
||||
strict=True,
|
||||
),
|
||||
)
|
||||
for t in tools
|
||||
]
|
||||
|
||||
|
||||
def test_get_vllm_registry_structural_tag_returns_structural_tag(
|
||||
sample_tools: list[ChatCompletionToolsParam],
|
||||
) -> None:
|
||||
parser = make_parser()
|
||||
strict_tools = _with_strict(sample_tools)
|
||||
req = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=strict_tools,
|
||||
tool_choice="auto",
|
||||
)
|
||||
tag = parser.get_structural_tag(req)
|
||||
assert isinstance(tag, StructuralTag)
|
||||
|
||||
req = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=sample_tools,
|
||||
tool_choice="required",
|
||||
)
|
||||
tag = parser.get_structural_tag(req)
|
||||
assert isinstance(tag, StructuralTag)
|
||||
|
||||
if sample_tools:
|
||||
tool = sample_tools[0]
|
||||
req = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="m",
|
||||
tools=sample_tools,
|
||||
)
|
||||
req.tool_choice = ChatCompletionNamedToolChoiceParam(
|
||||
function=ChatCompletionNamedFunction(name=tool.function.name)
|
||||
)
|
||||
tag = parser.get_structural_tag(req)
|
||||
assert isinstance(tag, StructuralTag)
|
||||
|
||||
|
||||
def test_extract_tool_calls_arguments_wrapper():
|
||||
mock_tokenizer = MagicMock()
|
||||
mock_tokenizer.get_vocab.return_value = {}
|
||||
|
||||
tool = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
parser = DeepSeekV4EngineToolParser(mock_tokenizer, tools=[tool])
|
||||
request = MagicMock()
|
||||
request.tools = [tool]
|
||||
|
||||
model_output = (
|
||||
f"{TC_START}"
|
||||
f'{INV_START}get_weather">'
|
||||
f'{PARAM_START}arguments" string="false">{{"location":"Beijing"}}{PARAM_END}'
|
||||
f"{INV_END}"
|
||||
f"{TC_END}"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
assert result.tools_called
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "Beijing"}
|
||||
|
||||
|
||||
_ANGLE_BRACKET_TOOL = ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="run_command",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"command": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tools",
|
||||
[[_ANGLE_BRACKET_TOOL], None],
|
||||
ids=["with_tools", "without_tools"],
|
||||
)
|
||||
def test_no_dsml_closing_tag_leak_in_streamed_args(tools):
|
||||
"""Streaming must not leak </|DSML|parameter> into argument values.
|
||||
|
||||
When a parameter value contains '>' (e.g. shell redirects like
|
||||
'2>&1'), certain chunk boundaries cause the incremental lexer to
|
||||
emit the closing delimiter text as part of the content token. The
|
||||
partial regex then captures it as part of the value, violating the
|
||||
prefix invariant and corrupting the streamed JSON.
|
||||
"""
|
||||
full_text = build_tool_call("run_command", {"command": "git --version 2>&1"})
|
||||
expected = {"command": "git --version 2>&1"}
|
||||
|
||||
for chunk_size in range(1, len(full_text) + 1):
|
||||
parser = make_parser(tools=tools)
|
||||
deltas = stream(parser, full_text, chunk_size=chunk_size)
|
||||
args_str = reconstruct_args(deltas)
|
||||
assert args_str, f"No args emitted at chunk_size={chunk_size}"
|
||||
assert "DSML" not in args_str, (
|
||||
f"DSML marker leaked into args at chunk_size={chunk_size}: {args_str!r}"
|
||||
)
|
||||
parsed = json.loads(args_str)
|
||||
assert parsed == expected, (
|
||||
f"Args mismatch at chunk_size={chunk_size}: "
|
||||
f"got {parsed!r}, expected {expected!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_non_streaming_extract_with_angle_brackets():
|
||||
"""Non-streaming extraction must correctly handle '>' in values."""
|
||||
parser = make_parser()
|
||||
full_text = build_tool_call("run_command", {"command": "git --version 2>&1"})
|
||||
result = parser.extract_tool_calls(full_text, make_request())
|
||||
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 1
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"command": "git --version 2>&1"}
|
||||
assert "DSML" not in result.tool_calls[0].function.arguments
|
||||
|
||||
|
||||
def test_composed_schema_converts_object_and_array_params():
|
||||
tool = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "set_timer",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"wait": {
|
||||
"anyOf": [
|
||||
{"type": "object"},
|
||||
{"type": "null"},
|
||||
],
|
||||
},
|
||||
"patches": {
|
||||
"allOf": [
|
||||
{"type": "array", "items": {"type": "object"}},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
parser = make_parser(tools=[tool])
|
||||
request = make_request(tools=[tool])
|
||||
model_output = (
|
||||
f"{TC_START}\n"
|
||||
f'{INV_START}set_timer">\n'
|
||||
f'{PARAM_START}wait" string="false">'
|
||||
f'{{"type":"for","minutes":2880}}'
|
||||
f"{PARAM_END}\n"
|
||||
f'{PARAM_START}patches" string="false">'
|
||||
f'[{{"op":"replace","path":"/schedule","value":"quiet"}}]'
|
||||
f"{PARAM_END}\n"
|
||||
f"{INV_END}\n"
|
||||
f"{TC_END}"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
|
||||
assert result.tools_called
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {
|
||||
"wait": {"type": "for", "minutes": 2880},
|
||||
"patches": [{"op": "replace", "path": "/schedule", "value": "quiet"}],
|
||||
}
|
||||
@@ -1,275 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.dots_tool_parser import DotsToolParser
|
||||
|
||||
|
||||
def _tool(name: str, properties: dict) -> ChatCompletionToolsParam:
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"description": "test tool",
|
||||
"parameters": {"type": "object", "properties": properties},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
)
|
||||
|
||||
|
||||
def _stream(
|
||||
parser: DotsToolParser,
|
||||
chunks: Iterable[str],
|
||||
request: ChatCompletionRequest,
|
||||
) -> list[DeltaMessage]:
|
||||
previous_text = ""
|
||||
messages: list[DeltaMessage] = []
|
||||
for chunk in chunks:
|
||||
current_text = previous_text + chunk
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
chunk,
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
request,
|
||||
)
|
||||
if delta is not None:
|
||||
messages.append(delta)
|
||||
previous_text = current_text
|
||||
return messages
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser() -> DotsToolParser:
|
||||
return DotsToolParser(MagicMock())
|
||||
|
||||
|
||||
def test_registered_in_tool_parser_manager() -> None:
|
||||
assert ToolParserManager.get_tool_parser("dots") is DotsToolParser
|
||||
|
||||
|
||||
def test_non_stream_xml_converts_schema_types_and_resolves_ref(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
tool = ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": "set_location",
|
||||
"description": "Set location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"$ref": "#/$defs/Location"},
|
||||
"days": {"type": "integer"},
|
||||
"include_weather": {"type": "boolean"},
|
||||
},
|
||||
"$defs": {
|
||||
"Location": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
request = _request([tool])
|
||||
text = (
|
||||
"ok<dots_function_call>"
|
||||
'<invoke name="set_location">'
|
||||
'<parameter name="location">{"city": "Shanghai"}</parameter>'
|
||||
'<parameter name="days">3</parameter>'
|
||||
'<parameter name="include_weather">true</parameter>'
|
||||
"</invoke>"
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert result.content == "ok"
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "set_location"
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"location": {"city": "Shanghai"},
|
||||
"days": 3,
|
||||
"include_weather": True,
|
||||
}
|
||||
|
||||
|
||||
def test_non_stream_supports_multiple_invokes_and_json_fallback(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
tools = [
|
||||
_tool("search", {"query": {"type": "string"}}),
|
||||
_tool("open", {"id": {"type": "integer"}}),
|
||||
]
|
||||
request = _request(tools)
|
||||
text = (
|
||||
"<dots_function_call>"
|
||||
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
|
||||
'<invoke name="open"><parameter name="id">7</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
'<dots_function_call>{"name":"search","arguments":{"query":"tables"}}'
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert [call.function.name for call in result.tool_calls] == [
|
||||
"search",
|
||||
"open",
|
||||
"search",
|
||||
]
|
||||
assert [json.loads(call.function.arguments) for call in result.tool_calls] == [
|
||||
{"query": "chairs"},
|
||||
{"id": 7},
|
||||
{"query": "tables"},
|
||||
]
|
||||
|
||||
|
||||
def test_non_stream_unknown_tool_is_left_as_content(parser: DotsToolParser) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
text = (
|
||||
"<dots_function_call>"
|
||||
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert not result.tools_called
|
||||
assert result.tool_calls == []
|
||||
assert result.content == text
|
||||
|
||||
|
||||
def test_streaming_buffers_partial_marker_and_emits_all_complete_calls(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
chunks = [
|
||||
"visible<dots_func",
|
||||
(
|
||||
"tion_call>"
|
||||
'<invoke name="search"><parameter name="query">chairs</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
"<dots_function_call>"
|
||||
'<invoke name="search"><parameter name="query">tables</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
),
|
||||
]
|
||||
|
||||
messages = _stream(parser, chunks, request)
|
||||
|
||||
assert "".join(message.content or "" for message in messages) == "visible"
|
||||
calls = [call for message in messages for call in message.tool_calls]
|
||||
assert [call.index for call in calls] == [0, 1]
|
||||
assert [json.loads(call.function.arguments or "") for call in calls] == [
|
||||
{"query": "chairs"},
|
||||
{"query": "tables"},
|
||||
]
|
||||
|
||||
|
||||
def test_streaming_filters_unknown_tools_and_surfaces_content(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
text = (
|
||||
"<dots_function_call>"
|
||||
'<invoke name="ghost"><parameter name="query">chairs</parameter></invoke>'
|
||||
"</dots_function_call>"
|
||||
)
|
||||
|
||||
messages = _stream(parser, [text], request)
|
||||
|
||||
assert len(messages) == 1
|
||||
assert messages[0].tool_calls == []
|
||||
assert "ghost" in (messages[0].content or "")
|
||||
assert parser._buffer == ""
|
||||
|
||||
|
||||
def test_streaming_malformed_block_does_not_block_later_valid_call(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
messages = _stream(
|
||||
parser,
|
||||
[
|
||||
"<dots_function_call>garbage</dots_function_call>",
|
||||
(
|
||||
"<dots_function_call>"
|
||||
'<invoke name="search">'
|
||||
'<parameter name="query">chairs</parameter>'
|
||||
"</invoke></dots_function_call>"
|
||||
),
|
||||
],
|
||||
request,
|
||||
)
|
||||
|
||||
assert messages[0].content == "garbage"
|
||||
assert messages[0].tool_calls == []
|
||||
assert messages[1].tool_calls[0].function.name == "search"
|
||||
|
||||
|
||||
def test_streaming_strips_stray_end_marker(parser: DotsToolParser) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
messages = _stream(parser, ["some text </dots_function_call>"], request)
|
||||
|
||||
assert len(messages) == 1
|
||||
assert messages[0].content == "some text "
|
||||
assert messages[0].tool_calls == []
|
||||
|
||||
|
||||
def test_streaming_flushes_partial_opening_marker_at_eof(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
|
||||
messages = _stream(parser, ["answer <dots_func"], request)
|
||||
|
||||
assert messages[0].content == "answer "
|
||||
assert parser.flush_pending_normal_text() == "<dots_func"
|
||||
assert parser.flush_pending_normal_text() == ""
|
||||
|
||||
|
||||
def test_streaming_emits_complete_json_before_end_marker_without_duplication(
|
||||
parser: DotsToolParser,
|
||||
) -> None:
|
||||
request = _request([_tool("search", {"query": {"type": "string"}})])
|
||||
|
||||
messages = _stream(
|
||||
parser,
|
||||
[
|
||||
"<dots_function_call>",
|
||||
'{"name":"search","arguments":{"query":"chairs"}}',
|
||||
"</dots_function_call>",
|
||||
],
|
||||
request,
|
||||
)
|
||||
|
||||
calls = [call for message in messages for call in message.tool_calls]
|
||||
assert [call.function.name for call in calls] == ["search", None]
|
||||
assert "".join(call.function.arguments or "" for call in calls) == (
|
||||
'{"query": "chairs"}'
|
||||
)
|
||||
assert messages[-1].tool_calls == calls
|
||||
@@ -1,359 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
import json
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tool_parsers.ernie45_tool_parser import Ernie45ToolParser
|
||||
|
||||
# Use a common model that is likely to be available
|
||||
MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ernie45_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ernie45_tool_parser(ernie45_tokenizer):
|
||||
return Ernie45ToolParser(ernie45_tokenizer)
|
||||
|
||||
|
||||
def assert_tool_calls(
|
||||
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
|
||||
):
|
||||
assert len(actual_tool_calls) == len(expected_tool_calls)
|
||||
|
||||
for actual_tool_call, expected_tool_call in zip(
|
||||
actual_tool_calls, expected_tool_calls
|
||||
):
|
||||
assert isinstance(actual_tool_call.id, str)
|
||||
assert len(actual_tool_call.id) > 0
|
||||
|
||||
assert actual_tool_call.type == "function"
|
||||
assert actual_tool_call.function.name == expected_tool_call.function.name
|
||||
# Compare arguments as JSON objects to handle formatting differences
|
||||
actual_args = json.loads(actual_tool_call.function.arguments)
|
||||
expected_args = json.loads(expected_tool_call.function.arguments)
|
||||
assert actual_args == expected_args
|
||||
|
||||
|
||||
def test_extract_tool_calls_no_tools(ernie45_tool_parser):
|
||||
model_output = "This is a test"
|
||||
extracted_tool_calls = ernie45_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert not extracted_tool_calls.tools_called
|
||||
assert extracted_tool_calls.tool_calls == []
|
||||
assert extracted_tool_calls.content == model_output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
"single_tool_call",
|
||||
"multiple_tool_calls",
|
||||
"tool_call_with_content_before",
|
||||
],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
(
|
||||
"""<tool_call>
|
||||
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
|
||||
</tool_call>
|
||||
""",
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_temperature",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Beijing",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"""<tool_call>
|
||||
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
|
||||
</tool_call>
|
||||
<tool_call>
|
||||
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
|
||||
</tool_call>
|
||||
""",
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_temperature",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Beijing",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_temperature_unit",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Guangzhou",
|
||||
"unit": "c",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"""I need to call two tools to handle these two issues separately.
|
||||
</think>
|
||||
|
||||
<tool_call>
|
||||
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
|
||||
</tool_call>
|
||||
<tool_call>
|
||||
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
|
||||
</tool_call>
|
||||
""",
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_temperature",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Beijing",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_temperature_unit",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Guangzhou",
|
||||
"unit": "c",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
"I need to call two tools to handle these two issues separately.\n</think>",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls(
|
||||
ernie45_tool_parser, model_output, expected_tool_calls, expected_content
|
||||
):
|
||||
extracted_tool_calls = ernie45_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert extracted_tool_calls.tools_called
|
||||
|
||||
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
|
||||
|
||||
assert extracted_tool_calls.content == expected_content
|
||||
|
||||
|
||||
def stream_delta_message_generator(
|
||||
ernie45_tool_parser: Ernie45ToolParser,
|
||||
ernie45_tokenizer: TokenizerLike,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> Generator[DeltaMessage, None, None]:
|
||||
all_token_ids = ernie45_tokenizer.encode(model_output, add_special_tokens=False)
|
||||
|
||||
previous_text = ""
|
||||
previous_tokens = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, delta_token in enumerate(all_token_ids):
|
||||
delta_token_ids = [delta_token]
|
||||
previous_token_ids = all_token_ids[:i]
|
||||
current_token_ids = all_token_ids[: i + 1]
|
||||
|
||||
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
|
||||
detokenize_incrementally(
|
||||
tokenizer=ernie45_tokenizer,
|
||||
all_input_ids=current_token_ids,
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
)
|
||||
|
||||
current_text = previous_text + delta_text
|
||||
|
||||
delta_message = ernie45_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request=request,
|
||||
)
|
||||
if delta_message:
|
||||
yield delta_message
|
||||
|
||||
previous_text = current_text
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
prefix_offset = new_prefix_offset
|
||||
read_offset = new_read_offset
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
"single_tool_call",
|
||||
"multiple_tool_calls",
|
||||
"tool_call_with_content_before",
|
||||
],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
(
|
||||
"""<tool_call>
|
||||
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
|
||||
</tool_call>
|
||||
""",
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_temperature",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Beijing",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"""<tool_call>
|
||||
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
|
||||
</tool_call>
|
||||
<tool_call>
|
||||
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
|
||||
</tool_call>
|
||||
""",
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_temperature",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Beijing",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_temperature_unit",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Guangzhou",
|
||||
"unit": "c",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"""I need to call two tools to handle these two issues separately.
|
||||
</think>
|
||||
|
||||
<tool_call>
|
||||
{"name": "get_current_temperature", "arguments": {"location": "Beijing"}}
|
||||
</tool_call>
|
||||
<tool_call>
|
||||
{"name": "get_temperature_unit", "arguments": {"location": "Guangzhou", "unit": "c"}}
|
||||
</tool_call>
|
||||
""",
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_temperature",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Beijing",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_temperature_unit",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"location": "Guangzhou",
|
||||
"unit": "c",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
"I need to call two tools to handle these two issues separately.\n</think>",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls_streaming_incremental(
|
||||
ernie45_tool_parser,
|
||||
ernie45_tokenizer,
|
||||
model_output,
|
||||
expected_tool_calls,
|
||||
expected_content,
|
||||
):
|
||||
"""Verify the Ernie45 Parser streaming behavior by verifying each chunk is as expected.""" # noqa: E501
|
||||
request = ChatCompletionRequest(model=MODEL, messages=[])
|
||||
|
||||
tool_calls_dict = {}
|
||||
for delta_message in stream_delta_message_generator(
|
||||
ernie45_tool_parser, ernie45_tokenizer, model_output, request
|
||||
):
|
||||
if (
|
||||
delta_message.role is None
|
||||
and delta_message.content is None
|
||||
and delta_message.reasoning is None
|
||||
and len(delta_message.tool_calls) == 0
|
||||
):
|
||||
continue
|
||||
tool_calls = delta_message.tool_calls
|
||||
for tool_call_chunk in tool_calls:
|
||||
index = tool_call_chunk.index
|
||||
if index not in tool_calls_dict:
|
||||
if tool_call_chunk.function.arguments is None:
|
||||
tool_call_chunk.function.arguments = ""
|
||||
tool_calls_dict[index] = tool_call_chunk
|
||||
else:
|
||||
tool_calls_dict[
|
||||
index
|
||||
].function.arguments += tool_call_chunk.function.arguments
|
||||
actual_tool_calls = list(tool_calls_dict.values())
|
||||
|
||||
assert len(actual_tool_calls) > 0
|
||||
# check tool call format
|
||||
assert_tool_calls(actual_tool_calls, expected_tool_calls)
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tool_parsers.functiongemma_tool_parser import FunctionGemmaToolParser
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.encode.return_value = [1, 2, 3]
|
||||
tokenizer.get_vocab.return_value = {}
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return FunctionGemmaToolParser(mock_tokenizer)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request():
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = []
|
||||
request.tool_choice = "auto"
|
||||
request.parallel_tool_calls = True
|
||||
return request
|
||||
|
||||
|
||||
class TestExtractToolCalls:
|
||||
def test_no_tool_calls(self, parser, mock_request):
|
||||
model_output = "Hello, how can I help you today?"
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert result.tool_calls == []
|
||||
assert result.content == model_output
|
||||
|
||||
def test_single_tool_call(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<start_function_call>call:get_weather{location:<escape>London<escape>}"
|
||||
"<end_function_call>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert '"location": "London"' in result.tool_calls[0].function.arguments
|
||||
|
||||
def test_multiple_arguments(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<start_function_call>call:get_weather{"
|
||||
"location:<escape>San Francisco<escape>,"
|
||||
"unit:<escape>celsius<escape>}"
|
||||
"<end_function_call>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = result.tool_calls[0].function.arguments
|
||||
assert "San Francisco" in args
|
||||
assert "celsius" in args
|
||||
|
||||
def test_text_before_tool_call(self, parser, mock_request):
|
||||
model_output = (
|
||||
"Let me check the weather for you. "
|
||||
"<start_function_call>call:get_weather{location:<escape>Paris<escape>}"
|
||||
"<end_function_call>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.content == "Let me check the weather for you."
|
||||
|
||||
def test_multiple_tool_calls(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<start_function_call>call:get_weather{location:<escape>London<escape>}"
|
||||
"<end_function_call>"
|
||||
"<start_function_call>call:get_time{timezone:<escape>UTC<escape>}"
|
||||
"<end_function_call>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert result.tool_calls[1].function.name == "get_time"
|
||||
|
||||
|
||||
class TestParseArguments:
|
||||
def test_empty_arguments(self, parser):
|
||||
result = parser._parse_arguments("")
|
||||
assert result == {}
|
||||
|
||||
def test_single_string_argument(self, parser):
|
||||
result = parser._parse_arguments("city:<escape>Tokyo<escape>")
|
||||
assert result == {"city": "Tokyo"}
|
||||
|
||||
def test_multiple_arguments(self, parser):
|
||||
args_str = "city:<escape>Tokyo<escape>,country:<escape>Japan<escape>"
|
||||
result = parser._parse_arguments(args_str)
|
||||
assert result == {"city": "Tokyo", "country": "Japan"}
|
||||
|
||||
def test_numeric_argument(self, parser):
|
||||
result = parser._parse_arguments("count:<escape>42<escape>")
|
||||
assert result == {"count": 42}
|
||||
|
||||
def test_boolean_argument(self, parser):
|
||||
result = parser._parse_arguments("enabled:<escape>true<escape>")
|
||||
assert result == {"enabled": True}
|
||||
|
||||
def test_argument_with_spaces(self, parser):
|
||||
result = parser._parse_arguments("message:<escape>Hello World<escape>")
|
||||
assert result == {"message": "Hello World"}
|
||||
|
||||
|
||||
class TestAdjustRequest:
|
||||
def test_skip_special_tokens_disabled(self, parser, mock_request):
|
||||
mock_request.tools = [{"type": "function", "function": {"name": "test"}}]
|
||||
mock_request.tool_choice = "auto"
|
||||
mock_request.skip_special_tokens = True
|
||||
|
||||
result = parser.adjust_request(mock_request)
|
||||
assert result.skip_special_tokens is False
|
||||
|
||||
def test_skip_special_tokens_when_tool_choice_none(self, parser, mock_request):
|
||||
mock_request.tools = [{"type": "function", "function": {"name": "test"}}]
|
||||
mock_request.tool_choice = "none"
|
||||
mock_request.skip_special_tokens = True
|
||||
|
||||
result = parser.adjust_request(mock_request)
|
||||
assert result.skip_special_tokens is True
|
||||
|
||||
|
||||
class TestBufferDeltaText:
|
||||
def test_regular_text_not_buffered(self, parser):
|
||||
result = parser._buffer_delta_text("hello")
|
||||
assert result == "hello"
|
||||
assert parser.buffered_delta_text == ""
|
||||
|
||||
def test_complete_tag_flushed(self, parser):
|
||||
parser.buffered_delta_text = "<start_function_"
|
||||
result = parser._buffer_delta_text("call>")
|
||||
assert "<start_function_call>" in result
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,350 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def default_tokenizer() -> TokenizerLike:
|
||||
"""Override module-scoped default_tokenizer because gigachat tests
|
||||
mutate the tokenizer via ``add_tokens``."""
|
||||
return AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
|
||||
|
||||
MSG_SEP_TOKEN = "<|message_sep|>\n\n"
|
||||
ROLE_SEP_TOKEN = "<|role_sep|>\n"
|
||||
EOS_TOKEN = "</s>"
|
||||
TOOL_HEADER_GIGACHAT3 = f"function call{ROLE_SEP_TOKEN}"
|
||||
TOOL_HEADER_GIGACHAT31 = "<|function_call|>"
|
||||
|
||||
|
||||
SIMPLE_ARGS_DICT = {
|
||||
"action": "create",
|
||||
"id": "preferences",
|
||||
}
|
||||
SIMPLE_FUNCTION_JSON = json.dumps(
|
||||
{
|
||||
"name": "manage_user_memory",
|
||||
"arguments": SIMPLE_ARGS_DICT,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
SIMPLE_FUNCTION_OUTPUT_GIGACHAT3 = (
|
||||
f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{SIMPLE_FUNCTION_JSON}"
|
||||
)
|
||||
SIMPLE_FUNCTION_OUTPUT_GIGACHAT31 = f"{TOOL_HEADER_GIGACHAT31}{SIMPLE_FUNCTION_JSON}"
|
||||
SIMPLE_FUNCTION_CALL = FunctionCall(
|
||||
name="manage_user_memory",
|
||||
arguments=json.dumps(SIMPLE_ARGS_DICT, ensure_ascii=False),
|
||||
)
|
||||
|
||||
|
||||
PARAMETERLESS_FUNCTION_JSON = json.dumps(
|
||||
{
|
||||
"name": "manage_user_memory",
|
||||
"arguments": {},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3 = (
|
||||
f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{PARAMETERLESS_FUNCTION_JSON}"
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31 = (
|
||||
f"{TOOL_HEADER_GIGACHAT31}{PARAMETERLESS_FUNCTION_JSON}"
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
|
||||
name="manage_user_memory",
|
||||
arguments=json.dumps({}, ensure_ascii=False),
|
||||
)
|
||||
|
||||
|
||||
COMPLEX_ARGS_DICT = {
|
||||
"action": "create",
|
||||
"id": "preferences",
|
||||
"content": {
|
||||
"short_answers": True,
|
||||
"hate_emojis": True,
|
||||
"english_ui": False,
|
||||
"russian_math_explanations": True,
|
||||
},
|
||||
}
|
||||
COMPLEX_FUNCTION_JSON = json.dumps(
|
||||
{
|
||||
"name": "manage_user_memory",
|
||||
"arguments": COMPLEX_ARGS_DICT,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
COMPLEX_FUNCTION_OUTPUT_GIGACHAT3 = (
|
||||
f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{COMPLEX_FUNCTION_JSON}"
|
||||
)
|
||||
COMPLEX_FUNCTION_OUTPUT_GIGACHAT31 = f"{TOOL_HEADER_GIGACHAT31}{COMPLEX_FUNCTION_JSON}"
|
||||
COMPLEX_FUNCTION_CALL = FunctionCall(
|
||||
name="manage_user_memory",
|
||||
arguments=json.dumps(COMPLEX_ARGS_DICT, ensure_ascii=False),
|
||||
)
|
||||
|
||||
|
||||
CONTENT_TEXT = "I'll check that for you."
|
||||
MIXED_OUTPUT_GIGACHAT3 = f"{CONTENT_TEXT}{SIMPLE_FUNCTION_OUTPUT_GIGACHAT3}"
|
||||
MIXED_OUTPUT_GIGACHAT31 = f"{CONTENT_TEXT}{SIMPLE_FUNCTION_OUTPUT_GIGACHAT31}"
|
||||
|
||||
|
||||
@pytest.fixture(name="gigachat_tokenizer")
|
||||
def fixture_gigachat_tokenizer(default_tokenizer: TokenizerLike):
|
||||
default_tokenizer.add_tokens(
|
||||
[
|
||||
MSG_SEP_TOKEN,
|
||||
ROLE_SEP_TOKEN,
|
||||
TOOL_HEADER_GIGACHAT31,
|
||||
EOS_TOKEN,
|
||||
]
|
||||
)
|
||||
return default_tokenizer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_no_tool_call(streaming: bool, gigachat_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
|
||||
gigachat_tokenizer
|
||||
)
|
||||
model_output = "How can I help you today?"
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
assert content == model_output
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
TEST_CASES = [
|
||||
pytest.param(
|
||||
True,
|
||||
SIMPLE_FUNCTION_OUTPUT_GIGACHAT3,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
None,
|
||||
id="simple_streaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
SIMPLE_FUNCTION_OUTPUT_GIGACHAT3,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
None,
|
||||
id="simple_nonstreaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3,
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
None,
|
||||
id="parameterless_streaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3,
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
None,
|
||||
id="parameterless_nonstreaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
COMPLEX_FUNCTION_OUTPUT_GIGACHAT3,
|
||||
[COMPLEX_FUNCTION_CALL],
|
||||
None,
|
||||
id="complex_streaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
COMPLEX_FUNCTION_OUTPUT_GIGACHAT3,
|
||||
[COMPLEX_FUNCTION_CALL],
|
||||
None,
|
||||
id="complex_nonstreaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
MIXED_OUTPUT_GIGACHAT3,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_streaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
MIXED_OUTPUT_GIGACHAT3,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_nonstreaming_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
MIXED_OUTPUT_GIGACHAT3 + EOS_TOKEN,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_streaming_with_eos_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
MIXED_OUTPUT_GIGACHAT3 + EOS_TOKEN,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_nonstreaming_with_eos_gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
SIMPLE_FUNCTION_OUTPUT_GIGACHAT31,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
None,
|
||||
id="simple_streaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
SIMPLE_FUNCTION_OUTPUT_GIGACHAT31,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
None,
|
||||
id="simple_nonstreaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31,
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
None,
|
||||
id="parameterless_streaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31,
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
None,
|
||||
id="parameterless_nonstreaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
COMPLEX_FUNCTION_OUTPUT_GIGACHAT31,
|
||||
[COMPLEX_FUNCTION_CALL],
|
||||
None,
|
||||
id="complex_streaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
COMPLEX_FUNCTION_OUTPUT_GIGACHAT31,
|
||||
[COMPLEX_FUNCTION_CALL],
|
||||
None,
|
||||
id="complex_nonstreaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
MIXED_OUTPUT_GIGACHAT31,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_streaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
MIXED_OUTPUT_GIGACHAT31,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_nonstreaming_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
MIXED_OUTPUT_GIGACHAT31 + EOS_TOKEN,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_streaming_with_eos_gigachat31",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
MIXED_OUTPUT_GIGACHAT31 + EOS_TOKEN,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
CONTENT_TEXT,
|
||||
id="mixed_content_nonstreaming_with_eos_gigachat31",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"streaming, model_output, expected_tool_calls, expected_content", TEST_CASES
|
||||
)
|
||||
def test_tool_call(
|
||||
streaming: bool,
|
||||
model_output: str,
|
||||
expected_tool_calls: list[FunctionCall],
|
||||
expected_content: str | None,
|
||||
gigachat_tokenizer: TokenizerLike,
|
||||
):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
|
||||
gigachat_tokenizer
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
if content == "":
|
||||
content = None
|
||||
assert content == expected_content
|
||||
assert len(tool_calls) == len(expected_tool_calls)
|
||||
for actual, expected in zip(tool_calls, expected_tool_calls):
|
||||
assert actual.type == "function"
|
||||
assert actual.function.name == expected.name
|
||||
actual_args = json.loads(actual.function.arguments)
|
||||
expected_args = json.loads(expected.arguments)
|
||||
assert actual_args == expected_args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_output_deltas",
|
||||
[
|
||||
pytest.param(
|
||||
[
|
||||
CONTENT_TEXT[:3],
|
||||
CONTENT_TEXT[3:5],
|
||||
CONTENT_TEXT[5:],
|
||||
MSG_SEP_TOKEN,
|
||||
TOOL_HEADER_GIGACHAT3,
|
||||
COMPLEX_FUNCTION_JSON[:40],
|
||||
COMPLEX_FUNCTION_JSON[40:-1],
|
||||
COMPLEX_FUNCTION_JSON[-1],
|
||||
],
|
||||
id="gigachat3",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
CONTENT_TEXT[:3],
|
||||
CONTENT_TEXT[3:5],
|
||||
CONTENT_TEXT[5:],
|
||||
TOOL_HEADER_GIGACHAT31,
|
||||
COMPLEX_FUNCTION_JSON[:40],
|
||||
COMPLEX_FUNCTION_JSON[40:-1],
|
||||
COMPLEX_FUNCTION_JSON[-1],
|
||||
],
|
||||
id="gigachat31",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_streaming_tool_call_with_large_steps(
|
||||
model_output_deltas: list[str],
|
||||
gigachat_tokenizer: TokenizerLike,
|
||||
):
|
||||
"""Test that the closing braces are streamed correctly."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
|
||||
gigachat_tokenizer
|
||||
)
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser,
|
||||
model_output_deltas,
|
||||
assert_one_tool_per_delta=False,
|
||||
)
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
call = reconstructor.tool_calls[0]
|
||||
assert call.type == "function"
|
||||
assert call.function.name == "manage_user_memory"
|
||||
args_dict = json.loads(call.function.arguments)
|
||||
assert args_dict == COMPLEX_ARGS_DICT
|
||||
@@ -1,268 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""Tests for the GLM-4.7 tool call parser."""
|
||||
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.entrypoints.openai.responses.utils import build_response_output_items
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
|
||||
|
||||
MODEL = "zai-org/GLM-4.7"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def glm47_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_tools():
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(name="get_current_date", parameters={}),
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="get_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"date": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def glm47_tool_parser(glm47_tokenizer, sample_tools):
|
||||
return Glm47MoeModelToolParser(glm47_tokenizer, tools=sample_tools)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request(sample_tools) -> ChatCompletionRequest:
|
||||
request = Mock(spec=ChatCompletionRequest)
|
||||
request.tools = sample_tools
|
||||
request.tool_choice = "auto"
|
||||
return request
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def namespace_tool_request() -> ResponsesRequest:
|
||||
return ResponsesRequest.model_validate(
|
||||
{
|
||||
"input": "hi",
|
||||
"tools": [
|
||||
{
|
||||
"type": "namespace",
|
||||
"name": "mcp__computer_use",
|
||||
"description": "Computer use tools.",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_app_state",
|
||||
"description": "Get app state.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"app": {"type": "string"},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestGlm47ExtractToolCalls:
|
||||
def test_namespace_tool_call_round_trip_to_responses_output(
|
||||
self, glm47_tokenizer, namespace_tool_request
|
||||
):
|
||||
parser = Glm47MoeModelToolParser(
|
||||
glm47_tokenizer, tools=namespace_tool_request.tools
|
||||
)
|
||||
out = (
|
||||
"<tool_call>mcp__computer_use__get_app_state"
|
||||
"<arg_key>app</arg_key>"
|
||||
"<arg_value>Google Chrome</arg_value>"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(out, request=namespace_tool_request)
|
||||
|
||||
assert result.tools_called
|
||||
tool_call = result.tool_calls[0].function
|
||||
assert tool_call == FunctionCall(
|
||||
name="mcp__computer_use__get_app_state",
|
||||
arguments='{"app": "Google Chrome"}',
|
||||
)
|
||||
|
||||
output_items = build_response_output_items(
|
||||
reasoning=None,
|
||||
content=None,
|
||||
tool_calls=[tool_call],
|
||||
tools=namespace_tool_request.tools,
|
||||
)
|
||||
output_tool_call = output_items[0]
|
||||
assert isinstance(output_tool_call, ResponseFunctionToolCall)
|
||||
assert output_tool_call.name == "get_app_state"
|
||||
assert output_tool_call.namespace == "mcp__computer_use"
|
||||
|
||||
def test_no_tool_call(self, glm47_tool_parser, mock_request):
|
||||
out = "This is a plain response."
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert not r.tools_called
|
||||
assert r.content == out
|
||||
|
||||
def test_zero_arg_inline(self, glm47_tool_parser, mock_request):
|
||||
out = "<tool_call>get_current_date</tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert r.tool_calls[0].function.name == "get_current_date"
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {}
|
||||
assert r.content is None
|
||||
|
||||
def test_zero_arg_newline(self, glm47_tool_parser, mock_request):
|
||||
out = "<tool_call>get_current_date\n</tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert r.tool_calls[0].function.name == "get_current_date"
|
||||
|
||||
def test_args_same_line(self, glm47_tool_parser, mock_request):
|
||||
out = "<tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value></tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {"city": "Beijing"}
|
||||
|
||||
def test_args_with_newlines(self, glm47_tool_parser, mock_request):
|
||||
out = "<tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>Beijing</arg_value>\n</tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {"city": "Beijing"}
|
||||
|
||||
def test_whitespace_preserved_in_arg_values(self, glm47_tool_parser, mock_request):
|
||||
out = "<tool_call>get_weather<arg_key>city</arg_key><arg_value> Beijing </arg_value></tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {"city": " Beijing "}
|
||||
|
||||
def test_content_before(self, glm47_tool_parser, mock_request):
|
||||
out = "Checking.<tool_call>get_current_date</tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert r.content == "Checking."
|
||||
|
||||
def test_multiple(self, glm47_tool_parser, mock_request):
|
||||
out = (
|
||||
"<tool_call>get_weather<arg_key>city</arg_key><arg_value>Beijing</arg_value></tool_call>"
|
||||
"<tool_call>get_weather<arg_key>city</arg_key><arg_value>Shanghai</arg_value></tool_call>"
|
||||
)
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert len(r.tool_calls) == 2
|
||||
|
||||
def test_empty_content_none(self, glm47_tool_parser, mock_request):
|
||||
out = "<tool_call>get_current_date</tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.content is None
|
||||
|
||||
def test_whitespace_content_none(self, glm47_tool_parser, mock_request):
|
||||
out = " \n <tool_call>get_current_date</tool_call>"
|
||||
r = glm47_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.content is None
|
||||
|
||||
|
||||
def _reset(parser):
|
||||
parser.current_tool_name_sent = False
|
||||
parser.prev_tool_call_arr = []
|
||||
parser.current_tool_id = -1
|
||||
parser.streamed_args_for_tool = []
|
||||
parser._tool_call_ids = []
|
||||
parser._sent_content_idx = 0
|
||||
|
||||
|
||||
class TestGlm47Streaming:
|
||||
def test_no_args(self, glm47_tool_parser, mock_request):
|
||||
_reset(glm47_tool_parser)
|
||||
chunks = ["<tool_call>", "get_current_date", "</tool_call>"]
|
||||
current_text = ""
|
||||
deltas = []
|
||||
for chunk in chunks:
|
||||
current_text += chunk
|
||||
delta = glm47_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=mock_request,
|
||||
)
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
tool_calls = [
|
||||
tool_call for delta in deltas for tool_call in (delta.tool_calls or [])
|
||||
]
|
||||
names = [
|
||||
tool_call.function.name
|
||||
for tool_call in tool_calls
|
||||
if tool_call.function and tool_call.function.name
|
||||
]
|
||||
arguments = [
|
||||
tool_call.function.arguments
|
||||
for tool_call in tool_calls
|
||||
if tool_call.function and tool_call.function.arguments
|
||||
]
|
||||
assert names == ["get_current_date"]
|
||||
assert "".join(arguments) == "{}"
|
||||
|
||||
def test_with_args(self, glm47_tool_parser, mock_request):
|
||||
_reset(glm47_tool_parser)
|
||||
chunks = [
|
||||
"<tool_call>",
|
||||
"get_weather\n",
|
||||
"<arg_key>city</arg_key>",
|
||||
"<arg_value>",
|
||||
"Beijing",
|
||||
"</arg_value>",
|
||||
"</tool_call>",
|
||||
]
|
||||
current_text = ""
|
||||
deltas = []
|
||||
for chunk in chunks:
|
||||
current_text += chunk
|
||||
delta = glm47_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=mock_request,
|
||||
)
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
arguments = [
|
||||
tool_call.function.arguments
|
||||
for delta in deltas
|
||||
for tool_call in (delta.tool_calls or [])
|
||||
if tool_call.function and tool_call.function.arguments
|
||||
]
|
||||
args = json.loads("".join(arguments))
|
||||
assert args["city"] == "Beijing"
|
||||
@@ -1,228 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compatibility tests for GLM-4.5 using the shared GLM XML parser."""
|
||||
|
||||
import json
|
||||
from typing import Any, TypedDict
|
||||
|
||||
from tests.parser.engine.replay_harness import MockTokenizer
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
|
||||
|
||||
MODEL = "zai-org/GLM-4.5"
|
||||
|
||||
_GLM_VOCAB = {
|
||||
"<think>": 50,
|
||||
"</think>": 51,
|
||||
"<tool_call>": 60,
|
||||
"</tool_call>": 61,
|
||||
"<arg_key>": 62,
|
||||
"</arg_key>": 63,
|
||||
"<arg_value>": 64,
|
||||
"</arg_value>": 65,
|
||||
}
|
||||
|
||||
|
||||
class _CollectedToolDelta(TypedDict):
|
||||
name: str | None
|
||||
args_fragments: list[str]
|
||||
|
||||
|
||||
def _mock_tokenizer() -> MockTokenizer:
|
||||
return MockTokenizer(vocab=_GLM_VOCAB, tokens=[])
|
||||
|
||||
|
||||
def _tools() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="get_current_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"state": {"type": "string"},
|
||||
"unit": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="calculate",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operation": {"type": "string"},
|
||||
"a": {"type": "number"},
|
||||
"b": {"type": "number"},
|
||||
"enabled": {"type": "boolean"},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(name="get_time", parameters={}),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _request(tools: list[ChatCompletionToolsParam]) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
|
||||
|
||||
|
||||
def _parser(tools: list[ChatCompletionToolsParam] | None = None):
|
||||
return Glm47MoeModelToolParser(_mock_tokenizer(), tools=tools)
|
||||
|
||||
|
||||
def _collect_tool_deltas(deltas: Any) -> dict[int, _CollectedToolDelta]:
|
||||
calls: dict[int, _CollectedToolDelta] = {}
|
||||
for delta in deltas:
|
||||
if delta is None or not delta.tool_calls:
|
||||
continue
|
||||
for tool_call in delta.tool_calls:
|
||||
entry = calls.setdefault(
|
||||
tool_call.index,
|
||||
{"name": None, "args_fragments": []},
|
||||
)
|
||||
function = tool_call.function
|
||||
if function is None:
|
||||
continue
|
||||
if isinstance(function, dict):
|
||||
name = function.get("name")
|
||||
arguments = function.get("arguments")
|
||||
else:
|
||||
name = function.name
|
||||
arguments = function.arguments
|
||||
if isinstance(name, str) and name:
|
||||
entry["name"] = name
|
||||
if isinstance(arguments, str) and arguments:
|
||||
entry["args_fragments"].append(arguments)
|
||||
return calls
|
||||
|
||||
|
||||
def test_glm45_uses_shared_glm47_parser():
|
||||
assert ToolParserManager.get_tool_parser("glm45") is Glm47MoeModelToolParser
|
||||
assert ToolParserManager.get_tool_parser("glm47") is Glm47MoeModelToolParser
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_glm45_newline_format():
|
||||
tools = _tools()
|
||||
parser = _parser(tools)
|
||||
model_output = """I'll check it. <tool_call>get_current_weather
|
||||
<arg_key>city</arg_key>
|
||||
<arg_value>Dallas</arg_value>
|
||||
<arg_key>state</arg_key>
|
||||
<arg_value>TX</arg_value>
|
||||
<arg_key>unit</arg_key>
|
||||
<arg_value>fahrenheit</arg_value>
|
||||
</tool_call>"""
|
||||
|
||||
extracted = parser.extract_tool_calls(model_output, request=_request(tools))
|
||||
|
||||
assert extracted.tools_called
|
||||
assert extracted.content == "I'll check it."
|
||||
assert len(extracted.tool_calls) == 1
|
||||
tool_call = extracted.tool_calls[0]
|
||||
assert tool_call.function.name == "get_current_weather"
|
||||
assert json.loads(tool_call.function.arguments) == {
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
|
||||
|
||||
def test_extract_multiple_tool_calls_with_glm45_newline_format():
|
||||
tools = _tools()
|
||||
parser = _parser(tools)
|
||||
model_output = """<tool_call>get_current_weather
|
||||
<arg_key>city</arg_key><arg_value>Dallas</arg_value>
|
||||
</tool_call>
|
||||
<tool_call>get_current_weather
|
||||
<arg_key>city</arg_key><arg_value>Orlando</arg_value>
|
||||
</tool_call>"""
|
||||
|
||||
extracted = parser.extract_tool_calls(model_output, request=_request(tools))
|
||||
|
||||
assert extracted.tools_called
|
||||
assert [tc.function.name for tc in extracted.tool_calls] == [
|
||||
"get_current_weather",
|
||||
"get_current_weather",
|
||||
]
|
||||
assert [
|
||||
json.loads(tc.function.arguments)["city"] for tc in extracted.tool_calls
|
||||
] == ["Dallas", "Orlando"]
|
||||
|
||||
|
||||
def test_extract_tool_calls_coerces_schema_types():
|
||||
tools = _tools()
|
||||
parser = _parser(tools)
|
||||
model_output = """<tool_call>calculate
|
||||
<arg_key>operation</arg_key><arg_value>add</arg_value>
|
||||
<arg_key>a</arg_key><arg_value>42</arg_value>
|
||||
<arg_key>b</arg_key><arg_value>3.14</arg_value>
|
||||
<arg_key>enabled</arg_key><arg_value>true</arg_value>
|
||||
</tool_call>"""
|
||||
|
||||
extracted = parser.extract_tool_calls(model_output, request=_request(tools))
|
||||
|
||||
assert extracted.tools_called
|
||||
assert json.loads(extracted.tool_calls[0].function.arguments) == {
|
||||
"operation": "add",
|
||||
"a": 42,
|
||||
"b": 3.14,
|
||||
"enabled": True,
|
||||
}
|
||||
|
||||
|
||||
def test_extract_zero_argument_tool_call_with_glm45_newline_format():
|
||||
tools = _tools()
|
||||
parser = _parser(tools)
|
||||
|
||||
extracted = parser.extract_tool_calls(
|
||||
"<tool_call>get_time\n</tool_call>",
|
||||
request=_request(tools),
|
||||
)
|
||||
|
||||
assert extracted.tools_called
|
||||
assert extracted.tool_calls[0].function.name == "get_time"
|
||||
assert json.loads(extracted.tool_calls[0].function.arguments) == {}
|
||||
|
||||
|
||||
def test_streaming_tool_call_with_glm45_newline_format():
|
||||
tools = _tools()
|
||||
parser = _parser(tools)
|
||||
request = _request(tools)
|
||||
chunks = [
|
||||
"<tool_call>",
|
||||
"get_current_weather\n",
|
||||
"<arg_key>city</arg_key>",
|
||||
"<arg_value>Bei",
|
||||
"jing</arg_value>",
|
||||
"</tool_call>",
|
||||
]
|
||||
deltas = []
|
||||
current_text = ""
|
||||
|
||||
for chunk in chunks:
|
||||
current_text += chunk
|
||||
deltas.append(
|
||||
parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request,
|
||||
)
|
||||
)
|
||||
|
||||
calls = _collect_tool_deltas(deltas)
|
||||
assert calls[0]["name"] == "get_current_weather"
|
||||
assert json.loads("".join(calls[0]["args_fragments"])) == {"city": "Beijing"}
|
||||
@@ -1,147 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import random
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
|
||||
|
||||
MODEL = "ibm-granite/granite-4.0-h-tiny"
|
||||
|
||||
|
||||
def create_complex_input(create_string_args: bool):
|
||||
coord_arg: dict | str = {
|
||||
"coordinates": [[23.54, 43.1], [-12.2, 54.3], [4, 5]],
|
||||
"coordinate_type": "latlong",
|
||||
}
|
||||
if create_string_args:
|
||||
# test granite behavior
|
||||
coord_arg = json.dumps(coord_arg)
|
||||
return [
|
||||
{"name": "find_bbox", "arguments": coord_arg},
|
||||
{
|
||||
"name": "get_stock_price",
|
||||
"arguments": {
|
||||
"symbol": "AAPL",
|
||||
"start_date": "2021-01-01",
|
||||
"end_date": "2021-12-31",
|
||||
},
|
||||
},
|
||||
{"name": "find_bbox", "arguments": coord_arg},
|
||||
]
|
||||
|
||||
|
||||
def random_chunks(s: str, min_len: int, max_len: int):
|
||||
chunks = []
|
||||
i = 0
|
||||
n = len(s)
|
||||
|
||||
while i < n:
|
||||
size = random.randint(min_len, max_len)
|
||||
chunks.append(s[i : i + size])
|
||||
i += size
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tokenizer():
|
||||
return AutoTokenizer.from_pretrained(MODEL)
|
||||
|
||||
|
||||
# create a variety of input chunk sizes
|
||||
@pytest.mark.parametrize(
|
||||
"min_chunk, max_chunk",
|
||||
[
|
||||
(1, 1),
|
||||
(1, 2),
|
||||
(5, 7),
|
||||
(6, 20),
|
||||
],
|
||||
)
|
||||
def test_tool_call_parser_complex(min_chunk: int, max_chunk: int, tokenizer):
|
||||
input_dicts = create_complex_input(True)
|
||||
|
||||
formatted_tcs = [
|
||||
"<tool_call> " + json.dumps(call) + " </tool_call>" for call in input_dicts
|
||||
]
|
||||
|
||||
text_messages = [
|
||||
"Here goes the bbox call: \n",
|
||||
" Now the stock price call: \n ",
|
||||
" Now another bbox call: \n ",
|
||||
" See? I'm a helpful assistant.",
|
||||
]
|
||||
|
||||
test_input = (
|
||||
text_messages[0]
|
||||
+ formatted_tcs[0]
|
||||
+ text_messages[1]
|
||||
+ formatted_tcs[1]
|
||||
+ text_messages[2]
|
||||
+ formatted_tcs[2]
|
||||
+ text_messages[3]
|
||||
)
|
||||
|
||||
any_chat_request = ChatCompletionRequest(
|
||||
seed=42,
|
||||
model=MODEL,
|
||||
messages=[],
|
||||
)
|
||||
|
||||
parser = Granite4ToolParser(tokenizer=tokenizer)
|
||||
|
||||
delta_messages = list[DeltaMessage]()
|
||||
for text in random_chunks(test_input, min_chunk, max_chunk):
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text="",
|
||||
delta_text=text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=any_chat_request,
|
||||
)
|
||||
if delta is not None:
|
||||
delta_messages.append(delta)
|
||||
|
||||
content = ""
|
||||
tool_calls = list[dict[str, Any]]()
|
||||
|
||||
current_name = "__start__"
|
||||
current_args = ""
|
||||
|
||||
for msg in delta_messages:
|
||||
if msg.content:
|
||||
content += msg.content
|
||||
for tool_call in msg.tool_calls:
|
||||
if delta_func := tool_call.function:
|
||||
if delta_func.name is not None:
|
||||
if current_name == "__start__":
|
||||
current_name = delta_func.name
|
||||
|
||||
if delta_func.name != current_name:
|
||||
tool_calls.append(
|
||||
{
|
||||
"name": current_name,
|
||||
"arguments": json.loads(current_args),
|
||||
}
|
||||
)
|
||||
current_name = delta_func.name
|
||||
current_args = ""
|
||||
|
||||
if delta_func.arguments:
|
||||
current_args += delta_func.arguments
|
||||
|
||||
if current_name != "__start__":
|
||||
tool_calls.append({"name": current_name, "arguments": json.loads(current_args)})
|
||||
|
||||
assert content == "".join(text_messages)
|
||||
assert tool_calls == create_complex_input(False)
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
split_string_into_token_deltas,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.granite_engine_tool_parser import GraniteEngineToolParser
|
||||
|
||||
|
||||
class TestGraniteToolParser(ToolParserTests):
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="granite",
|
||||
# Test data
|
||||
no_tool_calls_output="This is a regular response without any tool calls.",
|
||||
single_tool_call_output=(
|
||||
'<|tool_call|> [{"name": "get_weather", '
|
||||
'"arguments": {"city": "Tokyo"}}]'
|
||||
),
|
||||
parallel_tool_calls_output="""<|tool_call|> [
|
||||
{"name": "get_weather", "arguments": {"city": "Tokyo"}},
|
||||
{"name": "get_time", "arguments": {"timezone": "Asia/Tokyo"}}
|
||||
]""",
|
||||
various_data_types_output="""<tool_call> [{
|
||||
"name": "test_function",
|
||||
"arguments": {
|
||||
"string_field": "hello",
|
||||
"int_field": 42,
|
||||
"float_field": 3.14,
|
||||
"bool_field": true,
|
||||
"null_field": null,
|
||||
"array_field": ["a", "b", "c"],
|
||||
"object_field": {"nested": "value"},
|
||||
"empty_array": [],
|
||||
"empty_object": {}
|
||||
}
|
||||
}]""",
|
||||
empty_arguments_output=(
|
||||
'<|tool_call|> [{"name": "refresh", "arguments": {}}]'
|
||||
),
|
||||
surrounding_text_output="""Let me check the weather for you.
|
||||
<|tool_call|> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]
|
||||
I'll get that information.""",
|
||||
escaped_strings_output="""<tool_call> [{
|
||||
"name": "test_function",
|
||||
"arguments": {
|
||||
"quoted": "He said \\"hello\\"",
|
||||
"path": "C:\\\\Users\\\\file.txt",
|
||||
"newline": "line1\\nline2",
|
||||
"unicode": "emoji: 🎉"
|
||||
}
|
||||
}]""",
|
||||
malformed_input_outputs=[
|
||||
'<|tool_call|> [{"name": "func", "arguments": {',
|
||||
'<|tool_call|> {"name": "func", "arguments": {}}', # Not an array
|
||||
'[{"name": "func", "arguments": "not a dict"}]',
|
||||
'Some text [{"name": "func"}]', # JSON but not tool call format
|
||||
],
|
||||
# Expected results
|
||||
single_tool_call_expected_name="get_weather",
|
||||
single_tool_call_expected_args={"city": "Tokyo"},
|
||||
# Granite strips content when tool calls present
|
||||
single_tool_call_expected_content=None,
|
||||
parallel_tool_calls_count=2,
|
||||
parallel_tool_calls_names=["get_weather", "get_time"],
|
||||
)
|
||||
|
||||
# Granite-Specific Tests
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_granite_token_prefix_format(self, tool_parser, streaming):
|
||||
"""Verify parser handles Granite 3.0 <|tool_call|> token format."""
|
||||
single_tool_call_token = (
|
||||
'<|tool_call|> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, single_tool_call_token, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) == 1, (
|
||||
f"Expected 1 tool call from token format, got {len(tool_calls)}"
|
||||
)
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_granite_string_prefix_format(self, tool_parser, streaming):
|
||||
"""Verify parser handles Granite 3.1 <tool_call> string format."""
|
||||
single_tool_call_string = (
|
||||
'<tool_call> [{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, single_tool_call_string, streaming=streaming
|
||||
)
|
||||
assert len(tool_calls) == 1, (
|
||||
f"Expected 1 tool call from string format, got {len(tool_calls)}"
|
||||
)
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
|
||||
|
||||
# granite emits arguments before name and its own tokenizer (not gpt2) is used
|
||||
# here so the token boundaries match production; get_tokenizer only fetches the
|
||||
# small tokenizer files, not the model weights.
|
||||
@pytest.fixture(scope="module")
|
||||
def granite_tokenizer():
|
||||
return get_tokenizer(tokenizer_name="ibm-granite/granite-3.1-8b-instruct")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [2, 3, 4, 5])
|
||||
def test_streaming_parallel_calls_batched_deltas(granite_tokenizer, chunk_size):
|
||||
"""A batched delta (multiple tokens) spanning the boundary between two
|
||||
parallel calls must not drop the first call's name. granite streams
|
||||
arguments before name, so the name only completes as the next call appears.
|
||||
"""
|
||||
parser = GraniteEngineToolParser(granite_tokenizer)
|
||||
model_output = (
|
||||
'<|tool_call|> [{"arguments": {"city": "Tokyo"}, "name": "get_weather"}, '
|
||||
'{"arguments": {"timezone": "Asia/Tokyo"}, "name": "get_time"}]'
|
||||
)
|
||||
token_deltas = split_string_into_token_deltas(granite_tokenizer, model_output)
|
||||
batched = [
|
||||
"".join(token_deltas[i : i + chunk_size])
|
||||
for i in range(0, len(token_deltas), chunk_size)
|
||||
]
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser, batched, assert_one_tool_per_delta=False
|
||||
)
|
||||
names = [tc.function.name for tc in reconstructor.tool_calls]
|
||||
assert names == ["get_weather", "get_time"]
|
||||
# trailing args of the final call are flushed by the serving layer
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"city": "Tokyo"
|
||||
}
|
||||
@@ -1,414 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.granite4_tool_parser import Granite4ToolParser
|
||||
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
|
||||
|
||||
CONFIGS = {
|
||||
"llama": {
|
||||
"tool_parser": Hermes2ProToolParser,
|
||||
},
|
||||
"granite4": {
|
||||
"tool_parser": Granite4ToolParser,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def qwen_tokenizer() -> TokenizerLike:
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
return get_tokenizer("Qwen/Qwen3-32B")
|
||||
|
||||
|
||||
@pytest.fixture(params=CONFIGS.keys())
|
||||
def hermes_parser(request, qwen_tokenizer: TokenizerLike) -> ToolParser:
|
||||
config = CONFIGS[request.param]
|
||||
return config["tool_parser"](qwen_tokenizer)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def any_chat_request() -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
seed=42,
|
||||
model="Qwen/Qwen3-32B",
|
||||
messages=[],
|
||||
)
|
||||
|
||||
|
||||
def test_hermes_parser_streaming_just_forward_text(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
text = """This is some prior text that has nothing to do with tool calling."""
|
||||
tokens = qwen_tokenizer.encode(text)
|
||||
previous_text = ""
|
||||
delta_messages = []
|
||||
for token in tokens:
|
||||
delta_text = qwen_tokenizer.decode([token])
|
||||
current_text = previous_text + delta_text
|
||||
delta = hermes_parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=any_chat_request,
|
||||
)
|
||||
previous_text = current_text
|
||||
delta_messages.append(delta)
|
||||
|
||||
for delta in delta_messages:
|
||||
assert delta is not None
|
||||
assert not delta.tool_calls
|
||||
|
||||
print(delta_messages)
|
||||
assert "".join([delta.content for delta in delta_messages]) == text
|
||||
|
||||
|
||||
def test_hermes_parser_streaming_failure_case_bug_19056(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
text = """<tool_call>
|
||||
{"name": "final_answer", "arguments": {"trigger": true}}
|
||||
</tool_call>"""
|
||||
tokens = qwen_tokenizer.encode(text)
|
||||
previous_text = ""
|
||||
delta_messages = []
|
||||
for token in tokens:
|
||||
text = qwen_tokenizer.decode([token])
|
||||
current_text = previous_text + text
|
||||
delta = hermes_parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=any_chat_request,
|
||||
)
|
||||
previous_text = current_text
|
||||
if delta is not None:
|
||||
delta_messages.append(delta)
|
||||
|
||||
assert delta_messages[0].tool_calls[0].function.name == "final_answer"
|
||||
tool_call_args = "".join(
|
||||
delta.tool_calls[0].function.arguments or "" for delta in delta_messages
|
||||
)
|
||||
assert tool_call_args == '{"trigger": true}'
|
||||
|
||||
|
||||
def test_hermes_parser_streaming(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
text = '<tool_call>\
|
||||
{"name": "get_current_temperature",\
|
||||
"arguments": {"location":\
|
||||
"San Francisco, California, United States", "unit": "celsius"}}\
|
||||
</tool_call>'
|
||||
|
||||
tokens = qwen_tokenizer.encode(text)
|
||||
previous_text = ""
|
||||
delta_messages = []
|
||||
for token in tokens:
|
||||
text = qwen_tokenizer.decode([token])
|
||||
current_text = previous_text + text
|
||||
delta = hermes_parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=any_chat_request,
|
||||
)
|
||||
previous_text = current_text
|
||||
if delta is not None:
|
||||
delta_messages.append(delta)
|
||||
print(delta_messages)
|
||||
assert delta_messages[0].tool_calls[0].function.name == "get_current_temperature"
|
||||
# load to normalize whitespace
|
||||
tool_call_args = json.loads(
|
||||
"".join(
|
||||
delta.tool_calls[0].function.arguments or "" for delta in delta_messages
|
||||
)
|
||||
)
|
||||
assert tool_call_args == {
|
||||
"location": "San Francisco, California, United States",
|
||||
"unit": "celsius",
|
||||
}
|
||||
|
||||
|
||||
def _simulate_streaming(
|
||||
tokenizer: TokenizerLike,
|
||||
parser: ToolParser,
|
||||
request: ChatCompletionRequest,
|
||||
text: str,
|
||||
stream_interval: int = 1,
|
||||
) -> list:
|
||||
"""Simulate streaming with a given stream_interval.
|
||||
|
||||
Tokens are batched into chunks of `stream_interval` tokens,
|
||||
mimicking how the output processor delivers them.
|
||||
Returns a list of non-None DeltaMessages.
|
||||
"""
|
||||
tokens = tokenizer.encode(text)
|
||||
previous_text = ""
|
||||
delta_messages = []
|
||||
for i in range(0, len(tokens), stream_interval):
|
||||
chunk_ids = tokens[i : i + stream_interval]
|
||||
delta_text = tokenizer.decode(chunk_ids)
|
||||
current_text = previous_text + delta_text
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=chunk_ids,
|
||||
request=request,
|
||||
)
|
||||
previous_text = current_text
|
||||
if delta is not None:
|
||||
delta_messages.append(delta)
|
||||
return delta_messages
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [2, 3, 5, 8])
|
||||
def test_hermes_streaming_tool_call_with_stream_interval(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
stream_interval: int,
|
||||
) -> None:
|
||||
"""Tool call streaming must produce correct name + args at any interval."""
|
||||
text = (
|
||||
'<tool_call>{"name": "get_current_temperature", '
|
||||
'"arguments": {"location": "San Francisco", "unit": "celsius"}}'
|
||||
"</tool_call>"
|
||||
)
|
||||
parser = Hermes2ProToolParser(qwen_tokenizer)
|
||||
deltas = _simulate_streaming(
|
||||
qwen_tokenizer, parser, any_chat_request, text, stream_interval
|
||||
)
|
||||
|
||||
# Flatten all DeltaToolCalls across all deltas.
|
||||
tool_deltas = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
|
||||
assert tool_deltas, "Expected at least one tool call delta"
|
||||
assert tool_deltas[0].function.name == "get_current_temperature"
|
||||
|
||||
# Concatenated arguments must be valid JSON matching the original.
|
||||
args_str = "".join(tc.function.arguments or "" for tc in tool_deltas)
|
||||
assert json.loads(args_str) == {
|
||||
"location": "San Francisco",
|
||||
"unit": "celsius",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [2, 3, 5, 8])
|
||||
def test_hermes_streaming_content_then_tool_call_with_stream_interval(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
stream_interval: int,
|
||||
) -> None:
|
||||
"""Content before a tool call must be fully streamed, then tool call."""
|
||||
text = (
|
||||
"Sure, let me check the weather."
|
||||
'<tool_call>{"name": "get_weather", '
|
||||
'"arguments": {"city": "NYC"}}</tool_call>'
|
||||
)
|
||||
parser = Hermes2ProToolParser(qwen_tokenizer)
|
||||
deltas = _simulate_streaming(
|
||||
qwen_tokenizer, parser, any_chat_request, text, stream_interval
|
||||
)
|
||||
|
||||
content_deltas = [d for d in deltas if d.content]
|
||||
tool_deltas = [d for d in deltas if d.tool_calls]
|
||||
|
||||
# Content must reconstruct the prefix.
|
||||
content_str = "".join(d.content for d in content_deltas)
|
||||
assert content_str == "Sure, let me check the weather."
|
||||
|
||||
# Tool call must be correct.
|
||||
tool_calls = [tc for d in tool_deltas for tc in d.tool_calls]
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
args_str = "".join(tc.function.arguments or "" for tc in tool_calls)
|
||||
assert json.loads(args_str) == {"city": "NYC"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [1, 2, 4])
|
||||
def test_hermes_streaming_multiple_tool_calls_with_stream_interval(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
stream_interval: int,
|
||||
) -> None:
|
||||
"""Multiple sequential tool calls must each be streamed correctly."""
|
||||
text = (
|
||||
'<tool_call>{"name": "search", "arguments": {"q": "cats"}}</tool_call>'
|
||||
'<tool_call>{"name": "search", "arguments": {"q": "dogs"}}</tool_call>'
|
||||
)
|
||||
parser = Hermes2ProToolParser(qwen_tokenizer)
|
||||
deltas = _simulate_streaming(
|
||||
qwen_tokenizer, parser, any_chat_request, text, stream_interval
|
||||
)
|
||||
|
||||
# Flatten all DeltaToolCalls across all deltas.
|
||||
all_tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
|
||||
|
||||
# Separate by tool index.
|
||||
tool0 = [tc for tc in all_tool_calls if tc.index == 0]
|
||||
tool1 = [tc for tc in all_tool_calls if tc.index == 1]
|
||||
|
||||
assert tool0[0].function.name == "search"
|
||||
args0 = "".join(tc.function.arguments or "" for tc in tool0)
|
||||
assert json.loads(args0) == {"q": "cats"}
|
||||
|
||||
assert tool1[0].function.name == "search"
|
||||
args1 = "".join(tc.function.arguments or "" for tc in tool1)
|
||||
assert json.loads(args1) == {"q": "dogs"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [2, 5])
|
||||
def test_hermes_streaming_boolean_args_with_stream_interval(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
stream_interval: int,
|
||||
) -> None:
|
||||
"""Regression test for bug #19056 with stream_interval > 1."""
|
||||
text = (
|
||||
"<tool_call>\n"
|
||||
'{"name": "final_answer", "arguments": {"trigger": true}}\n'
|
||||
"</tool_call>"
|
||||
)
|
||||
parser = Hermes2ProToolParser(qwen_tokenizer)
|
||||
deltas = _simulate_streaming(
|
||||
qwen_tokenizer, parser, any_chat_request, text, stream_interval
|
||||
)
|
||||
|
||||
tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
|
||||
assert tool_calls[0].function.name == "final_answer"
|
||||
args_str = "".join(tc.function.arguments or "" for tc in tool_calls)
|
||||
assert json.loads(args_str) == {"trigger": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [2, 3, 5])
|
||||
def test_hermes_streaming_just_forward_text_with_stream_interval(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
stream_interval: int,
|
||||
) -> None:
|
||||
"""Plain text with no tool calls must be fully forwarded."""
|
||||
text = "This is plain text with no tool calling involved."
|
||||
parser = Hermes2ProToolParser(qwen_tokenizer)
|
||||
deltas = _simulate_streaming(
|
||||
qwen_tokenizer, parser, any_chat_request, text, stream_interval
|
||||
)
|
||||
|
||||
for d in deltas:
|
||||
assert not d.tool_calls
|
||||
assert "".join(d.content for d in deltas) == text
|
||||
|
||||
|
||||
def test_hermes_parser_non_streaming_no_tool_call(
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
text = """This is not a tool call."""
|
||||
tool_call = hermes_parser.extract_tool_calls(
|
||||
model_output=text,
|
||||
request=any_chat_request,
|
||||
)
|
||||
|
||||
assert tool_call is not None
|
||||
assert not tool_call.tools_called
|
||||
|
||||
|
||||
def test_hermes_parser_non_streaming_tool_call_between_tags(
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
text = """<tool_call>
|
||||
{"name": "final_answer", "arguments": {"trigger": true}}
|
||||
</tool_call>"""
|
||||
tool_call = hermes_parser.extract_tool_calls(
|
||||
model_output=text,
|
||||
request=any_chat_request,
|
||||
)
|
||||
|
||||
assert tool_call is not None
|
||||
assert tool_call.tools_called
|
||||
assert tool_call.tool_calls[0].function.name == "final_answer"
|
||||
assert tool_call.tool_calls[0].function.arguments == '{"trigger": true}'
|
||||
|
||||
|
||||
def test_hermes_parser_non_streaming_tool_call_until_eos(
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
if isinstance(hermes_parser, Granite4ToolParser):
|
||||
pytest.skip(reason="The Granite4 tool parser enforces a complete response")
|
||||
|
||||
text = """<tool_call>
|
||||
{"name": "final_answer", "arguments": {"trigger": true}}"""
|
||||
tool_call = hermes_parser.extract_tool_calls(
|
||||
model_output=text,
|
||||
request=any_chat_request,
|
||||
)
|
||||
|
||||
assert tool_call is not None
|
||||
assert tool_call.tools_called
|
||||
assert tool_call.tool_calls[0].function.name == "final_answer"
|
||||
assert tool_call.tool_calls[0].function.arguments == '{"trigger": true}'
|
||||
|
||||
|
||||
def test_hermes_parser_non_streaming_tool_call_invalid_json(
|
||||
hermes_parser: ToolParser,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
# Missing closing brace to trigger exception
|
||||
text = """<tool_call>
|
||||
{"name": "final_answer", "arguments": {"trigger": true}"""
|
||||
tool_call = hermes_parser.extract_tool_calls(
|
||||
model_output=text,
|
||||
request=any_chat_request,
|
||||
)
|
||||
|
||||
assert tool_call is not None
|
||||
assert not tool_call.tools_called
|
||||
|
||||
|
||||
def test_hermes_streaming_content_and_tool_call_in_single_chunk(
|
||||
qwen_tokenizer: TokenizerLike,
|
||||
any_chat_request: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Content + complete tool call in one chunk must both be emitted."""
|
||||
text = 'Hi!<tool_call>{"name": "f", "arguments": {"x": 1}}</tool_call>'
|
||||
# Use a stream_interval large enough to guarantee a single chunk.
|
||||
parser = Hermes2ProToolParser(qwen_tokenizer)
|
||||
deltas = _simulate_streaming(
|
||||
qwen_tokenizer,
|
||||
parser,
|
||||
any_chat_request,
|
||||
text,
|
||||
stream_interval=9999,
|
||||
)
|
||||
|
||||
content_parts = [d.content for d in deltas if d.content]
|
||||
tool_parts = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
|
||||
|
||||
assert "".join(content_parts) == "Hi!"
|
||||
assert tool_parts[0].function.name == "f"
|
||||
args_str = "".join(tc.function.arguments or "" for tc in tool_parts)
|
||||
assert json.loads(args_str) == {"x": 1}
|
||||
@@ -1,191 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall, ToolCall
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
|
||||
|
||||
def make_tool_call(name, arguments):
|
||||
return ToolCall(
|
||||
type="function",
|
||||
function=FunctionCall(name=name, arguments=json.dumps(arguments)),
|
||||
)
|
||||
|
||||
|
||||
# TODO: add reason prefix and suffix.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_output,expected_tool_calls,expected_content",
|
||||
[
|
||||
# No tool call
|
||||
("How can I help you today?", [], "How can I help you today?"),
|
||||
# Single tool call, no content
|
||||
(
|
||||
'<tool_calls>[{"name": "get_weather", "arguments": {"city": "San Francisco", "metric": "celsius"}}]</tool_calls>', # noqa: E501
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather", {"city": "San Francisco", "metric": "celsius"}
|
||||
)
|
||||
],
|
||||
None,
|
||||
),
|
||||
# Multiple tool calls
|
||||
(
|
||||
'<tool_calls>[{"name": "get_weather", "arguments": {"city": "San Francisco", "metric": "celsius"}}, {"name": "register_user", "arguments": {"name": "John Doe", "age": 37, "address": {"city": "San Francisco", "state": "CA"}, "role": null, "passed_test": true, "aliases": ["John", "Johnny"]}}]</tool_calls>', # noqa: E501
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather", {"city": "San Francisco", "metric": "celsius"}
|
||||
),
|
||||
make_tool_call(
|
||||
"register_user",
|
||||
{
|
||||
"name": "John Doe",
|
||||
"age": 37,
|
||||
"address": {"city": "San Francisco", "state": "CA"},
|
||||
"role": None,
|
||||
"passed_test": True,
|
||||
"aliases": ["John", "Johnny"],
|
||||
},
|
||||
),
|
||||
],
|
||||
None,
|
||||
),
|
||||
# Content before tool call
|
||||
(
|
||||
'I will call the tool now. <tool_calls>[{"name": "get_weather", "arguments": {"city": "Boston"}}]</tool_calls>', # noqa: E501
|
||||
[make_tool_call("get_weather", {"city": "Boston"})],
|
||||
"I will call the tool now. ",
|
||||
),
|
||||
# Content after tool call (should be stripped)
|
||||
(
|
||||
'<tool_calls>[{"name": "get_weather", "arguments": {"city": "Seattle"}}]</tool_calls>\nThank you!', # noqa: E501
|
||||
[make_tool_call("get_weather", {"city": "Seattle"})],
|
||||
None,
|
||||
),
|
||||
(
|
||||
'<tool_calls>[{"name": "complex_tool", "arguments": {"level1": {"level2": {"level3": {"value": 123}}}}}]</tool_calls>',
|
||||
[
|
||||
make_tool_call(
|
||||
"complex_tool", {"level1": {"level2": {"level3": {"value": 123}}}}
|
||||
)
|
||||
],
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hunyuan_a13b_tool_parser_extract(
|
||||
model_output, expected_tool_calls, expected_content
|
||||
):
|
||||
mock_tokenizer = MagicMock()
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("hunyuan_a13b")(
|
||||
mock_tokenizer
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=False
|
||||
)
|
||||
|
||||
# align the random id.
|
||||
for idx in range(len(tool_calls)):
|
||||
tool_calls[idx].id = expected_tool_calls[idx].id
|
||||
assert tool_calls == expected_tool_calls
|
||||
assert content == expected_content
|
||||
|
||||
|
||||
# Streaming test: simulate incremental output
|
||||
@pytest.mark.parametrize(
|
||||
"model_deltas,expected_tool_calls",
|
||||
[
|
||||
(
|
||||
[
|
||||
'<tool_calls>[{"name": "get_weather", ',
|
||||
'"arguments": {"city": "San Francisco", ',
|
||||
'"metric": "celsius"}}]',
|
||||
"</tool_calls>",
|
||||
],
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather", {"city": "San Francisco", "metric": "celsius"}
|
||||
)
|
||||
],
|
||||
),
|
||||
(
|
||||
[
|
||||
'<tool_calls>[{"name":',
|
||||
' "get_weather",',
|
||||
' "arguments":',
|
||||
' {"city": "Boston"}',
|
||||
"}]",
|
||||
"</tool_calls>",
|
||||
],
|
||||
[make_tool_call("get_weather", {"city": "Boston"})],
|
||||
),
|
||||
(
|
||||
[
|
||||
"",
|
||||
'<tool_calls>[{"name":',
|
||||
' "get_weather",',
|
||||
' "arguments":',
|
||||
' {"city": "Boston"}',
|
||||
"}]",
|
||||
"</tool_calls>",
|
||||
"\n</answer>",
|
||||
],
|
||||
[make_tool_call("get_weather", {"city": "Boston"})],
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
'<tool_calls>[{"name": "complex_tool",',
|
||||
' "arguments": ',
|
||||
' {"level1": {"level2": ',
|
||||
'{"level3": {"value": 123}}}}}',
|
||||
"]</tool_calls>",
|
||||
],
|
||||
[
|
||||
make_tool_call(
|
||||
"complex_tool", {"level1": {"level2": {"level3": {"value": 123}}}}
|
||||
)
|
||||
],
|
||||
marks=pytest.mark.xfail(
|
||||
reason="stream parsing not support nested json yet."
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_hunyuan_a13b_tool_parser_streaming(model_deltas, expected_tool_calls):
|
||||
mock_tokenizer = MagicMock()
|
||||
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("hunyuan_a13b")(
|
||||
mock_tokenizer
|
||||
)
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser, model_deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
# align the random id.
|
||||
for idx in range(len(reconstructor.tool_calls)):
|
||||
reconstructor.tool_calls[idx].id = expected_tool_calls[idx].id
|
||||
|
||||
assert reconstructor.tool_calls == expected_tool_calls
|
||||
|
||||
|
||||
def test_hunyuan_a13b_tool_parser_non_ascii():
|
||||
mock_tokenizer = MagicMock()
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("hunyuan_a13b")(
|
||||
mock_tokenizer
|
||||
)
|
||||
model_output = '<tool_calls>[{"name": "get_weather", "arguments": {"city": "北京"}}]</tool_calls>'
|
||||
_, tool_calls = run_tool_extraction(tool_parser, model_output, streaming=False)
|
||||
args = tool_calls[0].function.arguments
|
||||
assert "北京" in args
|
||||
assert "\\u" not in args
|
||||
@@ -1,274 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
"""Tests for the HYV3 tool call parser."""
|
||||
|
||||
import json
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.hy_v3_tool_parser import HYV3ToolParser
|
||||
|
||||
parser_name = "hy_v3"
|
||||
MODEL = "tencent/Hy3-preview"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def hy_v3_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hy_v3_tool_parser(hy_v3_tokenizer):
|
||||
return HYV3ToolParser(hy_v3_tokenizer)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request() -> ChatCompletionRequest:
|
||||
request = Mock(spec=ChatCompletionRequest)
|
||||
request.tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(name="get_current_date", parameters={}),
|
||||
),
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="get_weather",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"date": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
]
|
||||
request.tool_choice = "auto"
|
||||
return request
|
||||
|
||||
|
||||
class TestHYV3ExtractToolCalls:
|
||||
def test_no_tool_call(self, hy_v3_tool_parser, mock_request):
|
||||
out = "This is a plain response."
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert not r.tools_called
|
||||
assert r.content == out
|
||||
|
||||
def test_zero_arg_inline(self, hy_v3_tool_parser, mock_request):
|
||||
out = (
|
||||
"<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>"
|
||||
)
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert r.tool_calls[0].function.name == "get_current_date"
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {}
|
||||
assert r.content is None
|
||||
|
||||
def test_zero_arg_newline(self, hy_v3_tool_parser, mock_request):
|
||||
out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert r.tool_calls[0].function.name == "get_current_date"
|
||||
|
||||
def test_args_same_line(self, hy_v3_tool_parser, mock_request):
|
||||
out = (
|
||||
"<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing"
|
||||
"</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>"
|
||||
)
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {
|
||||
"city": "Beijing",
|
||||
"date": "2026-03-30",
|
||||
}
|
||||
|
||||
def test_args_with_newlines(self, hy_v3_tool_parser, mock_request):
|
||||
out = (
|
||||
"<tool_calls>\n<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Beijing"
|
||||
"</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n</tool_calls>"
|
||||
)
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert json.loads(r.tool_calls[0].function.arguments) == {
|
||||
"city": "Beijing",
|
||||
"date": "2026-03-30",
|
||||
}
|
||||
|
||||
def test_content_before(self, hy_v3_tool_parser, mock_request):
|
||||
out = "Checking.<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.tools_called
|
||||
assert r.content == "Checking."
|
||||
|
||||
def test_multiple(self, hy_v3_tool_parser, mock_request):
|
||||
out = (
|
||||
"<tool_calls>\n<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Beijing"
|
||||
"</arg_value>\n<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n"
|
||||
"<tool_call>get_weather<tool_sep>\n<arg_key>city</arg_key>\n<arg_value>Hangzhou</arg_value>\n"
|
||||
"<arg_key>date</arg_key>\n<arg_value>2026-03-30</arg_value>\n</tool_call>\n</tool_calls>"
|
||||
)
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert len(r.tool_calls) == 2
|
||||
|
||||
def test_empty_content_none(self, hy_v3_tool_parser, mock_request):
|
||||
out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
|
||||
r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request)
|
||||
assert r.content is None
|
||||
|
||||
|
||||
def _simulate_streaming(
|
||||
parser: HYV3ToolParser,
|
||||
deltas: list[str],
|
||||
request: ChatCompletionRequest,
|
||||
) -> list[DeltaMessage | None]:
|
||||
results: list[DeltaMessage | None] = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
vocab = parser.vocab
|
||||
for delta_text in deltas:
|
||||
current_text = previous_text + delta_text
|
||||
delta_token_ids = [tid for tok, tid in vocab.items() if tok in delta_text]
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
result = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=previous_token_ids,
|
||||
current_token_ids=current_token_ids,
|
||||
delta_token_ids=delta_token_ids,
|
||||
request=request,
|
||||
)
|
||||
results.append(result)
|
||||
previous_text = current_text
|
||||
previous_token_ids = current_token_ids
|
||||
return results
|
||||
|
||||
|
||||
def _collect_streaming_tool_calls(results: list[DeltaMessage | None]) -> list[dict]:
|
||||
tool_calls: dict[int, dict] = {}
|
||||
for result in results:
|
||||
if result is None or not result.tool_calls:
|
||||
continue
|
||||
for tc in result.tool_calls:
|
||||
idx = tc.index
|
||||
if idx not in tool_calls:
|
||||
tool_calls[idx] = {
|
||||
"name": tc.function.name or "",
|
||||
"arguments": tc.function.arguments or "",
|
||||
}
|
||||
else:
|
||||
if tc.function.name:
|
||||
tool_calls[idx]["name"] += tc.function.name
|
||||
if tc.function.arguments:
|
||||
tool_calls[idx]["arguments"] += tc.function.arguments
|
||||
return [tool_calls[i] for i in sorted(tool_calls.keys())]
|
||||
|
||||
|
||||
def _collect_streaming_content(results: list[DeltaMessage | None]) -> str:
|
||||
parts = []
|
||||
for result in results:
|
||||
if result is not None and result.content:
|
||||
parts.append(result.content)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
class TestHYV3ExtractToolCallsStreaming:
|
||||
def test_no_tool_call_streaming(self, hy_v3_tool_parser, mock_request):
|
||||
deltas = ["This is ", "a plain ", "response."]
|
||||
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
|
||||
content = _collect_streaming_content(results)
|
||||
assert content == "This is a plain response."
|
||||
assert len(_collect_streaming_tool_calls(results)) == 0
|
||||
|
||||
def test_zero_arg_streaming(self, hy_v3_tool_parser, mock_request):
|
||||
deltas = [
|
||||
"<tool_calls>",
|
||||
"\n<tool_call>",
|
||||
"get_current_date",
|
||||
"<tool_sep>",
|
||||
"\n</tool_call>",
|
||||
"\n</tool_calls>",
|
||||
]
|
||||
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
|
||||
tc = _collect_streaming_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert tc[0]["name"] == "get_current_date"
|
||||
assert json.loads(tc[0]["arguments"]) == {}
|
||||
|
||||
def test_args_streaming(self, hy_v3_tool_parser, mock_request):
|
||||
deltas = [
|
||||
"<tool_calls>",
|
||||
"\n<tool_call>",
|
||||
"get_weather",
|
||||
"<tool_sep>",
|
||||
"\n<arg_key>city</arg_key>",
|
||||
"\n<arg_value>Beijing</arg_value>",
|
||||
"\n<arg_key>date</arg_key>",
|
||||
"\n<arg_value>2026-03-30</arg_value>",
|
||||
"\n</tool_call>",
|
||||
"\n</tool_calls>",
|
||||
]
|
||||
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
|
||||
tc = _collect_streaming_tool_calls(results)
|
||||
assert len(tc) == 1 and tc[0]["name"] == "get_weather"
|
||||
assert json.loads(tc[0]["arguments"]) == {
|
||||
"city": "Beijing",
|
||||
"date": "2026-03-30",
|
||||
}
|
||||
|
||||
def test_content_before_streaming(self, hy_v3_tool_parser, mock_request):
|
||||
deltas = [
|
||||
"Checking.",
|
||||
"<tool_calls>",
|
||||
"\n<tool_call>",
|
||||
"get_current_date",
|
||||
"<tool_sep>",
|
||||
"\n</tool_call>",
|
||||
"\n</tool_calls>",
|
||||
]
|
||||
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
|
||||
assert "Checking." in _collect_streaming_content(results)
|
||||
tc = _collect_streaming_tool_calls(results)
|
||||
assert len(tc) == 1 and tc[0]["name"] == "get_current_date"
|
||||
|
||||
def test_multiple_streaming(self, hy_v3_tool_parser, mock_request):
|
||||
deltas = [
|
||||
"<tool_calls>",
|
||||
"\n<tool_call>",
|
||||
"get_weather",
|
||||
"<tool_sep>",
|
||||
"\n<arg_key>city</arg_key>",
|
||||
"\n<arg_value>Beijing</arg_value>",
|
||||
"\n<arg_key>date</arg_key>",
|
||||
"\n<arg_value>2026-03-30</arg_value>",
|
||||
"\n</tool_call>",
|
||||
"\n<tool_call>",
|
||||
"get_weather",
|
||||
"<tool_sep>",
|
||||
"\n<arg_key>city</arg_key>",
|
||||
"\n<arg_value>Hangzhou</arg_value>",
|
||||
"\n<arg_key>date</arg_key>",
|
||||
"\n<arg_value>2026-03-30</arg_value>",
|
||||
"\n</tool_call>",
|
||||
"\n</tool_calls>",
|
||||
]
|
||||
results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request)
|
||||
tc = _collect_streaming_tool_calls(results)
|
||||
assert len(tc) == 2
|
||||
assert json.loads(tc[0]["arguments"])["city"] == "Beijing"
|
||||
assert json.loads(tc[1]["arguments"])["city"] == "Hangzhou"
|
||||
|
||||
def test_all_in_one_delta_streaming(self, hy_v3_tool_parser, mock_request):
|
||||
out = "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>"
|
||||
results = _simulate_streaming(hy_v3_tool_parser, [out], mock_request)
|
||||
tc = _collect_streaming_tool_calls(results)
|
||||
assert len(tc) == 1 and tc[0]["name"] == "get_current_date"
|
||||
assert json.loads(tc[0]["arguments"]) == {}
|
||||
@@ -1,192 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from tests.tool_parsers.utils import run_tool_extraction_streaming
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.internlm2_tool_parser import Internlm2ToolParser
|
||||
|
||||
|
||||
class TestInternLM2ToolParser(ToolParserTests):
|
||||
@pytest.fixture
|
||||
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""Add some internlm2 specific tokens to the default vocab."""
|
||||
tokenizer_vocab = default_tokenizer.get_vocab()
|
||||
default_tokenizer.get_vocab = MagicMock()
|
||||
tokenizer_vocab.update(
|
||||
{
|
||||
"<|action_start|>": 92540,
|
||||
"<|plugin|>": 92541,
|
||||
"<|action_end|>": 92542,
|
||||
}
|
||||
)
|
||||
default_tokenizer.get_vocab.return_value = tokenizer_vocab
|
||||
return default_tokenizer
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="internlm",
|
||||
# Test data
|
||||
no_tool_calls_output="This is a regular response without any tool calls.",
|
||||
single_tool_call_output=(
|
||||
'<|action_start|><|plugin|>{"name": "get_weather", '
|
||||
'"parameters": {"city": "Tokyo"}}<|action_end|>'
|
||||
),
|
||||
# InternLM2 doesn't support parallel calls
|
||||
parallel_tool_calls_output=(
|
||||
'<|action_start|><|plugin|>{"name": "get_weather", '
|
||||
'"parameters": {"city": "Tokyo"}}<|action_end|>'
|
||||
),
|
||||
various_data_types_output="""<|action_start|><|plugin|>{
|
||||
"name": "test_function",
|
||||
"parameters": {
|
||||
"string_field": "hello",
|
||||
"int_field": 42,
|
||||
"float_field": 3.14,
|
||||
"bool_field": true,
|
||||
"null_field": null,
|
||||
"array_field": ["a", "b", "c"],
|
||||
"object_field": {"nested": "value"},
|
||||
"empty_array": [],
|
||||
"empty_object": {}
|
||||
}
|
||||
}<|action_end|>""",
|
||||
empty_arguments_output=(
|
||||
'<|action_start|><|plugin|>{"name": "refresh", '
|
||||
'"parameters": {}}<|action_end|>'
|
||||
),
|
||||
surrounding_text_output=(
|
||||
"Let me check the weather for you. "
|
||||
'<|action_start|><|plugin|>{"name": "get_weather", '
|
||||
'"parameters": {"city": "Tokyo"}}<|action_end|>'
|
||||
),
|
||||
escaped_strings_output="""<|action_start|><|plugin|>{
|
||||
"name": "test_function",
|
||||
"parameters": {
|
||||
"quoted": "He said \\"hello\\"",
|
||||
"path": "C:\\\\Users\\\\file.txt",
|
||||
"newline": "line1\\nline2",
|
||||
"unicode": "emoji: 🎉"
|
||||
}
|
||||
}<|action_end|>""",
|
||||
malformed_input_outputs=[
|
||||
'<|action_start|><|plugin|>{"name": "func", "parameters": {',
|
||||
(
|
||||
'<|action_start|><|plugin|>{"name": "func", '
|
||||
'"parameters": "not a dict"}<|action_end|>'
|
||||
),
|
||||
"<|action_start|><|plugin|>not json<|action_end|>",
|
||||
"<|action_start|><|plugin|>",
|
||||
'<|action_start|>{"name": "func"}',
|
||||
],
|
||||
# Expected results
|
||||
single_tool_call_expected_name="get_weather",
|
||||
single_tool_call_expected_args={"city": "Tokyo"},
|
||||
single_tool_call_expected_content=None,
|
||||
parallel_tool_calls_count=1, # InternLM2 only supports single tool calls
|
||||
parallel_tool_calls_names=["get_weather"],
|
||||
# Parser-specific settings
|
||||
allow_empty_or_json_empty_args=True,
|
||||
# xfail markers
|
||||
xfail_streaming={
|
||||
"test_single_tool_call_simple_args": (
|
||||
"InternLM2 streaming not fully implemented"
|
||||
),
|
||||
"test_parallel_tool_calls": (
|
||||
"InternLM2 streaming not fully implemented"
|
||||
),
|
||||
"test_various_data_types": (
|
||||
"InternLM2 streaming not fully implemented"
|
||||
),
|
||||
"test_empty_arguments": ("InternLM2 streaming not fully implemented"),
|
||||
"test_surrounding_text": ("InternLM2 streaming not fully implemented"),
|
||||
"test_escaped_strings": ("InternLM2 streaming not fully implemented"),
|
||||
"test_streaming_reconstruction": (
|
||||
"InternLM2 streaming parser returns '<|action_start|' as "
|
||||
"content instead of None - streaming/non-streaming inconsistency"
|
||||
),
|
||||
},
|
||||
xfail_nonstreaming={
|
||||
"test_malformed_input": (
|
||||
"InternLM2 parser raises JSONDecodeError on malformed JSON "
|
||||
"instead of gracefully handling it"
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_streaming_arguments_in_single_delta(default_tokenizer: TokenizerLike) -> None:
|
||||
"""Arguments arriving whole in one delta must not be dropped."""
|
||||
tokenizer_vocab = default_tokenizer.get_vocab()
|
||||
default_tokenizer.get_vocab = MagicMock()
|
||||
tokenizer_vocab.update(
|
||||
{
|
||||
"<|action_start|>": 92540,
|
||||
"<|plugin|>": 92541,
|
||||
"<|action_end|>": 92542,
|
||||
}
|
||||
)
|
||||
default_tokenizer.get_vocab.return_value = tokenizer_vocab
|
||||
parser = Internlm2ToolParser(default_tokenizer)
|
||||
|
||||
deltas = [
|
||||
'<|action_start|><|plugin|>{"name": "get_weather"',
|
||||
', "parameters": {"city": "Dallas", "state": "TX"}}<|action_end|>',
|
||||
]
|
||||
|
||||
streamed = ""
|
||||
current_text = ""
|
||||
for delta_text in deltas:
|
||||
previous_text = current_text
|
||||
current_text += delta_text
|
||||
delta_message = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=None,
|
||||
)
|
||||
if delta_message and delta_message.tool_calls:
|
||||
arguments = delta_message.tool_calls[0].function.arguments
|
||||
if arguments:
|
||||
streamed += arguments
|
||||
|
||||
assert json.loads(streamed) == {"city": "Dallas", "state": "TX"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arguments_deltas",
|
||||
[
|
||||
['{"location": "Beijing', '", "days": 3}}<|action_end|>'],
|
||||
[*'{"location": "Beijing", "days": 3}}', "<|action_end|>"],
|
||||
],
|
||||
ids=["split_inside_string", "char_by_char"],
|
||||
)
|
||||
def test_streaming_arguments_match_final_arguments(
|
||||
default_tokenizer: TokenizerLike, arguments_deltas: list[str]
|
||||
) -> None:
|
||||
"""Streamed argument deltas must add up to the final arguments, wherever
|
||||
the chunk boundaries fall."""
|
||||
parser = Internlm2ToolParser(default_tokenizer)
|
||||
deltas = [
|
||||
'<|action_start|><|plugin|>{"name": "get_weather", "parameters": ',
|
||||
*arguments_deltas,
|
||||
]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
streamed = reconstructor.tool_calls[0].function.arguments
|
||||
streamed += parser.get_remaining_unstreamed_args()
|
||||
assert streamed == '{"location": "Beijing", "days": 3}'
|
||||
@@ -1,339 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Generator
|
||||
|
||||
import partial_json_parser
|
||||
import pytest
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage, FunctionCall, ToolCall
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tool_parsers.jamba_tool_parser import JambaToolParser
|
||||
|
||||
MODEL = "ai21labs/Jamba-tiny-dev"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def jamba_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def jamba_tool_parser(jamba_tokenizer):
|
||||
return JambaToolParser(jamba_tokenizer)
|
||||
|
||||
|
||||
def assert_tool_calls(
|
||||
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
|
||||
):
|
||||
assert len(actual_tool_calls) == len(expected_tool_calls)
|
||||
|
||||
for actual_tool_call, expected_tool_call in zip(
|
||||
actual_tool_calls, expected_tool_calls
|
||||
):
|
||||
assert isinstance(actual_tool_call.id, str)
|
||||
assert len(actual_tool_call.id) > 16
|
||||
|
||||
assert actual_tool_call.type == "function"
|
||||
assert actual_tool_call.function == expected_tool_call.function
|
||||
|
||||
|
||||
def stream_delta_message_generator(
|
||||
jamba_tool_parser: JambaToolParser,
|
||||
jamba_tokenizer: TokenizerLike,
|
||||
model_output: str,
|
||||
) -> Generator[DeltaMessage, None, None]:
|
||||
all_token_ids = jamba_tokenizer.encode(model_output, add_special_tokens=False)
|
||||
|
||||
previous_text = ""
|
||||
previous_tokens = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, delta_token in enumerate(all_token_ids):
|
||||
delta_token_ids = [delta_token]
|
||||
previous_token_ids = all_token_ids[:i]
|
||||
current_token_ids = all_token_ids[: i + 1]
|
||||
|
||||
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
|
||||
detokenize_incrementally(
|
||||
tokenizer=jamba_tokenizer,
|
||||
all_input_ids=current_token_ids,
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
)
|
||||
|
||||
current_text = previous_text + delta_text
|
||||
|
||||
delta_message = jamba_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request=None,
|
||||
)
|
||||
if delta_message:
|
||||
yield delta_message
|
||||
|
||||
previous_text = current_text
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
prefix_offset = new_prefix_offset
|
||||
read_offset = new_read_offset
|
||||
|
||||
|
||||
def test_extract_tool_calls_no_tools(jamba_tool_parser):
|
||||
model_output = "This is a test"
|
||||
extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert not extracted_tool_calls.tools_called
|
||||
assert extracted_tool_calls.tool_calls == []
|
||||
assert extracted_tool_calls.content == model_output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
"single_tool",
|
||||
"single_tool_with_content",
|
||||
"parallel_tools",
|
||||
],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
(
|
||||
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
""" Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
" Sure! let me call the tool for you.",
|
||||
),
|
||||
(
|
||||
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Orlando", "state": "FL", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls(
|
||||
jamba_tool_parser, model_output, expected_tool_calls, expected_content
|
||||
):
|
||||
extracted_tool_calls = jamba_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert extracted_tool_calls.tools_called
|
||||
|
||||
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
|
||||
|
||||
assert extracted_tool_calls.content == expected_content
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
"no_tools",
|
||||
"single_tool",
|
||||
"single_tool_with_content",
|
||||
"parallel_tools",
|
||||
],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
("""This is a test""", [], """This is a test"""),
|
||||
(
|
||||
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
" ",
|
||||
),
|
||||
(
|
||||
""" Sure! let me call the tool for you.<tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
" Sure! let me call the tool for you.",
|
||||
),
|
||||
(
|
||||
""" <tool_calls>[\n {"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}},\n {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}\n]</tool_calls>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{"city": "Orlando", "state": "FL", "unit": "fahrenheit"}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
" ",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls_streaming(
|
||||
jamba_tool_parser,
|
||||
jamba_tokenizer,
|
||||
model_output,
|
||||
expected_tool_calls,
|
||||
expected_content,
|
||||
):
|
||||
other_content: str = ""
|
||||
function_names: list[str] = []
|
||||
function_args_strs: list[str] = []
|
||||
tool_call_idx: int = -1
|
||||
tool_call_ids: list[str | None] = []
|
||||
|
||||
for delta_message in stream_delta_message_generator(
|
||||
jamba_tool_parser, jamba_tokenizer, model_output
|
||||
):
|
||||
# role should never be streamed from tool parser
|
||||
assert not delta_message.role
|
||||
|
||||
if delta_message.content:
|
||||
other_content += delta_message.content
|
||||
|
||||
streamed_tool_calls = delta_message.tool_calls
|
||||
|
||||
if streamed_tool_calls and len(streamed_tool_calls) > 0:
|
||||
# make sure only one diff is present - correct even for parallel
|
||||
assert len(streamed_tool_calls) == 1
|
||||
tool_call = streamed_tool_calls[0]
|
||||
|
||||
# if a new tool is being called, set up empty arguments
|
||||
if tool_call.index != tool_call_idx:
|
||||
tool_call_idx = tool_call.index
|
||||
function_args_strs.append("")
|
||||
tool_call_ids.append(None)
|
||||
|
||||
# if a tool call ID is streamed, make sure one hasn't been already
|
||||
if tool_call.id and not tool_call_ids[tool_call.index]:
|
||||
tool_call_ids[tool_call.index] = tool_call.id
|
||||
|
||||
# if parts of the function start being streamed
|
||||
if tool_call.function:
|
||||
# if the function name is defined, set it. it should be streamed
|
||||
# IN ENTIRETY, exactly one time.
|
||||
if tool_call.function.name:
|
||||
assert isinstance(tool_call.function.name, str)
|
||||
function_names.append(tool_call.function.name)
|
||||
|
||||
if tool_call.function.arguments:
|
||||
# make sure they're a string and then add them to the list
|
||||
assert isinstance(tool_call.function.arguments, str)
|
||||
|
||||
function_args_strs[tool_call.index] += tool_call.function.arguments
|
||||
|
||||
assert other_content == expected_content
|
||||
|
||||
actual_tool_calls = [
|
||||
ToolCall(
|
||||
id=tool_call_id,
|
||||
function=FunctionCall(
|
||||
name=function_name,
|
||||
arguments=partial_json_parser.ensure_json(
|
||||
function_args_str, Allow.OBJ | Allow.STR
|
||||
),
|
||||
),
|
||||
)
|
||||
for tool_call_id, function_name, function_args_str in zip(
|
||||
tool_call_ids, function_names, function_args_strs
|
||||
)
|
||||
]
|
||||
assert_tool_calls(actual_tool_calls, expected_tool_calls)
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_arguments_in_single_delta(jamba_tool_parser):
|
||||
"""Arguments delivered whole in one coarse delta must not be dropped."""
|
||||
deltas = [
|
||||
'<tool_calls>[{"name": "get_current_weather"',
|
||||
",",
|
||||
' "arguments": {"city": "Dallas", "state": "TX"}}]',
|
||||
"</tool_calls>",
|
||||
]
|
||||
|
||||
streamed_arguments = ""
|
||||
current_text = ""
|
||||
for delta_text in deltas:
|
||||
previous_text = current_text
|
||||
current_text += delta_text
|
||||
delta_message = jamba_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=None,
|
||||
)
|
||||
if delta_message and delta_message.tool_calls:
|
||||
arguments = delta_message.tool_calls[0].function.arguments
|
||||
if arguments:
|
||||
streamed_arguments += arguments
|
||||
|
||||
assert json.loads(streamed_arguments) == {"city": "Dallas", "state": "TX"}
|
||||
@@ -1,330 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.k2_horizon_tool_parser import K2HorizonToolParser
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "lookup",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"limit": {"type": "integer"},
|
||||
"enabled": {"type": "boolean"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "ping",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.get_vocab.return_value = {}
|
||||
return tokenizer
|
||||
|
||||
|
||||
def _request(
|
||||
tool_format: str | None = None,
|
||||
tool_choice="auto",
|
||||
) -> ChatCompletionRequest:
|
||||
chat_template_kwargs = (
|
||||
{"tool_call_format": tool_format} if tool_format is not None else None
|
||||
)
|
||||
return ChatCompletionRequest(
|
||||
model="k2-horizon",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
tools=TOOLS,
|
||||
tool_choice=tool_choice,
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _group(*calls: str, prefix: str = "", suffix: str = "") -> str:
|
||||
return prefix + "<ifm|tool_calls>" + "\n".join(calls) + "</ifm|tool_calls>" + suffix
|
||||
|
||||
|
||||
def _json_call(name: str, arguments: dict) -> str:
|
||||
payload = json.dumps({"name": name, "arguments": arguments})
|
||||
return f"<ifm|tool_call>{payload}</ifm|tool_call>"
|
||||
|
||||
|
||||
def _xml_call(
|
||||
name: str,
|
||||
arguments: list[tuple[str, str]],
|
||||
*,
|
||||
typed: bool = False,
|
||||
) -> str:
|
||||
parts = [f"<ifm|tool_call>{name}"]
|
||||
for key, value in arguments:
|
||||
parts.append(f"<ifm|arg_key>{key}</ifm|arg_key>")
|
||||
if typed:
|
||||
arg_type = "integer" if key == "limit" else "string"
|
||||
parts.append(f"<ifm|arg_type>{arg_type}</ifm|arg_type>")
|
||||
parts.append(f"<ifm|arg_value>{value}</ifm|arg_value>")
|
||||
parts.append("</ifm|tool_call>")
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _collect_stream(
|
||||
parser: K2HorizonToolParser,
|
||||
request: ChatCompletionRequest,
|
||||
output: str,
|
||||
) -> tuple[str, list[tuple[str, dict]]]:
|
||||
messages: list[DeltaMessage | None] = []
|
||||
for char in output:
|
||||
messages.append(
|
||||
parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text="",
|
||||
delta_text=char,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request,
|
||||
)
|
||||
)
|
||||
messages.append(parser.finish_streaming())
|
||||
|
||||
content: list[str] = []
|
||||
calls: list[tuple[str, dict]] = []
|
||||
for message in messages:
|
||||
if message is None:
|
||||
continue
|
||||
if message.content is not None:
|
||||
content.append(message.content)
|
||||
for tool_call in message.tool_calls:
|
||||
assert tool_call.function is not None
|
||||
assert tool_call.function.name is not None
|
||||
assert tool_call.function.arguments is not None
|
||||
calls.append(
|
||||
(
|
||||
tool_call.function.name,
|
||||
json.loads(tool_call.function.arguments),
|
||||
)
|
||||
)
|
||||
return "".join(content), calls
|
||||
|
||||
|
||||
def test_xml_default_parses_multiple_calls_and_coerces_schema(tokenizer):
|
||||
parser = K2HorizonToolParser(tokenizer, _request().tools)
|
||||
output = _group(
|
||||
_xml_call("lookup", [("query", "123"), ("limit", "3")]),
|
||||
_xml_call("ping", []),
|
||||
prefix="Before ",
|
||||
suffix=" after.",
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(output, _request())
|
||||
|
||||
assert parser.tool_format == "xml"
|
||||
assert result.tools_called
|
||||
assert result.content == "Before after."
|
||||
assert [call.function.name for call in result.tool_calls] == ["lookup", "ping"]
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == {
|
||||
"query": "123",
|
||||
"limit": 3,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_format", "call", "expected"),
|
||||
[
|
||||
("json", _json_call("lookup", {"limit": "2"}), {"limit": 2}),
|
||||
(
|
||||
"xml_typed",
|
||||
_xml_call("lookup", [("query", "weather"), ("limit", "2")], typed=True),
|
||||
{"query": "weather", "limit": 2},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_request_level_format_overrides(tokenizer, tool_format, call, expected):
|
||||
request = _request(tool_format)
|
||||
parser = K2HorizonToolParser(tokenizer, request.tools)
|
||||
|
||||
result = parser.extract_tool_calls(_group(call), request)
|
||||
|
||||
assert result.tools_called
|
||||
assert json.loads(result.tool_calls[0].function.arguments) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_format",
|
||||
["yaml", "python", "xml_untyped", "", None, 1],
|
||||
)
|
||||
def test_invalid_request_format_rejected_before_generation(tokenizer, tool_format):
|
||||
parser = K2HorizonToolParser(tokenizer, _request().tools)
|
||||
request = _request().model_copy(
|
||||
update={"chat_template_kwargs": {"tool_call_format": tool_format}}
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported tool_call_format"):
|
||||
parser.adjust_request(request)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_request", "call"),
|
||||
[
|
||||
(_request(), _json_call("ping", {})),
|
||||
(_request("json"), _xml_call("ping", [])),
|
||||
(_request("xml"), _xml_call("lookup", [("query", "x")], typed=True)),
|
||||
(_request("xml_typed"), _xml_call("lookup", [("query", "x")])),
|
||||
],
|
||||
)
|
||||
def test_mismatched_formats_fail_closed(tokenizer, tool_request, call):
|
||||
parser = K2HorizonToolParser(tokenizer, tool_request.tools)
|
||||
output = _group(call)
|
||||
|
||||
result = parser.extract_tool_calls(output, tool_request)
|
||||
|
||||
assert not result.tools_called
|
||||
assert result.content == output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
[
|
||||
"auto",
|
||||
"required",
|
||||
{"type": "function", "function": {"name": "lookup"}},
|
||||
],
|
||||
)
|
||||
def test_auto_required_and_named_use_ifm_parser(tokenizer, tool_choice):
|
||||
request = _request("json", tool_choice)
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name="k2_horizon",
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert parser_cls is not None
|
||||
parser = parser_cls(tokenizer, request.tools)
|
||||
|
||||
assert parser.adjust_request(request) is request
|
||||
assert request.structured_outputs is None
|
||||
_, content, tool_calls = parser.parse(
|
||||
_group(_json_call("lookup", {"enabled": "true"})),
|
||||
request,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert tool_calls is not None
|
||||
assert json.loads(tool_calls[0].arguments) == {"enabled": True}
|
||||
|
||||
|
||||
def test_unknown_or_wrong_named_tool_fails_closed(tokenizer):
|
||||
named_request = _request(
|
||||
"json", {"type": "function", "function": {"name": "lookup"}}
|
||||
)
|
||||
parser = K2HorizonToolParser(tokenizer, named_request.tools)
|
||||
|
||||
unknown = parser.extract_tool_calls(
|
||||
_group(_json_call("missing", {})), _request("json")
|
||||
)
|
||||
wrong_named = parser.extract_tool_calls(
|
||||
_group(_json_call("ping", {})), named_request
|
||||
)
|
||||
|
||||
assert not unknown.tools_called
|
||||
assert not wrong_named.tools_called
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output",
|
||||
[
|
||||
"<ifm|tool_calls><ifm|tool_call>ping",
|
||||
"<ifm|tool_calls><ifm|tool_call>ping</ifm|tool_call>",
|
||||
_group("junk"),
|
||||
],
|
||||
)
|
||||
def test_malformed_and_incomplete_input_is_content(tokenizer, output):
|
||||
request = _request()
|
||||
parser = K2HorizonToolParser(tokenizer, request.tools)
|
||||
|
||||
result = parser.extract_tool_calls(output, request)
|
||||
|
||||
assert not result.tools_called
|
||||
assert result.content == output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tool_request", "call", "expected"),
|
||||
[
|
||||
(_request(), _xml_call("lookup", [("limit", "2")]), {"limit": 2}),
|
||||
(_request("json"), _json_call("lookup", {"limit": "2"}), {"limit": 2}),
|
||||
(
|
||||
_request("xml_typed"),
|
||||
_xml_call("lookup", [("limit", "2")], typed=True),
|
||||
{"limit": 2},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_character_split_streaming(tokenizer, tool_request, call, expected):
|
||||
parser = K2HorizonToolParser(tokenizer, tool_request.tools)
|
||||
output = _group(call, prefix="Before", suffix="after")
|
||||
|
||||
content, calls = _collect_stream(parser, tool_request, output)
|
||||
|
||||
assert content == "Beforeafter"
|
||||
assert calls == [("lookup", expected)]
|
||||
|
||||
|
||||
def test_streaming_flushes_incomplete_markup_as_content(tokenizer):
|
||||
request = _request()
|
||||
parser = K2HorizonToolParser(tokenizer, request.tools)
|
||||
output = "Before<ifm|tool_calls><ifm|tool_call>ping"
|
||||
|
||||
content, calls = _collect_stream(parser, request, output)
|
||||
|
||||
assert content == output
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_whitespace_only_surrounding_content_is_not_preserved(tokenizer):
|
||||
request = _request()
|
||||
parser = K2HorizonToolParser(tokenizer, request.tools)
|
||||
|
||||
result = parser.extract_tool_calls(
|
||||
_group(_xml_call("ping", []), prefix=" \n", suffix="\t "), request
|
||||
)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content is None
|
||||
|
||||
|
||||
def test_tool_parser_registered():
|
||||
assert ToolParserManager.get_tool_parser("k2_horizon") is K2HorizonToolParser
|
||||
assert K2HorizonToolParser.supports_required_and_named is False
|
||||
|
||||
|
||||
def test_named_tool_resolves_custom_choice():
|
||||
# Custom tools ride the function shim; a ToolChoiceCustom must resolve to
|
||||
# its name like ToolChoiceFunction does.
|
||||
request = ResponsesRequest(
|
||||
input="hi",
|
||||
tools=[{"type": "custom", "name": "emit_command"}],
|
||||
tool_choice={"type": "custom", "name": "emit_command"},
|
||||
)
|
||||
assert K2HorizonToolParser._named_tool(request) == "emit_command"
|
||||
@@ -1,583 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
|
||||
|
||||
MODEL = "moonshotai/Kimi-K2-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def kimi_k2_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(kimi_k2_tokenizer):
|
||||
return KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
|
||||
|
||||
SECTION_BEGIN = "<|tool_calls_section_begin|>"
|
||||
SECTION_END = "<|tool_calls_section_end|>"
|
||||
TOOL_BEGIN = "<|tool_call_begin|>"
|
||||
TOOL_END = "<|tool_call_end|>"
|
||||
ARG_BEGIN = "<|tool_call_argument_begin|>"
|
||||
|
||||
|
||||
def _tool(tool_id: str, args: str) -> str:
|
||||
return f"{TOOL_BEGIN}{tool_id} {ARG_BEGIN}{args}{TOOL_END}"
|
||||
|
||||
|
||||
def _wrap(*tool_strs: str) -> str:
|
||||
return SECTION_BEGIN + "".join(tool_strs) + SECTION_END
|
||||
|
||||
|
||||
class TestExtractToolCalls:
|
||||
def test_no_tools(self, parser):
|
||||
content, tool_calls = run_tool_extraction(
|
||||
parser, "This is a test", streaming=False
|
||||
)
|
||||
assert content == "This is a test"
|
||||
assert tool_calls == []
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_output, expected_names, expected_args_list, expected_content",
|
||||
[
|
||||
pytest.param(
|
||||
"I'll check. "
|
||||
+ _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')),
|
||||
["get_weather"],
|
||||
[{"city": "Beijing"}],
|
||||
"I'll check. ",
|
||||
id="single_tool_call",
|
||||
),
|
||||
pytest.param(
|
||||
"Compare weather. "
|
||||
+ _wrap(
|
||||
_tool("functions.get_weather:0", '{"city": "Beijing"}'),
|
||||
_tool("functions.get_weather:1", '{"city": "Shanghai"}'),
|
||||
),
|
||||
["get_weather", "get_weather"],
|
||||
[{"city": "Beijing"}, {"city": "Shanghai"}],
|
||||
"Compare weather. ",
|
||||
id="parallel_tool_calls",
|
||||
),
|
||||
pytest.param(
|
||||
"Multiple tasks. "
|
||||
+ _wrap(
|
||||
_tool("functions.get_weather:0", '{"city": "New York"}'),
|
||||
_tool("functions.get_news:1", '{"topic": "technology"}'),
|
||||
_tool(
|
||||
"functions.send_email:2",
|
||||
'{"to": "user@example.com", "subject": "Daily Update"}',
|
||||
),
|
||||
),
|
||||
["get_weather", "get_news", "send_email"],
|
||||
[
|
||||
{"city": "New York"},
|
||||
{"topic": "technology"},
|
||||
{"to": "user@example.com", "subject": "Daily Update"},
|
||||
],
|
||||
"Multiple tasks. ",
|
||||
id="three_tool_calls",
|
||||
),
|
||||
pytest.param(
|
||||
"Process HTML. "
|
||||
+ _wrap(
|
||||
_tool("functions.process_html:0", '{"html": "<div>content</div>"}')
|
||||
),
|
||||
["process_html"],
|
||||
[{"html": "<div>content</div>"}],
|
||||
"Process HTML. ",
|
||||
id="angle_brackets_in_json",
|
||||
),
|
||||
pytest.param(
|
||||
"Formatted. "
|
||||
+ _wrap(
|
||||
_tool(
|
||||
"functions.process_data:0",
|
||||
'{\n "name": "test",\n "value": 123\n}',
|
||||
)
|
||||
),
|
||||
["process_data"],
|
||||
[{"name": "test", "value": 123}],
|
||||
"Formatted. ",
|
||||
id="multiline_json",
|
||||
),
|
||||
pytest.param(
|
||||
"No prefix. " + _wrap(_tool("get_weather:0", '{"city": "Tokyo"}')),
|
||||
["get_weather"],
|
||||
[{"city": "Tokyo"}],
|
||||
"No prefix. ",
|
||||
id="no_functions_prefix",
|
||||
),
|
||||
pytest.param(
|
||||
"Empty args. " + _wrap(_tool("functions.test:0", "{}")),
|
||||
["test"],
|
||||
[{}],
|
||||
"Empty args. ",
|
||||
id="empty_arguments",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls(
|
||||
self, parser, model_output, expected_names, expected_args_list, expected_content
|
||||
):
|
||||
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
|
||||
assert content == expected_content
|
||||
assert len(tool_calls) == len(expected_names)
|
||||
for tc, name, expected_args in zip(
|
||||
tool_calls, expected_names, expected_args_list
|
||||
):
|
||||
assert tc.type == "function"
|
||||
assert tc.function.name == name
|
||||
assert json.loads(tc.function.arguments) == expected_args
|
||||
# id format: "something:digit"
|
||||
assert tc.id.split(":")[-1].isdigit()
|
||||
|
||||
def test_invalid_json_still_extracted(self, parser):
|
||||
"""Tool calls with invalid JSON are still returned (arguments as-is)."""
|
||||
model_output = (
|
||||
"Help. "
|
||||
+ SECTION_BEGIN
|
||||
+ _tool("functions.bad:0", '{"city": "Beijing"')
|
||||
+ _tool("functions.good:1", '{"city": "Shanghai"}')
|
||||
+ SECTION_END
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
|
||||
assert len(tool_calls) == 2
|
||||
assert tool_calls[0].function.name == "bad"
|
||||
assert tool_calls[1].function.name == "good"
|
||||
|
||||
def test_invalid_funcall_id_skipped(self, parser):
|
||||
"""Tool calls with malformed id (no colon+digit) are skipped."""
|
||||
model_output = (
|
||||
"Help. "
|
||||
+ SECTION_BEGIN
|
||||
+ _tool("functions.invalid.0", '{"city": "Beijing"}')
|
||||
+ _tool("functions.valid:1", '{"city": "Shanghai"}')
|
||||
+ SECTION_END
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function.name == "valid"
|
||||
|
||||
def test_native_id_extracted(self, parser):
|
||||
"""Regression: parser extracts native ID onto ToolCall (PR #32768)."""
|
||||
model_output = "Checking weather. " + _wrap(
|
||||
_tool("functions.get_weather:0", '{"city": "Tokyo"}')
|
||||
)
|
||||
content, tool_calls = run_tool_extraction(parser, model_output, streaming=False)
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].id == "functions.get_weather:0"
|
||||
assert tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(tool_calls[0].function.arguments) == {"city": "Tokyo"}
|
||||
|
||||
def test_multi_turn_native_id_continuity(self, kimi_k2_tokenizer):
|
||||
"""Regression: native IDs from turn 1 preserved across turns (PR #32768)."""
|
||||
turn1_parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
turn1_output = "Let me check. " + _wrap(
|
||||
_tool("functions.get_weather:0", '{"city": "Beijing"}')
|
||||
)
|
||||
_, turn1_tools = run_tool_extraction(
|
||||
turn1_parser, turn1_output, streaming=False
|
||||
)
|
||||
assert len(turn1_tools) == 1
|
||||
assert turn1_tools[0].id == "functions.get_weather:0"
|
||||
|
||||
# Fresh parser for turn 2
|
||||
turn2_parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
turn2_output = "Now let me get news. " + _wrap(
|
||||
_tool("functions.get_news:0", '{"topic": "weather in Beijing"}')
|
||||
)
|
||||
_, turn2_tools = run_tool_extraction(
|
||||
turn2_parser, turn2_output, streaming=False
|
||||
)
|
||||
assert len(turn2_tools) == 1
|
||||
assert turn2_tools[0].id == "functions.get_news:0"
|
||||
|
||||
|
||||
def _split_tool_output_to_deltas(
|
||||
content: str, tool_strs: list[tuple[str, str]]
|
||||
) -> list[str]:
|
||||
"""Build a list of string deltas with special tokens as separate chunks.
|
||||
|
||||
Args:
|
||||
content: text before tool section
|
||||
tool_strs: list of (tool_id, args_json)
|
||||
|
||||
"""
|
||||
deltas = [content, SECTION_BEGIN]
|
||||
for tool_id, args_json in tool_strs:
|
||||
deltas.extend(
|
||||
[
|
||||
TOOL_BEGIN,
|
||||
f"{tool_id} ",
|
||||
ARG_BEGIN,
|
||||
f"{args_json} ",
|
||||
TOOL_END,
|
||||
]
|
||||
)
|
||||
deltas.append(SECTION_END)
|
||||
return deltas
|
||||
|
||||
|
||||
class TestStreamingHappyPath:
|
||||
def test_single_tool_call(self, parser):
|
||||
"""Verify DeltaToolCall output: name, id, arguments for one tool."""
|
||||
deltas = _split_tool_output_to_deltas(
|
||||
"I'll help. ",
|
||||
[("functions.get_weather:0", '{"city": "Beijing"}')],
|
||||
)
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert len(rec.tool_calls) == 1
|
||||
tc = rec.tool_calls[0]
|
||||
assert tc.function.name == "get_weather"
|
||||
assert tc.id == "functions.get_weather:0"
|
||||
assert json.loads(tc.function.arguments) == {"city": "Beijing"}
|
||||
|
||||
def test_multiple_tool_calls(self, parser):
|
||||
"""Two tool calls emitted with correct indices, names, arguments."""
|
||||
deltas = _split_tool_output_to_deltas(
|
||||
"Compare weather. ",
|
||||
[
|
||||
("functions.get_weather:0", '{"city": "Tokyo"}'),
|
||||
("functions.get_weather:1", '{"city": "NYC"}'),
|
||||
],
|
||||
)
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert len(rec.tool_calls) == 2
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert rec.tool_calls[0].id == "functions.get_weather:0"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"}
|
||||
|
||||
assert rec.tool_calls[1].function.name == "get_weather"
|
||||
assert rec.tool_calls[1].id == "functions.get_weather:1"
|
||||
assert json.loads(rec.tool_calls[1].function.arguments) == {"city": "NYC"}
|
||||
|
||||
def test_content_before_tools(self, parser):
|
||||
"""Content before section is streamed; markers/args don't leak."""
|
||||
deltas = _split_tool_output_to_deltas(
|
||||
"I'll check the weather. ",
|
||||
[("functions.get_weather:0", '{"city": "Tokyo"}')],
|
||||
)
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert "check the weather" in rec.other_content
|
||||
# No markers or tool content leaked
|
||||
for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]:
|
||||
assert marker not in rec.other_content
|
||||
assert "get_weather" not in rec.other_content
|
||||
assert "Tokyo" not in rec.other_content
|
||||
|
||||
def test_no_tool_calls(self, parser):
|
||||
"""Plain text streaming returns content only."""
|
||||
deltas = ["This is just ", "regular text ", "without tools."]
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert rec.other_content == "This is just regular text without tools."
|
||||
assert rec.tool_calls == []
|
||||
|
||||
def test_incremental_arguments(self, parser):
|
||||
"""Arguments split across small chunks accumulate correctly."""
|
||||
deltas = [
|
||||
"Help. ",
|
||||
SECTION_BEGIN,
|
||||
TOOL_BEGIN,
|
||||
"functions.get_weather:0 ",
|
||||
ARG_BEGIN,
|
||||
'{"ci',
|
||||
'ty": "Be',
|
||||
'ijing"}',
|
||||
" ",
|
||||
TOOL_END,
|
||||
SECTION_END,
|
||||
]
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert len(rec.tool_calls) == 1
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"}
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_output",
|
||||
[
|
||||
pytest.param(
|
||||
"Single. "
|
||||
+ _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}')),
|
||||
id="single_tool",
|
||||
),
|
||||
pytest.param(
|
||||
"Multi. "
|
||||
+ _wrap(
|
||||
_tool("functions.get_weather:0", '{"city": "Tokyo"}'),
|
||||
_tool("functions.get_news:1", '{"topic": "tech"}'),
|
||||
),
|
||||
id="parallel_tools",
|
||||
),
|
||||
pytest.param(
|
||||
"No prefix id. " + _wrap(_tool("get_weather:0", '{"city": "NYC"}')),
|
||||
id="no_functions_prefix",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_streaming_matches_nonstreaming(self, parser, model_output):
|
||||
"""Streaming reconstruction matches non-streaming extraction."""
|
||||
content_non, tools_non = run_tool_extraction(
|
||||
parser, model_output, streaming=False
|
||||
)
|
||||
content_stream, tools_stream = run_tool_extraction(
|
||||
parser, model_output, streaming=True
|
||||
)
|
||||
|
||||
assert len(tools_non) == len(tools_stream)
|
||||
for tc_non, tc_stream in zip(tools_non, tools_stream):
|
||||
assert tc_non.function.name == tc_stream.function.name
|
||||
assert json.loads(tc_non.function.arguments) == json.loads(
|
||||
tc_stream.function.arguments
|
||||
)
|
||||
|
||||
|
||||
class TestStreamingEdgeCases:
|
||||
def test_marker_suppression(self, parser):
|
||||
"""No special-token markers appear in reconstructed content."""
|
||||
deltas = _split_tool_output_to_deltas(
|
||||
"I'll check. ",
|
||||
[("functions.get_weather:0", '{"city": "Tokyo"}')],
|
||||
)
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
forbidden = [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]
|
||||
for marker in forbidden:
|
||||
assert marker not in rec.other_content, (
|
||||
f"Marker leaked: {marker!r} in {rec.other_content!r}"
|
||||
)
|
||||
|
||||
def test_noise_between_markers_suppressed(self, parser):
|
||||
"""Text between section_begin and tool_call_begin doesn't leak."""
|
||||
deltas = [
|
||||
"Reasoning. ",
|
||||
SECTION_BEGIN,
|
||||
" spurious noise ",
|
||||
TOOL_BEGIN,
|
||||
"functions.test:0 ",
|
||||
ARG_BEGIN,
|
||||
'{"k": "v"} ',
|
||||
TOOL_END,
|
||||
SECTION_END,
|
||||
]
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert "spurious" not in rec.other_content
|
||||
assert "noise" not in rec.other_content
|
||||
|
||||
def test_empty_tool_section(self, parser):
|
||||
"""Empty section (begin immediately followed by end) doesn't crash."""
|
||||
deltas = ["Reasoning. ", SECTION_BEGIN, SECTION_END]
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert rec.tool_calls == []
|
||||
|
||||
def test_three_different_tools(self, parser):
|
||||
"""Three tool calls with different functions stream correctly."""
|
||||
deltas = _split_tool_output_to_deltas(
|
||||
"Multiple tasks. ",
|
||||
[
|
||||
("functions.get_weather:0", '{"city": "NYC"}'),
|
||||
("functions.get_news:1", '{"topic": "tech"}'),
|
||||
("functions.send_email:2", '{"to": "a@b.com"}'),
|
||||
],
|
||||
)
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
assert len(rec.tool_calls) == 3
|
||||
names = [tc.function.name for tc in rec.tool_calls]
|
||||
assert names == ["get_weather", "get_news", "send_email"]
|
||||
ids = [tc.id for tc in rec.tool_calls]
|
||||
assert len(set(ids)) == 3 # unique ids
|
||||
|
||||
def test_truncated_tool_call_no_end_marker(self, parser):
|
||||
"""Stream ending mid-tool-call (max_tokens) doesn't crash."""
|
||||
deltas = [
|
||||
"I'll check. ",
|
||||
SECTION_BEGIN,
|
||||
TOOL_BEGIN,
|
||||
"functions.get_weather:0 ",
|
||||
ARG_BEGIN,
|
||||
'{"city": "Bei',
|
||||
# Stream ends here — no TOOL_END, no SECTION_END
|
||||
]
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
# Should not crash; tool name and partial args extracted
|
||||
assert len(rec.tool_calls) == 1
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert rec.tool_calls[0].id == "functions.get_weather:0"
|
||||
assert rec.tool_calls[0].function.arguments == '{"city": "Bei'
|
||||
# No markers leaked into content
|
||||
for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]:
|
||||
assert marker not in rec.other_content
|
||||
|
||||
def test_content_after_tool_section(self, parser):
|
||||
"""Trailing text after section_end doesn't crash or leak markers."""
|
||||
deltas = [
|
||||
"Before. ",
|
||||
SECTION_BEGIN,
|
||||
TOOL_BEGIN,
|
||||
"functions.get_weather:0 ",
|
||||
ARG_BEGIN,
|
||||
'{"city": "Tokyo"} ',
|
||||
TOOL_END,
|
||||
SECTION_END,
|
||||
" After tools.",
|
||||
]
|
||||
rec = run_tool_extraction_streaming(parser, deltas)
|
||||
|
||||
# Tool call extracted correctly
|
||||
assert len(rec.tool_calls) == 1
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"}
|
||||
# Trailing content after tool section is dropped
|
||||
assert "After tools." not in rec.other_content
|
||||
# No markers leaked into content
|
||||
for marker in [SECTION_BEGIN, SECTION_END, TOOL_BEGIN, TOOL_END, ARG_BEGIN]:
|
||||
assert marker not in rec.other_content
|
||||
|
||||
|
||||
class TestAdjustRequest:
|
||||
def test_sets_skip_special_tokens_false(self, parser):
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = [{"type": "function", "function": {"name": "test"}}]
|
||||
request.tool_choice = "auto"
|
||||
request.skip_special_tokens = True
|
||||
|
||||
result = parser.adjust_request(request)
|
||||
assert result.skip_special_tokens is False
|
||||
|
||||
def test_no_change_when_tool_choice_none(self, parser):
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = [{"type": "function", "function": {"name": "test"}}]
|
||||
request.tool_choice = "none"
|
||||
request.skip_special_tokens = True
|
||||
|
||||
result = parser.adjust_request(request)
|
||||
assert result.skip_special_tokens is True
|
||||
|
||||
def test_no_change_when_no_tools(self, parser):
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = None
|
||||
request.tool_choice = "auto"
|
||||
request.skip_special_tokens = True
|
||||
|
||||
result = parser.adjust_request(request)
|
||||
assert result.skip_special_tokens is True
|
||||
|
||||
|
||||
def _chunk_tokenized_deltas(tokenizer, text: str, stream_interval: int) -> list[str]:
|
||||
"""Encode text, group tokens into chunks of stream_interval, decode each."""
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
deltas = []
|
||||
prev = ""
|
||||
for i in range(0, len(token_ids), stream_interval):
|
||||
decoded = tokenizer.decode(
|
||||
token_ids[: i + stream_interval], skip_special_tokens=False
|
||||
)
|
||||
deltas.append(decoded[len(prev) :])
|
||||
prev = decoded
|
||||
return deltas
|
||||
|
||||
|
||||
class TestStreamingIntervals:
|
||||
"""Test streaming at various token-chunk sizes to catch boundary bugs."""
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
|
||||
def test_single_tool_call_at_interval(self, kimi_k2_tokenizer, stream_interval):
|
||||
text = "Help. " + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}'))
|
||||
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
|
||||
parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
rec = run_tool_extraction_streaming(
|
||||
parser, deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert len(rec.tool_calls) == 1
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"}
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
|
||||
def test_content_then_tool_call_at_interval(
|
||||
self, kimi_k2_tokenizer, stream_interval
|
||||
):
|
||||
text = "Sure, let me check. " + _wrap(
|
||||
_tool("functions.get_weather:0", '{"city": "Tokyo"}')
|
||||
)
|
||||
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
|
||||
parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
rec = run_tool_extraction_streaming(
|
||||
parser, deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert "let me check" in rec.other_content
|
||||
assert "get_weather" not in rec.other_content
|
||||
assert len(rec.tool_calls) == 1
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Tokyo"}
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
|
||||
def test_multiple_tool_calls_at_interval(self, kimi_k2_tokenizer, stream_interval):
|
||||
text = "Compare. " + _wrap(
|
||||
_tool("functions.search:0", '{"q": "cats"}'),
|
||||
_tool("functions.search:1", '{"q": "dogs"}'),
|
||||
)
|
||||
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
|
||||
parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
rec = run_tool_extraction_streaming(
|
||||
parser, deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert len(rec.tool_calls) == 2
|
||||
assert rec.tool_calls[0].function.name == "search"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"q": "cats"}
|
||||
assert rec.tool_calls[1].function.name == "search"
|
||||
assert json.loads(rec.tool_calls[1].function.arguments) == {"q": "dogs"}
|
||||
|
||||
@pytest.mark.parametrize("stream_interval", [1, 2, 3, 5, 8])
|
||||
def test_plain_text_at_interval(self, kimi_k2_tokenizer, stream_interval):
|
||||
text = "This is plain text with no tool calling involved."
|
||||
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval)
|
||||
parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
rec = run_tool_extraction_streaming(
|
||||
parser, deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert rec.other_content == text
|
||||
assert rec.tool_calls == []
|
||||
|
||||
def test_content_and_tool_call_in_single_chunk(self, kimi_k2_tokenizer):
|
||||
"""Content + complete tool call in one chunk must both be emitted."""
|
||||
text = "Hi! " + _wrap(_tool("functions.get_weather:0", '{"city": "Beijing"}'))
|
||||
deltas = _chunk_tokenized_deltas(kimi_k2_tokenizer, text, stream_interval=9999)
|
||||
parser = KimiK2ToolParser(kimi_k2_tokenizer)
|
||||
rec = run_tool_extraction_streaming(
|
||||
parser, deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert "Hi!" in rec.other_content
|
||||
assert "get_weather" not in rec.other_content
|
||||
assert len(rec.tool_calls) == 1
|
||||
assert rec.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(rec.tool_calls[0].function.arguments) == {"city": "Beijing"}
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Named tool choice for Kimi K3: allowed when the XTML structural tag is
|
||||
attached (strict tool calling), rejected otherwise."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
|
||||
class _DummyTokenizer:
|
||||
def get_vocab(self):
|
||||
return {}
|
||||
|
||||
def encode(self, text, add_special_tokens=False):
|
||||
return [ord(c) for c in text]
|
||||
|
||||
|
||||
def _request(with_tag: bool) -> ChatCompletionRequest:
|
||||
req = ChatCompletionRequest(
|
||||
model="k3",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
tool_choice={"type": "function", "function": {"name": "get_weather"}},
|
||||
)
|
||||
if with_tag:
|
||||
req.structured_outputs = StructuredOutputsParams(
|
||||
structural_tag='{"type": "structural_tag", "format": {}}'
|
||||
)
|
||||
return req
|
||||
|
||||
|
||||
def _parser():
|
||||
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
|
||||
|
||||
return KimiK3ToolParser(_DummyTokenizer())
|
||||
|
||||
|
||||
def test_named_choice_allowed_with_structural_tag():
|
||||
req = _parser().adjust_request(_request(with_tag=True))
|
||||
assert req.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_named_choice_rejected_without_structural_tag():
|
||||
with pytest.raises(VLLMValidationError):
|
||||
_parser().adjust_request(_request(with_tag=False))
|
||||
|
||||
|
||||
def _custom_request(with_tag: bool) -> ResponsesRequest:
|
||||
req = ResponsesRequest(
|
||||
input="hi",
|
||||
tools=[{"type": "custom", "name": "emit_command"}],
|
||||
tool_choice={"type": "custom", "name": "emit_command"},
|
||||
)
|
||||
if with_tag:
|
||||
req.structured_outputs = StructuredOutputsParams(
|
||||
structural_tag='{"type": "structural_tag", "format": {}}'
|
||||
)
|
||||
return req
|
||||
|
||||
|
||||
def test_named_custom_choice_allowed_with_structural_tag():
|
||||
req = _parser().adjust_request(_custom_request(with_tag=True))
|
||||
assert req.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_named_custom_choice_rejected_without_structural_tag():
|
||||
with pytest.raises(VLLMValidationError):
|
||||
_parser().adjust_request(_custom_request(with_tag=False))
|
||||
@@ -1,666 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
|
||||
TOOL_CALL_START = "<|tool_call_start|>"
|
||||
TOOL_CALL_END = "<|tool_call_end|>"
|
||||
|
||||
SIMPLE_FUNCTION_OUTPUT = "get_candidate_status(candidate_id='12345')"
|
||||
SIMPLE_FUNCTION_CALL = FunctionCall(
|
||||
name="get_candidate_status",
|
||||
arguments='{"candidate_id": "12345"}',
|
||||
)
|
||||
MORE_TYPES_FUNCTION_OUTPUT = (
|
||||
"register_user(name='John Doe', "
|
||||
"age=37, "
|
||||
"address={'city': 'San Francisco', 'state': 'CA'}, "
|
||||
"role=None, "
|
||||
"passed_test=True, "
|
||||
"aliases=['John', 'Johnny'])"
|
||||
)
|
||||
MORE_TYPES_FUNCTION_CALL = FunctionCall(
|
||||
name="register_user",
|
||||
arguments='{"name": "John Doe", '
|
||||
'"age": 37, '
|
||||
'"address": {"city": "San Francisco", "state": "CA"}, '
|
||||
'"role": null, '
|
||||
'"passed_test": true, '
|
||||
'"aliases": ["John", "Johnny"]}',
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
|
||||
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments="{}",
|
||||
)
|
||||
EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
|
||||
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"additional_data": {}}',
|
||||
)
|
||||
EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
|
||||
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"steps": []}',
|
||||
)
|
||||
SET_ARG_FUNCTION_OUTPUT = "label(tags={'urgent', 'bug'})"
|
||||
SET_ARG_FUNCTION_CALL = FunctionCall(
|
||||
name="label",
|
||||
arguments='{"tags": ["urgent", "bug"]}',
|
||||
)
|
||||
LEADING_ZERO_FUNCTION_OUTPUT = "set_date(month=07, day=05)"
|
||||
LEADING_ZERO_FUNCTION_CALL = FunctionCall(
|
||||
name="set_date",
|
||||
arguments='{"month": 7, "day": 5}',
|
||||
)
|
||||
NESTED_QUOTE_FUNCTION_OUTPUT = (
|
||||
"bash(command='sed -n '360,450p' /testbed/sympy/matrices/common.py')"
|
||||
)
|
||||
NESTED_QUOTE_FUNCTION_CALL = FunctionCall(
|
||||
name="bash",
|
||||
arguments=('{"command": "sed -n \'360,450p\' /testbed/sympy/matrices/common.py"}'),
|
||||
)
|
||||
NESTED_PYTHON_C_FUNCTION_OUTPUT = (
|
||||
"bash(command='cd /testbed && python3 -c \"from sympy import latex\n"
|
||||
"print(latex(3*x, mul_symbol='\\,'))\"')"
|
||||
)
|
||||
NESTED_PYTHON_C_FUNCTION_CALL = FunctionCall(
|
||||
name="bash",
|
||||
arguments=(
|
||||
'{"command": "cd /testbed && python3 -c \\"from sympy import latex\\n'
|
||||
"print(latex(3*x, mul_symbol='\\\\,'))\\\"\"}"
|
||||
),
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT = (
|
||||
r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
|
||||
)
|
||||
DOTTED_NAME_FUNCTION_OUTPUT = (
|
||||
"grocery.orderIngredients("
|
||||
"ingredientList=[{'name': 'Lasagna noodles', 'amount': 250, 'unit': 'g'}], "
|
||||
"deliveryAddress='845 Willow Lane, Springfield, IL 62704')"
|
||||
)
|
||||
DOTTED_NAME_FUNCTION_CALL = FunctionCall(
|
||||
name="grocery.orderIngredients",
|
||||
arguments=(
|
||||
'{"ingredientList": ['
|
||||
'{"name": "Lasagna noodles", "amount": 250, "unit": "g"}], '
|
||||
'"deliveryAddress": "845 Willow Lane, Springfield, IL 62704"}'
|
||||
),
|
||||
)
|
||||
# Agentic shell commands: a bracket inside a string argument must not corrupt
|
||||
# the streaming bracket tracker.
|
||||
BRACKET_IN_STRING_FUNCTION_OUTPUT = "exec(command='grep -F \"]\" log.txt')"
|
||||
BRACKET_IN_STRING_FUNCTION_CALL = FunctionCall(
|
||||
name="exec",
|
||||
arguments='{"command": "grep -F \\"]\\" log.txt"}',
|
||||
)
|
||||
# A raw newline inside a string argument (multi-line shell command) is invalid
|
||||
# Python; the parser must recover the full value instead of dropping the call.
|
||||
MULTILINE_FUNCTION_OUTPUT = (
|
||||
"exec(command='cat > f.py << EOF\nimport csv\nprint(1)\nEOF')"
|
||||
)
|
||||
MULTILINE_FUNCTION_CALL = FunctionCall(
|
||||
name="exec",
|
||||
arguments='{"command": "cat > f.py << EOF\\nimport csv\\nprint(1)\\nEOF"}',
|
||||
)
|
||||
# A NUL byte inside a string argument makes ast.parse raise ValueError (not
|
||||
# SyntaxError) for the whole source; the escape path must recover the call.
|
||||
NUL_BYTE_FUNCTION_OUTPUT = "exec(command='printf a\x00b')"
|
||||
NUL_BYTE_FUNCTION_CALL = FunctionCall(
|
||||
name="exec",
|
||||
arguments='{"command": "printf a\\u0000b"}',
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def lfm2_tokenizer() -> TokenizerLike:
|
||||
return AutoTokenizer.from_pretrained("LiquidAI/LFM2.5-1.2B-Instruct")
|
||||
|
||||
|
||||
def _wrap(tool_text: str, content_after: str = "") -> str:
|
||||
"""Wrap pythonic tool call in LFM2.5 sentinel tokens."""
|
||||
result = f"{TOOL_CALL_START}[{tool_text}]{TOOL_CALL_END}"
|
||||
if content_after:
|
||||
result += f"\n{content_after}"
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_no_tool_call(streaming: bool, lfm2_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
model_output = "How can I help you today?"
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert content == model_output
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
TEST_CASES = [
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(SIMPLE_FUNCTION_OUTPUT),
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
None,
|
||||
id="simple_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(SIMPLE_FUNCTION_OUTPUT),
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
None,
|
||||
id="simple_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(MORE_TYPES_FUNCTION_OUTPUT),
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
None,
|
||||
id="more_types_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(MORE_TYPES_FUNCTION_OUTPUT),
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
None,
|
||||
id="more_types_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(PARAMETERLESS_FUNCTION_OUTPUT),
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
None,
|
||||
id="parameterless_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(PARAMETERLESS_FUNCTION_OUTPUT),
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
None,
|
||||
id="parameterless_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(EMPTY_DICT_FUNCTION_OUTPUT),
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
None,
|
||||
id="empty_dict_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(EMPTY_DICT_FUNCTION_OUTPUT),
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
None,
|
||||
id="empty_dict_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(EMPTY_LIST_FUNCTION_OUTPUT),
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
None,
|
||||
id="empty_list_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(EMPTY_LIST_FUNCTION_OUTPUT),
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
None,
|
||||
id="empty_list_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(ESCAPED_STRING_FUNCTION_OUTPUT),
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
None,
|
||||
id="escaped_string_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(ESCAPED_STRING_FUNCTION_OUTPUT),
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
None,
|
||||
id="escaped_string_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"),
|
||||
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
|
||||
None,
|
||||
id="parallel_calls_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(f"{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}"),
|
||||
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
|
||||
None,
|
||||
id="parallel_calls_nonstreaming",
|
||||
),
|
||||
# LFM2.5 specific: content AFTER tool call
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(
|
||||
SIMPLE_FUNCTION_OUTPUT,
|
||||
content_after="Checking the current status of candidate ID 12345.",
|
||||
),
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
"Checking the current status of candidate ID 12345.",
|
||||
id="content_after_tool_call_nonstreaming",
|
||||
),
|
||||
# Dotted / class-method function names: grocery.orderIngredients(...)
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(DOTTED_NAME_FUNCTION_OUTPUT),
|
||||
[DOTTED_NAME_FUNCTION_CALL],
|
||||
None,
|
||||
id="dotted_name_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(DOTTED_NAME_FUNCTION_OUTPUT),
|
||||
[DOTTED_NAME_FUNCTION_CALL],
|
||||
None,
|
||||
id="dotted_name_nonstreaming",
|
||||
),
|
||||
# Messy agentic shell commands: literal bracket inside a string argument
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(BRACKET_IN_STRING_FUNCTION_OUTPUT),
|
||||
[BRACKET_IN_STRING_FUNCTION_CALL],
|
||||
None,
|
||||
id="bracket_in_string_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(BRACKET_IN_STRING_FUNCTION_OUTPUT),
|
||||
[BRACKET_IN_STRING_FUNCTION_CALL],
|
||||
None,
|
||||
id="bracket_in_string_nonstreaming",
|
||||
),
|
||||
# Multi-line string argument (raw newlines in a shell command)
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(MULTILINE_FUNCTION_OUTPUT),
|
||||
[MULTILINE_FUNCTION_CALL],
|
||||
None,
|
||||
id="multiline_string_arg_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(MULTILINE_FUNCTION_OUTPUT),
|
||||
[MULTILINE_FUNCTION_CALL],
|
||||
None,
|
||||
id="multiline_string_arg_nonstreaming",
|
||||
),
|
||||
# Set argument decoded as a list (JSON has no set type)
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(SET_ARG_FUNCTION_OUTPUT),
|
||||
[SET_ARG_FUNCTION_CALL],
|
||||
None,
|
||||
id="set_arg_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(SET_ARG_FUNCTION_OUTPUT),
|
||||
[SET_ARG_FUNCTION_CALL],
|
||||
None,
|
||||
id="set_arg_nonstreaming",
|
||||
),
|
||||
# Unescaped same-style quotes nested in a shell command argument
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(NESTED_QUOTE_FUNCTION_OUTPUT),
|
||||
[NESTED_QUOTE_FUNCTION_CALL],
|
||||
None,
|
||||
id="nested_quote_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(NESTED_QUOTE_FUNCTION_OUTPUT),
|
||||
[NESTED_QUOTE_FUNCTION_CALL],
|
||||
None,
|
||||
id="nested_quote_nonstreaming",
|
||||
),
|
||||
# Doubly nested: multi-line python -c payload with its own inner quotes
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(NESTED_PYTHON_C_FUNCTION_OUTPUT),
|
||||
[NESTED_PYTHON_C_FUNCTION_CALL],
|
||||
None,
|
||||
id="nested_python_c_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(NESTED_PYTHON_C_FUNCTION_OUTPUT),
|
||||
[NESTED_PYTHON_C_FUNCTION_CALL],
|
||||
None,
|
||||
id="nested_python_c_nonstreaming",
|
||||
),
|
||||
# Zero-padded integer arguments (SyntaxError: leading zeros)
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(LEADING_ZERO_FUNCTION_OUTPUT),
|
||||
[LEADING_ZERO_FUNCTION_CALL],
|
||||
None,
|
||||
id="leading_zero_int_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(LEADING_ZERO_FUNCTION_OUTPUT),
|
||||
[LEADING_ZERO_FUNCTION_CALL],
|
||||
None,
|
||||
id="leading_zero_int_nonstreaming",
|
||||
),
|
||||
# NUL byte in a string argument (ValueError from ast.parse, not
|
||||
# SyntaxError)
|
||||
pytest.param(
|
||||
True,
|
||||
_wrap(NUL_BYTE_FUNCTION_OUTPUT),
|
||||
[NUL_BYTE_FUNCTION_CALL],
|
||||
None,
|
||||
id="nul_byte_string_arg_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
_wrap(NUL_BYTE_FUNCTION_OUTPUT),
|
||||
[NUL_BYTE_FUNCTION_CALL],
|
||||
None,
|
||||
id="nul_byte_string_arg_nonstreaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"streaming, model_output, expected_tool_calls, expected_content",
|
||||
TEST_CASES,
|
||||
)
|
||||
def test_tool_call(
|
||||
streaming: bool,
|
||||
model_output: str,
|
||||
expected_tool_calls: list[FunctionCall],
|
||||
expected_content: str | None,
|
||||
lfm2_tokenizer: TokenizerLike,
|
||||
):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
if expected_content and not streaming:
|
||||
assert content == expected_content
|
||||
assert len(tool_calls) == len(expected_tool_calls)
|
||||
for actual, expected in zip(tool_calls, expected_tool_calls):
|
||||
assert actual.type == "function"
|
||||
assert actual.function == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", ["", " "])
|
||||
def test_empty_tool_call_block(body: str, lfm2_tokenizer: TokenizerLike):
|
||||
"""An empty block ([] or [ ]) means the model called no tools; it must
|
||||
not report tools_called=True with zero tool calls (the invariant
|
||||
run_tool_extraction asserts)."""
|
||||
cls = ToolParserManager.get_tool_parser("lfm2")
|
||||
model_output = f"{TOOL_CALL_START}[{body}]{TOOL_CALL_END}"
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
cls(lfm2_tokenizer), model_output, streaming=False
|
||||
)
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
cls(lfm2_tokenizer), [model_output], assert_one_tool_per_delta=False
|
||||
)
|
||||
assert len(reconstructor.tool_calls) == 0
|
||||
|
||||
|
||||
def test_whitespace_after_start_token(lfm2_tokenizer: TokenizerLike):
|
||||
"""Whitespace between <|tool_call_start|> and the opening bracket must
|
||||
not break parsing. The streaming path used to feed the indented text to
|
||||
ast.parse verbatim, so every completion candidate raised
|
||||
IndentationError and the call was silently dropped (non-streaming
|
||||
stripped it and succeeded)."""
|
||||
cls = ToolParserManager.get_tool_parser("lfm2")
|
||||
model_output = f"{TOOL_CALL_START} [{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
cls(lfm2_tokenizer), model_output, streaming=False
|
||||
)
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
cls(lfm2_tokenizer), [model_output], assert_one_tool_per_delta=False
|
||||
)
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
|
||||
|
||||
def test_streaming_tool_call_with_large_steps(lfm2_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
model_output_deltas = [
|
||||
f"{TOOL_CALL_START}[get_candidate_status(candidate_id='12345'), "
|
||||
f"{PARAMETERLESS_FUNCTION_OUTPUT}, "
|
||||
f"{EMPTY_LIST_FUNCTION_OUTPUT}]{TOOL_CALL_END}",
|
||||
]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert len(reconstructor.tool_calls) == 3
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
|
||||
|
||||
|
||||
def test_streaming_full_block_and_trailing_in_single_delta(
|
||||
lfm2_tokenizer: TokenizerLike,
|
||||
):
|
||||
"""The entire <|tool_call_start|>[...]<|tool_call_end|> block plus
|
||||
trailing assistant text arrive in one delta. Trailing content must
|
||||
still be emitted — not silently dropped."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
full_text = f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone."
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert "Done." in reconstructor.other_content
|
||||
|
||||
|
||||
def test_streaming_leading_content_and_full_block_in_single_delta(
|
||||
lfm2_tokenizer: TokenizerLike,
|
||||
):
|
||||
"""Leading assistant text plus the entire tool block arrive in one
|
||||
delta. Leading content must be emitted — not silently dropped."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
full_text = (
|
||||
f"Let me check. {TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
|
||||
)
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert "Let me check." in reconstructor.other_content
|
||||
|
||||
|
||||
def test_streaming_leading_block_and_trailing_in_single_delta(
|
||||
lfm2_tokenizer: TokenizerLike,
|
||||
):
|
||||
"""Leading text + complete tool block + trailing text in one delta.
|
||||
Both leading and trailing content must be preserved."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
full_text = (
|
||||
"Let me check. "
|
||||
f"{TOOL_CALL_START}[{SIMPLE_FUNCTION_OUTPUT}]{TOOL_CALL_END}\nDone."
|
||||
)
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert "Let me check." in reconstructor.other_content
|
||||
assert "Done." in reconstructor.other_content
|
||||
|
||||
|
||||
def test_echoed_tool_call_body_not_leaked_to_content(
|
||||
lfm2_tokenizer: TokenizerLike,
|
||||
):
|
||||
"""LFM2 sometimes emits the tool call body again after the first
|
||||
<|tool_call_end|>, capped with a second <|tool_call_end|>. The
|
||||
echoed body must not surface as assistant content — neither in
|
||||
streaming nor non-streaming paths."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
body = (
|
||||
"[grocery.orderIngredients("
|
||||
"ingredientList=[{'name': 'apple', 'quantity': '2'}], "
|
||||
"deliveryAddress='123 Main St')]"
|
||||
)
|
||||
model_output = f"{TOOL_CALL_START}{body}{TOOL_CALL_END}{body}{TOOL_CALL_END}"
|
||||
|
||||
# Non-streaming
|
||||
content_ns, tool_calls_ns = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=False
|
||||
)
|
||||
assert len(tool_calls_ns) == 1
|
||||
assert tool_calls_ns[0].function.name == "grocery.orderIngredients"
|
||||
assert content_ns in (None, "")
|
||||
|
||||
# Streaming: re-fetch a fresh parser since state was mutated above.
|
||||
tool_parser2: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
content_s, tool_calls_s = run_tool_extraction(
|
||||
tool_parser2, model_output, streaming=True
|
||||
)
|
||||
assert len(tool_calls_s) == 1
|
||||
assert tool_calls_s[0].function.name == "grocery.orderIngredients"
|
||||
# Echoed body must not leak as content.
|
||||
assert content_s in (None, "")
|
||||
assert "grocery.orderIngredients" not in (content_s or "")
|
||||
assert TOOL_CALL_END not in (content_s or "")
|
||||
|
||||
|
||||
def test_streaming_char_by_char_multi_dict_list(lfm2_tokenizer: TokenizerLike):
|
||||
"""Stream a tool call containing a list of multiple dicts one
|
||||
character at a time. Every prefix lands in some partial-parse state
|
||||
(mid-key, mid-value, open quote inside dict, empty dict, etc.). The
|
||||
parser must not raise — incomplete prefixes should silently wait for
|
||||
more text instead of logging exceptions."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
full_text = (
|
||||
f"{TOOL_CALL_START}[grocery.orderIngredients("
|
||||
"ingredientList=["
|
||||
'{"name": "apple", "quantity": "2"}, '
|
||||
'{"name": "bread", "quantity": "1"}'
|
||||
f"])]{TOOL_CALL_END}"
|
||||
)
|
||||
deltas = [c for c in full_text]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser, deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function.name == "grocery.orderIngredients"
|
||||
import json
|
||||
|
||||
args = json.loads(reconstructor.tool_calls[0].function.arguments)
|
||||
assert args == {
|
||||
"ingredientList": [
|
||||
{"name": "apple", "quantity": "2"},
|
||||
{"name": "bread", "quantity": "1"},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def test_streaming_dotted_name_in_single_delta(lfm2_tokenizer: TokenizerLike):
|
||||
"""A pythonic call with a dotted/attribute function name (e.g.
|
||||
``domain.method(arg=...)``) must be parsed correctly in streaming mode
|
||||
just as in non-streaming mode."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
full_text = f"{TOOL_CALL_START}[{DOTTED_NAME_FUNCTION_OUTPUT}]{TOOL_CALL_END}"
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(tool_parser, [full_text])
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function == DOTTED_NAME_FUNCTION_CALL
|
||||
|
||||
|
||||
def test_adjust_request_disables_skip_special_tokens(
|
||||
lfm2_tokenizer: TokenizerLike,
|
||||
):
|
||||
"""When tools are present, the parser must force
|
||||
``skip_special_tokens=False`` so the engine does not strip the
|
||||
<|tool_call_start|>/<|tool_call_end|> sentinels before they reach the
|
||||
parser."""
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
|
||||
request_with_tools = ChatCompletionRequest(
|
||||
messages=[],
|
||||
model="test-model",
|
||||
tools=[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
},
|
||||
},
|
||||
}
|
||||
],
|
||||
)
|
||||
assert request_with_tools.skip_special_tokens is True
|
||||
adjusted = tool_parser.adjust_request(request_with_tools)
|
||||
assert adjusted.skip_special_tokens is False
|
||||
|
||||
# No tools → no override; default behaviour preserved.
|
||||
request_no_tools = ChatCompletionRequest(messages=[], model="test-model")
|
||||
assert request_no_tools.skip_special_tokens is True
|
||||
adjusted_no_tools = tool_parser.adjust_request(request_no_tools)
|
||||
assert adjusted_no_tools.skip_special_tokens is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False])
|
||||
def test_regex_timeout_handling(streaming: bool, lfm2_tokenizer: TokenizerLike):
|
||||
"""Test regex timeout is handled gracefully."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("lfm2")(lfm2_tokenizer)
|
||||
|
||||
fake_input = f"{TOOL_CALL_START}[A(A=" + "\t)A(A=,\t" * 2
|
||||
fake_input += f"]{TOOL_CALL_END}"
|
||||
|
||||
mock_regex = MagicMock()
|
||||
mock_regex.match.side_effect = TimeoutError("Regex timeout")
|
||||
|
||||
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, fake_input, streaming=streaming
|
||||
)
|
||||
|
||||
assert content == fake_input
|
||||
assert len(tool_calls) == 0
|
||||
mock_regex.match.assert_called_once()
|
||||
@@ -1,269 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import ExtractedToolCallInformation
|
||||
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
|
||||
|
||||
LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llama_tokenizer():
|
||||
return AutoTokenizer.from_pretrained(LLAMA_MODEL)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(llama_tokenizer):
|
||||
return Llama3JsonToolParser(llama_tokenizer)
|
||||
|
||||
|
||||
def test_extract_tool_calls_simple(parser):
|
||||
# Test with a simple tool call
|
||||
model_output = (
|
||||
'Here is the result: {"name": "getOpenIncidentsTool", '
|
||||
'"parameters": {}} Would you like to know more?'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert isinstance(result, ExtractedToolCallInformation)
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].type == "function"
|
||||
assert result.tool_calls[0].function.name == "getOpenIncidentsTool"
|
||||
assert result.tool_calls[0].function.arguments == "{}"
|
||||
assert result.content is None
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_arguments(parser):
|
||||
# Test with a tool call that has arguments
|
||||
model_output = (
|
||||
'{"name": "searchTool", "parameters": {"query": "test query", "limit": 10}}'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "searchTool"
|
||||
assert '"query": "test query"' in result.tool_calls[0].function.arguments
|
||||
assert '"limit": 10' in result.tool_calls[0].function.arguments
|
||||
|
||||
|
||||
def test_extract_tool_calls_no_json(parser):
|
||||
# Test with text that doesn't contain a JSON object
|
||||
model_output = "This is just some text without any tool calls"
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert len(result.tool_calls) == 0
|
||||
assert result.content == model_output
|
||||
|
||||
|
||||
def test_extract_tool_calls_invalid_json(parser):
|
||||
# Test with invalid JSON
|
||||
model_output = '{"name": "invalidTool", "parameters": {invalid json}'
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert len(result.tool_calls) == 0
|
||||
assert result.content == model_output
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_arguments_key(parser):
|
||||
# Test with a tool call that uses "arguments" instead of "parameters"
|
||||
model_output = '{"name": "searchTool", "arguments": {"query": "test"}}'
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "searchTool"
|
||||
assert '"query": "test"' in result.tool_calls[0].function.arguments
|
||||
|
||||
|
||||
def test_extract_tool_calls_multiple_json(parser):
|
||||
# Test with multiple JSONs separated by semicolons
|
||||
model_output = (
|
||||
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
|
||||
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
|
||||
'{"name": "searchTool", "parameters": {"query": "test2"}}'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 3
|
||||
|
||||
# Check first tool call
|
||||
assert result.tool_calls[0].function.name == "searchTool"
|
||||
assert '"query": "test1"' in result.tool_calls[0].function.arguments
|
||||
|
||||
# Check second tool call
|
||||
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
|
||||
assert result.tool_calls[1].function.arguments == "{}"
|
||||
|
||||
# Check third tool call
|
||||
assert result.tool_calls[2].function.name == "searchTool"
|
||||
assert '"query": "test2"' in result.tool_calls[2].function.arguments
|
||||
|
||||
|
||||
def test_extract_tool_calls_multiple_json_with_whitespace(parser):
|
||||
# Test with multiple JSONs separated by semicolons and extra whitespace
|
||||
model_output = (
|
||||
'{"name": "searchTool", "parameters": {"query": "test1"}} ; '
|
||||
'{"name": "getOpenIncidentsTool", "parameters": {}} ; '
|
||||
'{"name": "searchTool", "parameters": {"query": "test2"}}'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 3
|
||||
assert result.tool_calls[0].function.name == "searchTool"
|
||||
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
|
||||
assert result.tool_calls[2].function.name == "searchTool"
|
||||
|
||||
|
||||
def test_extract_tool_calls_multiple_json_with_surrounding_text(parser):
|
||||
# Test with multiple JSONs and surrounding text
|
||||
model_output = (
|
||||
"Here are the results: "
|
||||
'{"name": "searchTool", "parameters": {"query": "test1"}}; '
|
||||
'{"name": "getOpenIncidentsTool", "parameters": {}}; '
|
||||
'{"name": "searchTool", "parameters": {"query": "test2"}} '
|
||||
"Would you like to know more?"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 3
|
||||
assert result.tool_calls[0].function.name == "searchTool"
|
||||
assert result.tool_calls[1].function.name == "getOpenIncidentsTool"
|
||||
assert result.tool_calls[2].function.name == "searchTool"
|
||||
|
||||
|
||||
def test_extract_tool_calls_deeply_nested_json(parser):
|
||||
# Test with deeply nested JSON parameters (5 levels)
|
||||
model_output = (
|
||||
'{"name": "complexTool", '
|
||||
'"parameters": {'
|
||||
'"level1": {'
|
||||
'"level2": {'
|
||||
'"level3": {'
|
||||
'"level4": {'
|
||||
'"value": "deep"'
|
||||
"}}}}}}"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "complexTool"
|
||||
# Verify the nested structure is preserved in the arguments
|
||||
import json
|
||||
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["level1"]["level2"]["level3"]["level4"]["value"] == "deep"
|
||||
|
||||
|
||||
def test_extract_tool_calls_multiple_with_deep_nesting(parser):
|
||||
# Test with multiple tool calls where some have deeply nested parameters
|
||||
model_output = (
|
||||
'{"name": "simpleTool", "parameters": {"value": "test"}}; '
|
||||
'{"name": "complexTool", "parameters": '
|
||||
'{"config": {"database": {"connection": {"pool": {"size": 10}}}}}}'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
|
||||
# Check first tool call
|
||||
assert result.tool_calls[0].function.name == "simpleTool"
|
||||
import json
|
||||
|
||||
args0 = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args0["value"] == "test"
|
||||
|
||||
# Check second tool call with deep nesting
|
||||
assert result.tool_calls[1].function.name == "complexTool"
|
||||
args1 = json.loads(result.tool_calls[1].function.arguments)
|
||||
assert args1["config"]["database"]["connection"]["pool"]["size"] == 10
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_quotes_and_brackets_in_string(parser):
|
||||
# Test with quotes and brackets inside quoted string values
|
||||
model_output = (
|
||||
'{"name": "searchTool", '
|
||||
'"parameters": {'
|
||||
'"query": "test {value} [complex]",'
|
||||
'"nested": {"inner": "more {brackets}"}'
|
||||
"}}"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "searchTool"
|
||||
# Verify the string values are preserved including brackets and quotes
|
||||
import json
|
||||
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["query"] == "test {value} [complex]"
|
||||
assert args["nested"]["inner"] == "more {brackets}"
|
||||
|
||||
|
||||
def test_extract_tool_calls_with_escaped_quotes_in_nested_json(parser):
|
||||
# Test with escaped quotes in deeply nested JSON
|
||||
model_output = (
|
||||
'{"name": "parserTool", "parameters": {"text": "He said \\"Hello {world}\\""}}'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "parserTool"
|
||||
# Verify escaped quotes are preserved
|
||||
import json
|
||||
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["text"] == 'He said "Hello {world}"'
|
||||
|
||||
|
||||
def test_extract_tool_calls_missing_name_key(parser):
|
||||
# Test that missing "name" key returns content
|
||||
model_output = '{"parameters": {}}'
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert len(result.tool_calls) == 0
|
||||
assert result.content == model_output
|
||||
|
||||
|
||||
def test_extract_tool_calls_missing_parameters_and_arguments_key(parser):
|
||||
# Test that missing both "parameters" and "arguments" keys returns content
|
||||
model_output = '{"name": "toolWithoutParams"}'
|
||||
result = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert len(result.tool_calls) == 0
|
||||
assert result.content == model_output
|
||||
|
||||
|
||||
def test_regex_timeout_handling(parser):
|
||||
"""Test regex timeout is handled gracefully."""
|
||||
fake_problematic_input = "{hello world[A(A=" + "\t)A(A=,\t" * 2
|
||||
|
||||
# create a mock regex that raises TimeoutError
|
||||
mock_regex = MagicMock()
|
||||
mock_regex.finditer.side_effect = TimeoutError("Regex timeout")
|
||||
|
||||
with patch.object(parser, "tool_call_start_regex", mock_regex):
|
||||
result = parser.extract_tool_calls(fake_problematic_input, None)
|
||||
|
||||
# should treat as regular text when regex times out
|
||||
assert result.content == fake_problematic_input
|
||||
assert result.tools_called is False
|
||||
assert len(result.tool_calls) == 0
|
||||
mock_regex.finditer.assert_called_once()
|
||||
@@ -1,269 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
|
||||
# Test cases similar to pythonic parser but with Llama4 specific format
|
||||
SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]"
|
||||
SIMPLE_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "LA", "metric": "C"}',
|
||||
)
|
||||
MORE_TYPES_FUNCTION_OUTPUT = (
|
||||
"[register_user(name='Doe', "
|
||||
"age=9, "
|
||||
"address={'city': 'LA', 'state': 'CA'}, "
|
||||
"role=None, "
|
||||
"passed_test=True, "
|
||||
"aliases=['John', 'Johnny'])]"
|
||||
)
|
||||
MORE_TYPES_FUNCTION_CALL = FunctionCall(
|
||||
name="register_user",
|
||||
arguments='{"name": "Doe", '
|
||||
'"age": 9, '
|
||||
'"address": {"city": "LA", "state": "CA"}, '
|
||||
'"role": null, '
|
||||
'"passed_test": true, '
|
||||
'"aliases": ["John", "Johnny"]}',
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_OUTPUT = "[get_weather()]"
|
||||
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments="{}",
|
||||
)
|
||||
EMPTY_DICT_FUNCTION_OUTPUT = "[do_something_cool(additional_data={})]"
|
||||
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"additional_data": {}}',
|
||||
)
|
||||
EMPTY_LIST_FUNCTION_OUTPUT = "[do_something_cool(steps=[])]"
|
||||
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"steps": []}',
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT = (
|
||||
r"[get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')]"
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
|
||||
)
|
||||
PYTHON_TAG_FUNCTION_OUTPUT = (
|
||||
"<|python_start|>[get_weather(city='LA', metric='C')]<|python_end|>"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
model_output = "How can I help you today?"
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert content == model_output
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
test_str = "<|python_start|>"
|
||||
test_str += "[get_weather(city='LA', metric='C'),"
|
||||
test_str += "register_user(name='Doe', age=9)]"
|
||||
TEST_CASES = [
|
||||
pytest.param(
|
||||
True,
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT,
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="simple_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False, SIMPLE_FUNCTION_OUTPUT, [SIMPLE_FUNCTION_CALL], id="simple_nonstreaming"
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
MORE_TYPES_FUNCTION_OUTPUT,
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
MORE_TYPES_FUNCTION_OUTPUT,
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
PARAMETERLESS_FUNCTION_OUTPUT,
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
id="parameterless_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
PARAMETERLESS_FUNCTION_OUTPUT,
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
id="parameterless_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
EMPTY_DICT_FUNCTION_OUTPUT,
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
id="empty_dict_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
EMPTY_DICT_FUNCTION_OUTPUT,
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
id="empty_dict_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
EMPTY_LIST_FUNCTION_OUTPUT,
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
id="empty_list_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
EMPTY_LIST_FUNCTION_OUTPUT,
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
id="empty_list_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT,
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="escaped_string_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT,
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="escaped_string_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
"[get_weather(city='LA',metric='C'),register_user(name='Doe',age=9)]",
|
||||
[
|
||||
SIMPLE_FUNCTION_CALL,
|
||||
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
|
||||
],
|
||||
id="parallel_calls_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
"[get_weather(city='LA',metric='C'),register_user(name='Doe',age=9)]",
|
||||
[
|
||||
SIMPLE_FUNCTION_CALL,
|
||||
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
|
||||
],
|
||||
id="parallel_calls_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
PYTHON_TAG_FUNCTION_OUTPUT,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
id="python_tag_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
PYTHON_TAG_FUNCTION_OUTPUT,
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
id="python_tag_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
test_str,
|
||||
[
|
||||
SIMPLE_FUNCTION_CALL,
|
||||
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
|
||||
],
|
||||
id="parallel_calls_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
"<|python_start|>[get_weather(city='LA', metric='C'), "
|
||||
"register_user(name='Doe', age=9)]",
|
||||
[
|
||||
SIMPLE_FUNCTION_CALL,
|
||||
FunctionCall(name="register_user", arguments='{"name": "Doe", "age": 9}'),
|
||||
],
|
||||
id="parallel_calls_nonstreaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
|
||||
def test_tool_call(
|
||||
streaming: bool,
|
||||
model_output: str,
|
||||
expected_tool_calls: list[FunctionCall],
|
||||
default_tokenizer: TokenizerLike,
|
||||
):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert len(tool_calls) == len(expected_tool_calls)
|
||||
for actual, expected in zip(tool_calls, expected_tool_calls):
|
||||
assert actual.type == "function"
|
||||
assert actual.function == expected
|
||||
|
||||
|
||||
def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
model_output_deltas = [
|
||||
"<|python_start|>[get_weather(city='LA', metric='C'), "
|
||||
"get_weather(), "
|
||||
"do_something_cool(steps=[])]<|python_end|>",
|
||||
]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert reconstructor.other_content == ""
|
||||
assert len(reconstructor.tool_calls) == 3
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False])
|
||||
def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):
|
||||
"""Test regex timeout is handled gracefully."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
|
||||
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
|
||||
|
||||
# create a mock regex that raises TimeoutError
|
||||
mock_regex = MagicMock()
|
||||
mock_regex.match.side_effect = TimeoutError("Regex timeout")
|
||||
|
||||
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, fake_problematic_input, streaming=streaming
|
||||
)
|
||||
|
||||
# should treat as regular text when regex times out
|
||||
assert content == fake_problematic_input
|
||||
assert len(tool_calls) == 0
|
||||
mock_regex.match.assert_called_once()
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
class TestLongCatToolParser(ToolParserTests):
|
||||
@pytest.fixture
|
||||
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""Add some longcat specific tokens to the default vocab."""
|
||||
tokenizer = default_tokenizer
|
||||
tokenizer_vocab = tokenizer.get_vocab()
|
||||
tokenizer.get_vocab = MagicMock()
|
||||
tokenizer_vocab.update(
|
||||
{
|
||||
"<longcat_tool_call>": 32000,
|
||||
"</longcat_tool_call>": 32001,
|
||||
}
|
||||
)
|
||||
tokenizer.get_vocab.return_value = tokenizer_vocab
|
||||
return tokenizer
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="longcat",
|
||||
# Test data
|
||||
no_tool_calls_output="This is a regular response without any tool calls.",
|
||||
single_tool_call_output=(
|
||||
'<longcat_tool_call>{"name": "get_weather", '
|
||||
'"arguments": {"city": "Tokyo"}}</longcat_tool_call>'
|
||||
),
|
||||
parallel_tool_calls_output=(
|
||||
'<longcat_tool_call>{"name": "get_weather", '
|
||||
'"arguments": {"city": "Tokyo"}}</longcat_tool_call>\n'
|
||||
'<longcat_tool_call>{"name": "get_time", '
|
||||
'"arguments": {"timezone": "Asia/Tokyo"}}</longcat_tool_call>'
|
||||
),
|
||||
various_data_types_output="""<longcat_tool_call>{
|
||||
"name": "test_function",
|
||||
"arguments": {
|
||||
"string_field": "hello",
|
||||
"int_field": 42,
|
||||
"float_field": 3.14,
|
||||
"bool_field": true,
|
||||
"null_field": null,
|
||||
"array_field": ["a", "b", "c"],
|
||||
"object_field": {"nested": "value"},
|
||||
"empty_array": [],
|
||||
"empty_object": {}
|
||||
}
|
||||
}</longcat_tool_call>""",
|
||||
empty_arguments_output=(
|
||||
'<longcat_tool_call>{"name": "refresh", "arguments": {}}'
|
||||
"</longcat_tool_call>"
|
||||
),
|
||||
surrounding_text_output=(
|
||||
"Let me check the weather for you.\n"
|
||||
'<longcat_tool_call>{"name": "get_weather", '
|
||||
'"arguments": {"city": "Tokyo"}}</longcat_tool_call>\n'
|
||||
"Here is the result."
|
||||
),
|
||||
escaped_strings_output="""<longcat_tool_call>{
|
||||
"name": "test_function",
|
||||
"arguments": {
|
||||
"quoted": "He said \\"hello\\"",
|
||||
"path": "C:\\\\Users\\\\file.txt",
|
||||
"newline": "line1\\nline2",
|
||||
"unicode": "emoji: 🎉"
|
||||
}
|
||||
}</longcat_tool_call>""",
|
||||
malformed_input_outputs=[
|
||||
'<longcat_tool_call>{"name": "func", "arguments": {',
|
||||
(
|
||||
'<longcat_tool_call>{"name": "func", '
|
||||
'"arguments": "not a dict"}</longcat_tool_call>'
|
||||
),
|
||||
"Some text with <longcat_tool_call>invalid json",
|
||||
],
|
||||
# Expected results
|
||||
single_tool_call_expected_name="get_weather",
|
||||
single_tool_call_expected_args={"city": "Tokyo"},
|
||||
single_tool_call_expected_content=None,
|
||||
parallel_tool_calls_count=2,
|
||||
parallel_tool_calls_names=["get_weather", "get_time"],
|
||||
# xfail markers
|
||||
xfail_streaming={
|
||||
"test_malformed_input": "Streaming has complex buffering behavior",
|
||||
},
|
||||
xfail_nonstreaming={},
|
||||
# Configuration
|
||||
allow_empty_or_json_empty_args=True,
|
||||
)
|
||||
@@ -1,787 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# ruff: noqa: E501
|
||||
|
||||
import json
|
||||
import random
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import run_tool_extraction_streaming
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall, ToolCall
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
from vllm.tool_parsers.minicpm5xml_tool_parser import MiniCPM5XMLToolParser
|
||||
|
||||
|
||||
def _tool(name: str, parameters: dict) -> ChatCompletionToolsParam:
|
||||
return ChatCompletionToolsParam(
|
||||
type="function",
|
||||
function={
|
||||
"name": name,
|
||||
"parameters": parameters,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def make_tools_weather() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
_tool(
|
||||
"get_weather",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"date": {"type": "string"},
|
||||
},
|
||||
"required": ["city"],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def make_tools_sum() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
_tool(
|
||||
"sum_values",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"nums": {"type": "array"},
|
||||
"exact": {"type": "boolean"},
|
||||
},
|
||||
"required": ["nums"],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def make_tools_no_required() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
_tool(
|
||||
"noop",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"note": {"type": "string"}},
|
||||
"required": [],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def make_request(
|
||||
tools: list[ChatCompletionToolsParam],
|
||||
tool_choice: str = "auto",
|
||||
) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
tools=tools,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
|
||||
|
||||
def make_tool_call(name: str, arguments: dict) -> ToolCall:
|
||||
return ToolCall(
|
||||
type="function",
|
||||
function=FunctionCall(
|
||||
name=name,
|
||||
arguments=json.dumps(arguments, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def assert_tool_calls(
|
||||
actual: list[ToolCall],
|
||||
expected: list[ToolCall],
|
||||
) -> None:
|
||||
assert len(actual) == len(expected)
|
||||
for act, exp in zip(actual, expected):
|
||||
assert act.type == "function"
|
||||
assert act.function.name == exp.function.name
|
||||
assert json.loads(act.function.arguments) == json.loads(exp.function.arguments)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser() -> ToolParser:
|
||||
mock_tokenizer = MagicMock()
|
||||
return MiniCPM5XMLToolParser(mock_tokenizer)
|
||||
|
||||
|
||||
def test_registered_in_tool_parser_manager() -> None:
|
||||
cls = ToolParserManager.get_tool_parser("minicpm5")
|
||||
assert cls is MiniCPM5XMLToolParser
|
||||
|
||||
|
||||
def test_adjust_request_skip_special_tokens(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
assert request.skip_special_tokens is True
|
||||
adjusted = parser.adjust_request(request)
|
||||
assert adjusted.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_adjust_request_tool_choice_none(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather(), tool_choice="none")
|
||||
adjusted = parser.adjust_request(request)
|
||||
assert adjusted.skip_special_tokens is True
|
||||
|
||||
|
||||
def test_no_tool_call(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
out = parser.extract_tool_calls("How can I help you?", request)
|
||||
assert not out.tools_called
|
||||
assert out.tool_calls == []
|
||||
assert out.content == "How can I help you?"
|
||||
|
||||
|
||||
def test_single_call_with_surrounding_text(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
"Intro before.\n"
|
||||
'<function name="get_weather">'
|
||||
'<param name="city">上海</param>'
|
||||
'<param name="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
"Outro after.\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert out.tools_called
|
||||
assert_tool_calls(
|
||||
out.tool_calls,
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather",
|
||||
{
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
assert out.content is None
|
||||
|
||||
|
||||
def test_cdata_multiline(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
'<param name="city"><![CDATA[北\n京]]></param>'
|
||||
'<param name="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args["city"] == "北\n京"
|
||||
assert args["date"] == "2024-06-27"
|
||||
|
||||
|
||||
def test_tokenizer_space_marker(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function\u0120name="get_weather">'
|
||||
'<param\u0120name="city">上海</param>'
|
||||
'<param\u0120name="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert out.tools_called
|
||||
assert_tool_calls(
|
||||
out.tool_calls,
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather",
|
||||
{
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_collapsed_function_and_param_tags(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<functionname="get_weather">'
|
||||
'<paramname="city">上海</param>'
|
||||
'<paramname="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert out.tools_called
|
||||
assert_tool_calls(
|
||||
out.tool_calls,
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather",
|
||||
{
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_collapsed_param_tags_with_tokenizer_space_function(
|
||||
parser: ToolParser,
|
||||
) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function\u0120name="get_weather">'
|
||||
'<paramname="city">上海</param>'
|
||||
'<paramname="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert out.tools_called
|
||||
assert_tool_calls(
|
||||
out.tool_calls,
|
||||
[
|
||||
make_tool_call(
|
||||
"get_weather",
|
||||
{
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_partial_chunks(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
chunks = [
|
||||
'<function name="get_weather">',
|
||||
'<param name="city">',
|
||||
'上海</param><param name="date">2024-06-27</param></function>\n',
|
||||
]
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser,
|
||||
chunks,
|
||||
request,
|
||||
assert_one_tool_per_delta=False,
|
||||
)
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
}
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_tokenizer_space_marker(
|
||||
parser: ToolParser,
|
||||
) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
chunks = [
|
||||
'<function\u0120name="get_weather">',
|
||||
'<param\u0120name="city">',
|
||||
'上海</param><param\u0120name="date">2024-06-27</param></function>\n',
|
||||
]
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser,
|
||||
chunks,
|
||||
request,
|
||||
assert_one_tool_per_delta=False,
|
||||
)
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
}
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_collapsed_tags_weather(
|
||||
parser: ToolParser,
|
||||
) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<functionname="get_weather">'
|
||||
'<paramname="city">上海</param>'
|
||||
'<paramname="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
random.seed(2)
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser,
|
||||
_random_chunks(text, 1, 4),
|
||||
request,
|
||||
)
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
}
|
||||
|
||||
|
||||
def test_collapsed_tags_current_weather(parser: ToolParser) -> None:
|
||||
request = make_request(
|
||||
[
|
||||
_tool(
|
||||
"get_current_weather",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"state": {"type": "string"},
|
||||
"unit": {"type": "string"},
|
||||
},
|
||||
"required": ["city", "state", "unit"],
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
text = (
|
||||
'<functionname="get_current_weather">'
|
||||
'<paramname="city">Dallas</param>'
|
||||
'<paramname="state">TX</param>'
|
||||
'<paramname="unit">fahrenheit</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert out.tools_called
|
||||
assert json.loads(out.tool_calls[0].function.arguments) == {
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_incremental_arguments(
|
||||
parser: ToolParser,
|
||||
) -> None:
|
||||
request = make_request(
|
||||
[
|
||||
_tool(
|
||||
"get_current_weather",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"state": {"type": "string"},
|
||||
"unit": {"type": "string"},
|
||||
},
|
||||
"required": ["city", "state", "unit"],
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
text = (
|
||||
'<function name="get_current_weather">'
|
||||
'<param name="city">Dallas</param>'
|
||||
'<param name="state">TX</param>'
|
||||
'<param name="unit">fahrenheit</param>'
|
||||
"</function>"
|
||||
)
|
||||
prev = ""
|
||||
arguments = ""
|
||||
for chunk in [text[:37], text[37:70], text[70:]]:
|
||||
current = prev + chunk
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
prev,
|
||||
current,
|
||||
chunk,
|
||||
[],
|
||||
[],
|
||||
[],
|
||||
request,
|
||||
)
|
||||
if delta and delta.tool_calls:
|
||||
arg_delta = delta.tool_calls[0].function.arguments
|
||||
if arg_delta:
|
||||
arguments += arg_delta
|
||||
prev = current
|
||||
|
||||
assert json.loads(arguments) == {
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
|
||||
|
||||
def test_unknown_tool_block_preserved(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = '<function name="unknown"><param name="x">1</param></function>\n'
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
assert "unknown" in (out.content or "")
|
||||
|
||||
|
||||
def test_non_string_types(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_sum())
|
||||
text = (
|
||||
'<function name="sum_values">'
|
||||
'<param name="nums">[1, 2, 3]</param>'
|
||||
'<param name="exact">true</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args["nums"] == [1, 2, 3]
|
||||
assert args["exact"] is True
|
||||
|
||||
|
||||
def test_multiple_calls_interleaved_text(parser: ToolParser) -> None:
|
||||
tools = make_tools_weather() + make_tools_sum()
|
||||
request = make_request(tools)
|
||||
text = (
|
||||
"Head\n"
|
||||
'<function name="get_weather"><param name="city">北京</param></function>\n'
|
||||
"TXT\n"
|
||||
'<function name="sum_values"><param name="nums">[7,8,9]</param>'
|
||||
'<param name="exact">false</param></function>\n'
|
||||
"Tail\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 2
|
||||
args0 = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args0["city"] == "北京"
|
||||
args1 = json.loads(out.tool_calls[1].function.arguments)
|
||||
assert args1["nums"] == [7, 8, 9]
|
||||
assert args1["exact"] is False
|
||||
assert out.content is None
|
||||
|
||||
|
||||
def test_incomplete_missing_function_end(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = '<function name="get_weather"><param name="city">北京</param>'
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
assert "get_weather" in (out.content or "")
|
||||
|
||||
|
||||
def test_param_missing_name_invalid(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
"<param>北京</param>"
|
||||
'<param name="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
assert "<param>北京</param>" in (out.content or "")
|
||||
|
||||
|
||||
def test_duplicate_param_names_invalid(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
'<param name="city">北京</param>'
|
||||
'<param name="city">上海</param>'
|
||||
"</function>\n"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
|
||||
|
||||
def test_case_sensitive_param_name_invalid(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = '<function name="get_weather"><param name="City">北京</param></function>\n'
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
|
||||
|
||||
def test_no_required_and_zero_param_valid(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_no_required())
|
||||
text = '<function name="noop"></function>\n'
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {}
|
||||
|
||||
|
||||
def test_thinking_only_sentencepiece_normalized_in_content(
|
||||
parser: ToolParser,
|
||||
) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = "\u010aFirst,\u0120I\u0120need\u0120to\u0120check\u0120the\u0120weather."
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
assert out.content is not None
|
||||
assert "\u0120" not in out.content
|
||||
assert "\u010a" not in out.content
|
||||
assert "First, I need to check the weather." in out.content
|
||||
|
||||
|
||||
def test_properties_wrapped_arguments(parser: ToolParser) -> None:
|
||||
tools = [
|
||||
_tool(
|
||||
"get_customer_by_phone",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"phone_number": {"type": "string"}},
|
||||
"required": ["phone_number"],
|
||||
},
|
||||
)
|
||||
]
|
||||
request = make_request(tools)
|
||||
text = (
|
||||
'<function name="get_customer_by_phone">'
|
||||
"<param name=\"properties\">{'phone_number': '555-123-2002'}</param>"
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "get_customer_by_phone"
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"phone_number": "555-123-2002"}
|
||||
|
||||
|
||||
def test_arguments_wrapped_arguments(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
'<param name="arguments">{"city": "上海", "date": "2024-06-27"}</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"city": "上海", "date": "2024-06-27"}
|
||||
|
||||
|
||||
def test_wrapped_arguments_still_validate_schema(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
'<param name="properties">{"unknown": "x"}</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
|
||||
|
||||
def test_extra_arguments_ignored_when_required_present(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather">'
|
||||
'<param name="city">上海</param>'
|
||||
'<param name="unknown">ignored</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"city": "上海"}
|
||||
|
||||
|
||||
def test_extra_arguments_do_not_satisfy_required(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
'<function name="get_weather"><param name="unknown">ignored</param></function>'
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert not out.tools_called
|
||||
|
||||
|
||||
def test_zero_arg_tool_ignores_extra_arguments(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_no_required())
|
||||
text = (
|
||||
'<function name="noop">'
|
||||
'<param name="note">ignored</param>'
|
||||
'<param name="extra">ignored</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"note": "ignored"}
|
||||
|
||||
|
||||
def test_alias_get_details_by_phone(parser: ToolParser) -> None:
|
||||
tools = [
|
||||
_tool(
|
||||
"get_customer_by_phone",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"phone_number": {"type": "string"}},
|
||||
"required": ["phone_number"],
|
||||
},
|
||||
)
|
||||
]
|
||||
request = make_request(tools)
|
||||
text = (
|
||||
'<function name="get_details_by_phone">'
|
||||
'<param name="phone_number">555-123-2002</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "get_customer_by_phone"
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"phone_number": "555-123-2002"}
|
||||
|
||||
|
||||
def test_alias_get_line_details(parser: ToolParser) -> None:
|
||||
tools = [
|
||||
_tool(
|
||||
"get_details_by_id",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"id": {"type": "string"}},
|
||||
"required": ["id"],
|
||||
},
|
||||
)
|
||||
]
|
||||
request = make_request(tools)
|
||||
text = (
|
||||
'<function name="get_line_details">'
|
||||
'<param name="customer_id">C1001</param>'
|
||||
'<param name="line_id">L1001</param>'
|
||||
"</function>"
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "get_details_by_id"
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"id": "L1001"}
|
||||
|
||||
|
||||
def test_alias_enable_roaming(parser: ToolParser) -> None:
|
||||
tools = [
|
||||
_tool(
|
||||
"toggle_roaming",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"line_id": {"type": "string"},
|
||||
"enabled": {"type": "boolean"},
|
||||
},
|
||||
"required": ["line_id", "enabled"],
|
||||
},
|
||||
)
|
||||
]
|
||||
request = make_request(tools)
|
||||
text = (
|
||||
'<function name="enable_roaming"><param name="line_id">L1001</param></function>'
|
||||
)
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
assert len(out.tool_calls) == 1
|
||||
assert out.tool_calls[0].function.name == "toggle_roaming"
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args == {"line_id": "L1001", "enabled": True}
|
||||
|
||||
|
||||
def _random_chunks(text: str, min_len: int, max_len: int) -> list[str]:
|
||||
chunks: list[str] = []
|
||||
index = 0
|
||||
while index < len(text):
|
||||
size = random.randint(min_len, max_len)
|
||||
chunks.append(text[index : index + size])
|
||||
index += size
|
||||
return chunks
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_single(parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_weather())
|
||||
text = (
|
||||
"Intro before.\n"
|
||||
'<function name="get_weather">'
|
||||
'<param name="city">上海</param>'
|
||||
'<param name="date">2024-06-27</param>'
|
||||
"</function>\n"
|
||||
"Outro after.\n"
|
||||
)
|
||||
random.seed(0)
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser,
|
||||
_random_chunks(text, 1, 4),
|
||||
request,
|
||||
)
|
||||
assert "Intro before." in reconstructor.other_content
|
||||
assert "Outro after." in reconstructor.other_content
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert reconstructor.tool_calls[0].function.name == "get_weather"
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"city": "上海",
|
||||
"date": "2024-06-27",
|
||||
}
|
||||
|
||||
|
||||
def test_extract_tool_calls_streaming_multiple(parser: ToolParser) -> None:
|
||||
tools = make_tools_weather() + make_tools_sum()
|
||||
request = make_request(tools)
|
||||
text = (
|
||||
"Head\n"
|
||||
'<function name="get_weather"><param name="city">北京</param></function>\n'
|
||||
"TXT\n"
|
||||
'<function name="sum_values"><param name="nums">[7,8,9]</param>'
|
||||
'<param name="exact">false</param></function>\n'
|
||||
"Tail\n"
|
||||
)
|
||||
random.seed(1)
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser,
|
||||
_random_chunks(text, 1, 5),
|
||||
request,
|
||||
)
|
||||
assert "Head" in reconstructor.other_content
|
||||
assert "TXT" in reconstructor.other_content
|
||||
assert "Tail" in reconstructor.other_content
|
||||
assert len(reconstructor.tool_calls) == 2
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments)["city"] == "北京"
|
||||
assert json.loads(reconstructor.tool_calls[1].function.arguments) == {
|
||||
"nums": [7, 8, 9],
|
||||
"exact": False,
|
||||
}
|
||||
|
||||
|
||||
def make_tools_write() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
_tool(
|
||||
"write_file",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"content": {"type": "string"}},
|
||||
"required": ["content"],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class TestParameterWhitespace:
|
||||
"""CDATA is verbatim, so its whitespace must survive."""
|
||||
|
||||
def test_cdata_whitespace_preserved(self, parser: ToolParser) -> None:
|
||||
request = make_request(make_tools_write())
|
||||
text = (
|
||||
'<function name="write_file">'
|
||||
'<param name="content"><![CDATA[ def foo():\n pass\n]]></param>'
|
||||
"</function>\n"
|
||||
)
|
||||
|
||||
out = parser.extract_tool_calls(text, request)
|
||||
|
||||
assert len(out.tool_calls) == 1
|
||||
args = json.loads(out.tool_calls[0].function.arguments)
|
||||
assert args["content"] == " def foo():\n pass\n"
|
||||
|
||||
def test_cdata_whitespace_preserved_streaming(self, parser: ToolParser) -> None:
|
||||
"""Value split across chunks, i.e. the partial path."""
|
||||
request = make_request(make_tools_write())
|
||||
chunks = [
|
||||
'<function name="write_file">',
|
||||
'<param name="content"><![CDATA[ def foo():\n',
|
||||
" pass\n]]></param></function>\n",
|
||||
]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
parser,
|
||||
chunks,
|
||||
request,
|
||||
assert_one_tool_per_delta=False,
|
||||
)
|
||||
|
||||
assert len(reconstructor.tool_calls) == 1
|
||||
assert json.loads(reconstructor.tool_calls[0].function.arguments) == {
|
||||
"content": " def foo():\n pass\n"
|
||||
}
|
||||
@@ -1,603 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tool_parsers.minimax_m2_tool_parser import (
|
||||
MinimaxM2ToolParser,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# Token IDs matching FakeTokenizer.vocab
|
||||
TC_START_ID = 1
|
||||
TC_END_ID = 2
|
||||
|
||||
|
||||
class FakeTokenizer:
|
||||
"""Minimal fake tokenizer for unit tests."""
|
||||
|
||||
def __init__(self):
|
||||
self.model_tokenizer = True
|
||||
self.vocab = {
|
||||
"<minimax:tool_call>": TC_START_ID,
|
||||
"</minimax:tool_call>": TC_END_ID,
|
||||
}
|
||||
|
||||
def get_vocab(self):
|
||||
return self.vocab
|
||||
|
||||
def decode(self, token_ids):
|
||||
id_to_token = {v: k for k, v in self.vocab.items()}
|
||||
return "".join(id_to_token.get(token_id, "") for token_id in token_ids)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser():
|
||||
return MinimaxM2ToolParser(FakeTokenizer())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _feed(parser, chunks, request=None):
|
||||
"""Feed chunks through the streaming parser and collect results.
|
||||
|
||||
Each element in *chunks* is either:
|
||||
- a ``str``: used as delta_text (current_text accumulates automatically)
|
||||
- a ``(delta_text, delta_token_ids)`` tuple for special-token scenarios
|
||||
|
||||
Returns a list of non-None DeltaMessage objects.
|
||||
"""
|
||||
previous = ""
|
||||
results = []
|
||||
for chunk in chunks:
|
||||
if isinstance(chunk, tuple):
|
||||
delta, delta_ids = chunk
|
||||
else:
|
||||
delta = chunk
|
||||
delta_ids = []
|
||||
|
||||
current = previous + delta
|
||||
result = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous,
|
||||
current_text=current,
|
||||
delta_text=delta,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=delta_ids,
|
||||
request=request,
|
||||
)
|
||||
if result is not None:
|
||||
results.append(result)
|
||||
previous = current
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _collect_content(results):
|
||||
"""Join all content strings from a list of DeltaMessages."""
|
||||
return "".join(r.content for r in results if r.content)
|
||||
|
||||
|
||||
def _collect_tool_calls(results):
|
||||
"""Aggregate tool calls by index from a list of DeltaMessages.
|
||||
|
||||
Returns a dict: index -> {"id": ..., "name": ..., "arguments": ...}
|
||||
"""
|
||||
tool_calls = {}
|
||||
for r in results:
|
||||
for tc in r.tool_calls or []:
|
||||
if tc.index not in tool_calls:
|
||||
tool_calls[tc.index] = {
|
||||
"id": None,
|
||||
"name": "",
|
||||
"arguments": "",
|
||||
}
|
||||
if tc.id:
|
||||
tool_calls[tc.index]["id"] = tc.id
|
||||
if tc.function:
|
||||
if tc.function.name:
|
||||
tool_calls[tc.index]["name"] += tc.function.name
|
||||
if tc.function.arguments:
|
||||
tool_calls[tc.index]["arguments"] += tc.function.arguments
|
||||
return tool_calls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 1: content before tool calls
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestContentStreaming:
|
||||
"""Tests for plain content (no tool calls)."""
|
||||
|
||||
def test_plain_content(self, parser):
|
||||
"""No tool call tokens — all text is streamed as content."""
|
||||
results = _feed(parser, ["Hello ", "world"])
|
||||
assert _collect_content(results) == "Hello world"
|
||||
|
||||
def test_content_before_tool_call(self, parser):
|
||||
"""Text before <minimax:tool_call> is streamed as content."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"Let me check. ",
|
||||
'<minimax:tool_call><invoke name="get_weather">'
|
||||
'<parameter name="city">Seattle</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
assert _collect_content(results) == "Let me check. "
|
||||
|
||||
def test_empty_delta_no_crash(self, parser):
|
||||
"""Empty delta_text with no token IDs returns None."""
|
||||
results = _feed(parser, [("", [])])
|
||||
assert results == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase 2: tool call parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSingleInvoke:
|
||||
"""Tests for a single <invoke> block."""
|
||||
|
||||
def test_incremental_chunks(self, parser):
|
||||
"""Each XML element arrives in a separate chunk."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
'<parameter name="city">Seattle</parameter>',
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert tc[0]["name"] == "get_weather"
|
||||
assert json.loads(tc[0]["arguments"]) == {"city": "Seattle"}
|
||||
assert tc[0]["id"] is not None
|
||||
|
||||
def test_single_chunk_complete(self, parser):
|
||||
"""Entire tool call arrives in one delta."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="get_weather">'
|
||||
'<parameter name="city">Seattle</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert json.loads(tc[0]["arguments"]) == {"city": "Seattle"}
|
||||
|
||||
def test_multiple_params(self, parser):
|
||||
"""Multiple parameters in one invoke."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="get_weather">',
|
||||
'<parameter name="city">Seattle</parameter>',
|
||||
'<parameter name="days">5</parameter>',
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert json.loads(tc[0]["arguments"]) == {
|
||||
"city": "Seattle",
|
||||
"days": "5",
|
||||
}
|
||||
|
||||
|
||||
class TestMultipleInvokes:
|
||||
"""Tests for multiple <invoke> blocks in one tool call."""
|
||||
|
||||
def test_two_invokes_incremental(self, parser):
|
||||
"""Two invokes arriving one chunk at a time."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="search_web">'
|
||||
'<parameter name="query">OpenAI</parameter>'
|
||||
"</invoke>",
|
||||
'<invoke name="search_web">'
|
||||
'<parameter name="query">Gemini</parameter>'
|
||||
"</invoke>",
|
||||
"</minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 2
|
||||
assert tc[0]["name"] == "search_web"
|
||||
assert tc[1]["name"] == "search_web"
|
||||
assert json.loads(tc[0]["arguments"]) == {"query": "OpenAI"}
|
||||
assert json.loads(tc[1]["arguments"]) == {"query": "Gemini"}
|
||||
|
||||
def test_two_invokes_in_single_delta(self, parser):
|
||||
"""Both invokes close in the same delta — loop must emit both."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="fn_a"><parameter name="x">1</parameter></invoke>'
|
||||
'<invoke name="fn_b"><parameter name="y">2</parameter></invoke>',
|
||||
"</minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 2
|
||||
assert tc[0]["name"] == "fn_a"
|
||||
assert tc[1]["name"] == "fn_b"
|
||||
|
||||
def test_different_functions(self, parser):
|
||||
"""Parallel calls to different functions."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="get_weather">'
|
||||
'<parameter name="city">NYC</parameter>'
|
||||
"</invoke>",
|
||||
'<invoke name="get_stock">'
|
||||
'<parameter name="ticker">AAPL</parameter>'
|
||||
"</invoke>",
|
||||
"</minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert tc[0]["name"] == "get_weather"
|
||||
assert tc[1]["name"] == "get_stock"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DeltaMessage structure
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDeltaMessageFormat:
|
||||
"""Verify the shape of emitted DeltaMessage / DeltaToolCall."""
|
||||
|
||||
def test_tool_call_fields(self, parser):
|
||||
"""Each emitted tool call has id, name, arguments, type, index."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="fn">'
|
||||
'<parameter name="k">v</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc_deltas = [tc for r in results for tc in (r.tool_calls or [])]
|
||||
assert len(tc_deltas) == 1
|
||||
tc = tc_deltas[0]
|
||||
assert tc.index == 0
|
||||
assert tc.type == "function"
|
||||
assert tc.id is not None
|
||||
assert tc.function.name == "fn"
|
||||
assert json.loads(tc.function.arguments) == {"k": "v"}
|
||||
|
||||
def test_multi_invoke_indices(self, parser):
|
||||
"""Multiple invokes get sequential indices."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"<minimax:tool_call>",
|
||||
'<invoke name="a"><parameter name="x">1</parameter></invoke>',
|
||||
'<invoke name="b"><parameter name="x">2</parameter></invoke>',
|
||||
"</minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc_deltas = [tc for r in results for tc in (r.tool_calls or [])]
|
||||
indices = [tc.index for tc in tc_deltas]
|
||||
assert indices == [0, 1]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Large chunks (stream_interval > 1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestLargeChunks:
|
||||
"""Simulate stream_interval > 1 where many tokens arrive at once."""
|
||||
|
||||
def test_header_and_params_in_separate_chunks(self, parser):
|
||||
"""Header in chunk 1, all params + close in chunk 2."""
|
||||
chunk1 = '<minimax:tool_call><invoke name="get_weather">'
|
||||
chunk2 = (
|
||||
'<parameter name="city">Seattle</parameter>'
|
||||
'<parameter name="days">5</parameter>'
|
||||
"</invoke></minimax:tool_call>"
|
||||
)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
chunk1,
|
||||
chunk2,
|
||||
],
|
||||
)
|
||||
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed == {"city": "Seattle", "days": "5"}
|
||||
|
||||
|
||||
class TestAnyOfNullableParam:
|
||||
"""Regression: anyOf nullable parameter parsing (PR #32342)."""
|
||||
|
||||
def test_anyof_nullable_param_non_null_value(self):
|
||||
"""A valid non-null string should be preserved, not collapsed to None."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="update_profile",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"nickname": {
|
||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="update_profile">'
|
||||
'<parameter name="nickname">Alice</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["nickname"] == "Alice"
|
||||
|
||||
def test_anyof_nullable_param_null_value(self):
|
||||
"""An actual null-like value should be returned as None/null."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="update_profile",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"nickname": {
|
||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="update_profile">'
|
||||
'<parameter name="nickname">null</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["nickname"] is None
|
||||
|
||||
def test_anyof_nullable_param_object_value(self):
|
||||
"""A valid object value in anyOf with null should parse as dict."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="update_settings",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"config": {
|
||||
"anyOf": [{"type": "object"}, {"type": "null"}],
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="update_settings">'
|
||||
'<parameter name="config">{"theme": "dark", "fontSize": 14}'
|
||||
"</parameter>"
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["config"] == {"theme": "dark", "fontSize": 14}
|
||||
assert isinstance(parsed["config"], dict)
|
||||
|
||||
|
||||
class TestNoneStringPreservation:
|
||||
"""Regression tests for #39567: 'none' as a string must not become None."""
|
||||
|
||||
def test_none_string_preserved_in_enum(self):
|
||||
"""'none' in an enum must stay as the string 'none', not Python None."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="set_theme",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"theme": {
|
||||
"type": "string",
|
||||
"enum": ["dark", "light", "none"],
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="set_theme">'
|
||||
'<parameter name="theme">none</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["theme"] == "none"
|
||||
assert parsed["theme"] is not None
|
||||
|
||||
def test_none_string_preserved_plain_string(self):
|
||||
"""'none' as a plain string param must stay as 'none'."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="echo",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="echo">'
|
||||
'<parameter name="message">none</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["message"] == "none"
|
||||
|
||||
def test_null_still_converts_to_none(self):
|
||||
"""'null' in a nullable param must still become Python None."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="update_profile",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"nickname": {
|
||||
"anyOf": [{"type": "string"}, {"type": "null"}],
|
||||
},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="update_profile">'
|
||||
'<parameter name="nickname">null</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["nickname"] is None
|
||||
|
||||
def test_nil_string_preserved(self):
|
||||
"""'nil' must stay as the string 'nil', not become None."""
|
||||
tools = [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="echo",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"value": {"type": "string"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
parser = MinimaxM2ToolParser(FakeTokenizer(), tools=tools)
|
||||
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="echo">'
|
||||
'<parameter name="value">nil</parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
parsed = json.loads(tc[0]["arguments"])
|
||||
assert parsed["value"] == "nil"
|
||||
|
||||
|
||||
class TestParameterWhitespace:
|
||||
"""Parameter values must preserve surrounding whitespace."""
|
||||
|
||||
def test_whitespace_preserved(self, parser):
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="echo">'
|
||||
'<parameter name="msg"> hi </parameter>'
|
||||
"</invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert json.loads(tc[0]["arguments"]) == {"msg": " hi "}
|
||||
|
||||
def test_whitespace_preserved_across_chunks(self, parser):
|
||||
"""Value split before </parameter> arrives, i.e. the partial path."""
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
'<minimax:tool_call><invoke name="write">'
|
||||
'<parameter name="content"> def foo():\n',
|
||||
" return 1\n",
|
||||
"</parameter></invoke></minimax:tool_call>",
|
||||
],
|
||||
)
|
||||
tc = _collect_tool_calls(results)
|
||||
assert len(tc) == 1
|
||||
assert json.loads(tc[0]["arguments"]) == {
|
||||
"content": " def foo():\n return 1\n"
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionToolsParam,
|
||||
FunctionDefinition,
|
||||
)
|
||||
from vllm.tool_parsers import ToolParserManager
|
||||
from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
NS = "]<]minimax[>["
|
||||
EOS_ID = 99
|
||||
|
||||
|
||||
class FakeTokenizer:
|
||||
"""Minimal fake tokenizer for unit tests."""
|
||||
|
||||
def __init__(self):
|
||||
self.model_tokenizer = True
|
||||
self.vocab: dict[str, int] = {}
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return self.vocab
|
||||
|
||||
|
||||
def sample_tools() -> list[ChatCompletionToolsParam]:
|
||||
return [
|
||||
ChatCompletionToolsParam(
|
||||
function=FunctionDefinition(
|
||||
name="create_order",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"user_id": {"type": "integer"},
|
||||
"urgent": {"type": "boolean"},
|
||||
"note": {"type": "string"},
|
||||
"shipping": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {"type": "string"},
|
||||
"zip": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
"items": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"sku": {"type": "string"},
|
||||
"qty": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
"metadata": {
|
||||
"type": "object",
|
||||
"additionalProperties": {"type": "string"},
|
||||
},
|
||||
"duplicate_demo": {"type": "object"},
|
||||
},
|
||||
},
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser() -> MinimaxM3ToolParser:
|
||||
return MinimaxM3ToolParser(FakeTokenizer(), tools=sample_tools())
|
||||
|
||||
|
||||
def build_order_call() -> str:
|
||||
return (
|
||||
f"{NS}<tool_call>\n"
|
||||
f'{NS}<invoke name="create_order">'
|
||||
f"{NS}<user_id>42{NS}</user_id>"
|
||||
f"{NS}<urgent>true{NS}</urgent>"
|
||||
f"{NS}<note>Please leave at front desk.{NS}</note>"
|
||||
f"{NS}<shipping>"
|
||||
f"{NS}<city>Singapore{NS}</city>"
|
||||
f"{NS}<zip>018956{NS}</zip>"
|
||||
f"{NS}</shipping>"
|
||||
f"{NS}<items>"
|
||||
f"{NS}<item>{NS}<sku>book-001{NS}</sku>{NS}<qty>2{NS}</qty>{NS}</item>"
|
||||
f"{NS}<item>{NS}<sku>pen-007{NS}</sku>{NS}<qty>5{NS}</qty>{NS}</item>"
|
||||
f"{NS}</items>"
|
||||
f"{NS}<metadata>"
|
||||
f"{NS}<source>mobile{NS}</source>"
|
||||
f"{NS}<campaign>may-launch{NS}</campaign>"
|
||||
f"{NS}</metadata>"
|
||||
f"{NS}<duplicate_demo>"
|
||||
f"{NS}<tag>a{NS}</tag>"
|
||||
f"{NS}<tag>b{NS}</tag>"
|
||||
f"{NS}</duplicate_demo>"
|
||||
f"{NS}</invoke>\n"
|
||||
f"{NS}</tool_call>"
|
||||
)
|
||||
|
||||
|
||||
ORDER_ARGUMENTS = {
|
||||
"user_id": 42,
|
||||
"urgent": True,
|
||||
"note": "Please leave at front desk.",
|
||||
"shipping": {"city": "Singapore", "zip": 18956},
|
||||
"items": [
|
||||
{"sku": "book-001", "qty": 2},
|
||||
{"sku": "pen-007", "qty": 5},
|
||||
],
|
||||
"metadata": {
|
||||
"source": "mobile",
|
||||
"campaign": "may-launch",
|
||||
},
|
||||
"duplicate_demo": {"tag": ["a", "b"]},
|
||||
}
|
||||
|
||||
|
||||
def build_order_invocation(user_id: int) -> str:
|
||||
return (
|
||||
f'{NS}<invoke name="create_order">'
|
||||
f"{NS}<user_id>{user_id}{NS}</user_id>"
|
||||
f"{NS}</invoke>"
|
||||
)
|
||||
|
||||
|
||||
def build_multiple_order_call() -> str:
|
||||
return (
|
||||
f"{NS}<tool_call>\n"
|
||||
f"{build_order_invocation(1)}\n"
|
||||
f"{build_order_invocation(2)}\n"
|
||||
f"{NS}</tool_call>"
|
||||
)
|
||||
|
||||
|
||||
def _feed(
|
||||
parser: MinimaxM3ToolParser, chunks: list[str | tuple[str, list[int]]]
|
||||
) -> list[DeltaMessage]:
|
||||
previous = ""
|
||||
results: list[DeltaMessage] = []
|
||||
for chunk in chunks:
|
||||
if isinstance(chunk, tuple):
|
||||
delta, delta_ids = chunk
|
||||
else:
|
||||
delta = chunk
|
||||
delta_ids = []
|
||||
|
||||
current = previous + delta
|
||||
result = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous,
|
||||
current_text=current,
|
||||
delta_text=delta,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=delta_ids,
|
||||
request=None,
|
||||
)
|
||||
if result is not None:
|
||||
results.append(result)
|
||||
previous = current
|
||||
return results
|
||||
|
||||
|
||||
def _collect_content(results: list[DeltaMessage]) -> str:
|
||||
return "".join(result.content for result in results if result.content)
|
||||
|
||||
|
||||
def _collect_tool_calls(results: list[DeltaMessage]) -> dict[int, dict[str, Any]]:
|
||||
tool_calls: dict[int, dict[str, Any]] = {}
|
||||
for result in results:
|
||||
for tool_call in result.tool_calls or []:
|
||||
tool_calls.setdefault(
|
||||
tool_call.index,
|
||||
{"id": None, "name": "", "arguments": ""},
|
||||
)
|
||||
if tool_call.id:
|
||||
tool_calls[tool_call.index]["id"] = tool_call.id
|
||||
if tool_call.function:
|
||||
if tool_call.function.name:
|
||||
tool_calls[tool_call.index]["name"] += tool_call.function.name
|
||||
if tool_call.function.arguments:
|
||||
tool_calls[tool_call.index]["arguments"] += (
|
||||
tool_call.function.arguments
|
||||
)
|
||||
return tool_calls
|
||||
|
||||
|
||||
def test_minimax_m3_parser_registered():
|
||||
assert ToolParserManager.get_tool_parser("minimax_m3") is MinimaxM3ToolParser
|
||||
|
||||
|
||||
def test_required_tool_choice_keeps_native_syntax(parser):
|
||||
request = ChatCompletionRequest(
|
||||
model="minimax-m3",
|
||||
messages=[{"role": "user", "content": "order"}],
|
||||
tools=sample_tools(),
|
||||
tool_choice="required",
|
||||
)
|
||||
|
||||
adjusted = parser.adjust_request(request)
|
||||
|
||||
assert adjusted.structured_outputs is None
|
||||
assert adjusted.skip_special_tokens
|
||||
|
||||
|
||||
def test_non_streaming_nested_tool_call(parser):
|
||||
result = parser.extract_tool_calls(
|
||||
"I will create it.\n" + build_order_call(),
|
||||
request=None,
|
||||
)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content == "I will create it.\n"
|
||||
assert len(result.tool_calls) == 1
|
||||
tool_call = result.tool_calls[0]
|
||||
assert tool_call.function.name == "create_order"
|
||||
assert json.loads(tool_call.function.arguments) == ORDER_ARGUMENTS
|
||||
|
||||
|
||||
def test_non_streaming_without_tool_call_keeps_content(parser):
|
||||
result = parser.extract_tool_calls("plain response", request=None)
|
||||
|
||||
assert not result.tools_called
|
||||
assert result.tool_calls == []
|
||||
assert result.content == "plain response"
|
||||
|
||||
|
||||
def test_non_streaming_multiple_tool_calls(parser):
|
||||
result = parser.extract_tool_calls(build_multiple_order_call(), request=None)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.content is None
|
||||
assert [tool_call.function.name for tool_call in result.tool_calls] == [
|
||||
"create_order",
|
||||
"create_order",
|
||||
]
|
||||
assert [
|
||||
json.loads(tool_call.function.arguments)["user_id"]
|
||||
for tool_call in result.tool_calls
|
||||
] == [1, 2]
|
||||
|
||||
|
||||
def test_streaming_without_tool_call_emits_text(parser):
|
||||
results = _feed(parser, ["plain ", "response"])
|
||||
|
||||
assert _collect_content(results) == "plain response"
|
||||
assert _collect_tool_calls(results) == {}
|
||||
|
||||
|
||||
def test_streaming_nested_tool_call(parser):
|
||||
tool_call_text = build_order_call()
|
||||
results = _feed(
|
||||
parser,
|
||||
[
|
||||
"I will create it.\n",
|
||||
tool_call_text[:5],
|
||||
tool_call_text[5:17],
|
||||
tool_call_text[17:120],
|
||||
tool_call_text[120:],
|
||||
("", [EOS_ID]),
|
||||
],
|
||||
)
|
||||
|
||||
assert _collect_content(results) == "I will create it.\n"
|
||||
tool_calls = _collect_tool_calls(results)
|
||||
assert len(tool_calls) == 1
|
||||
assert tool_calls[0]["name"] == "create_order"
|
||||
assert tool_calls[0]["id"] is not None
|
||||
assert json.loads(tool_calls[0]["arguments"]) == ORDER_ARGUMENTS
|
||||
assert results[-1].content is None
|
||||
@@ -1,251 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
|
||||
# https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1
|
||||
SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"
|
||||
SIMPLE_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "San Francisco", "metric": "celsius"}',
|
||||
)
|
||||
MORE_TYPES_FUNCTION_OUTPUT = (
|
||||
"register_user(name='John Doe', "
|
||||
"age=37, "
|
||||
"address={'city': 'San Francisco', 'state': 'CA'}, "
|
||||
"role=None, "
|
||||
"passed_test=True, "
|
||||
"aliases=['John', 'Johnny'])"
|
||||
)
|
||||
MORE_TYPES_FUNCTION_OUTPUT_JSON_LITERALS = (
|
||||
"register_user(name='John Doe', "
|
||||
"age=37, "
|
||||
"address={'city': 'San Francisco', 'state': 'CA'}, "
|
||||
"role=null, "
|
||||
"passed_test=true, "
|
||||
"aliases=['John', 'Johnny'])"
|
||||
)
|
||||
MORE_TYPES_FUNCTION_CALL = FunctionCall(
|
||||
name="register_user",
|
||||
arguments='{"name": "John Doe", '
|
||||
'"age": 37, '
|
||||
'"address": {"city": "San Francisco", "state": "CA"}, '
|
||||
'"role": null, '
|
||||
'"passed_test": true, '
|
||||
'"aliases": ["John", "Johnny"]}',
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
|
||||
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments="{}",
|
||||
)
|
||||
EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
|
||||
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"additional_data": {}}',
|
||||
)
|
||||
EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
|
||||
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"steps": []}',
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT = (
|
||||
r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
|
||||
default_tokenizer
|
||||
)
|
||||
model_output = "How can I help you today?"
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert content == model_output
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
TEST_CASES = [
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}</function_calls>",
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
id="simple_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}</function_calls>",
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
id="simple_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT_JSON_LITERALS}</function_calls>",
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_streaming_json_literals",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{MORE_TYPES_FUNCTION_OUTPUT_JSON_LITERALS}</function_calls>",
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_nonstreaming_json_literals",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{PARAMETERLESS_FUNCTION_OUTPUT}</function_calls>",
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
id="parameterless_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{PARAMETERLESS_FUNCTION_OUTPUT}</function_calls>",
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
id="parameterless_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{EMPTY_DICT_FUNCTION_OUTPUT}</function_calls>",
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
id="empty_dict_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{EMPTY_DICT_FUNCTION_OUTPUT}</function_calls>",
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
id="empty_dict_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{EMPTY_LIST_FUNCTION_OUTPUT}</function_calls>",
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
id="empty_list_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{EMPTY_LIST_FUNCTION_OUTPUT}</function_calls>",
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
id="empty_list_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{ESCAPED_STRING_FUNCTION_OUTPUT}</function_calls>",
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="escaped_string_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{ESCAPED_STRING_FUNCTION_OUTPUT}</function_calls>",
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="escaped_string_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}\n{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
|
||||
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
|
||||
id="parallel_calls_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"<function_calls>{SIMPLE_FUNCTION_OUTPUT}\n{MORE_TYPES_FUNCTION_OUTPUT}</function_calls>",
|
||||
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
|
||||
id="parallel_calls_nonstreaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
|
||||
def test_tool_call(
|
||||
streaming: bool,
|
||||
model_output: str,
|
||||
expected_tool_calls: list[FunctionCall],
|
||||
default_tokenizer: TokenizerLike,
|
||||
):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
|
||||
default_tokenizer
|
||||
)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert len(tool_calls) == len(expected_tool_calls)
|
||||
for actual, expected in zip(tool_calls, expected_tool_calls):
|
||||
assert actual.type == "function"
|
||||
assert actual.function == expected
|
||||
|
||||
|
||||
def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
|
||||
default_tokenizer
|
||||
)
|
||||
model_output_deltas = [
|
||||
"<function_calls>get_weather(city='San",
|
||||
" Francisco', metric='celsius')\n"
|
||||
f"{PARAMETERLESS_FUNCTION_OUTPUT}\n"
|
||||
f"{EMPTY_LIST_FUNCTION_OUTPUT}</function_calls>",
|
||||
]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert reconstructor.other_content == ""
|
||||
assert len(reconstructor.tool_calls) == 3
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False])
|
||||
def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):
|
||||
"""Test regex timeout is handled gracefully."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
|
||||
default_tokenizer
|
||||
)
|
||||
|
||||
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
|
||||
|
||||
# create a mock regex that raises TimeoutError
|
||||
mock_regex = MagicMock()
|
||||
mock_regex.match.side_effect = TimeoutError("Regex timeout")
|
||||
|
||||
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, fake_problematic_input, streaming=streaming
|
||||
)
|
||||
|
||||
# should treat as regular text when regex times out
|
||||
assert content == fake_problematic_input
|
||||
assert len(tool_calls) == 0
|
||||
mock_regex.match.assert_called_once()
|
||||
@@ -1,109 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
|
||||
class TestPhi4MiniToolParser(ToolParserTests):
|
||||
@pytest.fixture
|
||||
def tokenizer(self, default_tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""Add some phi4mini specific tokens to the default vocab."""
|
||||
tokenizer = default_tokenizer
|
||||
tokenizer_vocab = tokenizer.get_vocab()
|
||||
tokenizer.get_vocab = MagicMock()
|
||||
tokenizer_vocab.update(
|
||||
{
|
||||
"functools": 32000,
|
||||
}
|
||||
)
|
||||
tokenizer.get_vocab.return_value = tokenizer_vocab
|
||||
return tokenizer
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="phi4_mini_json",
|
||||
# Test data
|
||||
no_tool_calls_output="This is a regular response without any tool calls.",
|
||||
single_tool_call_output=(
|
||||
'functools[{"name": "get_weather", "arguments": {"city": "Tokyo"}}]'
|
||||
),
|
||||
parallel_tool_calls_output="""functools[
|
||||
{"name": "get_weather", "arguments": {"city": "Tokyo"}},
|
||||
{"name": "get_time", "arguments": {"timezone": "Asia/Tokyo"}}
|
||||
]""",
|
||||
various_data_types_output="""functools[{
|
||||
"name": "test_function",
|
||||
"arguments": {
|
||||
"string_field": "hello",
|
||||
"int_field": 42,
|
||||
"float_field": 3.14,
|
||||
"bool_field": true,
|
||||
"null_field": null,
|
||||
"array_field": ["a", "b", "c"],
|
||||
"object_field": {"nested": "value"},
|
||||
"empty_array": [],
|
||||
"empty_object": {}
|
||||
}
|
||||
}]""",
|
||||
empty_arguments_output='functools[{"name": "refresh", "arguments": {}}]',
|
||||
surrounding_text_output="""Let me check the weather for you.
|
||||
functools[{"name": "get_weather", "arguments": {"city": "Tokyo"}}]
|
||||
Would you like to know more?""",
|
||||
escaped_strings_output="""functools[{
|
||||
"name": "test_function",
|
||||
"arguments": {
|
||||
"quoted": "He said \\"hello\\"",
|
||||
"path": "C:\\\\Users\\\\file.txt",
|
||||
"newline": "line1\\nline2",
|
||||
"unicode": "emoji: 🎉"
|
||||
}
|
||||
}]""",
|
||||
malformed_input_outputs=[
|
||||
'functools[{"name": "func", "arguments": {',
|
||||
'functools[{"name": "func", "arguments": "not a dict"}]',
|
||||
'functools{"name": "func"}', # Missing brackets
|
||||
'functools[{"name": "func"}]', # Missing arguments/parameters
|
||||
"functools[] This is just text", # Empty functools
|
||||
"functools[ This is just text ]", # functools with invalid JSON
|
||||
],
|
||||
# Expected results
|
||||
single_tool_call_expected_name="get_weather",
|
||||
single_tool_call_expected_args={"city": "Tokyo"},
|
||||
# Phi-4 Mini strips content when tool calls present
|
||||
single_tool_call_expected_content=None,
|
||||
parallel_tool_calls_count=2,
|
||||
parallel_tool_calls_names=["get_weather", "get_time"],
|
||||
parallel_tool_calls_expected_content=None,
|
||||
# xfail markers
|
||||
xfail_streaming={
|
||||
"test_no_tool_calls": "Phi4 Mini streaming not implemented",
|
||||
"test_single_tool_call_simple_args": (
|
||||
"Phi4 Mini streaming not implemented"
|
||||
),
|
||||
"test_parallel_tool_calls": "Phi4 Mini streaming not implemented",
|
||||
"test_various_data_types": "Phi4 Mini streaming not implemented",
|
||||
"test_empty_arguments": "Phi4 Mini streaming not implemented",
|
||||
"test_surrounding_text": "Phi4 Mini streaming not implemented",
|
||||
"test_escaped_strings": "Phi4 Mini streaming not implemented",
|
||||
"test_streaming_reconstruction": "Phi4 Mini streaming not implemented",
|
||||
},
|
||||
xfail_nonstreaming={
|
||||
"test_various_data_types": (
|
||||
"Phi4MiniJsonToolParser regex has nesting limitations "
|
||||
"with nested objects"
|
||||
),
|
||||
"test_malformed_input": (
|
||||
"Phi4MiniJsonToolParser incorrectly sets "
|
||||
"tools_called=True on empty array"
|
||||
),
|
||||
},
|
||||
)
|
||||
@@ -1,366 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression tests for ``PoolsideV1ToolParser``.
|
||||
|
||||
Covers two bugs:
|
||||
|
||||
1. ``adjust_request`` did not skip the forced ``structured_outputs`` JSON
|
||||
for ``required``/named tool choice. These models emit XML tool calls
|
||||
(``<tool_call>...<arg_value>...</arg_value></tool_call>``) per the chat
|
||||
template, so guided JSON decoding conflicts with the format: the call
|
||||
leaks as content with empty ``tool_calls``. ``adjust_request`` now skips
|
||||
the constraint for both ChatCompletion (``ChatCompletionNamedToolChoice``)
|
||||
and Responses (``ToolChoiceFunction``) named choices.
|
||||
|
||||
2. ``extract_tool_calls`` stripped string-typed argument values, corrupting
|
||||
content whose whitespace is significant (e.g. code/file bodies losing
|
||||
leading indent and trailing newline). String values are now kept verbatim;
|
||||
only non-string types are stripped/deserialized.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses.tool_param import FunctionToolParam
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.tool_parsers.poolside_v1_tool_parser import PoolsideV1ToolParser
|
||||
|
||||
|
||||
def _write_file_tool() -> dict[str, Any]:
|
||||
"""Tool with a string arg (``content``) and a non-string arg (``mode``)."""
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "write_file",
|
||||
"description": "Write content to a file",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {"type": "string"},
|
||||
"mode": {"type": "integer"},
|
||||
},
|
||||
"required": ["content"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _responses_write_file_tool() -> FunctionToolParam:
|
||||
return FunctionToolParam(
|
||||
type="function",
|
||||
name="write_file",
|
||||
description="Write content to a file",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"content": {"type": "string"},
|
||||
"mode": {"type": "integer"},
|
||||
},
|
||||
"required": ["content"],
|
||||
},
|
||||
strict=True,
|
||||
)
|
||||
|
||||
|
||||
def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"model": "poolside-test",
|
||||
"messages": [{"role": "user", "content": "write the file"}],
|
||||
"tools": [_write_file_tool()],
|
||||
"tool_choice": tool_choice,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_responses_request(
|
||||
*,
|
||||
tool_choice: str | dict[str, Any],
|
||||
include: list[str] | None = None,
|
||||
tools: list[dict[str, Any]] | None = None,
|
||||
) -> ResponsesRequest:
|
||||
return ResponsesRequest(
|
||||
model="poolside-test",
|
||||
input=[{"role": "user", "content": "write the file"}],
|
||||
tools=tools if tools is not None else [_responses_write_file_tool()],
|
||||
tool_choice=tool_choice,
|
||||
stream=True,
|
||||
max_output_tokens=200,
|
||||
include=include,
|
||||
)
|
||||
|
||||
|
||||
class _StubTokenizer:
|
||||
"""Minimal tokenizer stub to satisfy ``PoolsideV1ToolParser.__init__``."""
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {"<tool_call>": 151_657, "</tool_call>": 151_658}
|
||||
|
||||
|
||||
def _make_parser(request: ChatCompletionRequest) -> PoolsideV1ToolParser:
|
||||
return PoolsideV1ToolParser(_StubTokenizer(), tools=request.tools)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bug 1: required/named must skip forced structured_outputs (#39870 pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_required_skips_structured_outputs_chatcompletion() -> None:
|
||||
request = _build_chat_request(tool_choice="required")
|
||||
_make_parser(request).adjust_request(request)
|
||||
|
||||
assert request.structured_outputs is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_named_skips_structured_outputs_chatcompletion() -> None:
|
||||
request = _build_chat_request(
|
||||
tool_choice={"type": "function", "function": {"name": "write_file"}}
|
||||
)
|
||||
_make_parser(request).adjust_request(request)
|
||||
|
||||
assert request.structured_outputs is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_required_skips_structured_outputs_responses() -> None:
|
||||
request = _build_responses_request(tool_choice="required")
|
||||
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
|
||||
|
||||
assert request.text is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_named_skips_structured_outputs_responses() -> None:
|
||||
# Responses-API named choice parses to ToolChoiceFunction, a different
|
||||
# type than the ChatCompletion named choice; both must be handled.
|
||||
request = _build_responses_request(
|
||||
tool_choice={"type": "function", "name": "write_file"}
|
||||
)
|
||||
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
|
||||
|
||||
assert request.text is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_named_custom_tool_choice_skips_structured_outputs_responses() -> None:
|
||||
# A named custom choice parses to ToolChoiceCustom; custom tools ride the
|
||||
# function shim, so it must route like ToolChoiceFunction.
|
||||
request = _build_responses_request(
|
||||
tools=[{"type": "custom", "name": "emit_command"}],
|
||||
tool_choice={"type": "custom", "name": "emit_command"},
|
||||
)
|
||||
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
|
||||
|
||||
assert request.text is None
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_auto_still_keeps_special_tokens() -> None:
|
||||
request = _build_chat_request(tool_choice="auto")
|
||||
_make_parser(request).adjust_request(request)
|
||||
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bug 2: string arg whitespace must be preserved (#42026 pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_string_arg_preserves_whitespace() -> None:
|
||||
request = _build_chat_request(tool_choice="auto")
|
||||
parser = _make_parser(request)
|
||||
|
||||
content = " def f():\n return 1\n"
|
||||
model_output = (
|
||||
"<tool_call>write_file\n"
|
||||
"<arg_key>content</arg_key>\n"
|
||||
f"<arg_value>{content}</arg_value>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
|
||||
assert result.tools_called
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
# Leading indent and trailing newline must survive verbatim.
|
||||
assert args["content"] == content
|
||||
|
||||
|
||||
def test_non_string_arg_still_deserialized() -> None:
|
||||
request = _build_chat_request(tool_choice="auto")
|
||||
parser = _make_parser(request)
|
||||
|
||||
model_output = (
|
||||
"<tool_call>write_file\n"
|
||||
"<arg_key>content</arg_key>\n"
|
||||
"<arg_value>hi</arg_value>\n"
|
||||
"<arg_key>mode</arg_key>\n"
|
||||
"<arg_value> 420 </arg_value>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
|
||||
assert result.tools_called
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["content"] == "hi"
|
||||
# Non-string value is stripped and parsed to its native type.
|
||||
assert args["mode"] == 420
|
||||
|
||||
|
||||
def test_responses_extract_tool_calls_with_flat_tools() -> None:
|
||||
# required/named Responses calls route into extract_tool_calls with flat
|
||||
# FunctionTool (.name); _is_string_type must not raise.
|
||||
request = _build_responses_request(tool_choice="required")
|
||||
parser = PoolsideV1ToolParser(_StubTokenizer(), tools=request.tools)
|
||||
|
||||
content = " x = 1\n"
|
||||
model_output = (
|
||||
"<tool_call>write_file\n"
|
||||
"<arg_key>content</arg_key>\n"
|
||||
f"<arg_value>{content}</arg_value>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
|
||||
assert result.tools_called
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args["content"] == content
|
||||
|
||||
|
||||
def _weather_tool() -> dict[str, Any]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather for a city",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_weather_request(*, tool_choice: str) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest.model_validate(
|
||||
{
|
||||
"model": "poolside-test",
|
||||
"messages": [{"role": "user", "content": "weather in Paris?"}],
|
||||
"tools": [_weather_tool()],
|
||||
"tool_choice": tool_choice,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_no_newline_after_name_non_streaming() -> None:
|
||||
request = _build_weather_request(tool_choice="auto")
|
||||
parser = _make_parser(request)
|
||||
|
||||
model_output = (
|
||||
"<tool_call>get_weather"
|
||||
"<arg_key>city</arg_key><arg_value>Paris</arg_value>"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
|
||||
assert result.tools_called
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"city": "Paris"}
|
||||
|
||||
|
||||
def test_newline_after_name_still_parses_non_streaming() -> None:
|
||||
request = _build_weather_request(tool_choice="auto")
|
||||
parser = _make_parser(request)
|
||||
|
||||
model_output = (
|
||||
"<tool_call>get_weather\n"
|
||||
"<arg_key>city</arg_key>\n<arg_value>Paris</arg_value>\n"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
result = parser.extract_tool_calls(model_output, request)
|
||||
|
||||
assert result.tools_called
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"city": "Paris"}
|
||||
|
||||
|
||||
def test_no_newline_after_name_streaming() -> None:
|
||||
request = _build_weather_request(tool_choice="auto")
|
||||
parser = _make_parser(request)
|
||||
|
||||
model_output = (
|
||||
"<tool_call>get_weather"
|
||||
"<arg_key>city</arg_key><arg_value>Paris</arg_value>"
|
||||
"</tool_call>"
|
||||
)
|
||||
|
||||
name = ""
|
||||
args = ""
|
||||
prev = ""
|
||||
for ch in model_output:
|
||||
cur = prev + ch
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=prev,
|
||||
current_text=cur,
|
||||
delta_text=ch,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request,
|
||||
)
|
||||
prev = cur
|
||||
if delta is None:
|
||||
continue
|
||||
for tc in delta.tool_calls or []:
|
||||
if tc.function is None:
|
||||
continue
|
||||
if tc.function.name:
|
||||
name += tc.function.name
|
||||
if tc.function.arguments:
|
||||
args += tc.function.arguments
|
||||
|
||||
assert name == "get_weather"
|
||||
assert json.loads(args) == {"city": "Paris"}
|
||||
|
||||
|
||||
def _stream_partial_start_token(request: ResponsesRequest):
|
||||
parser = _make_parser(request)
|
||||
delta = parser.tool_call_start_token[0]
|
||||
return parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=delta,
|
||||
delta_text=delta,
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=request,
|
||||
)
|
||||
|
||||
|
||||
def test_streaming_responses_request_without_logprobs() -> None:
|
||||
request = _build_responses_request(tool_choice="auto")
|
||||
assert _stream_partial_start_token(request) is None
|
||||
|
||||
|
||||
def test_streaming_responses_request_with_logprobs_emits_empty_delta() -> None:
|
||||
request = _build_responses_request(
|
||||
tool_choice="auto", include=["message.output_text.logprobs"]
|
||||
)
|
||||
result = _stream_partial_start_token(request)
|
||||
assert result is not None
|
||||
assert result.content == ""
|
||||
@@ -1,231 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.utils import (
|
||||
run_tool_extraction,
|
||||
run_tool_extraction_streaming,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser, ToolParserManager
|
||||
|
||||
# https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1
|
||||
SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')"
|
||||
SIMPLE_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "San Francisco", "metric": "celsius"}',
|
||||
)
|
||||
MORE_TYPES_FUNCTION_OUTPUT = (
|
||||
"register_user(name='John Doe', "
|
||||
"age=37, "
|
||||
"address={'city': 'San Francisco', 'state': 'CA'}, "
|
||||
"role=None, "
|
||||
"passed_test=True, "
|
||||
"aliases=['John', 'Johnny'])"
|
||||
)
|
||||
MORE_TYPES_FUNCTION_CALL = FunctionCall(
|
||||
name="register_user",
|
||||
arguments='{"name": "John Doe", '
|
||||
'"age": 37, '
|
||||
'"address": {"city": "San Francisco", "state": "CA"}, '
|
||||
'"role": null, '
|
||||
'"passed_test": true, '
|
||||
'"aliases": ["John", "Johnny"]}',
|
||||
)
|
||||
PARAMETERLESS_FUNCTION_OUTPUT = "get_weather()"
|
||||
PARAMETERLESS_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments="{}",
|
||||
)
|
||||
EMPTY_DICT_FUNCTION_OUTPUT = "do_something_cool(additional_data={})"
|
||||
EMPTY_DICT_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"additional_data": {}}',
|
||||
)
|
||||
EMPTY_LIST_FUNCTION_OUTPUT = "do_something_cool(steps=[])"
|
||||
EMPTY_LIST_FUNCTION_CALL = FunctionCall(
|
||||
name="do_something_cool",
|
||||
arguments='{"steps": []}',
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_OUTPUT = (
|
||||
r"get_weather(city='Martha\'s Vineyard', metric='\"cool units\"')"
|
||||
)
|
||||
ESCAPED_STRING_FUNCTION_CALL = FunctionCall(
|
||||
name="get_weather",
|
||||
arguments='{"city": "Martha\'s Vineyard", "metric": "\\"cool units\\""}',
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [True, False])
|
||||
def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
model_output = "How can I help you today?"
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert content == model_output
|
||||
assert len(tool_calls) == 0
|
||||
|
||||
|
||||
TEST_CASES = [
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{SIMPLE_FUNCTION_OUTPUT}]",
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
id="simple_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{SIMPLE_FUNCTION_OUTPUT}]",
|
||||
[SIMPLE_FUNCTION_CALL],
|
||||
id="simple_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{MORE_TYPES_FUNCTION_OUTPUT}]",
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{MORE_TYPES_FUNCTION_OUTPUT}]",
|
||||
[MORE_TYPES_FUNCTION_CALL],
|
||||
id="more_types_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{PARAMETERLESS_FUNCTION_OUTPUT}]",
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
id="parameterless_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{PARAMETERLESS_FUNCTION_OUTPUT}]",
|
||||
[PARAMETERLESS_FUNCTION_CALL],
|
||||
id="parameterless_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{EMPTY_DICT_FUNCTION_OUTPUT}]",
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
id="empty_dict_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{EMPTY_DICT_FUNCTION_OUTPUT}]",
|
||||
[EMPTY_DICT_FUNCTION_CALL],
|
||||
id="empty_dict_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{EMPTY_LIST_FUNCTION_OUTPUT}]",
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
id="empty_list_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{EMPTY_LIST_FUNCTION_OUTPUT}]",
|
||||
[EMPTY_LIST_FUNCTION_CALL],
|
||||
id="empty_list_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{ESCAPED_STRING_FUNCTION_OUTPUT}]",
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="escaped_string_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{ESCAPED_STRING_FUNCTION_OUTPUT}]",
|
||||
[ESCAPED_STRING_FUNCTION_CALL],
|
||||
id="escaped_string_nonstreaming",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
f"[{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}]",
|
||||
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
|
||||
id="parallel_calls_streaming",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
f"[{SIMPLE_FUNCTION_OUTPUT}, {MORE_TYPES_FUNCTION_OUTPUT}]",
|
||||
[SIMPLE_FUNCTION_CALL, MORE_TYPES_FUNCTION_CALL],
|
||||
id="parallel_calls_nonstreaming",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming, model_output, expected_tool_calls", TEST_CASES)
|
||||
def test_tool_call(
|
||||
streaming: bool,
|
||||
model_output: str,
|
||||
expected_tool_calls: list[FunctionCall],
|
||||
default_tokenizer: TokenizerLike,
|
||||
):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, model_output, streaming=streaming
|
||||
)
|
||||
|
||||
assert content is None
|
||||
assert len(tool_calls) == len(expected_tool_calls)
|
||||
for actual, expected in zip(tool_calls, expected_tool_calls):
|
||||
assert actual.type == "function"
|
||||
assert actual.function == expected
|
||||
|
||||
|
||||
def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
model_output_deltas = [
|
||||
"[get_weather(city='San",
|
||||
" Francisco', metric='celsius'), "
|
||||
f"{PARAMETERLESS_FUNCTION_OUTPUT}, "
|
||||
f"{EMPTY_LIST_FUNCTION_OUTPUT}]",
|
||||
]
|
||||
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser, model_output_deltas, assert_one_tool_per_delta=False
|
||||
)
|
||||
|
||||
assert reconstructor.other_content == ""
|
||||
assert len(reconstructor.tool_calls) == 3
|
||||
assert reconstructor.tool_calls[0].function == SIMPLE_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[1].function == PARAMETERLESS_FUNCTION_CALL
|
||||
assert reconstructor.tool_calls[2].function == EMPTY_LIST_FUNCTION_CALL
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False])
|
||||
def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike):
|
||||
"""Test regex timeout is handled gracefully."""
|
||||
tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
|
||||
default_tokenizer
|
||||
)
|
||||
|
||||
fake_problematic_input = "hello world[A(A=" + "\t)A(A=,\t" * 2
|
||||
|
||||
# create a mock regex that raises TimeoutError
|
||||
mock_regex = MagicMock()
|
||||
mock_regex.match.side_effect = TimeoutError("Regex timeout")
|
||||
|
||||
with patch.object(tool_parser, "TOOL_CALL_REGEX", mock_regex):
|
||||
content, tool_calls = run_tool_extraction(
|
||||
tool_parser, fake_problematic_input, streaming=streaming
|
||||
)
|
||||
|
||||
# should treat as regular text when regex times out
|
||||
assert content == fake_problematic_input
|
||||
assert len(tool_calls) == 0
|
||||
mock_regex.match.assert_called_once()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,112 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.tool_parsers.common_tests import (
|
||||
ToolParserTestConfig,
|
||||
ToolParserTests,
|
||||
)
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
|
||||
|
||||
class TestStep3ToolParser(ToolParserTests):
|
||||
@pytest.fixture(scope="class")
|
||||
def tokenizer(self) -> TokenizerLike:
|
||||
return get_tokenizer("stepfun-ai/step3")
|
||||
|
||||
@pytest.fixture
|
||||
def test_config(self) -> ToolParserTestConfig:
|
||||
return ToolParserTestConfig(
|
||||
parser_name="step3",
|
||||
# Test data
|
||||
no_tool_calls_output="This is a regular response without any tool calls.",
|
||||
single_tool_call_output=(
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="get_weather">'
|
||||
'<steptml:parameter name="city">Tokyo</steptml:parameter>'
|
||||
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
|
||||
),
|
||||
parallel_tool_calls_output=(
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="get_weather">'
|
||||
'<steptml:parameter name="city">Tokyo</steptml:parameter>'
|
||||
"</steptml:invoke><|tool_call_end|><|tool_sep|>"
|
||||
'<|tool_call_begin|><steptml:invoke name="get_time">'
|
||||
'<steptml:parameter name="timezone">Asia/Tokyo</steptml:parameter>'
|
||||
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
|
||||
),
|
||||
various_data_types_output=(
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="test_function">'
|
||||
'<steptml:parameter name="string_field">hello</steptml:parameter>'
|
||||
'<steptml:parameter name="int_field">42</steptml:parameter>'
|
||||
'<steptml:parameter name="float_field">3.14</steptml:parameter>'
|
||||
'<steptml:parameter name="bool_field">true</steptml:parameter>'
|
||||
'<steptml:parameter name="null_field">null</steptml:parameter>'
|
||||
'<steptml:parameter name="array_field">'
|
||||
'["a", "b", "c"]</steptml:parameter>'
|
||||
'<steptml:parameter name="object_field">'
|
||||
'{"nested": "value"}</steptml:parameter>'
|
||||
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
|
||||
),
|
||||
empty_arguments_output=(
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="refresh"></steptml:invoke>'
|
||||
"<|tool_call_end|><|tool_calls_end|>"
|
||||
),
|
||||
surrounding_text_output=(
|
||||
"Let me check the weather for you.\n\n"
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="get_weather">'
|
||||
'<steptml:parameter name="city">Tokyo</steptml:parameter>'
|
||||
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>\n\n"
|
||||
"I'll get that information."
|
||||
),
|
||||
escaped_strings_output=(
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="test_function">'
|
||||
'<steptml:parameter name="quoted">He said "hello"</steptml:parameter>'
|
||||
'<steptml:parameter name="path">C:\\Users\\file.txt</steptml:parameter>'
|
||||
'<steptml:parameter name="newline">line1\nline2</steptml:parameter>'
|
||||
"</steptml:invoke><|tool_call_end|><|tool_calls_end|>"
|
||||
),
|
||||
malformed_input_outputs=[
|
||||
(
|
||||
"<|tool_calls_begin|><|tool_call_begin|>"
|
||||
'<steptml:invoke name="func">'
|
||||
),
|
||||
(
|
||||
'<|tool_call_begin|><steptml:invoke name="func">'
|
||||
"</steptml:invoke><|tool_call_end|>"
|
||||
),
|
||||
],
|
||||
# Expected results
|
||||
single_tool_call_expected_name="get_weather",
|
||||
single_tool_call_expected_args={"city": "Tokyo"},
|
||||
parallel_tool_calls_count=2,
|
||||
parallel_tool_calls_names=["get_weather", "get_time"],
|
||||
# xfail markers
|
||||
xfail_nonstreaming={
|
||||
"test_single_tool_call_simple_args": (
|
||||
"Step3 parser non-streaming has bugs"
|
||||
),
|
||||
"test_parallel_tool_calls": ("Step3 parser non-streaming has bugs"),
|
||||
"test_various_data_types": "Step3 parser non-streaming has bugs",
|
||||
"test_empty_arguments": "Step3 parser non-streaming has bugs",
|
||||
"test_surrounding_text": "Step3 parser non-streaming has bugs",
|
||||
"test_escaped_strings": "Step3 parser non-streaming has bugs",
|
||||
},
|
||||
xfail_streaming={
|
||||
"test_parallel_tool_calls": (
|
||||
"Step3 parser has significant bugs in both streaming "
|
||||
"and non-streaming"
|
||||
),
|
||||
"test_streaming_reconstruction": (
|
||||
"Step3 parser non-streaming has bugs, so streaming "
|
||||
"doesn't match non-streaming"
|
||||
),
|
||||
},
|
||||
supports_typed_arguments=False,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,557 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Generator
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
|
||||
from vllm.tool_parsers.xlam_tool_parser import xLAMToolParser
|
||||
|
||||
# Use a common model that is likely to be available
|
||||
MODEL = "Salesforce/Llama-xLAM-2-8B-fc-r"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def xlam_tokenizer():
|
||||
return get_tokenizer(tokenizer_name=MODEL)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def xlam_tool_parser(xlam_tokenizer):
|
||||
return xLAMToolParser(xlam_tokenizer)
|
||||
|
||||
|
||||
def assert_tool_calls(
|
||||
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
|
||||
):
|
||||
assert len(actual_tool_calls) == len(expected_tool_calls)
|
||||
|
||||
for actual_tool_call, expected_tool_call in zip(
|
||||
actual_tool_calls, expected_tool_calls
|
||||
):
|
||||
assert isinstance(actual_tool_call.id, str)
|
||||
assert len(actual_tool_call.id) > 16
|
||||
|
||||
assert actual_tool_call.type == "function"
|
||||
assert actual_tool_call.function == expected_tool_call.function
|
||||
|
||||
|
||||
def stream_delta_message_generator(
|
||||
xlam_tool_parser: xLAMToolParser,
|
||||
xlam_tokenizer: TokenizerLike,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> Generator[DeltaMessage, None, None]:
|
||||
all_token_ids = xlam_tokenizer.encode(model_output, add_special_tokens=False)
|
||||
|
||||
previous_text = ""
|
||||
previous_tokens = None
|
||||
prefix_offset = 0
|
||||
read_offset = 0
|
||||
for i, delta_token in enumerate(all_token_ids):
|
||||
delta_token_ids = [delta_token]
|
||||
previous_token_ids = all_token_ids[:i]
|
||||
current_token_ids = all_token_ids[: i + 1]
|
||||
|
||||
(new_tokens, delta_text, new_prefix_offset, new_read_offset) = (
|
||||
detokenize_incrementally(
|
||||
tokenizer=xlam_tokenizer,
|
||||
all_input_ids=current_token_ids,
|
||||
prev_tokens=previous_tokens,
|
||||
prefix_offset=prefix_offset,
|
||||
read_offset=read_offset,
|
||||
skip_special_tokens=False,
|
||||
spaces_between_special_tokens=True,
|
||||
)
|
||||
)
|
||||
|
||||
current_text = previous_text + delta_text
|
||||
|
||||
delta_message = xlam_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta_text,
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request=request,
|
||||
)
|
||||
if delta_message:
|
||||
yield delta_message
|
||||
|
||||
previous_text = current_text
|
||||
previous_tokens = (
|
||||
previous_tokens + new_tokens if previous_tokens else new_tokens
|
||||
)
|
||||
prefix_offset = new_prefix_offset
|
||||
read_offset = new_read_offset
|
||||
|
||||
|
||||
def test_extract_tool_calls_no_tools(xlam_tool_parser):
|
||||
model_output = "This is a test"
|
||||
extracted_tool_calls = xlam_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert not extracted_tool_calls.tools_called
|
||||
assert extracted_tool_calls.tool_calls == []
|
||||
assert extracted_tool_calls.content == model_output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
"parallel_tool_calls",
|
||||
"single_tool_with_think_tag",
|
||||
"single_tool_with_json_code_block",
|
||||
"single_tool_with_tool_calls_tag",
|
||||
"single_tool_with_tool_call_xml_tags",
|
||||
],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
(
|
||||
"""[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}, {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Orlando",
|
||||
"state": "FL",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
None,
|
||||
),
|
||||
(
|
||||
"""<think>I'll help you with that.</think>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"<think>I'll help you with that.</think>",
|
||||
),
|
||||
(
|
||||
"""I'll help you with that.\n```json\n[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\n```""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"I'll help you with that.",
|
||||
),
|
||||
(
|
||||
"""I'll check the weather for you.[TOOL_CALLS][{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"I'll check the weather for you.",
|
||||
),
|
||||
(
|
||||
"""I'll help you check the weather.<tool_call>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]</tool_call>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"I'll help you check the weather.",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls(
|
||||
xlam_tool_parser, model_output, expected_tool_calls, expected_content
|
||||
):
|
||||
extracted_tool_calls = xlam_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert extracted_tool_calls.tools_called
|
||||
|
||||
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
|
||||
|
||||
assert extracted_tool_calls.content == expected_content
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=["list_structured_tool_call"],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
(
|
||||
"""[{"name": "get_current_weather", "arguments": {"city": "Seattle", "state": "WA", "unit": "celsius"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Seattle",
|
||||
"state": "WA",
|
||||
"unit": "celsius",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls_list_structure(
|
||||
xlam_tool_parser, model_output, expected_tool_calls, expected_content
|
||||
):
|
||||
"""Test extraction of tool calls when the model outputs a list-structured tool call.""" # noqa: E501
|
||||
extracted_tool_calls = xlam_tool_parser.extract_tool_calls(
|
||||
model_output, request=None
|
||||
)
|
||||
assert extracted_tool_calls.tools_called
|
||||
|
||||
assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls)
|
||||
|
||||
assert extracted_tool_calls.content == expected_content
|
||||
|
||||
|
||||
# Test for preprocess_model_output method
|
||||
def test_preprocess_model_output(xlam_tool_parser):
|
||||
# Test with list structure
|
||||
model_output = (
|
||||
"""[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]""" # noqa: E501
|
||||
)
|
||||
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
|
||||
model_output
|
||||
)
|
||||
assert content is None
|
||||
assert potential_tool_calls == model_output
|
||||
|
||||
# Test with thinking tag
|
||||
model_output = """<think>I'll help you with that.</think>[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]""" # noqa: E501
|
||||
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
|
||||
model_output
|
||||
)
|
||||
assert content == "<think>I'll help you with that.</think>"
|
||||
assert (
|
||||
potential_tool_calls
|
||||
== '[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]'
|
||||
)
|
||||
|
||||
# Test with JSON code block
|
||||
model_output = """I'll help you with that.
|
||||
```json
|
||||
[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]
|
||||
```"""
|
||||
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
|
||||
model_output
|
||||
)
|
||||
assert content == "I'll help you with that."
|
||||
assert "get_current_weather" in potential_tool_calls
|
||||
|
||||
# Test with no tool calls
|
||||
model_output = """I'll help you with that."""
|
||||
content, potential_tool_calls = xlam_tool_parser.preprocess_model_output(
|
||||
model_output
|
||||
)
|
||||
assert content == model_output
|
||||
assert potential_tool_calls is None
|
||||
|
||||
|
||||
# Simulate streaming to test extract_tool_calls_streaming
|
||||
def test_streaming_with_list_structure(xlam_tool_parser):
|
||||
# Reset streaming state
|
||||
xlam_tool_parser.prev_tool_calls = []
|
||||
xlam_tool_parser.current_tools_sent = []
|
||||
xlam_tool_parser.streamed_args = []
|
||||
xlam_tool_parser.current_tool_id = -1
|
||||
|
||||
# Simulate receiving a message with list structure
|
||||
current_text = (
|
||||
"""[{"name": "get_current_weather", "arguments": {"city": "Seattle"}}]""" # noqa: E501
|
||||
)
|
||||
|
||||
# First call to set up the tool
|
||||
xlam_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=current_text,
|
||||
delta_text="]",
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=None,
|
||||
)
|
||||
|
||||
# Make sure the tool is set up correctly
|
||||
assert xlam_tool_parser.current_tool_id >= 0, "Tool index should be initialized"
|
||||
|
||||
# Manually set up the state for sending the tool name
|
||||
xlam_tool_parser.current_tools_sent = [False]
|
||||
|
||||
# Call to send the function name
|
||||
result = xlam_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text=current_text,
|
||||
current_text=current_text,
|
||||
delta_text="",
|
||||
previous_token_ids=[],
|
||||
current_token_ids=[],
|
||||
delta_token_ids=[],
|
||||
request=None,
|
||||
)
|
||||
|
||||
# Check that we get a result with the proper tool call
|
||||
if result is not None:
|
||||
assert hasattr(result, "tool_calls")
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_current_weather"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
ids=[
|
||||
"parallel_tool_calls",
|
||||
"single_tool_with_think_tag",
|
||||
"single_tool_with_json_code_block",
|
||||
"single_tool_with_tool_calls_tag",
|
||||
"single_tool_with_tool_call_xml_tags",
|
||||
],
|
||||
argnames=["model_output", "expected_tool_calls", "expected_content"],
|
||||
argvalues=[
|
||||
(
|
||||
"""[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}, {"name": "get_current_weather", "arguments": {"city": "Orlando", "state": "FL", "unit": "fahrenheit"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Orlando",
|
||||
"state": "FL",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
),
|
||||
],
|
||||
"",
|
||||
),
|
||||
(
|
||||
"""<think>I'll help you with that.</think>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"<think>I'll help you with that.</think>",
|
||||
),
|
||||
(
|
||||
"""```json\n[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\n```""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"",
|
||||
),
|
||||
(
|
||||
"""[TOOL_CALLS][{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"",
|
||||
),
|
||||
(
|
||||
"""I can help with that.<tool_call>[{"name": "get_current_weather", "arguments": {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]</tool_call>""", # noqa: E501
|
||||
[
|
||||
ToolCall(
|
||||
function=FunctionCall(
|
||||
name="get_current_weather",
|
||||
arguments=json.dumps(
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
}
|
||||
),
|
||||
)
|
||||
)
|
||||
],
|
||||
"I can help with that.",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_extract_tool_calls_streaming_incremental(
|
||||
xlam_tool_parser,
|
||||
xlam_tokenizer,
|
||||
model_output,
|
||||
expected_tool_calls,
|
||||
expected_content,
|
||||
):
|
||||
"""Verify the XLAM Parser streaming behavior by verifying each chunk is as expected.""" # noqa: E501
|
||||
request = ChatCompletionRequest(model=MODEL, messages=[])
|
||||
|
||||
chunks = []
|
||||
for delta_message in stream_delta_message_generator(
|
||||
xlam_tool_parser, xlam_tokenizer, model_output, request
|
||||
):
|
||||
chunks.append(delta_message)
|
||||
|
||||
# Should have multiple chunks
|
||||
assert len(chunks) >= 3
|
||||
|
||||
# Should have a chunk with tool header (id, name, type) for the first tool call # noqa: E501
|
||||
header_found = False
|
||||
expected_first_tool = expected_tool_calls[0]
|
||||
for chunk in chunks:
|
||||
if chunk.tool_calls and chunk.tool_calls[0].id:
|
||||
header_found = True
|
||||
assert (
|
||||
chunk.tool_calls[0].function.name == expected_first_tool.function.name
|
||||
)
|
||||
assert chunk.tool_calls[0].type == "function"
|
||||
# Arguments may be empty initially or None
|
||||
if chunk.tool_calls[0].function.arguments is not None:
|
||||
# If present, should be empty string initially
|
||||
assert chunk.tool_calls[0].function.arguments == ""
|
||||
break
|
||||
assert header_found
|
||||
|
||||
# Should have chunks with incremental arguments
|
||||
arg_chunks = []
|
||||
for chunk in chunks:
|
||||
if (
|
||||
chunk.tool_calls
|
||||
and chunk.tool_calls[0].function.arguments
|
||||
and chunk.tool_calls[0].function.arguments != ""
|
||||
and chunk.tool_calls[0].index
|
||||
== 0 # Only collect arguments from the first tool call
|
||||
):
|
||||
arg_chunks.append(chunk.tool_calls[0].function.arguments)
|
||||
|
||||
# Arguments should be streamed incrementally
|
||||
assert len(arg_chunks) > 1
|
||||
|
||||
# Concatenated arguments should form valid JSON for the first tool call
|
||||
full_args = "".join(arg_chunks)
|
||||
parsed_args = json.loads(full_args)
|
||||
expected_args = json.loads(expected_first_tool.function.arguments)
|
||||
assert parsed_args == expected_args
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
def test_extract_tool_calls_non_ascii(xlam_tool_parser, xlam_tokenizer, streaming):
|
||||
# Use parallel tool calls so the streaming path re-serializes arguments
|
||||
# (the ensure_ascii fix only runs when tool_count > 1).
|
||||
model_output = """[{"name": "get_current_weather", "arguments": {"city": "北京"}}, {"name": "get_current_weather", "arguments": {"city": "上海"}}]""" # noqa: E501
|
||||
|
||||
if streaming:
|
||||
request = ChatCompletionRequest(model=MODEL, messages=[])
|
||||
args = "".join(
|
||||
delta.tool_calls[0].function.arguments
|
||||
for delta in stream_delta_message_generator(
|
||||
xlam_tool_parser, xlam_tokenizer, model_output, request
|
||||
)
|
||||
if delta.tool_calls and delta.tool_calls[0].function.arguments
|
||||
)
|
||||
else:
|
||||
extracted = xlam_tool_parser.extract_tool_calls(model_output, request=None)
|
||||
args = "".join(tc.function.arguments for tc in extracted.tool_calls)
|
||||
|
||||
assert "北京" in args
|
||||
assert "\\u" not in args
|
||||
@@ -1,167 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers import ToolParser
|
||||
|
||||
|
||||
class StreamingToolReconstructor:
|
||||
def __init__(self, assert_one_tool_per_delta: bool = True):
|
||||
self.tool_calls: list[ToolCall] = []
|
||||
self.other_content: str = ""
|
||||
self._assert_one_tool_per_delta = assert_one_tool_per_delta
|
||||
|
||||
def append_delta(self, delta: DeltaMessage):
|
||||
if delta.content is not None:
|
||||
self.other_content += delta.content
|
||||
else:
|
||||
assert delta.tool_calls, (
|
||||
"Streaming results should have either content or tool calls (or both)"
|
||||
)
|
||||
if self._assert_one_tool_per_delta:
|
||||
# Note: This isn't strictly required by the API and may not be
|
||||
# possible to adhere to depending on the token space and number of
|
||||
# tokens per streamed response from the model, but it is required
|
||||
# by tool_use tests, so we enforce it here by default also.
|
||||
assert len(delta.tool_calls) < 2, (
|
||||
"Streaming should include only one tool call per update."
|
||||
)
|
||||
for call_delta in delta.tool_calls:
|
||||
assert call_delta.type is None or call_delta.type == "function", (
|
||||
"Streaming tool calls should only emit function calls. Got "
|
||||
f"{call_delta.type}"
|
||||
)
|
||||
current_tool_call = (
|
||||
self.tool_calls[call_delta.index]
|
||||
if call_delta.index < len(self.tool_calls)
|
||||
else None
|
||||
)
|
||||
if current_tool_call:
|
||||
assert not call_delta.function.name, (
|
||||
"Streaming tool calls should emit the full function name "
|
||||
f"exactly once. Got {call_delta.function.name}"
|
||||
)
|
||||
assert not call_delta.id, (
|
||||
"Streaming tool calls must emit function id only once. Got "
|
||||
f"{call_delta.id}"
|
||||
)
|
||||
assert call_delta.index == len(self.tool_calls) - 1, (
|
||||
f"Incorrect index for tool delta. Got {call_delta.index}, "
|
||||
f"expected {len(self.tool_calls) - 1}"
|
||||
)
|
||||
current_tool_call.function.arguments += call_delta.function.arguments
|
||||
else:
|
||||
assert call_delta.id is not None, (
|
||||
"Streaming tool calls must have an id on first appearance"
|
||||
)
|
||||
assert call_delta.function.name is not None, (
|
||||
"Streaming tool calls must have a function name on first appearance"
|
||||
)
|
||||
assert call_delta.index == len(self.tool_calls), (
|
||||
f"Incorrect index for tool delta. Got {call_delta.index}, "
|
||||
f"expected {len(self.tool_calls)}"
|
||||
)
|
||||
self.tool_calls.append(
|
||||
ToolCall(
|
||||
id=call_delta.id,
|
||||
function=FunctionCall(
|
||||
name=call_delta.function.name,
|
||||
arguments=call_delta.function.arguments or "",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run_tool_extraction(
|
||||
tool_parser: ToolParser,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | None = None,
|
||||
streaming: bool = False,
|
||||
assert_one_tool_per_delta: bool = True,
|
||||
) -> tuple[str | None, list[ToolCall]]:
|
||||
if streaming:
|
||||
reconstructor = run_tool_extraction_streaming(
|
||||
tool_parser,
|
||||
model_output,
|
||||
request,
|
||||
assert_one_tool_per_delta=assert_one_tool_per_delta,
|
||||
)
|
||||
return reconstructor.other_content or None, reconstructor.tool_calls
|
||||
else:
|
||||
extracted = run_tool_extraction_nonstreaming(tool_parser, model_output, request)
|
||||
assert extracted.tools_called == bool(extracted.tool_calls)
|
||||
return extracted.content, extracted.tool_calls
|
||||
|
||||
|
||||
def run_tool_extraction_nonstreaming(
|
||||
tool_parser: ToolParser,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | None = None,
|
||||
) -> ExtractedToolCallInformation:
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
return tool_parser.extract_tool_calls(model_output, request)
|
||||
|
||||
|
||||
def split_string_into_token_deltas(tokenizer: TokenizerLike, text: str) -> list[str]:
|
||||
# Split a string into a series of deltas using the provided tokenizer. Each
|
||||
# delta will be the string equivalent of a single token.
|
||||
token_ids = tokenizer.encode(text, add_special_tokens=False)
|
||||
previously_decoded_text = ""
|
||||
deltas = []
|
||||
for i in range(1, len(token_ids) + 1):
|
||||
current_tokens = token_ids[:i]
|
||||
current_text = tokenizer.decode(current_tokens)
|
||||
new_text = current_text[len(previously_decoded_text) :]
|
||||
previously_decoded_text = current_text
|
||||
deltas.append(new_text)
|
||||
return deltas
|
||||
|
||||
|
||||
def run_tool_extraction_streaming(
|
||||
tool_parser: ToolParser,
|
||||
model_deltas: Iterable[str],
|
||||
request: ChatCompletionRequest | None = None,
|
||||
assert_one_tool_per_delta: bool = True,
|
||||
) -> StreamingToolReconstructor:
|
||||
if isinstance(model_deltas, str):
|
||||
model_deltas = split_string_into_token_deltas(
|
||||
tool_parser.model_tokenizer, model_deltas
|
||||
)
|
||||
|
||||
request = request or ChatCompletionRequest(messages=[], model="test-model")
|
||||
reconstructor = StreamingToolReconstructor(
|
||||
assert_one_tool_per_delta=assert_one_tool_per_delta
|
||||
)
|
||||
previous_text = ""
|
||||
previous_tokens: list[int] = []
|
||||
for delta in model_deltas:
|
||||
token_delta = [
|
||||
tool_parser.vocab.get(token)
|
||||
for token in tool_parser.model_tokenizer.tokenize(delta)
|
||||
if token in tool_parser.vocab
|
||||
]
|
||||
current_text = previous_text + delta
|
||||
current_tokens = previous_tokens + token_delta
|
||||
delta_message = tool_parser.extract_tool_calls_streaming(
|
||||
previous_text,
|
||||
current_text,
|
||||
delta,
|
||||
previous_tokens,
|
||||
current_tokens,
|
||||
token_delta,
|
||||
request,
|
||||
)
|
||||
if delta_message is not None:
|
||||
reconstructor.append_delta(delta_message)
|
||||
previous_text = current_text
|
||||
previous_tokens = current_tokens
|
||||
return reconstructor
|
||||
Reference in New Issue
Block a user