feat(models): implement Google Gemma 4 architecture support (MoE, Multimodal, Reasoning, Tool-Use) (#38826)
Signed-off-by: Luciano Martins <lucianommartins@users.noreply.github.com> Signed-off-by: Luciano Martins <lucianomartins@google.com> Co-authored-by: Luciano Martins <lucianommartins@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
Luciano Martins
Isotr0py
parent
ecd5443dbc
commit
08ed2b9688
@@ -394,6 +394,22 @@ VLM_TEST_SETTINGS = {
|
||||
vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
|
||||
patch_hf_runner=model_utils.gemma3_patch_hf_runner,
|
||||
),
|
||||
"gemma4": VLMTestInfo(
|
||||
models=["google/gemma-4-E2B-it"],
|
||||
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "What's the content in the center of the image?",
|
||||
"cherry_blossom": "What is the season?",
|
||||
}
|
||||
),
|
||||
multi_image_prompt="Describe the two images in detail.",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_runner_kwargs={"limit_mm_per_prompt": {"image": 4}},
|
||||
),
|
||||
"granite_vision": VLMTestInfo(
|
||||
models=["ibm-granite/granite-vision-3.3-2b"],
|
||||
test_type=(VLMTestType.IMAGE),
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from ....conftest import ImageTestAssets
|
||||
from ...utils import build_model_context
|
||||
|
||||
# TODO: to be updated to "google/gemma-4-e2b-it" once the models are available
|
||||
GEMMA4_MODEL_ID = "google/gemma-4-E2B-it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [GEMMA4_MODEL_ID])
|
||||
def test_limit_mm_per_prompt(
|
||||
image_assets: ImageTestAssets,
|
||||
model_id: str,
|
||||
):
|
||||
"""Test that limit_mm_per_prompt accurately restricts multiple images."""
|
||||
# We only allow 1 image
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
mm_processor_kwargs={},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
# Provide 2 images in the prompt
|
||||
prompt = "<image><image>"
|
||||
# image_assets usually has multiple images
|
||||
images = [asset.pil_image for asset in image_assets][:2]
|
||||
if len(images) < 2:
|
||||
images = [images[0], images[0]]
|
||||
|
||||
mm_data = {"image": images}
|
||||
|
||||
# Expect ValueError when exceeding limit
|
||||
with pytest.raises(ValueError, match="At most 1 image"):
|
||||
processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
@@ -277,6 +277,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"google/gemma-2-9b", extras={"tiny": "google/gemma-2-2b-it"}
|
||||
),
|
||||
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
|
||||
"Gemma4ForCausalLM": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
|
||||
"Glm4ForCausalLM": _HfExamplesInfo("zai-org/GLM-4-9B-0414"),
|
||||
@@ -813,6 +817,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
|
||||
"Gemma3ForConditionalGeneration": _HfExamplesInfo("google/gemma-3-4b-it"),
|
||||
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.5.0",
|
||||
),
|
||||
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.reasoning.utils import run_reasoning_extraction
|
||||
from vllm.reasoning import ReasoningParser, ReasoningParserManager
|
||||
|
||||
# Using mistral tokenizer as a generic mock since the actual model is not on HF
|
||||
from vllm.tokenizers.registry import get_tokenizer
|
||||
|
||||
parser_name = "gemma4"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def generic_tokenizer():
|
||||
return get_tokenizer("google/gemma-4-E2B-it")
|
||||
|
||||
|
||||
INVALID_SIMPLE_NONSTREAMING = {
|
||||
"output": "This is a reasoning section<channel|>This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
INVALID_SIMPLE_STREAMING = {
|
||||
"output": "This is a reasoning section<channel|>This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is a reasoning sectionThis is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
INVALID_COMPLETE_NONSTREAMING = {
|
||||
"output": "This is a reasoning section<channel|>",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": None,
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
INVALID_COMPLETE_STREAMING = {
|
||||
"output": "This is a reasoning section<channel|>",
|
||||
"reasoning": None,
|
||||
"content": "This is a reasoning section",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
NO_CONTENT = {
|
||||
"output": "<|channel>This is reasoning",
|
||||
"reasoning": "This is reasoning",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
NO_REASONING = {
|
||||
"output": "This is content",
|
||||
"reasoning": None,
|
||||
"content": "This is content",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
REASONING_WITH_CHANNEL = {
|
||||
"output": "<|channel>This is a reasoning section<channel|>This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
COMPLETE_REASONING_WITH_CHANNEL = {
|
||||
"output": "<|channel>This is a reasoning section<channel|>",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": None,
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
MULTIPLE_LINES_WITH_CHANNEL = {
|
||||
"output": "<|channel>This\nThat<channel|>This is the rest\nThat",
|
||||
"reasoning": "This\nThat",
|
||||
"content": "This is the rest\nThat",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
CHANNEL_NO_END = {
|
||||
"output": "<|channel>This is a reasoning section",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
EMPTY = {
|
||||
"output": "",
|
||||
"reasoning": None,
|
||||
"content": "",
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
NEW_LINE_NONSTREAMING = {
|
||||
"output": (
|
||||
"Before\n<|channel>This is a reasoning section<channel|>\nThis is the rest"
|
||||
),
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "\nThis is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
NEW_LINE_STREAMING = {
|
||||
"output": (
|
||||
"Before\n<|channel>This is a reasoning section<channel|>\nThis is the rest"
|
||||
),
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "Before\n\nThis is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
|
||||
TEST_CASES = [
|
||||
pytest.param(False, INVALID_SIMPLE_NONSTREAMING, id="invalid_simple"),
|
||||
pytest.param(True, INVALID_SIMPLE_STREAMING, id="invalid_simple_streaming"),
|
||||
pytest.param(False, INVALID_COMPLETE_NONSTREAMING, id="invalid_complete"),
|
||||
pytest.param(True, INVALID_COMPLETE_STREAMING, id="invalid_complete_streaming"),
|
||||
pytest.param(False, NO_CONTENT, id="no_content"),
|
||||
pytest.param(False, NO_REASONING, id="no_reasoning"),
|
||||
pytest.param(False, REASONING_WITH_CHANNEL, id="reasoning"),
|
||||
pytest.param(True, REASONING_WITH_CHANNEL, id="reasoning_streaming"),
|
||||
pytest.param(False, COMPLETE_REASONING_WITH_CHANNEL, id="complete_reasoning"),
|
||||
pytest.param(
|
||||
True, COMPLETE_REASONING_WITH_CHANNEL, id="complete_reasoning_streaming"
|
||||
),
|
||||
pytest.param(False, MULTIPLE_LINES_WITH_CHANNEL, id="multiple_lines"),
|
||||
pytest.param(True, MULTIPLE_LINES_WITH_CHANNEL, id="multiple_lines_streaming"),
|
||||
pytest.param(False, CHANNEL_NO_END, id="no_end"),
|
||||
pytest.param(True, CHANNEL_NO_END, id="no_end_streaming"),
|
||||
pytest.param(False, EMPTY, id="empty"),
|
||||
pytest.param(False, NEW_LINE_NONSTREAMING, id="new_line"),
|
||||
pytest.param(True, NEW_LINE_STREAMING, id="new_line_streaming"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
|
||||
def test_gemma4_reasoning(
|
||||
streaming: bool,
|
||||
param_dict: dict,
|
||||
generic_tokenizer,
|
||||
):
|
||||
output = param_dict["output"]
|
||||
|
||||
# Resolve token IDs dynamically from the real tokenizer
|
||||
vocab = generic_tokenizer.get_vocab()
|
||||
start_token_id = vocab["<|channel>"]
|
||||
end_token_id = vocab["<channel|>"]
|
||||
|
||||
index_start = output.find("<|channel>")
|
||||
len_start = len("<|channel>")
|
||||
index_end = output.find("<channel|>")
|
||||
len_end = len("<channel|>")
|
||||
|
||||
output_tokens = []
|
||||
|
||||
def _encode(text: str) -> list[int]:
|
||||
if not text:
|
||||
return []
|
||||
# Handle both raw transformers and vLLM wrappers
|
||||
enc = getattr(generic_tokenizer, "tokenizer", generic_tokenizer)
|
||||
try:
|
||||
return enc.encode(text, add_special_tokens=False)
|
||||
except TypeError:
|
||||
return enc.encode(text)
|
||||
|
||||
if index_start != -1:
|
||||
output_before = output[:index_start]
|
||||
output_tokens += _encode(output_before)
|
||||
output_tokens += [start_token_id]
|
||||
|
||||
if index_end != -1:
|
||||
output_middle = output[index_start + len_start : index_end]
|
||||
output_after = output[index_end + len_end :]
|
||||
output_tokens += _encode(output_middle)
|
||||
output_tokens += [end_token_id]
|
||||
output_tokens += _encode(output_after)
|
||||
else:
|
||||
output_middle = output[index_start + len_start :]
|
||||
output_tokens += _encode(output_middle)
|
||||
elif index_end != -1:
|
||||
output_before = output[:index_end]
|
||||
output_after = output[index_end + len_end :]
|
||||
output_tokens += _encode(output_before)
|
||||
output_tokens += [end_token_id]
|
||||
output_tokens += _encode(output_after)
|
||||
else:
|
||||
output_tokens += _encode(output)
|
||||
|
||||
parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
|
||||
generic_tokenizer
|
||||
)
|
||||
|
||||
# We use the generic run_reasoning_extraction from utils
|
||||
# Use decode per token to get standard spaces instead of
|
||||
# SentencePiece space characters
|
||||
output_token_strings = [generic_tokenizer.decode([t]) for t in output_tokens]
|
||||
reasoning, content = run_reasoning_extraction(
|
||||
parser, output_token_strings, streaming=streaming
|
||||
)
|
||||
|
||||
assert reasoning == param_dict["reasoning"]
|
||||
assert content == param_dict["content"]
|
||||
|
||||
# Test is_reasoning_end
|
||||
is_reasoning_end = parser.is_reasoning_end(output_tokens)
|
||||
assert is_reasoning_end == param_dict["is_reasoning_end"]
|
||||
@@ -0,0 +1,504 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.tool_parsers.gemma4_tool_parser import (
|
||||
TOOL_CALL_END,
|
||||
TOOL_CALL_START,
|
||||
Gemma4ToolParser,
|
||||
_parse_gemma4_args,
|
||||
_parse_gemma4_array,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.encode.return_value = [1, 2, 3]
|
||||
# Include the tool call start token in the vocab for the parser
|
||||
tokenizer.get_vocab.return_value = {TOOL_CALL_START: 48, TOOL_CALL_END: 49}
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return Gemma4ToolParser(mock_tokenizer)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request():
|
||||
request = MagicMock(spec=ChatCompletionRequest)
|
||||
request.tools = []
|
||||
request.tool_choice = "auto"
|
||||
return request
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit tests for _parse_gemma4_args (shared parser logic)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseGemma4Args:
|
||||
def test_empty_string(self):
|
||||
assert _parse_gemma4_args("") == {}
|
||||
|
||||
def test_whitespace_only(self):
|
||||
assert _parse_gemma4_args(" ") == {}
|
||||
|
||||
def test_single_string_value(self):
|
||||
result = _parse_gemma4_args('location:<|"|>Paris<|"|>')
|
||||
assert result == {"location": "Paris"}
|
||||
|
||||
def test_string_value_with_comma(self):
|
||||
result = _parse_gemma4_args('location:<|"|>Paris, France<|"|>')
|
||||
assert result == {"location": "Paris, France"}
|
||||
|
||||
def test_multiple_string_values(self):
|
||||
result = _parse_gemma4_args(
|
||||
'location:<|"|>San Francisco<|"|>,unit:<|"|>celsius<|"|>'
|
||||
)
|
||||
assert result == {"location": "San Francisco", "unit": "celsius"}
|
||||
|
||||
def test_integer_value(self):
|
||||
result = _parse_gemma4_args("count:42")
|
||||
assert result == {"count": 42}
|
||||
|
||||
def test_float_value(self):
|
||||
result = _parse_gemma4_args("score:3.14")
|
||||
assert result == {"score": 3.14}
|
||||
|
||||
def test_boolean_true(self):
|
||||
result = _parse_gemma4_args("flag:true")
|
||||
assert result == {"flag": True}
|
||||
|
||||
def test_boolean_false(self):
|
||||
result = _parse_gemma4_args("flag:false")
|
||||
assert result == {"flag": False}
|
||||
|
||||
def test_mixed_types(self):
|
||||
result = _parse_gemma4_args(
|
||||
'name:<|"|>test<|"|>,count:42,active:true,score:3.14'
|
||||
)
|
||||
assert result == {
|
||||
"name": "test",
|
||||
"count": 42,
|
||||
"active": True,
|
||||
"score": 3.14,
|
||||
}
|
||||
|
||||
def test_nested_object(self):
|
||||
result = _parse_gemma4_args('nested:{inner:<|"|>value<|"|>}')
|
||||
assert result == {"nested": {"inner": "value"}}
|
||||
|
||||
def test_array_of_strings(self):
|
||||
result = _parse_gemma4_args('items:[<|"|>a<|"|>,<|"|>b<|"|>]')
|
||||
assert result == {"items": ["a", "b"]}
|
||||
|
||||
def test_unterminated_string(self):
|
||||
"""Unterminated strings should take everything after the delimiter."""
|
||||
result = _parse_gemma4_args('key:<|"|>unterminated')
|
||||
assert result == {"key": "unterminated"}
|
||||
|
||||
def test_empty_value(self):
|
||||
"""Key with no value after colon."""
|
||||
result = _parse_gemma4_args("key:")
|
||||
assert result == {"key": ""}
|
||||
|
||||
|
||||
class TestParseGemma4Array:
|
||||
def test_string_array(self):
|
||||
result = _parse_gemma4_array('<|"|>a<|"|>,<|"|>b<|"|>')
|
||||
assert result == ["a", "b"]
|
||||
|
||||
def test_empty_array(self):
|
||||
result = _parse_gemma4_array("")
|
||||
assert result == []
|
||||
|
||||
def test_bare_values(self):
|
||||
result = _parse_gemma4_array("42,true,3.14")
|
||||
assert result == [42, True, 3.14]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-streaming extraction tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestExtractToolCalls:
|
||||
def test_no_tool_calls(self, parser, mock_request):
|
||||
model_output = "Hello, how can I help you today?"
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is False
|
||||
assert result.tool_calls == []
|
||||
assert result.content == model_output
|
||||
|
||||
def test_single_tool_call(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}<tool_call|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "London"}
|
||||
|
||||
def test_multiple_arguments(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>call:get_weather{"
|
||||
'location:<|"|>San Francisco<|"|>,'
|
||||
'unit:<|"|>celsius<|"|>}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"location": "San Francisco", "unit": "celsius"}
|
||||
|
||||
def test_text_before_tool_call(self, parser, mock_request):
|
||||
model_output = (
|
||||
"Let me check the weather for you. "
|
||||
'<|tool_call>call:get_weather{location:<|"|>Paris<|"|>}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.content == "Let me check the weather for you."
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
|
||||
def test_multiple_tool_calls(self, parser, mock_request):
|
||||
model_output = (
|
||||
'<|tool_call>call:get_weather{location:<|"|>London<|"|>}'
|
||||
"<tool_call|>"
|
||||
'<|tool_call>call:get_time{location:<|"|>London<|"|>}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 2
|
||||
assert result.tool_calls[0].function.name == "get_weather"
|
||||
assert result.tool_calls[1].function.name == "get_time"
|
||||
|
||||
def test_nested_arguments(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>call:complex_function{"
|
||||
'nested:{inner:<|"|>value<|"|>},'
|
||||
'list:[<|"|>a<|"|>,<|"|>b<|"|>]}'
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "complex_function"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"nested": {"inner": "value"}, "list": ["a", "b"]}
|
||||
|
||||
def test_tool_call_with_number_and_boolean(self, parser, mock_request):
|
||||
model_output = (
|
||||
"<|tool_call>call:set_status{"
|
||||
"is_active:true,"
|
||||
"count:42,"
|
||||
"score:3.14}"
|
||||
"<tool_call|>"
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert len(result.tool_calls) == 1
|
||||
assert result.tool_calls[0].function.name == "set_status"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {"is_active": True, "count": 42, "score": 3.14}
|
||||
|
||||
def test_incomplete_tool_call(self, parser, mock_request):
|
||||
model_output = '<|tool_call>call:get_weather{location:<|"|>London'
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
# Incomplete — no <tool_call|> end marker, regex won't match
|
||||
assert result.tools_called is False
|
||||
assert result.content == model_output
|
||||
|
||||
def test_hyphenated_function_name(self, parser, mock_request):
|
||||
"""Ensure function names with hyphens are parsed correctly."""
|
||||
model_output = (
|
||||
'<|tool_call>call:get-weather{location:<|"|>London<|"|>}<tool_call|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "get-weather"
|
||||
|
||||
def test_dotted_function_name(self, parser, mock_request):
|
||||
"""Ensure function names with dots are parsed correctly."""
|
||||
model_output = (
|
||||
'<|tool_call>call:weather.get{location:<|"|>London<|"|>}<tool_call|>'
|
||||
)
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "weather.get"
|
||||
|
||||
def test_no_arguments(self, parser, mock_request):
|
||||
"""Tool calls with empty arguments."""
|
||||
model_output = "<|tool_call>call:get_status{}<tool_call|>"
|
||||
result = parser.extract_tool_calls(model_output, mock_request)
|
||||
|
||||
assert result.tools_called is True
|
||||
assert result.tool_calls[0].function.name == "get_status"
|
||||
args = json.loads(result.tool_calls[0].function.arguments)
|
||||
assert args == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Streaming extraction tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestStreamingExtraction:
|
||||
"""Tests for the streaming tool call extraction.
|
||||
|
||||
These simulate the token-by-token streaming that vLLM performs,
|
||||
feeding incremental text to extract_tool_calls_streaming() and
|
||||
verifying that the accumulated argument deltas form valid JSON.
|
||||
"""
|
||||
|
||||
def _simulate_streaming(
|
||||
self, parser: Gemma4ToolParser, mock_request: Any, chunks: list[str]
|
||||
) -> list[tuple[Any, str]]:
|
||||
"""Feed chunks through the streaming parser and collect results.
|
||||
|
||||
Returns a list of (delta_message, accumulated_text) tuples.
|
||||
"""
|
||||
results: list[tuple[Any, str]] = []
|
||||
previous_text: str = ""
|
||||
previous_token_ids: list[int] = []
|
||||
|
||||
for chunk in chunks:
|
||||
current_text = previous_text + chunk
|
||||
# Use token ID 48 for tool_call start, 49 for end, 0 otherwise
|
||||
delta_token_ids: list[int] = []
|
||||
if TOOL_CALL_START in chunk:
|
||||
delta_token_ids.append(48)
|
||||
elif TOOL_CALL_END in chunk:
|
||||
delta_token_ids.append(49)
|
||||
else:
|
||||
delta_token_ids.append(0)
|
||||
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=tuple(previous_token_ids),
|
||||
current_token_ids=tuple(current_token_ids),
|
||||
delta_token_ids=tuple(delta_token_ids),
|
||||
request=mock_request,
|
||||
)
|
||||
results.append((delta, current_text))
|
||||
previous_text = current_text
|
||||
previous_token_ids = list(current_token_ids)
|
||||
|
||||
return results
|
||||
|
||||
def _collect_arguments(self, results):
|
||||
"""Collect all argument deltas from streaming results into one string."""
|
||||
args_text = ""
|
||||
for delta, _ in results:
|
||||
if delta and delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
func = tc.function if isinstance(tc.function, dict) else tc.function
|
||||
if isinstance(func, dict):
|
||||
arg = func.get("arguments", "")
|
||||
else:
|
||||
arg = getattr(func, "arguments", "") or ""
|
||||
if arg:
|
||||
args_text += arg
|
||||
return args_text
|
||||
|
||||
def _collect_function_name(self, results):
|
||||
"""Extract the function name from streaming results."""
|
||||
for delta, _ in results:
|
||||
if delta and delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
func = tc.function if isinstance(tc.function, dict) else tc.function
|
||||
if isinstance(func, dict):
|
||||
name = func.get("name")
|
||||
else:
|
||||
name = getattr(func, "name", None)
|
||||
if name:
|
||||
return name
|
||||
return None
|
||||
|
||||
def test_basic_streaming_single_tool(self, parser, mock_request):
|
||||
"""Simulate the exact streaming scenario from the bug report.
|
||||
|
||||
Model generates:
|
||||
<|tool_call>call:get_weather{location:<|"|>Paris, France<|"|>}<tool_call|>
|
||||
|
||||
Expected: arguments should be valid JSON {"location": "Paris, France"}
|
||||
"""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:get_weather{",
|
||||
'location:<|"|>Paris',
|
||||
", France",
|
||||
'<|"|>}',
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
# Verify function name
|
||||
name = self._collect_function_name(results)
|
||||
assert name == "get_weather", f"Expected 'get_weather', got '{name}'"
|
||||
|
||||
# Verify arguments form valid JSON
|
||||
args_text = self._collect_arguments(results)
|
||||
assert args_text, "No arguments were streamed"
|
||||
parsed_args = json.loads(args_text)
|
||||
assert parsed_args == {"location": "Paris, France"}
|
||||
|
||||
def test_streaming_multi_arg(self, parser, mock_request):
|
||||
"""Streaming with multiple arguments."""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:get_weather{",
|
||||
'location:<|"|>Tokyo<|"|>,',
|
||||
'unit:<|"|>celsius<|"|>}',
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
name = self._collect_function_name(results)
|
||||
assert name == "get_weather"
|
||||
|
||||
args_text = self._collect_arguments(results)
|
||||
assert args_text
|
||||
parsed_args = json.loads(args_text)
|
||||
assert parsed_args == {"location": "Tokyo", "unit": "celsius"}
|
||||
|
||||
def test_streaming_no_extra_brace(self, parser, mock_request):
|
||||
"""Verify the closing } is NOT leaked into arguments (Bug #2)."""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:get_weather{",
|
||||
'location:<|"|>London<|"|>}',
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
args_text = self._collect_arguments(results)
|
||||
assert args_text
|
||||
|
||||
# The args text must be valid JSON (no extra })
|
||||
parsed = json.loads(args_text)
|
||||
assert parsed == {"location": "London"}
|
||||
|
||||
# Specifically assert no double-brace
|
||||
assert args_text.count("}") <= 1, (
|
||||
f"Arguments contain extra closing brace: {args_text!r}"
|
||||
)
|
||||
|
||||
def test_streaming_no_unquoted_keys(self, parser, mock_request):
|
||||
"""Verify keys are properly quoted in JSON (Bug #1)."""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:get_weather{",
|
||||
'location:<|"|>Paris<|"|>}',
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
args_text = self._collect_arguments(results)
|
||||
|
||||
# Must start with { and contain quoted key
|
||||
assert args_text.lstrip().startswith("{"), (
|
||||
f"Arguments don't start with '{{': {args_text!r}"
|
||||
)
|
||||
assert '"location"' in args_text, (
|
||||
f"Key 'location' not properly quoted: {args_text!r}"
|
||||
)
|
||||
|
||||
def test_streaming_name_no_call_prefix(self, parser, mock_request):
|
||||
"""Verify function name has no 'call:' prefix."""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:get_weather{",
|
||||
'location:<|"|>Paris<|"|>}',
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
name = self._collect_function_name(results)
|
||||
assert name == "get_weather"
|
||||
assert not name.startswith("call:"), f"Name has 'call:' prefix: {name!r}"
|
||||
|
||||
def test_streaming_text_before_tool_call(self, parser, mock_request):
|
||||
"""Text before tool call should be emitted as content."""
|
||||
chunks = [
|
||||
"Let me check ",
|
||||
"the weather. ",
|
||||
"<|tool_call>",
|
||||
"call:get_weather{",
|
||||
'location:<|"|>London<|"|>}',
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
|
||||
# First chunks should be content
|
||||
content_parts = []
|
||||
for delta, _ in results:
|
||||
if delta and delta.content:
|
||||
content_parts.append(delta.content)
|
||||
|
||||
assert "".join(content_parts).strip().startswith("Let me check")
|
||||
|
||||
def test_streaming_numeric_args(self, parser, mock_request):
|
||||
"""Streaming with numeric and boolean argument values."""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:set_config{",
|
||||
"count:42,",
|
||||
"active:true}",
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
args_text = self._collect_arguments(results)
|
||||
if args_text:
|
||||
parsed_args = json.loads(args_text)
|
||||
assert parsed_args["count"] == 42
|
||||
assert parsed_args["active"] is True
|
||||
|
||||
def test_streaming_empty_args(self, parser, mock_request):
|
||||
"""Tool call with no arguments."""
|
||||
chunks = [
|
||||
"<|tool_call>",
|
||||
"call:get_status{}",
|
||||
"<tool_call|>",
|
||||
]
|
||||
|
||||
results = self._simulate_streaming(parser, mock_request, chunks)
|
||||
name = self._collect_function_name(results)
|
||||
assert name == "get_status"
|
||||
Reference in New Issue
Block a user