[Bugfix][Parser] Stream Gemma4 string tool arguments incrementally (#61494)

Signed-off-by: Ben Browning <56071+bbrowning@users.noreply.github.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-10-14 21:25:41 +00:00
committed by GitHub
co-authored by Flora Feng
parent 984c242ede
commit 4c5ea86f53
5 changed files with 369 additions and 22 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the unified Gemma4 parser engine."""
import dataclasses
import json
from unittest.mock import MagicMock
@@ -17,8 +18,14 @@ from tests.parser.engine.streaming_helpers import (
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionToolsParam,
)
from vllm.parser.gemma4 import (
STRING_DELIM,
Gemma4Parser,
_gemma4_arg_value_extender,
gemma4_config,
)
from vllm.parser.gemma4 import Gemma4Parser
# ── Special token IDs (arbitrary but consistent) ─────────────────────
CHANNEL_START_ID = 50 # <|channel>
@@ -1649,3 +1656,219 @@ class TestGemma4IsReasoningEnd:
)
def test_is_reasoning_end(self, ids, thinking, ended):
assert self._parser(thinking).is_reasoning_end(ids) is ended
# ── Argument value extender ──────────────────────────────────────────
_D = STRING_DELIM
_EXTENDER_TOOL = ChatCompletionToolsParam(
type="function",
function={
"name": "f",
"parameters": {
"type": "object",
"properties": {
"a": {"type": "string"},
"b": {"type": "string"},
"location": {"type": "string"},
"n": {"type": "integer"},
},
},
},
)
_EXTENDER_BODIES = [
f"location:{_D}Paris, France{_D}",
f"a:{_D}x{_D},b:{_D}hello }} world{_D}",
f"a:{_D}smile :-) ok{_D},n:42",
f"a:{_D}trailing spaces {_D},n:42",
f'a:{_D}decoys <|x <|"y <|"|z{_D}',
f'a:{_D}quote " backslash \\ newline\nunicode 杭州{_D}',
f"a:{_D}x{_D},{_D}quoted key{_D}:{_D}val{_D}",
f"a:{_D}x{_D},nested:{{inner:{_D}deep value{_D},n:1}}",
f"items:[{_D}a{_D},{_D}b, c{_D}],a:{_D}last{_D}",
f"a:{_D}call:x{_D},b:{_D}brace {{ colon:{_D}",
f"a:{_D}{_D},b:{_D}}}}}{_D}",
]
_TRUNCATED_BODIES = [
f"a:{_D}hello }}",
f"a:{_D}hello )",
f"a:{_D}hello ",
f'a:{_D}hello<|"',
f"a:{_D}x{_D},b:{_D}open",
f"a:{_D}",
]
_TOOL_MODES = pytest.mark.parametrize(
"tools", [[], [_EXTENDER_TOOL]], ids=["no_schema", "schema"]
)
def _pieces(text: str, size: int) -> list[str]:
return [text[i : i + size] for i in range(0, len(text), size)]
def _stream_args(parser, request, chunks: list[str], finished: bool = True) -> str:
args = []
for i, chunk in enumerate(chunks):
last = finished and i == len(chunks) - 1
delta = parser.parse_delta(chunk, [], request, finished=last)
for tool_call in (delta and delta.tool_calls) or []:
if tool_call.function and tool_call.function.arguments:
args.append(tool_call.function.arguments)
return "".join(args)
class TestGemma4ArgValueExtender:
"""The extender streams exactly what ``_gemma4_arg_converter`` would, so
the converter-only path, which only differs in when it emits, must agree
with it byte for byte on the accumulated arguments."""
@staticmethod
def _parser(tokenizer, tools, extender: bool):
config = gemma4_config()
if not extender:
config = dataclasses.replace(config, arg_value_extender=None)
return Gemma4Parser(tokenizer, tools, parser_engine_config=config)
@_TOOL_MODES
@pytest.mark.parametrize("size", [1, 3, 7, 64])
@pytest.mark.parametrize("body", _EXTENDER_BODIES)
def test_matches_converter_at_every_chunking(
self, tool_call_tokenizer, mock_request, body, size, tools
):
mock_request.tools = tools
text = f"<|tool_call>call:f{{{body}}}<tool_call|>"
chunks = _pieces(text, size)
reference = self._parser(tool_call_tokenizer, tools, extender=True)
expected = reference.extract_tool_calls(text, mock_request)
expected = expected.tool_calls[0].function.arguments
with_extender = _stream_args(
self._parser(tool_call_tokenizer, tools, extender=True),
mock_request,
chunks,
)
converter_only = _stream_args(
self._parser(tool_call_tokenizer, tools, extender=False),
mock_request,
chunks,
)
assert with_extender == converter_only == expected
@_TOOL_MODES
@pytest.mark.parametrize("size", [1, 3, 7])
@pytest.mark.parametrize("body", _TRUNCATED_BODIES)
def test_truncated_stream_matches_converter(
self, tool_call_tokenizer, mock_request, body, size, tools
):
"""A stream cut inside the value ends with the converter's final
flush, so the extender must hold back the whitespace, closer or
partial delimiter that flush would drop."""
mock_request.tools = tools
chunks = _pieces(f"<|tool_call>call:f{{{body}", size)
with_extender = _stream_args(
self._parser(tool_call_tokenizer, tools, extender=True),
mock_request,
chunks,
)
converter_only = _stream_args(
self._parser(tool_call_tokenizer, tools, extender=False),
mock_request,
chunks,
)
assert with_extender == converter_only
json.loads(with_extender)
@pytest.mark.parametrize("offset", range(1, len(STRING_DELIM)))
def test_split_closing_delimiter(self, tool_call_parser, mock_request, offset):
"""``STRING_DELIM`` is not a lexer terminal, so the extender may see
the close split across deltas and must window across them."""
head = f"<|tool_call>call:f{{a:{_D}Paris"
tail = ",n:1}<tool_call|>"
chunks = [head, _D[:offset], _D[offset:] + tail]
streamed = _stream_args(tool_call_parser, mock_request, chunks)
assert streamed == '{"a": "Paris", "n": "1"}'
def test_string_value_streams_before_it_closes(
self, tool_call_tokenizer, mock_request
):
"""Body deltas carry no structural char, so without the extender the
gate holds the whole value until the close."""
head = [f"<|tool_call>call:f{{location:{_D}", "Paris"]
def streamed_before_close(extender: bool) -> str:
parser = self._parser(tool_call_tokenizer, [], extender)
return _stream_args(parser, mock_request, head, finished=False)
assert streamed_before_close(extender=True) == '{"location": "Paris'
assert streamed_before_close(extender=False) == '{"location": "'
def test_conversions_do_not_grow_with_value_length(
self, tool_call_tokenizer, mock_request
):
def streaming_conversions(words: int) -> int:
calls = 0
base = gemma4_config()
def counting_converter(raw_args: str, partial: bool) -> str:
nonlocal calls
if partial:
calls += 1
return base.arg_converter(raw_args, partial)
parser = Gemma4Parser(
tool_call_tokenizer,
parser_engine_config=dataclasses.replace(
base, arg_converter=counting_converter
),
)
value = "word, " * words
text = f"<|tool_call>call:f{{a:{_D}{value}{_D},n:1}}<tool_call|>"
_stream_args(parser, mock_request, _pieces(text, 4))
return calls
assert streaming_conversions(words=40) == streaming_conversions(words=400)
@pytest.mark.parametrize(
"raw_args",
[
f"a:{_D}closed{_D}",
f"a:{_D}x{_D},{_D}open ke",
f"nested:{{inner:{_D}deep",
f"a:{_D}x{_D},b:{_D}y{_D},a:{_D}dup",
"a:tr",
'a:<|"',
],
ids=["closed", "open_key", "nested", "duplicate_key", "bare", "opener"],
)
def test_factory_declines_without_open_last_string_value(self, raw_args):
assert _gemma4_arg_value_extender(raw_args) is None
def test_extender_holds_back_what_the_converter_hides(self):
extend = _gemma4_arg_value_extender(f"a:{_D}hello")
assert extend is not None
assert [extend(d) for d in [" }", "x", ")", "(", '<|"', "y", '<|"', "|>"]] == [
" ",
"}x",
"",
")(",
"",
'<|\\"y',
"",
None,
]
def test_factory_holds_back_the_tail_already_hidden(self):
extend = _gemma4_arg_value_extender(f"a:{_D}hello }} ")
assert extend is not None
assert extend("x") == "} x"
+51
View File
@@ -1685,6 +1685,57 @@ class TestArgDeltaWithConverter:
]
# ── TestArgValueExtender ──────────────────────────────────────────
def _extender_config(structural: frozenset[str] | None = None) -> ParserEngineConfig:
"""``_kv_converter`` plus an extender for its trailing ``k=v`` record,
which is open until whitespace follows it; mirrors
``_glm47_arg_value_extender``."""
def factory(raw_args: str):
if re.search(r"\w+=\S+$", raw_args) is None:
return None
def extend(delta: str) -> str | None:
if re.search(r"\s", delta):
return None
return json.dumps(delta, ensure_ascii=False)[1:-1]
return extend
return dataclasses.replace(
_converter_config(),
arg_value_extender=factory,
arg_structural_chars=structural,
)
class TestArgValueExtender:
"""Engine contract for ``ParserEngineConfig.arg_value_extender``."""
def test_live_extender_sees_deltas_without_structural_chars(self):
"""A delta without structural chars must still reach the live
extender: gating it out drops ``el`` from the streamed value and the
next accepted delta ``l=`` lands on the gap, so the converted JSON
no longer extends what was streamed."""
engine = _make_engine(_extender_config(structural=frozenset("= ")))
assert _run_streaming_tool(engine, "f", ["a", "=h", "el", "l=", "o "]) == {
"a": "hell=o"
}
@pytest.mark.parametrize(
"overrides",
[
{"arg_converter": None},
{"stream_arg_deltas": False},
],
)
def test_extender_requires_streaming_conversion(self, overrides):
with pytest.raises(ValueError, match="arg_value_extender"):
dataclasses.replace(_extender_config(), **overrides)
# ── TestSafeArgPrefix ────────────────────────────────────────────