[Bugfix][Parser] Stream Gemma4 string tool arguments incrementally (#61494)

Signed-off-by: Ben Browning <56071+bbrowning@users.noreply.github.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-10-14 21:25:41 +00:00
committed by GitHub
co-authored by Flora Feng
parent 984c242ede
commit 4c5ea86f53
5 changed files with 369 additions and 22 deletions
+7 -6
View File
@@ -342,8 +342,10 @@ class ParserEngine(Parser):
their unterminated content instead of buffering long arguments until
the closing tag arrives.
Also returns whether the prefix ends inside that streamed trailing
string value.
Also returns whether the returned prefix stops inside that trailing
string, so streamed text may be appended to it. This holds even when
*json_str* closes the string: converters serialise an open raw value
as a closed JSON string.
"""
last_colon = -1
last_key: str | None = None
@@ -1053,11 +1055,8 @@ class ParserEngine(Parser):
if not self._stream_arg_deltas:
return None
structural = self._arg_structural_chars
if structural is not None and structural.isdisjoint(raw_delta):
return None
slot = self._tool_slots[idx]
structural = self._arg_structural_chars
if slot.extend_open_value is not None:
diff = slot.extend_open_value(raw_delta)
if diff is not None:
@@ -1066,6 +1065,8 @@ class ParserEngine(Parser):
slot.append_streamed(diff)
return diff
slot.extend_open_value = None
elif structural is not None and structural.isdisjoint(raw_delta):
return None
try:
current_json = converter(slot.args, True)
+20 -3
View File
@@ -94,9 +94,17 @@ class ParserEngineConfig:
arg_structural_chars: frozenset[str] | None = None
# Called with the raw argument text when its converted JSON ends in an
# open value; returns an extender for that value, or None. Streaming
# skips ``arg_converter`` for every delta the extender accepts.
# Called with the raw argument text after each streaming conversion whose
# prefix-safe JSON extends the JSON streamed so far; returns an extender
# when the text ends in an open value that is the last value in the
# converted JSON, else None. While live, the extender receives every delta
# in place of ``arg_converter`` until it returns None, meaning the delta
# closed the value. Returned text is streamed as is, so it must match what
# ``arg_converter`` would serialise for those raw characters. Deltas are
# lexer output: a terminal is never split across deltas, but the lexer
# flushes a held partial terminal as text at the end of the stream or
# before a terminal matched by token ID, so a truncated value may end in
# a terminal prefix. Requires ``arg_converter`` and ``stream_arg_deltas``.
arg_value_extender: Callable[[str], ArgValueExtender | None] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
@@ -118,6 +126,15 @@ class ParserEngineConfig:
# Reject tool calls whose names are absent from the request tools.
validate_tool_names: bool = False
def __post_init__(self) -> None:
if self.arg_value_extender is not None and (
self.arg_converter is None or not self.stream_arg_deltas
):
raise ValueError(
f"{self.name}: arg_value_extender requires arg_converter "
"and stream_arg_deltas=True"
)
def terminal_literal(self, name: str) -> str | None:
"""Canonical spelling of terminal *name*, or ``None`` if undeclared."""
value = self.terminals.get(name)
+67 -12
View File
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import (
ArgValueExtender,
ParserEngineConfig,
ParserState,
Transition,
@@ -66,6 +67,12 @@ def _strip_partial_delim(value: str) -> str:
def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
return _parse_gemma4_fields(args_str, partial=partial)[0]
def _parse_gemma4_fields(
args_str: str, *, partial: bool = False
) -> tuple[dict, str | None]:
"""Parse Gemma4's custom key:value format into a Python dict.
Format examples::
@@ -82,13 +89,15 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
omitted because they may be incomplete and type-unstable
(e.g. partial boolean parsed as bare string).
Returns a dict ready for ``json.dumps()``.
Returns a dict ready for ``json.dumps()`` and the key of the trailing
top-level ``STRING_DELIM`` value left unterminated, if any.
"""
if not args_str or not args_str.strip():
return {}
return {}, None
result: dict = {}
open_key: str | None = None
i = 0
n = len(args_str)
@@ -130,6 +139,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
if partial:
value = _strip_partial_delim(value)
result[key] = value
open_key = key
break
result[key] = args_str[val_start:end_pos]
i = end_pos + _DELIM_LEN
@@ -199,7 +209,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
break
result[key] = raw_val
return result
return result, open_key
def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
@@ -283,16 +293,64 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
return items
def _drop_call_closer(text: str, paren_excess: bool) -> str:
"""Drop a trailing ``}`` or, with more ``)`` than ``(``, a trailing ``)``.
The closer of the call body is dropped before parsing, so a value that
itself ends in one loses that character until more text arrives.
"""
if text.endswith("}") or (text.endswith(")") and paren_excess):
return text[:-1]
return text
def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
"""Convert Gemma4 custom arg format to a JSON string."""
text = raw_args.strip()
if text.endswith("}") or text.endswith(")") and text.count("(") < text.count(")"):
text = text[:-1]
text = _drop_call_closer(text, text.count("(") < text.count(")"))
parsed = _parse_gemma4_args(text, partial=partial)
return json.dumps(parsed, ensure_ascii=False)
def _gemma4_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
"""Extend the open top-level ``STRING_DELIM`` value at the end of
*raw_args*, if it is the last value in the converted JSON.
The extender streams exactly what ``_gemma4_arg_converter`` would, so it
holds back the same tail: trailing whitespace, a dropped closer and a
partial ``STRING_DELIM``. ``STRING_DELIM`` is a special token but not a
lexer terminal, so unlike a tag it may reach the extender split across
deltas; the held tail doubles as the window that detects the close.
"""
text = raw_args.strip()
opens, closes = text.count("("), text.count(")")
body = _drop_call_closer(text, opens < closes)
params, open_key = _parse_gemma4_fields(body, partial=True)
if open_key is None or next(reversed(params)) != open_key:
return None
held_len = (
len(raw_args)
- len(raw_args.rstrip())
+ len(text)
- len(_strip_partial_delim(body))
)
held = raw_args[len(raw_args) - held_len :] if held_len else ""
def extend(delta: str) -> str | None:
nonlocal held, opens, closes
text = held + delta
if STRING_DELIM in text:
return None
opens += delta.count("(")
closes += delta.count(")")
visible = _strip_partial_delim(_drop_call_closer(text.rstrip(), opens < closes))
held = text[len(visible) :]
return json.dumps(visible, ensure_ascii=False)[1:-1]
return extend
@functools.cache
def gemma4_config(thinking: bool = False) -> ParserEngineConfig:
return ParserEngineConfig(
@@ -396,6 +454,7 @@ def gemma4_config(thinking: bool = False) -> ParserEngineConfig:
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
},
arg_converter=_gemma4_arg_converter,
arg_value_extender=_gemma4_arg_value_extender,
tool_args_json=False,
arg_structural_chars=frozenset(",:{}[]<"),
preserve_tokens=frozenset({STRING_DELIM}),
@@ -423,12 +482,8 @@ class Gemma4Parser(ParserEngine):
) -> None:
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
thinking = chat_kwargs.get("enable_thinking", False)
super().__init__(
tokenizer,
tools,
parser_engine_config=gemma4_config(thinking=thinking),
**kwargs,
)
kwargs.setdefault("parser_engine_config", gemma4_config(thinking=thinking))
super().__init__(tokenizer, tools, **kwargs)
vocab = self.vocab
self._tool_call_token_id: int | None = vocab.get("<|tool_call>")
self._new_turn_token_id: int | None = vocab.get("<|turn>")