[Bugfix][Parser] Stream Gemma4 string tool arguments incrementally (#61494)
Signed-off-by: Ben Browning <56071+bbrowning@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
984c242ede
commit
4c5ea86f53
@@ -342,8 +342,10 @@ class ParserEngine(Parser):
|
||||
their unterminated content instead of buffering long arguments until
|
||||
the closing tag arrives.
|
||||
|
||||
Also returns whether the prefix ends inside that streamed trailing
|
||||
string value.
|
||||
Also returns whether the returned prefix stops inside that trailing
|
||||
string, so streamed text may be appended to it. This holds even when
|
||||
*json_str* closes the string: converters serialise an open raw value
|
||||
as a closed JSON string.
|
||||
"""
|
||||
last_colon = -1
|
||||
last_key: str | None = None
|
||||
@@ -1053,11 +1055,8 @@ class ParserEngine(Parser):
|
||||
if not self._stream_arg_deltas:
|
||||
return None
|
||||
|
||||
structural = self._arg_structural_chars
|
||||
if structural is not None and structural.isdisjoint(raw_delta):
|
||||
return None
|
||||
|
||||
slot = self._tool_slots[idx]
|
||||
structural = self._arg_structural_chars
|
||||
if slot.extend_open_value is not None:
|
||||
diff = slot.extend_open_value(raw_delta)
|
||||
if diff is not None:
|
||||
@@ -1066,6 +1065,8 @@ class ParserEngine(Parser):
|
||||
slot.append_streamed(diff)
|
||||
return diff
|
||||
slot.extend_open_value = None
|
||||
elif structural is not None and structural.isdisjoint(raw_delta):
|
||||
return None
|
||||
|
||||
try:
|
||||
current_json = converter(slot.args, True)
|
||||
|
||||
@@ -94,9 +94,17 @@ class ParserEngineConfig:
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Called with the raw argument text when its converted JSON ends in an
|
||||
# open value; returns an extender for that value, or None. Streaming
|
||||
# skips ``arg_converter`` for every delta the extender accepts.
|
||||
# Called with the raw argument text after each streaming conversion whose
|
||||
# prefix-safe JSON extends the JSON streamed so far; returns an extender
|
||||
# when the text ends in an open value that is the last value in the
|
||||
# converted JSON, else None. While live, the extender receives every delta
|
||||
# in place of ``arg_converter`` until it returns None, meaning the delta
|
||||
# closed the value. Returned text is streamed as is, so it must match what
|
||||
# ``arg_converter`` would serialise for those raw characters. Deltas are
|
||||
# lexer output: a terminal is never split across deltas, but the lexer
|
||||
# flushes a held partial terminal as text at the end of the stream or
|
||||
# before a terminal matched by token ID, so a truncated value may end in
|
||||
# a terminal prefix. Requires ``arg_converter`` and ``stream_arg_deltas``.
|
||||
arg_value_extender: Callable[[str], ArgValueExtender | None] | None = None
|
||||
|
||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||
@@ -118,6 +126,15 @@ class ParserEngineConfig:
|
||||
# Reject tool calls whose names are absent from the request tools.
|
||||
validate_tool_names: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.arg_value_extender is not None and (
|
||||
self.arg_converter is None or not self.stream_arg_deltas
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.name}: arg_value_extender requires arg_converter "
|
||||
"and stream_arg_deltas=True"
|
||||
)
|
||||
|
||||
def terminal_literal(self, name: str) -> str | None:
|
||||
"""Canonical spelling of terminal *name*, or ``None`` if undeclared."""
|
||||
value = self.terminals.get(name)
|
||||
|
||||
+67
-12
@@ -22,6 +22,7 @@ from vllm.logger import init_logger
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
@@ -66,6 +67,12 @@ def _strip_partial_delim(value: str) -> str:
|
||||
|
||||
|
||||
def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
||||
return _parse_gemma4_fields(args_str, partial=partial)[0]
|
||||
|
||||
|
||||
def _parse_gemma4_fields(
|
||||
args_str: str, *, partial: bool = False
|
||||
) -> tuple[dict, str | None]:
|
||||
"""Parse Gemma4's custom key:value format into a Python dict.
|
||||
|
||||
Format examples::
|
||||
@@ -82,13 +89,15 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
||||
omitted because they may be incomplete and type-unstable
|
||||
(e.g. partial boolean parsed as bare string).
|
||||
|
||||
Returns a dict ready for ``json.dumps()``.
|
||||
Returns a dict ready for ``json.dumps()`` and the key of the trailing
|
||||
top-level ``STRING_DELIM`` value left unterminated, if any.
|
||||
|
||||
"""
|
||||
if not args_str or not args_str.strip():
|
||||
return {}
|
||||
return {}, None
|
||||
|
||||
result: dict = {}
|
||||
open_key: str | None = None
|
||||
i = 0
|
||||
n = len(args_str)
|
||||
|
||||
@@ -130,6 +139,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
||||
if partial:
|
||||
value = _strip_partial_delim(value)
|
||||
result[key] = value
|
||||
open_key = key
|
||||
break
|
||||
result[key] = args_str[val_start:end_pos]
|
||||
i = end_pos + _DELIM_LEN
|
||||
@@ -199,7 +209,7 @@ def _parse_gemma4_args(args_str: str, *, partial: bool = False) -> dict:
|
||||
break
|
||||
result[key] = raw_val
|
||||
|
||||
return result
|
||||
return result, open_key
|
||||
|
||||
|
||||
def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
|
||||
@@ -283,16 +293,64 @@ def _parse_gemma4_array(arr_str: str, *, partial: bool = False) -> list:
|
||||
return items
|
||||
|
||||
|
||||
def _drop_call_closer(text: str, paren_excess: bool) -> str:
|
||||
"""Drop a trailing ``}`` or, with more ``)`` than ``(``, a trailing ``)``.
|
||||
|
||||
The closer of the call body is dropped before parsing, so a value that
|
||||
itself ends in one loses that character until more text arrives.
|
||||
"""
|
||||
if text.endswith("}") or (text.endswith(")") and paren_excess):
|
||||
return text[:-1]
|
||||
return text
|
||||
|
||||
|
||||
def _gemma4_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
"""Convert Gemma4 custom arg format to a JSON string."""
|
||||
text = raw_args.strip()
|
||||
if text.endswith("}") or text.endswith(")") and text.count("(") < text.count(")"):
|
||||
text = text[:-1]
|
||||
|
||||
text = _drop_call_closer(text, text.count("(") < text.count(")"))
|
||||
parsed = _parse_gemma4_args(text, partial=partial)
|
||||
return json.dumps(parsed, ensure_ascii=False)
|
||||
|
||||
|
||||
def _gemma4_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
|
||||
"""Extend the open top-level ``STRING_DELIM`` value at the end of
|
||||
*raw_args*, if it is the last value in the converted JSON.
|
||||
|
||||
The extender streams exactly what ``_gemma4_arg_converter`` would, so it
|
||||
holds back the same tail: trailing whitespace, a dropped closer and a
|
||||
partial ``STRING_DELIM``. ``STRING_DELIM`` is a special token but not a
|
||||
lexer terminal, so unlike a tag it may reach the extender split across
|
||||
deltas; the held tail doubles as the window that detects the close.
|
||||
"""
|
||||
text = raw_args.strip()
|
||||
opens, closes = text.count("("), text.count(")")
|
||||
body = _drop_call_closer(text, opens < closes)
|
||||
params, open_key = _parse_gemma4_fields(body, partial=True)
|
||||
if open_key is None or next(reversed(params)) != open_key:
|
||||
return None
|
||||
|
||||
held_len = (
|
||||
len(raw_args)
|
||||
- len(raw_args.rstrip())
|
||||
+ len(text)
|
||||
- len(_strip_partial_delim(body))
|
||||
)
|
||||
held = raw_args[len(raw_args) - held_len :] if held_len else ""
|
||||
|
||||
def extend(delta: str) -> str | None:
|
||||
nonlocal held, opens, closes
|
||||
text = held + delta
|
||||
if STRING_DELIM in text:
|
||||
return None
|
||||
opens += delta.count("(")
|
||||
closes += delta.count(")")
|
||||
visible = _strip_partial_delim(_drop_call_closer(text.rstrip(), opens < closes))
|
||||
held = text[len(visible) :]
|
||||
return json.dumps(visible, ensure_ascii=False)[1:-1]
|
||||
|
||||
return extend
|
||||
|
||||
|
||||
@functools.cache
|
||||
def gemma4_config(thinking: bool = False) -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
@@ -396,6 +454,7 @@ def gemma4_config(thinking: bool = False) -> ParserEngineConfig:
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
arg_converter=_gemma4_arg_converter,
|
||||
arg_value_extender=_gemma4_arg_value_extender,
|
||||
tool_args_json=False,
|
||||
arg_structural_chars=frozenset(",:{}[]<"),
|
||||
preserve_tokens=frozenset({STRING_DELIM}),
|
||||
@@ -423,12 +482,8 @@ class Gemma4Parser(ParserEngine):
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("enable_thinking", False)
|
||||
super().__init__(
|
||||
tokenizer,
|
||||
tools,
|
||||
parser_engine_config=gemma4_config(thinking=thinking),
|
||||
**kwargs,
|
||||
)
|
||||
kwargs.setdefault("parser_engine_config", gemma4_config(thinking=thinking))
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
vocab = self.vocab
|
||||
self._tool_call_token_id: int | None = vocab.get("<|tool_call>")
|
||||
self._new_turn_token_id: int | None = vocab.get("<|turn>")
|
||||
|
||||
Reference in New Issue
Block a user