[Bugfix][Parser] Stop re-parsing the whole GLM tool argument per chunk (#61253)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Maroon Ayoub
2026-10-12 21:28:35 +00:00
committed by GitHub
co-authored by Flora Feng
parent 69d54eb078
commit 44f0ddfd7b
5 changed files with 226 additions and 26 deletions
+59 -22
View File
@@ -24,7 +24,11 @@ from vllm.entrypoints.generate.base.protocol import (
from vllm.logger import init_logger
from vllm.parser.abstract_parser import Parser, StreamState
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState
from vllm.parser.engine.parser_engine_config import (
ArgValueExtender,
ParserEngineConfig,
ParserState,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
from vllm.renderers.chat_utils import get_tool_call_id_type, make_tool_call_id
from vllm.tool_parsers.utils import (
@@ -60,7 +64,10 @@ class ToolCallSlot:
"_args_joined",
"name_sent",
"string_keys",
"streamed_json",
"_streamed_parts",
"_streamed_joined",
"extend_open_value",
"open_value_streams",
)
def __init__(self) -> None:
@@ -70,7 +77,10 @@ class ToolCallSlot:
self._args_joined: str | None = ""
self.name_sent: bool = False
self.string_keys: set[str] | None = None
self.streamed_json: str = ""
self._streamed_parts: list[str] = []
self._streamed_joined: str | None = ""
self.extend_open_value: ArgValueExtender | None = None
self.open_value_streams: bool = False
@property
def args(self) -> str:
@@ -82,6 +92,21 @@ class ToolCallSlot:
self._args_parts.append(value)
self._args_joined = None
@property
def streamed_json(self) -> str:
if self._streamed_joined is None:
self._streamed_joined = "".join(self._streamed_parts)
return self._streamed_joined
@streamed_json.setter
def streamed_json(self, value: str) -> None:
self._streamed_parts = [value]
self._streamed_joined = value
def append_streamed(self, value: str) -> None:
self._streamed_parts.append(value)
self._streamed_joined = None
class ParserEngine(Parser):
"""A :class:`Parser` backed by a single declarative engine config.
@@ -133,6 +158,7 @@ class ParserEngine(Parser):
self._arg_converter = parser_engine_config.arg_converter
self._arg_structural_chars = parser_engine_config.arg_structural_chars
self._arg_value_extender = parser_engine_config.arg_value_extender
self._stream_arg_deltas = parser_engine_config.stream_arg_deltas
self._strip_trailing_reasoning_ws = (
parser_engine_config.strip_trailing_reasoning_whitespace
@@ -303,7 +329,9 @@ class ParserEngine(Parser):
return args, changed
@staticmethod
def _safe_arg_prefix(json_str: str, string_keys: set[str] | None = None) -> str:
def _safe_arg_prefix(
json_str: str, string_keys: set[str] | None = None
) -> tuple[str, bool]:
"""Return the prefix of *json_str* up to the last top-level value.
Middle values (followed by a comma) are stable across streaming
@@ -313,6 +341,9 @@ class ParserEngine(Parser):
String values for keys in ``string_keys`` are prefix-stable, so stream
their unterminated content instead of buffering long arguments until
the closing tag arrives.
Also returns whether the prefix ends inside that streamed trailing
string value.
"""
last_colon = -1
last_key: str | None = None
@@ -345,14 +376,14 @@ class ParserEngine(Parser):
last_key = pending_key
pending_key = None
if last_colon < 0:
return ""
return "", False
end = last_colon + 1
while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):
end += 1
if end >= len(json_str) or json_str[end] != '"':
return json_str[:end]
return json_str[:end], False
if string_keys is not None and last_key not in string_keys:
return json_str[:end]
return json_str[:end], False
escape = False
for i in range(end + 1, len(json_str)):
@@ -364,8 +395,8 @@ class ParserEngine(Parser):
escape = True
continue
if c == '"':
return json_str[:i]
return json_str
return json_str[:i], True
return json_str, True
@staticmethod
def _streamable_string_keys(properties: dict) -> set[str] | None:
@@ -1027,6 +1058,15 @@ class ParserEngine(Parser):
return None
slot = self._tool_slots[idx]
if slot.extend_open_value is not None:
diff = slot.extend_open_value(raw_delta)
if diff is not None:
if not diff or not slot.open_value_streams:
return None
slot.append_streamed(diff)
return diff
slot.extend_open_value = None
try:
current_json = converter(slot.args, True)
except (json.JSONDecodeError, ValueError, TypeError):
@@ -1040,22 +1080,19 @@ class ParserEngine(Parser):
current_json = self._fix_arg_types(current_json, slot.name)
prev = slot.streamed_json
safe_json = self._safe_arg_prefix(current_json, slot.string_keys)
safe_json, value_streams = self._safe_arg_prefix(current_json, slot.string_keys)
if not safe_json or safe_json == prev:
if not safe_json:
return None
if prev and not safe_json.startswith(prev):
return None
if prev:
if not safe_json.startswith(prev):
return None
diff = safe_json[len(prev) :]
else:
diff = safe_json
if diff:
slot.streamed_json = safe_json
return diff
return None
slot.streamed_json = safe_json
if self._arg_value_extender is not None:
# Deltas that only extend the open value skip the conversion above.
slot.extend_open_value = self._arg_value_extender(slot.args)
slot.open_value_streams = value_streams
return safe_json[len(prev) :] or None
def _flush_arg_converter(self, idx: int) -> str | None:
converter = self._arg_converter
@@ -24,6 +24,10 @@ from functools import cached_property
from vllm.parser.engine.events import EventType
# Maps a raw argument delta to the JSON text it appends to the open trailing
# value, or returns None when the delta may change anything else.
ArgValueExtender = Callable[[str], str | None]
class ParserState(Enum):
CONTENT = auto()
@@ -90,6 +94,11 @@ class ParserEngineConfig:
arg_structural_chars: frozenset[str] | None = None
# Called with the raw argument text when its converted JSON ends in an
# open value; returns an extender for that value, or None. Streaming
# skips ``arg_converter`` for every delta the extender accepts.
arg_value_extender: Callable[[str], ArgValueExtender | None] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
preserve_tokens: frozenset[str] = field(default_factory=frozenset)