[Bugfix][Parser] Stop re-parsing the whole GLM tool argument per chunk (#61253)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Maroon Ayoub
2026-10-12 21:28:35 +00:00
committed by GitHub
co-authored by Flora Feng
parent 69d54eb078
commit 44f0ddfd7b
5 changed files with 226 additions and 26 deletions
+59 -22
View File
@@ -24,7 +24,11 @@ from vllm.entrypoints.generate.base.protocol import (
from vllm.logger import init_logger
from vllm.parser.abstract_parser import Parser, StreamState
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState
from vllm.parser.engine.parser_engine_config import (
ArgValueExtender,
ParserEngineConfig,
ParserState,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
from vllm.renderers.chat_utils import get_tool_call_id_type, make_tool_call_id
from vllm.tool_parsers.utils import (
@@ -60,7 +64,10 @@ class ToolCallSlot:
"_args_joined",
"name_sent",
"string_keys",
"streamed_json",
"_streamed_parts",
"_streamed_joined",
"extend_open_value",
"open_value_streams",
)
def __init__(self) -> None:
@@ -70,7 +77,10 @@ class ToolCallSlot:
self._args_joined: str | None = ""
self.name_sent: bool = False
self.string_keys: set[str] | None = None
self.streamed_json: str = ""
self._streamed_parts: list[str] = []
self._streamed_joined: str | None = ""
self.extend_open_value: ArgValueExtender | None = None
self.open_value_streams: bool = False
@property
def args(self) -> str:
@@ -82,6 +92,21 @@ class ToolCallSlot:
self._args_parts.append(value)
self._args_joined = None
@property
def streamed_json(self) -> str:
if self._streamed_joined is None:
self._streamed_joined = "".join(self._streamed_parts)
return self._streamed_joined
@streamed_json.setter
def streamed_json(self, value: str) -> None:
self._streamed_parts = [value]
self._streamed_joined = value
def append_streamed(self, value: str) -> None:
self._streamed_parts.append(value)
self._streamed_joined = None
class ParserEngine(Parser):
"""A :class:`Parser` backed by a single declarative engine config.
@@ -133,6 +158,7 @@ class ParserEngine(Parser):
self._arg_converter = parser_engine_config.arg_converter
self._arg_structural_chars = parser_engine_config.arg_structural_chars
self._arg_value_extender = parser_engine_config.arg_value_extender
self._stream_arg_deltas = parser_engine_config.stream_arg_deltas
self._strip_trailing_reasoning_ws = (
parser_engine_config.strip_trailing_reasoning_whitespace
@@ -303,7 +329,9 @@ class ParserEngine(Parser):
return args, changed
@staticmethod
def _safe_arg_prefix(json_str: str, string_keys: set[str] | None = None) -> str:
def _safe_arg_prefix(
json_str: str, string_keys: set[str] | None = None
) -> tuple[str, bool]:
"""Return the prefix of *json_str* up to the last top-level value.
Middle values (followed by a comma) are stable across streaming
@@ -313,6 +341,9 @@ class ParserEngine(Parser):
String values for keys in ``string_keys`` are prefix-stable, so stream
their unterminated content instead of buffering long arguments until
the closing tag arrives.
Also returns whether the prefix ends inside that streamed trailing
string value.
"""
last_colon = -1
last_key: str | None = None
@@ -345,14 +376,14 @@ class ParserEngine(Parser):
last_key = pending_key
pending_key = None
if last_colon < 0:
return ""
return "", False
end = last_colon + 1
while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):
end += 1
if end >= len(json_str) or json_str[end] != '"':
return json_str[:end]
return json_str[:end], False
if string_keys is not None and last_key not in string_keys:
return json_str[:end]
return json_str[:end], False
escape = False
for i in range(end + 1, len(json_str)):
@@ -364,8 +395,8 @@ class ParserEngine(Parser):
escape = True
continue
if c == '"':
return json_str[:i]
return json_str
return json_str[:i], True
return json_str, True
@staticmethod
def _streamable_string_keys(properties: dict) -> set[str] | None:
@@ -1027,6 +1058,15 @@ class ParserEngine(Parser):
return None
slot = self._tool_slots[idx]
if slot.extend_open_value is not None:
diff = slot.extend_open_value(raw_delta)
if diff is not None:
if not diff or not slot.open_value_streams:
return None
slot.append_streamed(diff)
return diff
slot.extend_open_value = None
try:
current_json = converter(slot.args, True)
except (json.JSONDecodeError, ValueError, TypeError):
@@ -1040,22 +1080,19 @@ class ParserEngine(Parser):
current_json = self._fix_arg_types(current_json, slot.name)
prev = slot.streamed_json
safe_json = self._safe_arg_prefix(current_json, slot.string_keys)
safe_json, value_streams = self._safe_arg_prefix(current_json, slot.string_keys)
if not safe_json or safe_json == prev:
if not safe_json:
return None
if prev and not safe_json.startswith(prev):
return None
if prev:
if not safe_json.startswith(prev):
return None
diff = safe_json[len(prev) :]
else:
diff = safe_json
if diff:
slot.streamed_json = safe_json
return diff
return None
slot.streamed_json = safe_json
if self._arg_value_extender is not None:
# Deltas that only extend the open value skip the conversion above.
slot.extend_open_value = self._arg_value_extender(slot.args)
slot.open_value_streams = value_streams
return safe_json[len(prev) :] or None
def _flush_arg_converter(self, idx: int) -> str | None:
converter = self._arg_converter
@@ -24,6 +24,10 @@ from functools import cached_property
from vllm.parser.engine.events import EventType
# Maps a raw argument delta to the JSON text it appends to the open trailing
# value, or returns None when the delta may change anything else.
ArgValueExtender = Callable[[str], str | None]
class ParserState(Enum):
CONTENT = auto()
@@ -90,6 +94,11 @@ class ParserEngineConfig:
arg_structural_chars: frozenset[str] | None = None
# Called with the raw argument text when its converted JSON ends in an
# open value; returns an extender for that value, or None. Streaming
# skips ``arg_converter`` for every delta the extender accepts.
arg_value_extender: Callable[[str], ArgValueExtender | None] | None = None
# Special tokens exempt from auto-drop but not state-machine terminals.
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
+34 -2
View File
@@ -24,6 +24,7 @@ from vllm.logger import init_logger
from vllm.parser.engine.events import EventType
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import (
ArgValueExtender,
ParserEngineConfig,
ParserState,
Transition,
@@ -63,12 +64,16 @@ _PARTIAL_ARG_RE = re.compile(
)
def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
def _glm47_parse_args(
raw_args: str, partial: bool
) -> tuple[dict[str, object], str | None]:
"""Return the arguments and the key of the open trailing value, if any."""
params: dict[str, object] = {}
for match in _ARG_RE.finditer(raw_args):
params[match.group("key").strip()] = match.group("value")
open_key = None
if partial:
remaining = _ARG_RE.sub("", raw_args)
match = _PARTIAL_ARG_RE.search(remaining)
@@ -76,8 +81,34 @@ def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
key = match.group("key").strip()
if key:
params[key] = match.group("value")
open_key = key
return json.dumps(params, ensure_ascii=False)
return params, open_key
def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
return json.dumps(_glm47_parse_args(raw_args, partial)[0], ensure_ascii=False)
def _glm47_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
# The open value runs to the end of raw_args and is the last JSON value,
# so a delta appends its JSON-escaped text unless it closes the value.
params, open_key = _glm47_parse_args(raw_args, partial=True)
if open_key is None or next(reversed(params)) != open_key:
return None
keep = len(ARG_VALUE_END) - 1
tail = raw_args[-keep:]
def extend(delta: str) -> str | None:
nonlocal tail
text = tail + delta
if ARG_VALUE_END in text:
return None
tail = text[-keep:]
return json.dumps(delta, ensure_ascii=False)[1:-1]
return extend
def _glm53_always_thinks(tokenizer: TokenizerLike) -> bool:
@@ -194,6 +225,7 @@ def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
},
turn_boundary_tokens=GLM_TURN_BOUNDARIES,
arg_converter=_glm47_arg_converter,
arg_value_extender=_glm47_arg_value_extender,
stream_arg_deltas=True,
tool_args_json=False,
validate_tool_names=True,