[Bugfix][Parser] Stop re-parsing the whole GLM tool argument per chunk (#61253)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
69d54eb078
commit
44f0ddfd7b
@@ -24,7 +24,11 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.parser.abstract_parser import Parser, StreamState
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ArgValueExtender,
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
from vllm.renderers.chat_utils import get_tool_call_id_type, make_tool_call_id
|
||||
from vllm.tool_parsers.utils import (
|
||||
@@ -60,7 +64,10 @@ class ToolCallSlot:
|
||||
"_args_joined",
|
||||
"name_sent",
|
||||
"string_keys",
|
||||
"streamed_json",
|
||||
"_streamed_parts",
|
||||
"_streamed_joined",
|
||||
"extend_open_value",
|
||||
"open_value_streams",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
@@ -70,7 +77,10 @@ class ToolCallSlot:
|
||||
self._args_joined: str | None = ""
|
||||
self.name_sent: bool = False
|
||||
self.string_keys: set[str] | None = None
|
||||
self.streamed_json: str = ""
|
||||
self._streamed_parts: list[str] = []
|
||||
self._streamed_joined: str | None = ""
|
||||
self.extend_open_value: ArgValueExtender | None = None
|
||||
self.open_value_streams: bool = False
|
||||
|
||||
@property
|
||||
def args(self) -> str:
|
||||
@@ -82,6 +92,21 @@ class ToolCallSlot:
|
||||
self._args_parts.append(value)
|
||||
self._args_joined = None
|
||||
|
||||
@property
|
||||
def streamed_json(self) -> str:
|
||||
if self._streamed_joined is None:
|
||||
self._streamed_joined = "".join(self._streamed_parts)
|
||||
return self._streamed_joined
|
||||
|
||||
@streamed_json.setter
|
||||
def streamed_json(self, value: str) -> None:
|
||||
self._streamed_parts = [value]
|
||||
self._streamed_joined = value
|
||||
|
||||
def append_streamed(self, value: str) -> None:
|
||||
self._streamed_parts.append(value)
|
||||
self._streamed_joined = None
|
||||
|
||||
|
||||
class ParserEngine(Parser):
|
||||
"""A :class:`Parser` backed by a single declarative engine config.
|
||||
@@ -133,6 +158,7 @@ class ParserEngine(Parser):
|
||||
|
||||
self._arg_converter = parser_engine_config.arg_converter
|
||||
self._arg_structural_chars = parser_engine_config.arg_structural_chars
|
||||
self._arg_value_extender = parser_engine_config.arg_value_extender
|
||||
self._stream_arg_deltas = parser_engine_config.stream_arg_deltas
|
||||
self._strip_trailing_reasoning_ws = (
|
||||
parser_engine_config.strip_trailing_reasoning_whitespace
|
||||
@@ -303,7 +329,9 @@ class ParserEngine(Parser):
|
||||
return args, changed
|
||||
|
||||
@staticmethod
|
||||
def _safe_arg_prefix(json_str: str, string_keys: set[str] | None = None) -> str:
|
||||
def _safe_arg_prefix(
|
||||
json_str: str, string_keys: set[str] | None = None
|
||||
) -> tuple[str, bool]:
|
||||
"""Return the prefix of *json_str* up to the last top-level value.
|
||||
|
||||
Middle values (followed by a comma) are stable across streaming
|
||||
@@ -313,6 +341,9 @@ class ParserEngine(Parser):
|
||||
String values for keys in ``string_keys`` are prefix-stable, so stream
|
||||
their unterminated content instead of buffering long arguments until
|
||||
the closing tag arrives.
|
||||
|
||||
Also returns whether the prefix ends inside that streamed trailing
|
||||
string value.
|
||||
"""
|
||||
last_colon = -1
|
||||
last_key: str | None = None
|
||||
@@ -345,14 +376,14 @@ class ParserEngine(Parser):
|
||||
last_key = pending_key
|
||||
pending_key = None
|
||||
if last_colon < 0:
|
||||
return ""
|
||||
return "", False
|
||||
end = last_colon + 1
|
||||
while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"):
|
||||
end += 1
|
||||
if end >= len(json_str) or json_str[end] != '"':
|
||||
return json_str[:end]
|
||||
return json_str[:end], False
|
||||
if string_keys is not None and last_key not in string_keys:
|
||||
return json_str[:end]
|
||||
return json_str[:end], False
|
||||
|
||||
escape = False
|
||||
for i in range(end + 1, len(json_str)):
|
||||
@@ -364,8 +395,8 @@ class ParserEngine(Parser):
|
||||
escape = True
|
||||
continue
|
||||
if c == '"':
|
||||
return json_str[:i]
|
||||
return json_str
|
||||
return json_str[:i], True
|
||||
return json_str, True
|
||||
|
||||
@staticmethod
|
||||
def _streamable_string_keys(properties: dict) -> set[str] | None:
|
||||
@@ -1027,6 +1058,15 @@ class ParserEngine(Parser):
|
||||
return None
|
||||
|
||||
slot = self._tool_slots[idx]
|
||||
if slot.extend_open_value is not None:
|
||||
diff = slot.extend_open_value(raw_delta)
|
||||
if diff is not None:
|
||||
if not diff or not slot.open_value_streams:
|
||||
return None
|
||||
slot.append_streamed(diff)
|
||||
return diff
|
||||
slot.extend_open_value = None
|
||||
|
||||
try:
|
||||
current_json = converter(slot.args, True)
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
@@ -1040,22 +1080,19 @@ class ParserEngine(Parser):
|
||||
current_json = self._fix_arg_types(current_json, slot.name)
|
||||
|
||||
prev = slot.streamed_json
|
||||
safe_json = self._safe_arg_prefix(current_json, slot.string_keys)
|
||||
safe_json, value_streams = self._safe_arg_prefix(current_json, slot.string_keys)
|
||||
|
||||
if not safe_json or safe_json == prev:
|
||||
if not safe_json:
|
||||
return None
|
||||
if prev and not safe_json.startswith(prev):
|
||||
return None
|
||||
|
||||
if prev:
|
||||
if not safe_json.startswith(prev):
|
||||
return None
|
||||
diff = safe_json[len(prev) :]
|
||||
else:
|
||||
diff = safe_json
|
||||
|
||||
if diff:
|
||||
slot.streamed_json = safe_json
|
||||
return diff
|
||||
return None
|
||||
slot.streamed_json = safe_json
|
||||
if self._arg_value_extender is not None:
|
||||
# Deltas that only extend the open value skip the conversion above.
|
||||
slot.extend_open_value = self._arg_value_extender(slot.args)
|
||||
slot.open_value_streams = value_streams
|
||||
return safe_json[len(prev) :] or None
|
||||
|
||||
def _flush_arg_converter(self, idx: int) -> str | None:
|
||||
converter = self._arg_converter
|
||||
|
||||
@@ -24,6 +24,10 @@ from functools import cached_property
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
|
||||
# Maps a raw argument delta to the JSON text it appends to the open trailing
|
||||
# value, or returns None when the delta may change anything else.
|
||||
ArgValueExtender = Callable[[str], str | None]
|
||||
|
||||
|
||||
class ParserState(Enum):
|
||||
CONTENT = auto()
|
||||
@@ -90,6 +94,11 @@ class ParserEngineConfig:
|
||||
|
||||
arg_structural_chars: frozenset[str] | None = None
|
||||
|
||||
# Called with the raw argument text when its converted JSON ends in an
|
||||
# open value; returns an extender for that value, or None. Streaming
|
||||
# skips ``arg_converter`` for every delta the extender accepts.
|
||||
arg_value_extender: Callable[[str], ArgValueExtender | None] | None = None
|
||||
|
||||
# Special tokens exempt from auto-drop but not state-machine terminals.
|
||||
preserve_tokens: frozenset[str] = field(default_factory=frozenset)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user