[Bugfix] Fix GLM tool parser streaming with MTP or stream interval (#39253)

Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-04-13 05:10:30 +00:00
committed by GitHub
parent 9dd5ee0117
commit 0e39202ca9
4 changed files with 785 additions and 413 deletions
+249 -293
View File
@@ -37,6 +37,7 @@ from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
from vllm.tool_parsers.utils import partial_tag_overlap
logger = init_logger(__name__)
@@ -44,9 +45,9 @@ logger = init_logger(__name__)
class Glm4MoeModelToolParser(ToolParser):
"""Tool parser for GLM-4 models with incremental string streaming.
This parser emits tool-call deltas incrementally as arguments arrive.
For string-type parameters, content is streamed character-by-character
rather than waiting for the complete </arg_value> tag.
On every streaming call the parser re-parses ``current_text`` to find
``<tool_call>`` regions, builds the JSON arguments string for each tool
call, and diffs against what was previously sent to emit only new content.
"""
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
@@ -82,17 +83,17 @@ class Glm4MoeModelToolParser(ToolParser):
self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
self._buffer: str = ""
# Streaming state for incremental tool-call streaming
self._in_tool_call: bool = False
self._current_tool_name: str | None = None
self._pending_key: str | None = None
self._streaming_string_value: bool = False
# Pre-compiled pattern for finding the last <arg_key>...</arg_key>
# before a partial <arg_value> (used in _build_args_json_so_far).
self._arg_key_pattern = re.compile(
re.escape(self.arg_key_start) + r"(.*?)" + re.escape(self.arg_key_end),
re.DOTALL,
)
# Streaming state for re-parse-and-diff approach
self._sent_content_idx: int = 0
self._tool_call_ids: list[str] = []
self._args_started: list[bool] = []
self._args_closed: list[bool] = []
self._seen_keys: list[set[str]] = []
@staticmethod
def _deserialize(value: str) -> Any:
@@ -222,6 +223,199 @@ class Glm4MoeModelToolParser(ToolParser):
tools_called=False, tool_calls=[], content=model_output
)
def _extract_content(self, current_text: str) -> str | None:
"""Return unsent non-tool-call text, or None.
Collects all text outside ``<tool_call>...</tool_call>`` regions,
including text between consecutive tool calls. Holds back any
suffix that could be a partial ``<tool_call>`` tag.
"""
# Build the "sendable index" — the furthest point we can send
# content up to. We scan through the text collecting segments
# that are outside tool-call regions.
content_segments: list[str] = []
pos = self._sent_content_idx
while pos < len(current_text):
start = current_text.find(self.tool_call_start_token, pos)
if start == -1:
# No more tool calls — send up to (len - partial-tag overlap)
tail = current_text[pos:]
overlap = partial_tag_overlap(tail, self.tool_call_start_token)
sendable = tail[: len(tail) - overlap] if overlap else tail
if sendable:
content_segments.append(sendable)
pos = len(current_text) - overlap
break
# Text before this <tool_call>
if start > pos:
content_segments.append(current_text[pos:start])
# Skip past the </tool_call> (or to end if incomplete)
end = current_text.find(self.tool_call_end_token, start)
if end != -1:
pos = end + len(self.tool_call_end_token)
else:
# Incomplete tool call — nothing more to send
pos = start
break
if content_segments:
self._sent_content_idx = pos
return "".join(content_segments)
# Even if no content, advance past completed tool-call regions
if pos > self._sent_content_idx:
self._sent_content_idx = pos
return None
def _extract_tool_call_regions(self, text: str) -> list[tuple[str, bool]]:
"""Extract ``(inner_text, is_complete)`` for each ``<tool_call>`` region."""
results: list[tuple[str, bool]] = []
pos = 0
while True:
start = text.find(self.tool_call_start_token, pos)
if start == -1:
break
inner_start = start + len(self.tool_call_start_token)
end = text.find(self.tool_call_end_token, inner_start)
if end != -1:
results.append((text[inner_start:end], True))
pos = end + len(self.tool_call_end_token)
else:
# Incomplete tool call — strip partial </tool_call> suffix
raw = text[inner_start:]
overlap = partial_tag_overlap(raw, self.tool_call_end_token)
if overlap:
raw = raw[:-overlap]
results.append((raw, False))
break
return results
def _extract_tool_name_from_region(self, inner_text: str) -> str | None:
"""Extract the tool name from the beginning of a tool-call region.
The name is everything before the first ``\\n`` or ``<arg_key>``.
Returns ``None`` if the name hasn't fully arrived yet.
"""
nl = inner_text.find("\n")
ak = inner_text.find(self.arg_key_start)
candidates = [i for i in [nl, ak] if i != -1]
if not candidates:
return None
cut = min(candidates)
name = inner_text[:cut].strip()
return name if name else None
def _build_args_json_so_far(
self,
tool_name: str,
inner_text: str,
is_complete: bool,
) -> str:
"""Build the JSON arguments string from the XML pairs seen so far.
For complete ``<arg_key>/<arg_value>`` pairs the value is fully
formatted. For the last argument whose ``<arg_value>`` has been
opened but not closed, the partial string content is included
(JSON-escaped, with an opening ``"`` but no closing ``"``).
The closing ``}`` is only appended when ``is_complete`` is True
(i.e. the ``</tool_call>`` tag has arrived).
"""
# Find all complete arg pairs
pairs = self.func_arg_regex.findall(inner_text)
parts: list[str] = []
for key, value in pairs:
key = key.strip()
key_json = json.dumps(key, ensure_ascii=False)
if self._is_string_type(tool_name, key, self.tools):
# Don't strip string values — whitespace is significant
# and must match the partial-value path for diffing.
val_json = json.dumps(value, ensure_ascii=False)
else:
val_json = json.dumps(
self._deserialize(value.strip()), ensure_ascii=False
)
parts.append(f"{key_json}: {val_json}")
# Check for a partial (incomplete) arg value
# Find the last <arg_value> that isn't closed
last_val_start = inner_text.rfind(self.arg_val_start)
last_val_end = inner_text.rfind(self.arg_val_end)
has_partial_value = last_val_start != -1 and (
last_val_end == -1 or last_val_end < last_val_start
)
if has_partial_value:
# Find the key for this partial value
# Look for the last <arg_key>...</arg_key> before this <arg_value>
last_key_match = None
for m in self._arg_key_pattern.finditer(inner_text[:last_val_start]):
last_key_match = m
if last_key_match:
partial_key = last_key_match.group(1).strip()
partial_content_start = last_val_start + len(self.arg_val_start)
partial_content = inner_text[partial_content_start:]
# Hold back any partial </arg_value> suffix
overlap = partial_tag_overlap(partial_content, self.arg_val_end)
if overlap:
partial_content = partial_content[:-overlap]
key_json = json.dumps(partial_key, ensure_ascii=False)
if is_complete:
# Tool call finished but </arg_value> is missing
# (malformed output). Treat partial as complete value
# so the diff naturally closes any open quotes.
if self._is_string_type(tool_name, partial_key, self.tools):
val_json = json.dumps(partial_content, ensure_ascii=False)
else:
val_json = json.dumps(
self._deserialize(partial_content.strip()),
ensure_ascii=False,
)
parts.append(f"{key_json}: {val_json}")
elif self._is_string_type(tool_name, partial_key, self.tools):
escaped = self._json_escape_string_content(partial_content)
# Open quote but no close — more content may arrive
parts.append(f'{key_json}: "{escaped}')
else:
# Non-string partial: include raw content, no wrapping
parts.append(f"{key_json}: {partial_content}")
if not parts:
return "{}" if is_complete else ""
joined = "{" + ", ".join(parts)
if is_complete:
joined += "}"
return joined
def _compute_args_diff(self, index: int, args_so_far: str) -> str | None:
"""Return new argument text not yet sent for tool *index*, or None."""
if not args_so_far or len(args_so_far) <= len(
self.streamed_args_for_tool[index]
):
return None
diff = args_so_far[len(self.streamed_args_for_tool[index]) :]
self.streamed_args_for_tool[index] = args_so_far
self.prev_tool_call_arr[index]["arguments"] = args_so_far
return diff
def _ensure_tool_state_for(self, index: int) -> None:
"""Grow state arrays so that *index* is valid."""
while len(self._tool_call_ids) <= index:
self._tool_call_ids.append(
make_tool_call_id(id_type="random", func_name=None, idx=None)
)
while len(self.streamed_args_for_tool) <= index:
self.streamed_args_for_tool.append("")
while len(self.prev_tool_call_arr) <= index:
self.prev_tool_call_arr.append({})
def extract_tool_calls_streaming(
self,
previous_text: str,
@@ -235,293 +429,55 @@ class Glm4MoeModelToolParser(ToolParser):
if not self._tools_enabled(request):
return DeltaMessage(content=delta_text) if delta_text else None
self._buffer += delta_text
content = self._extract_content(current_text)
regions = self._extract_tool_call_regions(current_text)
tool_call_deltas: list[DeltaToolCall] = []
while True:
if not self._in_tool_call:
start_idx = self._buffer.find(self.tool_call_start_token)
if start_idx == -1:
# Check for partial start token at end of buffer
for i in range(1, len(self.tool_call_start_token)):
if self._buffer.endswith(self.tool_call_start_token[:i]):
out = self._buffer[:-i]
self._buffer = self._buffer[-i:]
return DeltaMessage(content=out) if out else None
out = self._buffer
self._buffer = ""
return DeltaMessage(content=out) if out else None
for i, (inner_text, is_complete) in enumerate(regions):
self._ensure_tool_state_for(i)
if start_idx > 0:
out = self._buffer[:start_idx]
self._buffer = self._buffer[start_idx:]
return DeltaMessage(content=out) if out else None
# Extract tool name
tool_name = self._extract_tool_name_from_region(inner_text)
if not tool_name:
break
self._buffer = self._buffer[len(self.tool_call_start_token) :]
self._begin_tool_call()
continue
# Parse tool name first
if not self.current_tool_name_sent:
nl = self._buffer.find("\n")
ak = self._buffer.find(self.arg_key_start)
end = self._buffer.find(self.tool_call_end_token)
candidates = [i for i in [nl, ak, end] if i != -1]
if not candidates:
return None
cut = min(candidates)
tool_name = self._buffer[:cut].strip()
if tool_name == "" and cut == end:
# Handle empty tool call like `<tool_call></tool_call>`.
# Consume the tokens and reset state to avoid infinite loop.
self._buffer = self._buffer[end + len(self.tool_call_end_token) :]
self._finish_tool_call()
self._revert_last_tool_call_state()
continue
if cut == nl:
self._buffer = self._buffer[nl + 1 :]
else:
self._buffer = self._buffer[cut:]
self._current_tool_name = tool_name
self.current_tool_name_sent = True
return self._emit_tool_name_delta(tool_name)
assert self._current_tool_name is not None
# Handle incremental string value streaming
if self._streaming_string_value:
val_end = self._buffer.find(self.arg_val_end)
if val_end != -1:
raw_content = self._buffer[:val_end]
self._buffer = self._buffer[val_end + len(self.arg_val_end) :]
self._streaming_string_value = False
self._pending_key = None
escaped = self._json_escape_string_content(raw_content)
frag = escaped + '"'
self.streamed_args_for_tool[self.current_tool_id] += frag
return self._emit_tool_args_delta(frag)
else:
# Check for partial </arg_value> at end
safe_len = len(self._buffer)
for i in range(1, len(self.arg_val_end)):
if self._buffer.endswith(self.arg_val_end[:i]):
safe_len = len(self._buffer) - i
break
if safe_len > 0:
to_emit = self._buffer[:safe_len]
self._buffer = self._buffer[safe_len:]
escaped = self._json_escape_string_content(to_emit)
if escaped:
self.streamed_args_for_tool[self.current_tool_id] += escaped
return self._emit_tool_args_delta(escaped)
return None
# If we have a pending key, parse its value
if self._pending_key is not None:
val_pos = self._buffer.find(self.arg_val_start)
if val_pos == -1:
return None
if val_pos > 0:
self._buffer = self._buffer[val_pos:]
key = (self._pending_key or "").strip()
is_string = self._is_string_type(
self._current_tool_name, key, self.tools
# Emit tool name (once per tool call)
if "name" not in self.prev_tool_call_arr[i]:
self.prev_tool_call_arr[i]["name"] = tool_name
tool_call_deltas.append(
DeltaToolCall(
index=i,
id=self._tool_call_ids[i],
type="function",
function=DeltaFunctionCall(
name=tool_name,
arguments="",
).model_dump(exclude_none=True),
)
)
if is_string:
# String type: stream incrementally
self._buffer = self._buffer[len(self.arg_val_start) :]
if key in self._seen_keys[self.current_tool_id]:
self._pending_key = None
continue
self._seen_keys[self.current_tool_id].add(key)
key_json = json.dumps(key, ensure_ascii=False)
if not self._args_started[self.current_tool_id]:
frag = "{" + key_json + ': "'
self._args_started[self.current_tool_id] = True
else:
frag = ", " + key_json + ': "'
self.streamed_args_for_tool[self.current_tool_id] += frag
self._streaming_string_value = True
return self._emit_tool_args_delta(frag)
else:
# Non-string type: wait for complete value
val_end = self._buffer.find(self.arg_val_end)
if val_end == -1:
return None
raw_val = self._buffer[len(self.arg_val_start) : val_end].strip()
self._buffer = self._buffer[val_end + len(self.arg_val_end) :]
self._pending_key = None
frag_or_none = self._append_arg_fragment(key=key, raw_val=raw_val)
if frag_or_none:
return self._emit_tool_args_delta(frag_or_none)
continue
# Parse next arg or close
end_pos = self._buffer.find(self.tool_call_end_token)
key_pos = self._buffer.find(self.arg_key_start)
if end_pos != -1 and (key_pos == -1 or end_pos < key_pos):
self._buffer = self._buffer[end_pos + len(self.tool_call_end_token) :]
frag_or_none = self._close_args_if_needed()
# Finalize prev_tool_call_arr with complete parsed arguments
if self._current_tool_name:
try:
full_args_str = self.streamed_args_for_tool[
self.current_tool_id
]
args_dict = json.loads(full_args_str)
self.prev_tool_call_arr[self.current_tool_id] = {
"name": self._current_tool_name,
"arguments": args_dict,
}
except (json.JSONDecodeError, IndexError) as e:
logger.warning(
"Failed to finalize tool call state for tool %d: %s",
self.current_tool_id,
e,
)
self._finish_tool_call()
return (
self._emit_tool_args_delta(frag_or_none) if frag_or_none else None
)
if key_pos == -1:
return None
if key_pos > 0:
self._buffer = self._buffer[key_pos:]
key_end = self._buffer.find(self.arg_key_end)
if key_end == -1:
return None
key = self._buffer[len(self.arg_key_start) : key_end]
self._buffer = self._buffer[key_end + len(self.arg_key_end) :]
self._pending_key = key
continue
def _ensure_tool_state(self) -> None:
while len(self._tool_call_ids) <= self.current_tool_id:
self._tool_call_ids.append(
make_tool_call_id(id_type="random", func_name=None, idx=None)
# Build args JSON so far, diff, emit
args_so_far = self._build_args_json_so_far(
tool_name, inner_text, is_complete
)
while len(self.streamed_args_for_tool) <= self.current_tool_id:
self.streamed_args_for_tool.append("")
while len(self.prev_tool_call_arr) <= self.current_tool_id:
self.prev_tool_call_arr.append({})
while len(self._args_started) <= self.current_tool_id:
self._args_started.append(False)
while len(self._args_closed) <= self.current_tool_id:
self._args_closed.append(False)
while len(self._seen_keys) <= self.current_tool_id:
self._seen_keys.append(set())
def _begin_tool_call(self) -> None:
if self.current_tool_id == -1:
self.current_tool_id = 0
else:
self.current_tool_id += 1
self._ensure_tool_state()
self.current_tool_name_sent = False
self._current_tool_name = None
self._pending_key = None
self._streaming_string_value = False
self._in_tool_call = True
def _finish_tool_call(self) -> None:
self._in_tool_call = False
self._current_tool_name = None
self._pending_key = None
self._streaming_string_value = False
def _revert_last_tool_call_state(self) -> None:
"""Revert the state allocation for the last tool call."""
if self.current_tool_id < 0:
return
self._tool_call_ids.pop()
self.streamed_args_for_tool.pop()
self.prev_tool_call_arr.pop()
self._args_started.pop()
self._args_closed.pop()
self._seen_keys.pop()
self.current_tool_id -= 1
def _emit_tool_name_delta(self, tool_name: str) -> DeltaMessage:
self.prev_tool_call_arr[self.current_tool_id] = {
"name": self._current_tool_name,
"arguments": {},
}
return DeltaMessage(
tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
id=self._tool_call_ids[self.current_tool_id],
type="function",
function=DeltaFunctionCall(
name=tool_name,
arguments="",
).model_dump(exclude_none=True),
diff = self._compute_args_diff(i, args_so_far)
if diff:
tool_call_deltas.append(
DeltaToolCall(
index=i,
function=DeltaFunctionCall(arguments=diff).model_dump(
exclude_none=True
),
)
)
]
)
def _emit_tool_args_delta(self, fragment: str) -> DeltaMessage:
return DeltaMessage(
tool_calls=[
DeltaToolCall(
index=self.current_tool_id,
function=DeltaFunctionCall(arguments=fragment).model_dump(
exclude_none=True
),
)
]
)
# Update current_tool_id for serving layer compatibility
if regions:
self.current_tool_id = len(regions) - 1
def _append_arg_fragment(
self,
*,
key: str,
raw_val: str,
) -> str | None:
key = key.strip()
if not key:
return None
if key in self._seen_keys[self.current_tool_id]:
return None
# This function is only called for non-string types (already checked
# by _is_string_type in the caller), so we always deserialize.
val_obj: Any = self._deserialize(raw_val)
key_json = json.dumps(key, ensure_ascii=False)
val_json = json.dumps(val_obj, ensure_ascii=False)
if not self._args_started[self.current_tool_id]:
fragment = "{" + key_json + ": " + val_json
self._args_started[self.current_tool_id] = True
else:
fragment = "," + key_json + ": " + val_json
self._seen_keys[self.current_tool_id].add(key)
self.streamed_args_for_tool[self.current_tool_id] += fragment
return fragment
def _close_args_if_needed(self) -> str | None:
if self._args_closed[self.current_tool_id]:
return None
self._args_closed[self.current_tool_id] = True
if not self._args_started[self.current_tool_id]:
fragment = "{}"
self.streamed_args_for_tool[self.current_tool_id] = fragment
else:
fragment = "}"
self.streamed_args_for_tool[self.current_tool_id] += fragment
return fragment
if content or tool_call_deltas:
return DeltaMessage(
content=content,
tool_calls=tool_call_deltas,
)
return None
+13
View File
@@ -31,6 +31,19 @@ Tool: TypeAlias = ChatCompletionToolsParam | ResponsesTool
logger = init_logger(__name__)
def partial_tag_overlap(text: str, tag: str) -> int:
"""Length of the longest prefix of *tag* that matches a suffix of *text*.
E.g. text ending in ``"<tool_"`` returns 6 when tag is ``"<tool_call>"``.
Returns 0 when there is no overlap.
"""
max_check = min(len(tag) - 1, len(text))
for k in range(max_check, 0, -1):
if text.endswith(tag[:k]):
return k
return 0
def find_common_prefix(s1: str, s2: str) -> str:
"""
Finds a common prefix that is shared between two strings, if there is one.