[Frontend] Add Streaming Parser Engine and new GLM4.7/GLM5.1/GLM5.2 Parser (#45915)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-06-19 00:42:10 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5099474633
commit 6c379b9e54
11 changed files with 542 additions and 1956 deletions
+76
View File
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
)
from vllm.parser.engine.registered_adapters import (
Gemma4Parser,
Glm47MoeParser,
MinimaxM2Parser,
NemotronV3Parser,
Qwen3Parser,
@@ -571,6 +572,80 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample:
)
# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ──────────────
_GLM47_MOE_VOCAB: dict[str, int] = {
"<think>": 50,
"</think>": 51,
"<tool_call>": 60,
"</tool_call>": 61,
"<arg_key>": 62,
"</arg_key>": 63,
"<arg_value>": 64,
"</arg_value>": 65,
}
def _glm47_moe_arg_value(value: Any) -> str:
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, (int, float)):
return str(value)
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False)
def _glm47_moe_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = [
("<tool_call>", True),
(tc.name, False),
]
for key, value in tc.arguments.items():
segs.extend(
[
("<arg_key>", True),
(key, False),
("</arg_key>", True),
("<arg_value>", True),
(_glm47_moe_arg_value(value), False),
("</arg_value>", True),
]
)
segs.append(("</tool_call>", True))
return segs
def _glm47_moe_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if scenario.reasoning is not None:
segs.append((scenario.reasoning, False))
if scenario.content is not None or scenario.tool_calls:
segs.append(("</think>", True))
if scenario.content is not None:
segs.append((scenario.content, False))
if scenario.tool_calls:
for tc in scenario.tool_calls:
segs.extend(_glm47_moe_tool_segments(tc))
return segs
def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample:
sample = _make_sample(
sample_id=f"glm47_moe-{scenario.id}",
description=scenario.description,
vocab=_GLM47_MOE_VOCAB,
segments=_glm47_moe_segments(scenario),
expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "",
expected_content=_qwen3_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
)
if validate:
_validate_sample(sample, Glm47MoeParser)
return sample
# ── Registry and public API ──────────────────────────────────────────
_BUILDERS: dict[str, Any] = {
@@ -578,6 +653,7 @@ _BUILDERS: dict[str, Any] = {
"gemma4": _build_gemma4,
"minimax_m2": _build_minimax_m2,
"nemotron_v3": _build_nemotron_v3,
"glm47_moe": _build_glm47_moe,
}
@@ -11,7 +11,7 @@ parser_name = "glm45"
start_token = "<think>"
end_token = "</think>"
REASONING_MODEL_NAME = "zai-org/GLM-4.5"
REASONING_MODEL_NAME = "zai-org/GLM-4.7"
@pytest.fixture(scope="module")
@@ -35,18 +35,32 @@ WITH_THINK_STREAM = {
WITHOUT_THINK = {
"output": "This is the rest",
"reasoning": None,
"content": "This is the rest",
"reasoning": "This is the rest",
"content": None,
"is_reasoning_end": False,
}
WITHOUT_THINK_STREAM = {
"output": "This is the rest",
"reasoning": None,
"content": "This is the rest",
"reasoning": "This is the rest",
"content": None,
"is_reasoning_end": False,
}
WITHOUT_OPEN_THINK = {
"output": "This is a reasoning section</think>This is the rest",
"reasoning": "This is a reasoning section",
"content": "This is the rest",
"is_reasoning_end": True,
}
WITHOUT_OPEN_THINK_STREAM = {
"output": "This is a reasoning section</think>This is the rest",
"reasoning": "This is a reasoning section",
"content": "This is the rest",
"is_reasoning_end": True,
}
COMPLETE_REASONING = {
"output": "<think>This is a reasoning section</think>",
"reasoning": "This is a reasoning section",
@@ -61,8 +75,8 @@ MULTILINE_REASONING = {
}
ONLY_OPEN_TAG = {
"output": "<think>This is a reasoning section",
"reasoning": None,
"content": "<think>This is a reasoning section",
"reasoning": "This is a reasoning section",
"content": None,
"is_reasoning_end": False,
}
@@ -94,6 +108,16 @@ TEST_CASES = [
WITHOUT_THINK_STREAM,
id="without_think_stream",
),
pytest.param(
False,
WITHOUT_OPEN_THINK,
id="without_open_think",
),
pytest.param(
True,
WITHOUT_OPEN_THINK_STREAM,
id="without_open_think_stream",
),
pytest.param(
False,
COMPLETE_REASONING,
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
from vllm.tokenizers import get_tokenizer
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
MODEL = "zai-org/GLM-4.5"
MODEL = "zai-org/GLM-4.7"
@pytest.fixture(scope="module")
@@ -136,9 +136,10 @@ class TestGlm47Streaming:
_reset(glm47_tool_parser)
chunks = ["<tool_call>", "get_current_date", "</tool_call>"]
current_text = ""
deltas = []
for chunk in chunks:
current_text += chunk
glm47_tool_parser.extract_tool_calls_streaming(
delta = glm47_tool_parser.extract_tool_calls_streaming(
previous_text="",
current_text=current_text,
delta_text=chunk,
@@ -147,7 +148,23 @@ class TestGlm47Streaming:
delta_token_ids=[],
request=mock_request,
)
assert len(glm47_tool_parser.prev_tool_call_arr) >= 1
if delta:
deltas.append(delta)
tool_calls = [
tool_call for delta in deltas for tool_call in (delta.tool_calls or [])
]
names = [
tool_call.function.name
for tool_call in tool_calls
if tool_call.function and tool_call.function.name
]
arguments = [
tool_call.function.arguments
for tool_call in tool_calls
if tool_call.function and tool_call.function.arguments
]
assert names == ["get_current_date"]
assert "".join(arguments) == "{}"
def test_with_args(self, glm47_tool_parser, mock_request):
_reset(glm47_tool_parser)
@@ -161,9 +178,10 @@ class TestGlm47Streaming:
"</tool_call>",
]
current_text = ""
deltas = []
for chunk in chunks:
current_text += chunk
glm47_tool_parser.extract_tool_calls_streaming(
delta = glm47_tool_parser.extract_tool_calls_streaming(
previous_text="",
current_text=current_text,
delta_text=chunk,
@@ -172,5 +190,13 @@ class TestGlm47Streaming:
delta_token_ids=[],
request=mock_request,
)
args = json.loads(glm47_tool_parser.prev_tool_call_arr[0]["arguments"])
if delta:
deltas.append(delta)
arguments = [
tool_call.function.arguments
for delta in deltas
for tool_call in (delta.tool_calls or [])
if tool_call.function and tool_call.function.arguments
]
args = json.loads("".join(arguments))
assert args["city"] == "Beijing"
File diff suppressed because it is too large Load Diff
@@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and
from vllm.parser.engine.adapters import make_adapters
from vllm.parser.gemma4 import Gemma4Parser
from vllm.parser.glm47_moe import Glm47MoeParser
from vllm.parser.minimax_m2 import MinimaxM2Parser
from vllm.parser.nemotron_v3 import NemotronV3Parser
from vllm.parser.qwen3 import Qwen3Parser
@@ -32,3 +33,8 @@ from vllm.parser.qwen3 import Qwen3Parser
Qwen3ParserReasoningAdapter,
Qwen3ParserToolAdapter,
) = make_adapters(Qwen3Parser)
(
Glm47MoeParserReasoningAdapter,
Glm47MoeParserToolAdapter,
) = make_adapters(Glm47MoeParser)
+226
View File
@@ -0,0 +1,226 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-4.7 parser for reasoning and tool calls.
GLM-4.7 uses XML-like tool calls::
<tool_call>func_name<arg_key>key</arg_key><arg_value>value</arg_value></tool_call>
The function name can be followed directly by the first ``<arg_key>`` tag,
and tool calls may have no arguments.
"""
from __future__ import annotations
import functools
import json
from typing import TYPE_CHECKING
import regex as re
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.engine.events import EventType
from vllm.parser.engine.parser_engine import ParserEngine
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
if TYPE_CHECKING:
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
THINK_START = "<think>"
THINK_END = "</think>"
TOOL_CALL_START = "<tool_call>"
TOOL_CALL_END = "</tool_call>"
ARG_KEY_START = "<arg_key>"
ARG_KEY_END = "</arg_key>"
ARG_VALUE_START = "<arg_value>"
ARG_VALUE_END = "</arg_value>"
_ARG_RE = re.compile(
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
r"<arg_value>(?P<value>.*?)</arg_value>",
re.DOTALL,
)
_PARTIAL_ARG_RE = re.compile(
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
r"<arg_value>(?P<value>.*)$",
re.DOTALL,
)
def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
params: dict[str, object] = {}
for match in _ARG_RE.finditer(raw_args):
params[match.group("key").strip()] = match.group("value")
if partial:
remaining = _ARG_RE.sub("", raw_args)
match = _PARTIAL_ARG_RE.search(remaining)
if match:
key = match.group("key").strip()
if key:
params[key] = match.group("value")
return json.dumps(params, ensure_ascii=False)
@functools.cache
def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
arg_tag_transitions = {
(ParserState.TOOL_ARGS, terminal): Transition(
ParserState.TOOL_ARGS,
(EventType.ARG_VALUE_CHUNK,),
)
for terminal in (
"ARG_KEY_START",
"ARG_KEY_END",
"ARG_VALUE_START",
"ARG_VALUE_END",
)
}
reasoning_terminals = (
{
"THINK_START": THINK_START,
"THINK_END": THINK_END,
}
if thinking
else {}
)
reasoning_token_id_terminals = (
{
"THINK_START": THINK_START,
"THINK_END": THINK_END,
}
if thinking
else {}
)
reasoning_transitions = (
{
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
(ParserState.CONTENT, "THINK_END"): Transition(
ParserState.CONTENT,
(),
),
}
if thinking
else {}
)
return ParserEngineConfig(
name="glm47_moe",
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
terminals={
**reasoning_terminals,
"TOOL_START": TOOL_CALL_START,
"TOOL_END": TOOL_CALL_END,
"ARG_KEY_START": ARG_KEY_START,
"ARG_KEY_END": ARG_KEY_END,
"ARG_VALUE_START": ARG_VALUE_START,
"ARG_VALUE_END": ARG_VALUE_END,
},
token_id_terminals={
**reasoning_token_id_terminals,
"TOOL_START": TOOL_CALL_START,
"TOOL_END": TOOL_CALL_END,
},
transitions={
**reasoning_transitions,
(ParserState.REASONING, "THINK_START"): Transition(
ParserState.REASONING,
(),
),
(ParserState.REASONING, "TOOL_START"): Transition(
ParserState.TOOL_NAME,
(EventType.REASONING_END, EventType.TOOL_CALL_START),
),
(ParserState.CONTENT, "TOOL_START"): Transition(
ParserState.TOOL_NAME,
(EventType.TOOL_CALL_START,),
),
(ParserState.TOOL_NAME, "ARG_KEY_START"): Transition(
ParserState.TOOL_ARGS,
(EventType.ARG_VALUE_CHUNK,),
),
(ParserState.TOOL_NAME, "TOOL_END"): Transition(
ParserState.CONTENT,
(EventType.TOOL_CALL_END,),
),
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
ParserState.CONTENT,
(EventType.TOOL_CALL_END,),
),
**arg_tag_transitions,
},
arg_converter=_glm47_arg_converter,
stream_arg_deltas=True,
tool_args_json=False,
validate_tool_names=True,
)
class Glm47MoeParser(ParserEngine):
"""GLM-4.7 parser backed by the declarative parser engine."""
def __init__(
self,
tokenizer: TokenizerLike,
tools: list[Tool] | None = None,
**kwargs,
) -> None:
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
thinking = chat_kwargs.get("thinking", None)
enable_thinking = chat_kwargs.get("enable_thinking", None)
self.thinking_enabled = (
True
if thinking is None and enable_thinking is None
else bool(thinking) or bool(enable_thinking)
)
kwargs.setdefault(
"parser_engine_config",
glm47_moe_config(thinking=self.thinking_enabled),
)
super().__init__(tokenizer, tools, **kwargs)
def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None:
if name is not None:
name = name.strip()
super()._emit_name_delta(idx, deltas, name)
def _handle_tool_end(self, event, deltas) -> None:
idx = event.tool_index
if 0 <= idx < len(self._tool_slots):
self._tool_slots[idx].name = self._tool_slots[idx].name.strip()
super()._handle_tool_end(event, deltas)
def is_reasoning_end(self, input_ids: list[int]) -> bool:
if not self.thinking_enabled:
return True
return super().is_reasoning_end(input_ids)
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
if not self.thinking_enabled:
return input_ids
return super().extract_content_ids(input_ids)
def extract_reasoning(
self,
model_output: str,
request: ChatCompletionRequest | ResponsesRequest,
) -> tuple[str | None, str | None]:
if not self.thinking_enabled:
return None, model_output
return super().extract_reasoning(model_output, request)
+6 -2
View File
@@ -53,8 +53,12 @@ _REASONING_PARSERS_TO_REGISTER = {
"Gemma4ParserReasoningAdapter",
),
"glm45": (
"deepseek_v3_reasoning_parser",
"DeepSeekV3ReasoningWithThinkingParser",
"glm47_moe_reasoning_parser",
"Glm47MoeParserReasoningAdapter",
),
"glm47": (
"glm47_moe_reasoning_parser",
"Glm47MoeParserReasoningAdapter",
),
"openai_gptoss": (
"gptoss_reasoning_parser",
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.parser.engine.registered_adapters import Glm47MoeParserReasoningAdapter
__all__ = ["Glm47MoeParserReasoningAdapter"]
+2 -2
View File
@@ -51,8 +51,8 @@ _TOOL_PARSERS_TO_REGISTER = {
"Ernie45ToolParser",
),
"glm45": (
"glm4_moe_tool_parser",
"Glm4MoeModelToolParser",
"glm47_moe_tool_parser",
"Glm47MoeModelToolParser",
),
"glm47": (
"glm47_moe_tool_parser",
+3 -33
View File
@@ -1,41 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
GLM-4.7 Tool Call Parser.
GLM-4.7 uses a slightly different tool call format compared to GLM-4.5:
- The function name may appear on the same line as ``<tool_call>`` without
a newline separator before the first ``<arg_key>``.
- Tool calls may have zero arguments
(e.g. ``<tool_call>func</tool_call>``).
from __future__ import annotations
This parser overrides the parent regex patterns to handle both formats.
"""
import regex as re
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
from vllm.tool_parsers.glm4_moe_tool_parser import Glm4MoeModelToolParser
logger = init_logger(__name__)
from vllm.parser.engine.registered_adapters import Glm47MoeParserToolAdapter
class Glm47MoeModelToolParser(Glm4MoeModelToolParser):
class Glm47MoeModelToolParser(Glm47MoeParserToolAdapter): # type: ignore[valid-type, misc]
supports_required_and_named = False
structural_tag_model = "glm_4_7"
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
# GLM-4.7 format: <tool_call>func_name[<arg_key>...]*</tool_call>
# The function name can be followed by a newline, whitespace, or
# directly by <arg_key> tags (no separator). The arg section is
# optional so that zero-argument calls are supported.
self.func_detail_regex = re.compile(
r"<tool_call>\s*(\S+?)\s*(<arg_key>.*)?</tool_call>", re.DOTALL
)
self.func_arg_regex = re.compile(
r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>",
re.DOTALL,
)
-495
View File
@@ -1,495 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
GLM-4 Tool Call Parser with incremental string streaming support.
This parser fixes the streaming issue reported in Issue #32829 where long string
parameters (e.g., file content with 4000+ characters of code) are buffered until
complete, causing multi-second delays before the user sees any content.
The fix streams string values incrementally as they arrive, providing a true
streaming experience for long content.
"""
import json
from collections.abc import Sequence
from typing import Any
import regex as re
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import (
Tool,
ToolParser,
)
from vllm.tool_parsers.utils import (
extract_types_from_schema,
find_tool_properties,
partial_tag_overlap,
safe_literal_eval,
)
logger = init_logger(__name__)
class Glm4MoeModelToolParser(ToolParser):
"""Tool parser for GLM-4 models with incremental string streaming.
On every streaming call the parser re-parses ``current_text`` to find
``<tool_call>`` regions, builds the JSON arguments string for each tool
call, and diffs against what was previously sent to emit only new content.
"""
supports_required_and_named = False
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)
# Stateful streaming fields
self.current_tool_name_sent: bool = False
self.prev_tool_call_arr: list[dict[str, Any]] = []
self.current_tool_id: int = -1
self.streamed_args_for_tool: list[str] = []
self.tool_call_start_token: str = "<tool_call>"
self.tool_call_end_token: str = "</tool_call>"
self.arg_key_start: str = "<arg_key>"
self.arg_key_end: str = "</arg_key>"
self.arg_val_start: str = "<arg_value>"
self.arg_val_end: str = "</arg_value>"
self.tool_calls_start_token = self.tool_call_start_token
self.func_call_regex = re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL)
self.func_detail_regex = re.compile(
r"<tool_call>([^\n]*)\n(.*)</tool_call>", re.DOTALL
)
self.func_arg_regex = re.compile(
r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.DOTALL
)
if not self.model_tokenizer:
raise ValueError(
"The model tokenizer must be passed to the ToolParser "
"constructor during construction."
)
self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
# Pre-compiled pattern for finding the last <arg_key>...</arg_key>
# before a partial <arg_value> (used in _build_args_json_so_far).
self._arg_key_pattern = re.compile(
re.escape(self.arg_key_start) + r"(.*?)" + re.escape(self.arg_key_end),
re.DOTALL,
)
# Streaming state for re-parse-and-diff approach
self._sent_content_idx: int = 0
self._tool_call_ids: list[str] = []
@staticmethod
def _deserialize(value: str) -> Any:
try:
return json.loads(value)
except json.JSONDecodeError:
pass
try:
return safe_literal_eval(value)
except (ValueError, SyntaxError):
pass
return value
@staticmethod
def _json_escape_string_content(s: str) -> str:
"""JSON-escape string content for incremental streaming.
This escapes the content that goes INSIDE a JSON string (between quotes),
not including the surrounding quotes themselves.
"""
if not s:
return ""
return json.dumps(s, ensure_ascii=False)[1:-1]
def _is_string_type(self, tool_name: str, arg_name: str) -> bool:
tool_properties = find_tool_properties(self.tools, tool_name)
param_schema = tool_properties.get(arg_name)
if param_schema is None:
return False
param_types = extract_types_from_schema(param_schema)
return set(param_types) - {"null"} == {"string"}
@staticmethod
def _tools_enabled(request: ChatCompletionRequest) -> bool:
"""Return whether tool parsing should be applied for this request."""
try:
tools = getattr(request, "tools", None)
tool_choice = getattr(request, "tool_choice", None)
return bool(tools) and tool_choice != "none"
except Exception:
logger.exception("Failed to determine if tools are enabled.")
return False
def adjust_request(
self, request: ChatCompletionRequest | ResponsesRequest
) -> ChatCompletionRequest | ResponsesRequest:
"""Adjust request parameters for tool call token handling.
For required/named tool_choice, skip setting structured_outputs
because GLM models output tool calls in XML format (per chat
template). Guided decoding would force JSON output, conflicting
with the XML format and causing parsing failures.
"""
if request.tools:
tc = request.tool_choice
if tc == "required" or isinstance(tc, ChatCompletionNamedToolChoiceParam):
# Do NOT call super().adjust_request() for required/named,
# because it would set structured_outputs and force JSON
# output via guided decoding. GLM models use XML tool-call
# syntax (defined in the chat template), so guided decoding
# must be skipped to let the model output XML freely.
# The tool_parser handles extraction from XML output.
if request.tool_choice != "none":
request.skip_special_tokens = False
return request
request = super().adjust_request(request)
if request.tools and request.tool_choice != "none":
# Ensure tool call tokens (<tool_call>, </tool_call>) are not skipped
# during decoding. Even though they are not marked as special tokens,
# setting skip_special_tokens=False ensures proper handling in
# transformers 5.x where decoding behavior may have changed.
request.skip_special_tokens = False
return request
def extract_tool_calls(
self,
model_output: str,
request: ChatCompletionRequest,
) -> ExtractedToolCallInformation:
matched_tool_calls = self.func_call_regex.findall(model_output)
logger.debug("model_output: %s", model_output)
try:
tool_calls: list[ToolCall] = []
for match in matched_tool_calls:
tc_detail = self.func_detail_regex.search(match)
if not tc_detail:
logger.warning(
"Failed to parse tool call details from: %s",
match,
)
continue
tc_name = tc_detail.group(1).strip()
tc_args = tc_detail.group(2)
pairs = self.func_arg_regex.findall(tc_args) if tc_args else []
arg_dct: dict[str, Any] = {}
for key, value in pairs:
arg_key = key.strip()
if self._is_string_type(tc_name, arg_key):
arg_val = value
else:
arg_val = self._deserialize(value.strip())
logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val)
arg_dct[arg_key] = arg_val
tool_calls.append(
ToolCall(
type="function",
function=FunctionCall(
name=tc_name,
arguments=json.dumps(arg_dct, ensure_ascii=False),
),
)
)
except Exception:
logger.exception("Failed to extract tool call spec")
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
else:
if len(tool_calls) > 0:
content: str | None = model_output[
: model_output.find(self.tool_calls_start_token)
]
# Normalize empty/whitespace-only content to None
if not content or not content.strip():
content = None
return ExtractedToolCallInformation(
tools_called=True, tool_calls=tool_calls, content=content
)
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
def _extract_content(self, current_text: str) -> str | None:
"""Return unsent non-tool-call text, or None.
Collects all text outside ``<tool_call>...</tool_call>`` regions,
including text between consecutive tool calls. Holds back any
suffix that could be a partial ``<tool_call>`` tag.
"""
# Build the "sendable index" — the furthest point we can send
# content up to. We scan through the text collecting segments
# that are outside tool-call regions.
content_segments: list[str] = []
pos = self._sent_content_idx
while pos < len(current_text):
start = current_text.find(self.tool_call_start_token, pos)
if start == -1:
# No more tool calls — send up to (len - partial-tag overlap)
tail = current_text[pos:]
overlap = partial_tag_overlap(tail, self.tool_call_start_token)
sendable = tail[: len(tail) - overlap] if overlap else tail
if sendable:
content_segments.append(sendable)
pos = len(current_text) - overlap
break
# Text before this <tool_call>
if start > pos:
content_segments.append(current_text[pos:start])
# Skip past the </tool_call> (or to end if incomplete)
end = current_text.find(self.tool_call_end_token, start)
if end != -1:
pos = end + len(self.tool_call_end_token)
else:
# Incomplete tool call — nothing more to send
pos = start
break
if content_segments:
self._sent_content_idx = pos
return "".join(content_segments)
# Even if no content, advance past completed tool-call regions
if pos > self._sent_content_idx:
self._sent_content_idx = pos
return None
def _extract_tool_call_regions(self, text: str) -> list[tuple[str, bool]]:
"""Extract ``(inner_text, is_complete)`` for each ``<tool_call>`` region."""
results: list[tuple[str, bool]] = []
pos = 0
while True:
start = text.find(self.tool_call_start_token, pos)
if start == -1:
break
inner_start = start + len(self.tool_call_start_token)
end = text.find(self.tool_call_end_token, inner_start)
if end != -1:
results.append((text[inner_start:end], True))
pos = end + len(self.tool_call_end_token)
else:
# Incomplete tool call — strip partial </tool_call> suffix
raw = text[inner_start:]
overlap = partial_tag_overlap(raw, self.tool_call_end_token)
if overlap:
raw = raw[:-overlap]
results.append((raw, False))
break
return results
def _extract_tool_name_from_region(self, inner_text: str) -> str | None:
"""Extract the tool name from the beginning of a tool-call region.
The name is everything before the first ``\\n`` or ``<arg_key>``.
Returns ``None`` if the name hasn't fully arrived yet.
"""
nl = inner_text.find("\n")
ak = inner_text.find(self.arg_key_start)
candidates = [i for i in [nl, ak] if i != -1]
if not candidates:
return None
cut = min(candidates)
name = inner_text[:cut].strip()
return name if name else None
def _build_args_json_so_far(
self,
tool_name: str,
inner_text: str,
is_complete: bool,
) -> str:
"""Build the JSON arguments string from the XML pairs seen so far.
For complete ``<arg_key>/<arg_value>`` pairs the value is fully
formatted. For the last argument whose ``<arg_value>`` has been
opened but not closed, the partial string content is included
(JSON-escaped, with an opening ``"`` but no closing ``"``).
The closing ``}`` is only appended when ``is_complete`` is True
(i.e. the ``</tool_call>`` tag has arrived).
"""
# Find all complete arg pairs
pairs = self.func_arg_regex.findall(inner_text)
parts: list[str] = []
for key, value in pairs:
key = key.strip()
key_json = json.dumps(key, ensure_ascii=False)
if self._is_string_type(tool_name, key):
# Don't strip string values — whitespace is significant
# and must match the partial-value path for diffing.
val_json = json.dumps(value, ensure_ascii=False)
else:
val_json = json.dumps(
self._deserialize(value.strip()), ensure_ascii=False
)
parts.append(f"{key_json}: {val_json}")
# Check for a partial (incomplete) arg value
# Find the last <arg_value> that isn't closed
last_val_start = inner_text.rfind(self.arg_val_start)
last_val_end = inner_text.rfind(self.arg_val_end)
has_partial_value = last_val_start != -1 and (
last_val_end == -1 or last_val_end < last_val_start
)
if has_partial_value:
# Find the key for this partial value
# Look for the last <arg_key>...</arg_key> before this <arg_value>
last_key_match = None
for m in self._arg_key_pattern.finditer(inner_text[:last_val_start]):
last_key_match = m
if last_key_match:
partial_key = last_key_match.group(1).strip()
partial_content_start = last_val_start + len(self.arg_val_start)
partial_content = inner_text[partial_content_start:]
# Hold back any partial </arg_value> suffix
overlap = partial_tag_overlap(partial_content, self.arg_val_end)
if overlap:
partial_content = partial_content[:-overlap]
key_json = json.dumps(partial_key, ensure_ascii=False)
if is_complete:
# Tool call finished but </arg_value> is missing
# (malformed output). Treat partial as complete value
# so the diff naturally closes any open quotes.
if self._is_string_type(tool_name, partial_key):
val_json = json.dumps(partial_content, ensure_ascii=False)
else:
val_json = json.dumps(
self._deserialize(partial_content.strip()),
ensure_ascii=False,
)
parts.append(f"{key_json}: {val_json}")
elif self._is_string_type(tool_name, partial_key):
escaped = self._json_escape_string_content(partial_content)
# Open quote but no close — more content may arrive
parts.append(f'{key_json}: "{escaped}')
else:
# Non-string partial: include raw content, no wrapping
parts.append(f"{key_json}: {partial_content}")
if not parts:
return "{}" if is_complete else ""
joined = "{" + ", ".join(parts)
if is_complete:
joined += "}"
return joined
def _compute_args_diff(self, index: int, args_so_far: str) -> str | None:
"""Return new argument text not yet sent for tool *index*, or None."""
if not args_so_far or len(args_so_far) <= len(
self.streamed_args_for_tool[index]
):
return None
diff = args_so_far[len(self.streamed_args_for_tool[index]) :]
self.streamed_args_for_tool[index] = args_so_far
self.prev_tool_call_arr[index]["arguments"] = args_so_far
return diff
def _ensure_tool_state_for(self, index: int) -> None:
"""Grow state arrays so that *index* is valid."""
while len(self._tool_call_ids) <= index:
self._tool_call_ids.append(
make_tool_call_id(id_type="random", func_name=None, idx=None)
)
while len(self.streamed_args_for_tool) <= index:
self.streamed_args_for_tool.append("")
while len(self.prev_tool_call_arr) <= index:
self.prev_tool_call_arr.append({})
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage | None:
if not self._tools_enabled(request):
return DeltaMessage(content=delta_text) if delta_text else None
content = self._extract_content(current_text)
regions = self._extract_tool_call_regions(current_text)
tool_call_deltas: list[DeltaToolCall] = []
for i, (inner_text, is_complete) in enumerate(regions):
self._ensure_tool_state_for(i)
# Extract tool name
tool_name = self._extract_tool_name_from_region(inner_text)
if not tool_name:
break
# Emit tool name (once per tool call)
if "name" not in self.prev_tool_call_arr[i]:
self.prev_tool_call_arr[i]["name"] = tool_name
tool_call_deltas.append(
DeltaToolCall(
index=i,
id=self._tool_call_ids[i],
type="function",
function=DeltaFunctionCall(
name=tool_name,
arguments="",
).model_dump(exclude_none=True),
)
)
# Build args JSON so far, diff, emit
args_so_far = self._build_args_json_so_far(
tool_name, inner_text, is_complete
)
diff = self._compute_args_diff(i, args_so_far)
if diff:
tool_call_deltas.append(
DeltaToolCall(
index=i,
function=DeltaFunctionCall(arguments=diff).model_dump(
exclude_none=True
),
)
)
# Update current_tool_id for serving layer compatibility
if regions:
self.current_tool_id = len(regions) - 1
if content or tool_call_deltas:
return DeltaMessage(
content=content,
tool_calls=tool_call_deltas,
)
return None