[Frontend] Add Streaming Parser Engine and new GLM4.7/GLM5.1/GLM5.2 Parser (#45915)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5099474633
commit
6c379b9e54
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
)
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Gemma4Parser,
|
||||
Glm47MoeParser,
|
||||
MinimaxM2Parser,
|
||||
NemotronV3Parser,
|
||||
Qwen3Parser,
|
||||
@@ -571,6 +572,80 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
)
|
||||
|
||||
|
||||
# ── GLM-4.7 MoE (XML tool format, starts in REASONING) ──────────────
|
||||
|
||||
_GLM47_MOE_VOCAB: dict[str, int] = {
|
||||
"<think>": 50,
|
||||
"</think>": 51,
|
||||
"<tool_call>": 60,
|
||||
"</tool_call>": 61,
|
||||
"<arg_key>": 62,
|
||||
"</arg_key>": 63,
|
||||
"<arg_value>": 64,
|
||||
"</arg_value>": 65,
|
||||
}
|
||||
|
||||
|
||||
def _glm47_moe_arg_value(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _glm47_moe_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = [
|
||||
("<tool_call>", True),
|
||||
(tc.name, False),
|
||||
]
|
||||
for key, value in tc.arguments.items():
|
||||
segs.extend(
|
||||
[
|
||||
("<arg_key>", True),
|
||||
(key, False),
|
||||
("</arg_key>", True),
|
||||
("<arg_value>", True),
|
||||
(_glm47_moe_arg_value(value), False),
|
||||
("</arg_value>", True),
|
||||
]
|
||||
)
|
||||
segs.append(("</tool_call>", True))
|
||||
return segs
|
||||
|
||||
|
||||
def _glm47_moe_segments(scenario: Scenario) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = []
|
||||
if scenario.reasoning is not None:
|
||||
segs.append((scenario.reasoning, False))
|
||||
if scenario.content is not None or scenario.tool_calls:
|
||||
segs.append(("</think>", True))
|
||||
if scenario.content is not None:
|
||||
segs.append((scenario.content, False))
|
||||
if scenario.tool_calls:
|
||||
for tc in scenario.tool_calls:
|
||||
segs.extend(_glm47_moe_tool_segments(tc))
|
||||
return segs
|
||||
|
||||
|
||||
def _build_glm47_moe(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
sample = _make_sample(
|
||||
sample_id=f"glm47_moe-{scenario.id}",
|
||||
description=scenario.description,
|
||||
vocab=_GLM47_MOE_VOCAB,
|
||||
segments=_glm47_moe_segments(scenario),
|
||||
expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "",
|
||||
expected_content=_qwen3_expected_content(scenario),
|
||||
expected_tool_calls=_expected_tc(scenario),
|
||||
tools=_expected_tools(scenario),
|
||||
)
|
||||
if validate:
|
||||
_validate_sample(sample, Glm47MoeParser)
|
||||
return sample
|
||||
|
||||
|
||||
# ── Registry and public API ──────────────────────────────────────────
|
||||
|
||||
_BUILDERS: dict[str, Any] = {
|
||||
@@ -578,6 +653,7 @@ _BUILDERS: dict[str, Any] = {
|
||||
"gemma4": _build_gemma4,
|
||||
"minimax_m2": _build_minimax_m2,
|
||||
"nemotron_v3": _build_nemotron_v3,
|
||||
"glm47_moe": _build_glm47_moe,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ parser_name = "glm45"
|
||||
start_token = "<think>"
|
||||
end_token = "</think>"
|
||||
|
||||
REASONING_MODEL_NAME = "zai-org/GLM-4.5"
|
||||
REASONING_MODEL_NAME = "zai-org/GLM-4.7"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -35,18 +35,32 @@ WITH_THINK_STREAM = {
|
||||
|
||||
WITHOUT_THINK = {
|
||||
"output": "This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is the rest",
|
||||
"reasoning": "This is the rest",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
|
||||
WITHOUT_THINK_STREAM = {
|
||||
"output": "This is the rest",
|
||||
"reasoning": None,
|
||||
"content": "This is the rest",
|
||||
"reasoning": "This is the rest",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
|
||||
WITHOUT_OPEN_THINK = {
|
||||
"output": "This is a reasoning section</think>This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
|
||||
WITHOUT_OPEN_THINK_STREAM = {
|
||||
"output": "This is a reasoning section</think>This is the rest",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": "This is the rest",
|
||||
"is_reasoning_end": True,
|
||||
}
|
||||
|
||||
COMPLETE_REASONING = {
|
||||
"output": "<think>This is a reasoning section</think>",
|
||||
"reasoning": "This is a reasoning section",
|
||||
@@ -61,8 +75,8 @@ MULTILINE_REASONING = {
|
||||
}
|
||||
ONLY_OPEN_TAG = {
|
||||
"output": "<think>This is a reasoning section",
|
||||
"reasoning": None,
|
||||
"content": "<think>This is a reasoning section",
|
||||
"reasoning": "This is a reasoning section",
|
||||
"content": None,
|
||||
"is_reasoning_end": False,
|
||||
}
|
||||
|
||||
@@ -94,6 +108,16 @@ TEST_CASES = [
|
||||
WITHOUT_THINK_STREAM,
|
||||
id="without_think_stream",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
WITHOUT_OPEN_THINK,
|
||||
id="without_open_think",
|
||||
),
|
||||
pytest.param(
|
||||
True,
|
||||
WITHOUT_OPEN_THINK_STREAM,
|
||||
id="without_open_think_stream",
|
||||
),
|
||||
pytest.param(
|
||||
False,
|
||||
COMPLETE_REASONING,
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
|
||||
|
||||
MODEL = "zai-org/GLM-4.5"
|
||||
MODEL = "zai-org/GLM-4.7"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -136,9 +136,10 @@ class TestGlm47Streaming:
|
||||
_reset(glm47_tool_parser)
|
||||
chunks = ["<tool_call>", "get_current_date", "</tool_call>"]
|
||||
current_text = ""
|
||||
deltas = []
|
||||
for chunk in chunks:
|
||||
current_text += chunk
|
||||
glm47_tool_parser.extract_tool_calls_streaming(
|
||||
delta = glm47_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
@@ -147,7 +148,23 @@ class TestGlm47Streaming:
|
||||
delta_token_ids=[],
|
||||
request=mock_request,
|
||||
)
|
||||
assert len(glm47_tool_parser.prev_tool_call_arr) >= 1
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
tool_calls = [
|
||||
tool_call for delta in deltas for tool_call in (delta.tool_calls or [])
|
||||
]
|
||||
names = [
|
||||
tool_call.function.name
|
||||
for tool_call in tool_calls
|
||||
if tool_call.function and tool_call.function.name
|
||||
]
|
||||
arguments = [
|
||||
tool_call.function.arguments
|
||||
for tool_call in tool_calls
|
||||
if tool_call.function and tool_call.function.arguments
|
||||
]
|
||||
assert names == ["get_current_date"]
|
||||
assert "".join(arguments) == "{}"
|
||||
|
||||
def test_with_args(self, glm47_tool_parser, mock_request):
|
||||
_reset(glm47_tool_parser)
|
||||
@@ -161,9 +178,10 @@ class TestGlm47Streaming:
|
||||
"</tool_call>",
|
||||
]
|
||||
current_text = ""
|
||||
deltas = []
|
||||
for chunk in chunks:
|
||||
current_text += chunk
|
||||
glm47_tool_parser.extract_tool_calls_streaming(
|
||||
delta = glm47_tool_parser.extract_tool_calls_streaming(
|
||||
previous_text="",
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
@@ -172,5 +190,13 @@ class TestGlm47Streaming:
|
||||
delta_token_ids=[],
|
||||
request=mock_request,
|
||||
)
|
||||
args = json.loads(glm47_tool_parser.prev_tool_call_arr[0]["arguments"])
|
||||
if delta:
|
||||
deltas.append(delta)
|
||||
arguments = [
|
||||
tool_call.function.arguments
|
||||
for delta in deltas
|
||||
for tool_call in (delta.tool_calls or [])
|
||||
if tool_call.function and tool_call.function.arguments
|
||||
]
|
||||
args = json.loads("".join(arguments))
|
||||
assert args["city"] == "Beijing"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@ names so that :class:`ReasoningParserManager` and
|
||||
|
||||
from vllm.parser.engine.adapters import make_adapters
|
||||
from vllm.parser.gemma4 import Gemma4Parser
|
||||
from vllm.parser.glm47_moe import Glm47MoeParser
|
||||
from vllm.parser.minimax_m2 import MinimaxM2Parser
|
||||
from vllm.parser.nemotron_v3 import NemotronV3Parser
|
||||
from vllm.parser.qwen3 import Qwen3Parser
|
||||
@@ -32,3 +33,8 @@ from vllm.parser.qwen3 import Qwen3Parser
|
||||
Qwen3ParserReasoningAdapter,
|
||||
Qwen3ParserToolAdapter,
|
||||
) = make_adapters(Qwen3Parser)
|
||||
|
||||
(
|
||||
Glm47MoeParserReasoningAdapter,
|
||||
Glm47MoeParserToolAdapter,
|
||||
) = make_adapters(Glm47MoeParser)
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-4.7 parser for reasoning and tool calls.
|
||||
|
||||
GLM-4.7 uses XML-like tool calls::
|
||||
|
||||
<tool_call>func_name<arg_key>key</arg_key><arg_value>value</arg_value></tool_call>
|
||||
|
||||
The function name can be followed directly by the first ``<arg_key>`` tag,
|
||||
and tool calls may have no arguments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.parser_engine import ParserEngine
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
|
||||
THINK_START = "<think>"
|
||||
THINK_END = "</think>"
|
||||
TOOL_CALL_START = "<tool_call>"
|
||||
TOOL_CALL_END = "</tool_call>"
|
||||
ARG_KEY_START = "<arg_key>"
|
||||
ARG_KEY_END = "</arg_key>"
|
||||
ARG_VALUE_START = "<arg_value>"
|
||||
ARG_VALUE_END = "</arg_value>"
|
||||
|
||||
_ARG_RE = re.compile(
|
||||
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
|
||||
r"<arg_value>(?P<value>.*?)</arg_value>",
|
||||
re.DOTALL,
|
||||
)
|
||||
_PARTIAL_ARG_RE = re.compile(
|
||||
r"<arg_key>(?P<key>.*?)</arg_key>\s*"
|
||||
r"<arg_value>(?P<value>.*)$",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _glm47_arg_converter(raw_args: str, partial: bool) -> str:
|
||||
params: dict[str, object] = {}
|
||||
|
||||
for match in _ARG_RE.finditer(raw_args):
|
||||
params[match.group("key").strip()] = match.group("value")
|
||||
|
||||
if partial:
|
||||
remaining = _ARG_RE.sub("", raw_args)
|
||||
match = _PARTIAL_ARG_RE.search(remaining)
|
||||
if match:
|
||||
key = match.group("key").strip()
|
||||
if key:
|
||||
params[key] = match.group("value")
|
||||
|
||||
return json.dumps(params, ensure_ascii=False)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def glm47_moe_config(thinking: bool = True) -> ParserEngineConfig:
|
||||
arg_tag_transitions = {
|
||||
(ParserState.TOOL_ARGS, terminal): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.ARG_VALUE_CHUNK,),
|
||||
)
|
||||
for terminal in (
|
||||
"ARG_KEY_START",
|
||||
"ARG_KEY_END",
|
||||
"ARG_VALUE_START",
|
||||
"ARG_VALUE_END",
|
||||
)
|
||||
}
|
||||
|
||||
reasoning_terminals = (
|
||||
{
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
reasoning_token_id_terminals = (
|
||||
{
|
||||
"THINK_START": THINK_START,
|
||||
"THINK_END": THINK_END,
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
reasoning_transitions = (
|
||||
{
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
(ParserState.CONTENT, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
}
|
||||
if thinking
|
||||
else {}
|
||||
)
|
||||
|
||||
return ParserEngineConfig(
|
||||
name="glm47_moe",
|
||||
initial_state=ParserState.REASONING if thinking else ParserState.CONTENT,
|
||||
terminals={
|
||||
**reasoning_terminals,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
"ARG_KEY_START": ARG_KEY_START,
|
||||
"ARG_KEY_END": ARG_KEY_END,
|
||||
"ARG_VALUE_START": ARG_VALUE_START,
|
||||
"ARG_VALUE_END": ARG_VALUE_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
**reasoning_token_id_terminals,
|
||||
"TOOL_START": TOOL_CALL_START,
|
||||
"TOOL_END": TOOL_CALL_END,
|
||||
},
|
||||
transitions={
|
||||
**reasoning_transitions,
|
||||
(ParserState.REASONING, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(),
|
||||
),
|
||||
(ParserState.REASONING, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.REASONING_END, EventType.TOOL_CALL_START),
|
||||
),
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "ARG_KEY_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.ARG_VALUE_CHUNK,),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
**arg_tag_transitions,
|
||||
},
|
||||
arg_converter=_glm47_arg_converter,
|
||||
stream_arg_deltas=True,
|
||||
tool_args_json=False,
|
||||
validate_tool_names=True,
|
||||
)
|
||||
|
||||
|
||||
class Glm47MoeParser(ParserEngine):
|
||||
"""GLM-4.7 parser backed by the declarative parser engine."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
tools: list[Tool] | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {}
|
||||
thinking = chat_kwargs.get("thinking", None)
|
||||
enable_thinking = chat_kwargs.get("enable_thinking", None)
|
||||
self.thinking_enabled = (
|
||||
True
|
||||
if thinking is None and enable_thinking is None
|
||||
else bool(thinking) or bool(enable_thinking)
|
||||
)
|
||||
kwargs.setdefault(
|
||||
"parser_engine_config",
|
||||
glm47_moe_config(thinking=self.thinking_enabled),
|
||||
)
|
||||
super().__init__(tokenizer, tools, **kwargs)
|
||||
|
||||
def _emit_name_delta(self, idx: int, deltas, name: str | None) -> None:
|
||||
if name is not None:
|
||||
name = name.strip()
|
||||
super()._emit_name_delta(idx, deltas, name)
|
||||
|
||||
def _handle_tool_end(self, event, deltas) -> None:
|
||||
idx = event.tool_index
|
||||
if 0 <= idx < len(self._tool_slots):
|
||||
self._tool_slots[idx].name = self._tool_slots[idx].name.strip()
|
||||
super()._handle_tool_end(event, deltas)
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
if not self.thinking_enabled:
|
||||
return True
|
||||
return super().is_reasoning_end(input_ids)
|
||||
|
||||
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
|
||||
if not self.thinking_enabled:
|
||||
return input_ids
|
||||
return super().extract_content_ids(input_ids)
|
||||
|
||||
def extract_reasoning(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not self.thinking_enabled:
|
||||
return None, model_output
|
||||
return super().extract_reasoning(model_output, request)
|
||||
@@ -53,8 +53,12 @@ _REASONING_PARSERS_TO_REGISTER = {
|
||||
"Gemma4ParserReasoningAdapter",
|
||||
),
|
||||
"glm45": (
|
||||
"deepseek_v3_reasoning_parser",
|
||||
"DeepSeekV3ReasoningWithThinkingParser",
|
||||
"glm47_moe_reasoning_parser",
|
||||
"Glm47MoeParserReasoningAdapter",
|
||||
),
|
||||
"glm47": (
|
||||
"glm47_moe_reasoning_parser",
|
||||
"Glm47MoeParserReasoningAdapter",
|
||||
),
|
||||
"openai_gptoss": (
|
||||
"gptoss_reasoning_parser",
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.parser.engine.registered_adapters import Glm47MoeParserReasoningAdapter
|
||||
|
||||
__all__ = ["Glm47MoeParserReasoningAdapter"]
|
||||
@@ -51,8 +51,8 @@ _TOOL_PARSERS_TO_REGISTER = {
|
||||
"Ernie45ToolParser",
|
||||
),
|
||||
"glm45": (
|
||||
"glm4_moe_tool_parser",
|
||||
"Glm4MoeModelToolParser",
|
||||
"glm47_moe_tool_parser",
|
||||
"Glm47MoeModelToolParser",
|
||||
),
|
||||
"glm47": (
|
||||
"glm47_moe_tool_parser",
|
||||
|
||||
@@ -1,41 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
GLM-4.7 Tool Call Parser.
|
||||
|
||||
GLM-4.7 uses a slightly different tool call format compared to GLM-4.5:
|
||||
- The function name may appear on the same line as ``<tool_call>`` without
|
||||
a newline separator before the first ``<arg_key>``.
|
||||
- Tool calls may have zero arguments
|
||||
(e.g. ``<tool_call>func</tool_call>``).
|
||||
from __future__ import annotations
|
||||
|
||||
This parser overrides the parent regex patterns to handle both formats.
|
||||
"""
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import Tool
|
||||
from vllm.tool_parsers.glm4_moe_tool_parser import Glm4MoeModelToolParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
from vllm.parser.engine.registered_adapters import Glm47MoeParserToolAdapter
|
||||
|
||||
|
||||
class Glm47MoeModelToolParser(Glm4MoeModelToolParser):
|
||||
class Glm47MoeModelToolParser(Glm47MoeParserToolAdapter): # type: ignore[valid-type, misc]
|
||||
supports_required_and_named = False
|
||||
structural_tag_model = "glm_4_7"
|
||||
|
||||
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
|
||||
super().__init__(tokenizer, tools)
|
||||
# GLM-4.7 format: <tool_call>func_name[<arg_key>...]*</tool_call>
|
||||
# The function name can be followed by a newline, whitespace, or
|
||||
# directly by <arg_key> tags (no separator). The arg section is
|
||||
# optional so that zero-argument calls are supported.
|
||||
self.func_detail_regex = re.compile(
|
||||
r"<tool_call>\s*(\S+?)\s*(<arg_key>.*)?</tool_call>", re.DOTALL
|
||||
)
|
||||
self.func_arg_regex = re.compile(
|
||||
r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
@@ -1,495 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
GLM-4 Tool Call Parser with incremental string streaming support.
|
||||
|
||||
This parser fixes the streaming issue reported in Issue #32829 where long string
|
||||
parameters (e.g., file content with 4000+ characters of code) are buffered until
|
||||
complete, causing multi-second delays before the user sees any content.
|
||||
|
||||
The fix streams string values incrementally as they arrive, providing a true
|
||||
streaming experience for long content.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
import regex as re
|
||||
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
ExtractedToolCallInformation,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import (
|
||||
Tool,
|
||||
ToolParser,
|
||||
)
|
||||
from vllm.tool_parsers.utils import (
|
||||
extract_types_from_schema,
|
||||
find_tool_properties,
|
||||
partial_tag_overlap,
|
||||
safe_literal_eval,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class Glm4MoeModelToolParser(ToolParser):
|
||||
"""Tool parser for GLM-4 models with incremental string streaming.
|
||||
|
||||
On every streaming call the parser re-parses ``current_text`` to find
|
||||
``<tool_call>`` regions, builds the JSON arguments string for each tool
|
||||
call, and diffs against what was previously sent to emit only new content.
|
||||
"""
|
||||
|
||||
supports_required_and_named = False
|
||||
|
||||
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
|
||||
super().__init__(tokenizer, tools)
|
||||
# Stateful streaming fields
|
||||
self.current_tool_name_sent: bool = False
|
||||
self.prev_tool_call_arr: list[dict[str, Any]] = []
|
||||
self.current_tool_id: int = -1
|
||||
self.streamed_args_for_tool: list[str] = []
|
||||
|
||||
self.tool_call_start_token: str = "<tool_call>"
|
||||
self.tool_call_end_token: str = "</tool_call>"
|
||||
self.arg_key_start: str = "<arg_key>"
|
||||
self.arg_key_end: str = "</arg_key>"
|
||||
self.arg_val_start: str = "<arg_value>"
|
||||
self.arg_val_end: str = "</arg_value>"
|
||||
|
||||
self.tool_calls_start_token = self.tool_call_start_token
|
||||
|
||||
self.func_call_regex = re.compile(r"<tool_call>.*?</tool_call>", re.DOTALL)
|
||||
self.func_detail_regex = re.compile(
|
||||
r"<tool_call>([^\n]*)\n(.*)</tool_call>", re.DOTALL
|
||||
)
|
||||
self.func_arg_regex = re.compile(
|
||||
r"<arg_key>(.*?)</arg_key>\s*<arg_value>(.*?)</arg_value>", re.DOTALL
|
||||
)
|
||||
|
||||
if not self.model_tokenizer:
|
||||
raise ValueError(
|
||||
"The model tokenizer must be passed to the ToolParser "
|
||||
"constructor during construction."
|
||||
)
|
||||
|
||||
self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token)
|
||||
self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token)
|
||||
|
||||
# Pre-compiled pattern for finding the last <arg_key>...</arg_key>
|
||||
# before a partial <arg_value> (used in _build_args_json_so_far).
|
||||
self._arg_key_pattern = re.compile(
|
||||
re.escape(self.arg_key_start) + r"(.*?)" + re.escape(self.arg_key_end),
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
# Streaming state for re-parse-and-diff approach
|
||||
self._sent_content_idx: int = 0
|
||||
self._tool_call_ids: list[str] = []
|
||||
|
||||
@staticmethod
|
||||
def _deserialize(value: str) -> Any:
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
return safe_literal_eval(value)
|
||||
except (ValueError, SyntaxError):
|
||||
pass
|
||||
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _json_escape_string_content(s: str) -> str:
|
||||
"""JSON-escape string content for incremental streaming.
|
||||
|
||||
This escapes the content that goes INSIDE a JSON string (between quotes),
|
||||
not including the surrounding quotes themselves.
|
||||
"""
|
||||
if not s:
|
||||
return ""
|
||||
return json.dumps(s, ensure_ascii=False)[1:-1]
|
||||
|
||||
def _is_string_type(self, tool_name: str, arg_name: str) -> bool:
|
||||
tool_properties = find_tool_properties(self.tools, tool_name)
|
||||
param_schema = tool_properties.get(arg_name)
|
||||
if param_schema is None:
|
||||
return False
|
||||
param_types = extract_types_from_schema(param_schema)
|
||||
return set(param_types) - {"null"} == {"string"}
|
||||
|
||||
@staticmethod
|
||||
def _tools_enabled(request: ChatCompletionRequest) -> bool:
|
||||
"""Return whether tool parsing should be applied for this request."""
|
||||
try:
|
||||
tools = getattr(request, "tools", None)
|
||||
tool_choice = getattr(request, "tool_choice", None)
|
||||
return bool(tools) and tool_choice != "none"
|
||||
except Exception:
|
||||
logger.exception("Failed to determine if tools are enabled.")
|
||||
return False
|
||||
|
||||
def adjust_request(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> ChatCompletionRequest | ResponsesRequest:
|
||||
"""Adjust request parameters for tool call token handling.
|
||||
|
||||
For required/named tool_choice, skip setting structured_outputs
|
||||
because GLM models output tool calls in XML format (per chat
|
||||
template). Guided decoding would force JSON output, conflicting
|
||||
with the XML format and causing parsing failures.
|
||||
"""
|
||||
if request.tools:
|
||||
tc = request.tool_choice
|
||||
if tc == "required" or isinstance(tc, ChatCompletionNamedToolChoiceParam):
|
||||
# Do NOT call super().adjust_request() for required/named,
|
||||
# because it would set structured_outputs and force JSON
|
||||
# output via guided decoding. GLM models use XML tool-call
|
||||
# syntax (defined in the chat template), so guided decoding
|
||||
# must be skipped to let the model output XML freely.
|
||||
# The tool_parser handles extraction from XML output.
|
||||
if request.tool_choice != "none":
|
||||
request.skip_special_tokens = False
|
||||
return request
|
||||
request = super().adjust_request(request)
|
||||
if request.tools and request.tool_choice != "none":
|
||||
# Ensure tool call tokens (<tool_call>, </tool_call>) are not skipped
|
||||
# during decoding. Even though they are not marked as special tokens,
|
||||
# setting skip_special_tokens=False ensures proper handling in
|
||||
# transformers 5.x where decoding behavior may have changed.
|
||||
request.skip_special_tokens = False
|
||||
return request
|
||||
|
||||
def extract_tool_calls(
|
||||
self,
|
||||
model_output: str,
|
||||
request: ChatCompletionRequest,
|
||||
) -> ExtractedToolCallInformation:
|
||||
matched_tool_calls = self.func_call_regex.findall(model_output)
|
||||
logger.debug("model_output: %s", model_output)
|
||||
try:
|
||||
tool_calls: list[ToolCall] = []
|
||||
for match in matched_tool_calls:
|
||||
tc_detail = self.func_detail_regex.search(match)
|
||||
if not tc_detail:
|
||||
logger.warning(
|
||||
"Failed to parse tool call details from: %s",
|
||||
match,
|
||||
)
|
||||
continue
|
||||
tc_name = tc_detail.group(1).strip()
|
||||
tc_args = tc_detail.group(2)
|
||||
pairs = self.func_arg_regex.findall(tc_args) if tc_args else []
|
||||
arg_dct: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
arg_key = key.strip()
|
||||
if self._is_string_type(tc_name, arg_key):
|
||||
arg_val = value
|
||||
else:
|
||||
arg_val = self._deserialize(value.strip())
|
||||
logger.debug("arg_key = %s, arg_val = %s", arg_key, arg_val)
|
||||
arg_dct[arg_key] = arg_val
|
||||
tool_calls.append(
|
||||
ToolCall(
|
||||
type="function",
|
||||
function=FunctionCall(
|
||||
name=tc_name,
|
||||
arguments=json.dumps(arg_dct, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to extract tool call spec")
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=model_output
|
||||
)
|
||||
else:
|
||||
if len(tool_calls) > 0:
|
||||
content: str | None = model_output[
|
||||
: model_output.find(self.tool_calls_start_token)
|
||||
]
|
||||
# Normalize empty/whitespace-only content to None
|
||||
if not content or not content.strip():
|
||||
content = None
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=True, tool_calls=tool_calls, content=content
|
||||
)
|
||||
return ExtractedToolCallInformation(
|
||||
tools_called=False, tool_calls=[], content=model_output
|
||||
)
|
||||
|
||||
def _extract_content(self, current_text: str) -> str | None:
|
||||
"""Return unsent non-tool-call text, or None.
|
||||
|
||||
Collects all text outside ``<tool_call>...</tool_call>`` regions,
|
||||
including text between consecutive tool calls. Holds back any
|
||||
suffix that could be a partial ``<tool_call>`` tag.
|
||||
"""
|
||||
# Build the "sendable index" — the furthest point we can send
|
||||
# content up to. We scan through the text collecting segments
|
||||
# that are outside tool-call regions.
|
||||
content_segments: list[str] = []
|
||||
pos = self._sent_content_idx
|
||||
|
||||
while pos < len(current_text):
|
||||
start = current_text.find(self.tool_call_start_token, pos)
|
||||
if start == -1:
|
||||
# No more tool calls — send up to (len - partial-tag overlap)
|
||||
tail = current_text[pos:]
|
||||
overlap = partial_tag_overlap(tail, self.tool_call_start_token)
|
||||
sendable = tail[: len(tail) - overlap] if overlap else tail
|
||||
if sendable:
|
||||
content_segments.append(sendable)
|
||||
pos = len(current_text) - overlap
|
||||
break
|
||||
|
||||
# Text before this <tool_call>
|
||||
if start > pos:
|
||||
content_segments.append(current_text[pos:start])
|
||||
|
||||
# Skip past the </tool_call> (or to end if incomplete)
|
||||
end = current_text.find(self.tool_call_end_token, start)
|
||||
if end != -1:
|
||||
pos = end + len(self.tool_call_end_token)
|
||||
else:
|
||||
# Incomplete tool call — nothing more to send
|
||||
pos = start
|
||||
break
|
||||
|
||||
if content_segments:
|
||||
self._sent_content_idx = pos
|
||||
return "".join(content_segments)
|
||||
# Even if no content, advance past completed tool-call regions
|
||||
if pos > self._sent_content_idx:
|
||||
self._sent_content_idx = pos
|
||||
return None
|
||||
|
||||
def _extract_tool_call_regions(self, text: str) -> list[tuple[str, bool]]:
|
||||
"""Extract ``(inner_text, is_complete)`` for each ``<tool_call>`` region."""
|
||||
results: list[tuple[str, bool]] = []
|
||||
pos = 0
|
||||
while True:
|
||||
start = text.find(self.tool_call_start_token, pos)
|
||||
if start == -1:
|
||||
break
|
||||
inner_start = start + len(self.tool_call_start_token)
|
||||
end = text.find(self.tool_call_end_token, inner_start)
|
||||
if end != -1:
|
||||
results.append((text[inner_start:end], True))
|
||||
pos = end + len(self.tool_call_end_token)
|
||||
else:
|
||||
# Incomplete tool call — strip partial </tool_call> suffix
|
||||
raw = text[inner_start:]
|
||||
overlap = partial_tag_overlap(raw, self.tool_call_end_token)
|
||||
if overlap:
|
||||
raw = raw[:-overlap]
|
||||
results.append((raw, False))
|
||||
break
|
||||
return results
|
||||
|
||||
def _extract_tool_name_from_region(self, inner_text: str) -> str | None:
|
||||
"""Extract the tool name from the beginning of a tool-call region.
|
||||
|
||||
The name is everything before the first ``\\n`` or ``<arg_key>``.
|
||||
Returns ``None`` if the name hasn't fully arrived yet.
|
||||
"""
|
||||
nl = inner_text.find("\n")
|
||||
ak = inner_text.find(self.arg_key_start)
|
||||
candidates = [i for i in [nl, ak] if i != -1]
|
||||
if not candidates:
|
||||
return None
|
||||
cut = min(candidates)
|
||||
name = inner_text[:cut].strip()
|
||||
return name if name else None
|
||||
|
||||
def _build_args_json_so_far(
|
||||
self,
|
||||
tool_name: str,
|
||||
inner_text: str,
|
||||
is_complete: bool,
|
||||
) -> str:
|
||||
"""Build the JSON arguments string from the XML pairs seen so far.
|
||||
|
||||
For complete ``<arg_key>/<arg_value>`` pairs the value is fully
|
||||
formatted. For the last argument whose ``<arg_value>`` has been
|
||||
opened but not closed, the partial string content is included
|
||||
(JSON-escaped, with an opening ``"`` but no closing ``"``).
|
||||
|
||||
The closing ``}`` is only appended when ``is_complete`` is True
|
||||
(i.e. the ``</tool_call>`` tag has arrived).
|
||||
"""
|
||||
# Find all complete arg pairs
|
||||
pairs = self.func_arg_regex.findall(inner_text)
|
||||
|
||||
parts: list[str] = []
|
||||
for key, value in pairs:
|
||||
key = key.strip()
|
||||
key_json = json.dumps(key, ensure_ascii=False)
|
||||
if self._is_string_type(tool_name, key):
|
||||
# Don't strip string values — whitespace is significant
|
||||
# and must match the partial-value path for diffing.
|
||||
val_json = json.dumps(value, ensure_ascii=False)
|
||||
else:
|
||||
val_json = json.dumps(
|
||||
self._deserialize(value.strip()), ensure_ascii=False
|
||||
)
|
||||
parts.append(f"{key_json}: {val_json}")
|
||||
|
||||
# Check for a partial (incomplete) arg value
|
||||
# Find the last <arg_value> that isn't closed
|
||||
last_val_start = inner_text.rfind(self.arg_val_start)
|
||||
last_val_end = inner_text.rfind(self.arg_val_end)
|
||||
has_partial_value = last_val_start != -1 and (
|
||||
last_val_end == -1 or last_val_end < last_val_start
|
||||
)
|
||||
|
||||
if has_partial_value:
|
||||
# Find the key for this partial value
|
||||
# Look for the last <arg_key>...</arg_key> before this <arg_value>
|
||||
last_key_match = None
|
||||
for m in self._arg_key_pattern.finditer(inner_text[:last_val_start]):
|
||||
last_key_match = m
|
||||
|
||||
if last_key_match:
|
||||
partial_key = last_key_match.group(1).strip()
|
||||
partial_content_start = last_val_start + len(self.arg_val_start)
|
||||
partial_content = inner_text[partial_content_start:]
|
||||
|
||||
# Hold back any partial </arg_value> suffix
|
||||
overlap = partial_tag_overlap(partial_content, self.arg_val_end)
|
||||
if overlap:
|
||||
partial_content = partial_content[:-overlap]
|
||||
|
||||
key_json = json.dumps(partial_key, ensure_ascii=False)
|
||||
if is_complete:
|
||||
# Tool call finished but </arg_value> is missing
|
||||
# (malformed output). Treat partial as complete value
|
||||
# so the diff naturally closes any open quotes.
|
||||
if self._is_string_type(tool_name, partial_key):
|
||||
val_json = json.dumps(partial_content, ensure_ascii=False)
|
||||
else:
|
||||
val_json = json.dumps(
|
||||
self._deserialize(partial_content.strip()),
|
||||
ensure_ascii=False,
|
||||
)
|
||||
parts.append(f"{key_json}: {val_json}")
|
||||
elif self._is_string_type(tool_name, partial_key):
|
||||
escaped = self._json_escape_string_content(partial_content)
|
||||
# Open quote but no close — more content may arrive
|
||||
parts.append(f'{key_json}: "{escaped}')
|
||||
else:
|
||||
# Non-string partial: include raw content, no wrapping
|
||||
parts.append(f"{key_json}: {partial_content}")
|
||||
|
||||
if not parts:
|
||||
return "{}" if is_complete else ""
|
||||
|
||||
joined = "{" + ", ".join(parts)
|
||||
if is_complete:
|
||||
joined += "}"
|
||||
return joined
|
||||
|
||||
def _compute_args_diff(self, index: int, args_so_far: str) -> str | None:
|
||||
"""Return new argument text not yet sent for tool *index*, or None."""
|
||||
if not args_so_far or len(args_so_far) <= len(
|
||||
self.streamed_args_for_tool[index]
|
||||
):
|
||||
return None
|
||||
diff = args_so_far[len(self.streamed_args_for_tool[index]) :]
|
||||
self.streamed_args_for_tool[index] = args_so_far
|
||||
self.prev_tool_call_arr[index]["arguments"] = args_so_far
|
||||
return diff
|
||||
|
||||
def _ensure_tool_state_for(self, index: int) -> None:
|
||||
"""Grow state arrays so that *index* is valid."""
|
||||
while len(self._tool_call_ids) <= index:
|
||||
self._tool_call_ids.append(
|
||||
make_tool_call_id(id_type="random", func_name=None, idx=None)
|
||||
)
|
||||
while len(self.streamed_args_for_tool) <= index:
|
||||
self.streamed_args_for_tool.append("")
|
||||
while len(self.prev_tool_call_arr) <= index:
|
||||
self.prev_tool_call_arr.append({})
|
||||
|
||||
def extract_tool_calls_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
current_text: str,
|
||||
delta_text: str,
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
) -> DeltaMessage | None:
|
||||
if not self._tools_enabled(request):
|
||||
return DeltaMessage(content=delta_text) if delta_text else None
|
||||
|
||||
content = self._extract_content(current_text)
|
||||
regions = self._extract_tool_call_regions(current_text)
|
||||
tool_call_deltas: list[DeltaToolCall] = []
|
||||
|
||||
for i, (inner_text, is_complete) in enumerate(regions):
|
||||
self._ensure_tool_state_for(i)
|
||||
|
||||
# Extract tool name
|
||||
tool_name = self._extract_tool_name_from_region(inner_text)
|
||||
if not tool_name:
|
||||
break
|
||||
|
||||
# Emit tool name (once per tool call)
|
||||
if "name" not in self.prev_tool_call_arr[i]:
|
||||
self.prev_tool_call_arr[i]["name"] = tool_name
|
||||
tool_call_deltas.append(
|
||||
DeltaToolCall(
|
||||
index=i,
|
||||
id=self._tool_call_ids[i],
|
||||
type="function",
|
||||
function=DeltaFunctionCall(
|
||||
name=tool_name,
|
||||
arguments="",
|
||||
).model_dump(exclude_none=True),
|
||||
)
|
||||
)
|
||||
|
||||
# Build args JSON so far, diff, emit
|
||||
args_so_far = self._build_args_json_so_far(
|
||||
tool_name, inner_text, is_complete
|
||||
)
|
||||
diff = self._compute_args_diff(i, args_so_far)
|
||||
if diff:
|
||||
tool_call_deltas.append(
|
||||
DeltaToolCall(
|
||||
index=i,
|
||||
function=DeltaFunctionCall(arguments=diff).model_dump(
|
||||
exclude_none=True
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# Update current_tool_id for serving layer compatibility
|
||||
if regions:
|
||||
self.current_tool_id = len(regions) - 1
|
||||
|
||||
if content or tool_call_deltas:
|
||||
return DeltaMessage(
|
||||
content=content,
|
||||
tool_calls=tool_call_deltas,
|
||||
)
|
||||
return None
|
||||
Reference in New Issue
Block a user