[Perf][Parser] Stream Qwen3 tool argument values incrementally (12 KB call: 2.1 s to 11 ms) (#62027)

Signed-off-by: Afonso Ribeiro <afonso.diamantino2@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>

tests/parser/engine/test_qwen3.py

@@ -6,6 +6,7 @@ These validate that the engine-driven parser correctly handles
 Qwen3 XML-style tool calls.
 """
 
+import dataclasses
 import json
 from unittest.mock import MagicMock
 
@@ -808,6 +809,136 @@ class TestArgConverter:
         assert result == {"city": "Tokyo", "expr": "x<5"}
 
 
+WRITE_CALL_HEAD = (
+    "<tool_call>\n<function=write_file>\n"
+    "<parameter=path>\n/repo/app.py\n</parameter>\n"
+    "<parameter=mode>\n644\n</parameter>\n"
+    "<parameter=content>\n"
+)
+WRITE_CALL_TAIL = "\n</parameter>\n</function>\n</tool_call>"
+# Escapes, non-ASCII, blank lines and tag-like text that must stay inside
+# the value, including prefixes of the closing and next-parameter tags.
+CONTENT_LINE = 'if a < b > c: print("x\\y")  # </p> </param <parameter\té\n\n'
+
+
+class TestArgValueExtender:
+    """The open parameter value streams without re-running the converter."""
+
+    @pytest.fixture
+    def write_tool(self):
+        from vllm.entrypoints.openai.chat_completion.protocol import (
+            ChatCompletionToolsParam,
+        )
+
+        return ChatCompletionToolsParam(
+            type="function",
+            function={
+                "name": "write_file",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "path": {"type": "string"},
+                        "mode": {"type": "integer"},
+                        "content": {"type": "string"},
+                    },
+                },
+            },
+        )
+
+    @pytest.mark.parametrize("use_schema", [False, True], ids=["no_schema", "schema"])
+    @pytest.mark.parametrize("parser_name", ["qwen3", "mimo"])
+    def test_long_call_streams_non_streaming_arguments(
+        self, mock_tokenizer, mock_request, write_tool, use_schema, parser_name
+    ):
+        from vllm.parser.mimo import MiMoParser
+        from vllm.parser.qwen3 import Qwen3Parser
+
+        cls = MiMoParser if parser_name == "mimo" else Qwen3Parser
+        mock_request.tools = [write_tool] if use_schema else []
+        text = WRITE_CALL_HEAD + CONTENT_LINE * 200 + WRITE_CALL_TAIL
+
+        def make_parser():
+            return cls(mock_tokenizer, chat_template_kwargs={"enable_thinking": False})
+
+        results = simulate_tool_streaming(
+            make_parser(),
+            mock_request,
+            [text[i : i + 7] for i in range(0, len(text), 7)],
+        )
+        result = make_parser().extract_tool_calls(text, mock_request)
+
+        assert collect_tool_arguments(results) == (
+            result.tool_calls[0].function.arguments
+        )
+        # MiMo keeps the newlines around the value; the schema coerces both.
+        mode = '"\\n644\\n"' if parser_name == "mimo" else '"644"'
+        if use_schema:
+            mode = "644"
+        assert f'"mode": {mode},' in collect_tool_arguments(results)
+
+    def test_conversions_do_not_grow_with_argument_length(
+        self, mock_tokenizer, mock_request, write_tool
+    ):
+        mock_request.tools = [write_tool]
+        base = qwen3_config(thinking=False)
+        converter = base.arg_converter
+        assert converter is not None
+
+        def streaming_conversions(lines: int) -> int:
+            calls = 0
+
+            def counting_converter(raw_args: str, partial: bool) -> str:
+                nonlocal calls
+                calls += partial
+                return converter(raw_args, partial)
+
+            parser = ParserEngine(
+                mock_tokenizer,
+                parser_engine_config=dataclasses.replace(
+                    base, arg_converter=counting_converter
+                ),
+            )
+            text = WRITE_CALL_HEAD + CONTENT_LINE * lines + WRITE_CALL_TAIL
+            simulate_tool_streaming(
+                parser,
+                mock_request,
+                [text[i : i + 4] for i in range(0, len(text), 4)],
+            )
+            return calls
+
+        assert streaming_conversions(lines=20) == streaming_conversions(lines=200)
+
+    def test_extender_follows_newline_trimming_and_split_end(self):
+        from vllm.parser.qwen3 import _qwen3_arg_value_extender
+
+        extend = _qwen3_arg_value_extender("<parameter=k>")
+        assert extend is not None
+        deltas = ["\n", 'a"\n', "b", "\n", "</param", "eter>"]
+        expected = ["", 'a\\"', "\\nb", "", "\\n</param", None]
+        assert [extend(d) for d in deltas] == expected
+
+    def test_extender_stops_at_split_next_parameter(self):
+        from vllm.parser.qwen3 import _qwen3_arg_value_extender
+
+        extend = _qwen3_arg_value_extender("<parameter=k>x")
+        assert extend is not None
+        assert [extend(d) for d in ["< para", "meter ="]] == ["< para", None]
+
+    @pytest.mark.parametrize(
+        "raw_args",
+        [
+            "<parameter=k>x</parameter>",
+            "<parameter=k",
+            "<parameter=k>1</parameter><parameter=j>2</parameter><parameter=k>3",
+        ],
+        ids=["closed_value", "open_name", "repeated_key_not_last"],
+    )
+    def test_no_extender_unless_open_value_is_last(self, raw_args):
+        from vllm.parser.qwen3 import _qwen3_arg_value_extender
+
+        assert _qwen3_arg_value_extender(raw_args) is None
+
+
 class TestSchemaAwareTypeCoercion:
     """Verify that _fix_arg_types corrects miscoerced values using the
     tool schema."""

vllm/parser/mimo.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """MiMo's compact XML parameters contain verbatim string values."""
 
+import functools
 import json
 from dataclasses import replace
 
@@ -10,6 +11,7 @@ from vllm.parser.qwen3 import (
     _PARTIAL_PARAM_RE,
     CHATML_TURN_BOUNDARIES,
     Qwen3Parser,
+    _param_value_extender,
     qwen3_config,
 )
 
@@ -37,6 +39,9 @@ class MiMoParser(Qwen3Parser):
                     turn_boundary_tokens=CHATML_TURN_BOUNDARIES,
                 ),
                 arg_converter=_mimo_arg_converter,
+                arg_value_extender=functools.partial(
+                    _param_value_extender, trim_newlines=False
+                ),
             ),
         )
         super().__init__(tokenizer, tools, **kwargs)

vllm/parser/qwen3.py

@@ -25,6 +25,7 @@ import regex as re
 from vllm.parser.engine.events import EventType
 from vllm.parser.engine.parser_engine import ParserEngine
 from vllm.parser.engine.parser_engine_config import (
+    ArgValueExtender,
     ParserEngineConfig,
     ParserState,
     Transition,
@@ -86,6 +87,64 @@ def _qwen3_arg_converter(raw_args: str, partial: bool) -> str:
     return json.dumps(params, ensure_ascii=False)
 
 
+# What ends an open value in ``_PARAM_RE``: a closing tag or the next opener.
+_PARAM_VALUE_END_RE = re.compile(r"<\s*/\s*parameter\s*>|<\s*parameter\s*=")
+
+
+def _param_value_end_window(text: str) -> str:
+    """Return the suffix of *text* that may still grow into a value end."""
+    start = text.rfind("<")
+    if start == -1:
+        return ""
+    tail = text[start:]
+    return tail if _PARAM_VALUE_END_RE.fullmatch(tail, partial=True) else ""
+
+
+def _param_value_extender(
+    raw_args: str, trim_newlines: bool
+) -> ArgValueExtender | None:
+    """Extend the open trailing ``<parameter=...>`` value of *raw_args*.
+
+    The extender streams exactly what the converter would for the same text.
+    With *trim_newlines* it follows ``_trim_wrapping_newlines``: it drops a
+    leading newline of the value and holds back a trailing one.
+    """
+    match = _PARTIAL_PARAM_RE.search(_PARAM_RE.sub("", raw_args))
+    if match is None:
+        return None
+    name, value = match.group(1), match.group(2)
+    # A repeated key keeps its first position, so its value is not last.
+    if any(m.group(1) == name for m in _PARAM_RE.finditer(raw_args)):
+        return None
+
+    window = _param_value_end_window(value)
+    started = bool(value)
+    held = "\n" if trim_newlines and value.endswith("\n") and value != "\n" else ""
+
+    def extend(delta: str) -> str | None:
+        nonlocal window, started, held
+        text = window + delta
+        if _PARAM_VALUE_END_RE.search(text):
+            return None
+        window = _param_value_end_window(text)
+        if trim_newlines:
+            text = held + delta
+            if not started and text:
+                started = True
+                text = text.removeprefix("\n")
+            held = "\n" if text.endswith("\n") else ""
+            text = text[: len(text) - len(held)]
+        else:
+            text = delta
+        return json.dumps(text, ensure_ascii=False)[1:-1]
+
+    return extend
+
+
+def _qwen3_arg_value_extender(raw_args: str) -> ArgValueExtender | None:
+    return _param_value_extender(raw_args, trim_newlines=True)
+
+
 @functools.cache
 def qwen3_config(
     thinking: bool = True,
@@ -223,6 +282,7 @@ def qwen3_config(
             ),
         },
         arg_converter=_qwen3_arg_converter,
+        arg_value_extender=_qwen3_arg_value_extender,
         stream_arg_deltas=True,
         strip_trailing_reasoning_whitespace=False,
         tool_args_json=False,