From c4a3f9d137099023a93fbb2299f3c52477751176 Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Sun, 14 Jun 2026 23:59:05 -0400 Subject: [PATCH] [Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413) Signed-off-by: Ben Browning Co-authored-by: Flora Feng <4florafeng@gmail.com> --- tests/parser/engine/__init__.py | 0 tests/parser/engine/conftest.py | 40 + tests/parser/engine/replay_harness.py | 377 +++++ tests/parser/engine/streaming_helpers.py | 137 ++ tests/parser/engine/test_delegating_replay.py | 82 + tests/parser/engine/test_engine.py | 846 ++++++++++ tests/parser/engine/test_parser_engine.py | 1356 +++++++++++++++++ tests/parser/engine/test_qwen3.py | 1095 +++++++++++++ tests/parser/engine/test_qwen3_reasoning.py | 549 +++++++ tests/parser/engine/test_replay.py | 189 +++ tests/parser/engine/test_token_id_scanner.py | 631 ++++++++ tests/parser/engine/trace_builder.py | 410 +++++ .../test_qwen3coder_tool_parser.py | 158 +- .../test_structural_tag_registry.py | 8 +- vllm/parser/abstract_parser.py | 161 +- vllm/parser/engine/__init__.py | 17 + vllm/parser/engine/adapters.py | 199 +++ vllm/parser/engine/events.py | 26 + vllm/parser/engine/incremental_lexer.py | 210 +++ vllm/parser/engine/parser_engine.py | 969 ++++++++++++ vllm/parser/engine/parser_engine_config.py | 114 ++ vllm/parser/engine/registered_adapters.py | 16 + vllm/parser/engine/streaming_parser_engine.py | 408 +++++ vllm/parser/engine/token_id_scanner.py | 309 ++++ vllm/parser/qwen3.py | 218 +++ vllm/reasoning/__init__.py | 12 +- vllm/reasoning/abs_reasoning_parsers.py | 17 +- .../qwen3_engine_reasoning_parser.py | 6 + vllm/reasoning/qwen3_reasoning_parser.py | 231 --- vllm/tool_parsers/__init__.py | 12 +- vllm/tool_parsers/abstract_tool_parser.py | 1 + vllm/tool_parsers/qwen3_engine_tool_parser.py | 8 + vllm/tool_parsers/qwen3coder_tool_parser.py | 586 ------- 33 files changed, 8494 insertions(+), 904 deletions(-) create mode 100644 tests/parser/engine/__init__.py create mode 100644 tests/parser/engine/conftest.py create mode 100644 tests/parser/engine/replay_harness.py create mode 100644 tests/parser/engine/streaming_helpers.py create mode 100644 tests/parser/engine/test_delegating_replay.py create mode 100644 tests/parser/engine/test_engine.py create mode 100644 tests/parser/engine/test_parser_engine.py create mode 100644 tests/parser/engine/test_qwen3.py create mode 100644 tests/parser/engine/test_qwen3_reasoning.py create mode 100644 tests/parser/engine/test_replay.py create mode 100644 tests/parser/engine/test_token_id_scanner.py create mode 100644 tests/parser/engine/trace_builder.py create mode 100644 vllm/parser/engine/__init__.py create mode 100644 vllm/parser/engine/adapters.py create mode 100644 vllm/parser/engine/events.py create mode 100644 vllm/parser/engine/incremental_lexer.py create mode 100644 vllm/parser/engine/parser_engine.py create mode 100644 vllm/parser/engine/parser_engine_config.py create mode 100644 vllm/parser/engine/registered_adapters.py create mode 100644 vllm/parser/engine/streaming_parser_engine.py create mode 100644 vllm/parser/engine/token_id_scanner.py create mode 100644 vllm/parser/qwen3.py create mode 100644 vllm/reasoning/qwen3_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/qwen3_reasoning_parser.py create mode 100644 vllm/tool_parsers/qwen3_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/qwen3coder_tool_parser.py diff --git a/tests/parser/engine/__init__.py b/tests/parser/engine/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/parser/engine/conftest.py b/tests/parser/engine/conftest.py new file mode 100644 index 0000000000..47a2ad0b7d --- /dev/null +++ b/tests/parser/engine/conftest.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from unittest.mock import MagicMock + +import pytest + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) + + +@pytest.fixture() +def should_do_global_cleanup_after_test() -> bool: + return False + + +def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock: + """Create a mock tokenizer with the given special-token vocabulary. + + The returned mock supports get_vocab(), encode(), and decode(). + decode() maps known token IDs back to their text and falls back to + chr(id) for ASCII IDs or ```` for others. + """ + id_to_text = {v: k for k, v in vocab.items()} + tokenizer = MagicMock() + tokenizer.encode.return_value = [1, 2, 3] + tokenizer.get_vocab.return_value = dict(vocab) + tokenizer.decode.side_effect = lambda ids: "".join( + id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids + ) + return tokenizer + + +@pytest.fixture +def mock_request(): + req = MagicMock(spec=ChatCompletionRequest) + req.tools = [] + req.tool_choice = "auto" + return req diff --git a/tests/parser/engine/replay_harness.py b/tests/parser/engine/replay_harness.py new file mode 100644 index 0000000000..240d1ac18c --- /dev/null +++ b/tests/parser/engine/replay_harness.py @@ -0,0 +1,377 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Data-driven replay harness for parser engine testing. + +Replays token sequences through parsers at different chunk sizes to +verify chunk-size invariance: the same token sequence must produce +identical output regardless of how tokens are batched. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from dataclasses import dataclass, field + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) +from vllm.entrypoints.openai.engine.protocol import DeltaMessage + + +@dataclass +class Sample: + """One test sample loaded from a JSONL file.""" + + id: str + description: str + source: str + vocab: dict[str, int] + tokens: list[tuple[int, str]] + expected_reasoning: str | None + expected_content: str | None + expected_tool_calls: list[dict] | None + tools: list[dict] | None = None + chat_template_kwargs: dict | None = None + + +@dataclass +class ParseOutput: + """Accumulated parse output from replaying a token stream.""" + + reasoning: str = "" + content: str = "" + tool_calls: list[dict] = field(default_factory=list) + + +class MockTokenizer: + """Lightweight tokenizer mock that avoids unittest.mock overhead. + + Used by ``benchmarks/benchmark_parsers.py`` in tight timing loops, + so hot-path methods (``decode``, ``get_vocab``) must be cheap. + MagicMock's call-recording machinery added ~40% overhead to small- + sample benchmarks, inflating the per-token cost of the parser engine. + """ + + __slots__ = ( + "_vocab", + "_token_ids", + "_token_decode_map", + "_special_ids", + "eos_token_id", + "bos_token_id", + "pad_token_id", + ) + + def __init__( + self, + vocab: dict[str, int], + tokens: list[tuple[int, str]], + ) -> None: + self._vocab = vocab + self._token_ids = [tid for tid, _ in tokens] + self._token_decode_map = {tid: text for tid, text in tokens} + self._special_ids = set(vocab.values()) + self.eos_token_id = None + self.bos_token_id = None + self.pad_token_id = None + + def set_vocab(self, vocab: dict[str, int]) -> None: + self._vocab = vocab + + def get_vocab(self) -> dict[str, int]: + return self._vocab + + def encode(self, text: str, **kwargs) -> list[int]: + return self._token_ids + + def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str: + parts: list[str] = [] + for tid in ids: + if skip_special_tokens and tid in self._special_ids: + continue + text = self._token_decode_map.get(tid, f"?{tid}?") + parts.append(text) + return "".join(parts) + + +def make_mock_tokenizer(sample: Sample) -> MockTokenizer: + """Build a mock tokenizer from a sample's vocab and token data.""" + return MockTokenizer( + vocab=dict(sample.vocab), + tokens=sample.tokens, + ) + + +def _test_request( + tools: list[dict] | None = None, +) -> ChatCompletionRequest: + return ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "test"}], + tools=tools, + ) + + +def replay_streaming( + parser, + tokens: list[tuple[int, str]], + chunk_size: int | None = None, + holdback_chars: int = 0, + finished_on_last: bool = False, + tools: list[dict] | None = None, +) -> list[DeltaMessage | None]: + """Feed tokens through ``parser.parse_delta()`` at a given chunk size. + + Args: + parser: A :class:`Parser` instance with ``parse_delta()`` method. + tokens: List of ``(token_id, decoded_text)`` pairs. + chunk_size: Number of tokens per batch. ``None`` means all at once. + holdback_chars: Simulate detokenizer holdback by holding back + this many characters of decoded text between batches. + finished_on_last: When True, pass ``finished=True`` on the last + ``parse_delta()`` call, matching real server behavior. + tools: Optional tool definitions to include on the request, + matching the serving layer where tools set + ``tool_choice`` to ``"auto"``. + + Returns: + List of ``DeltaMessage`` results from each ``parse_delta()`` call. + """ + if chunk_size is None: + chunk_size = len(tokens) + + results: list[DeltaMessage | None] = [] + all_ids = [tid for tid, _ in tokens] + all_texts = [text for _, text in tokens] + + request = _test_request(tools=tools) + + if holdback_chars <= 0: + chunks = list(range(0, len(tokens), chunk_size)) + for i, start in enumerate(chunks): + batch_end = min(start + chunk_size, len(tokens)) + batch_ids = all_ids[start:batch_end] + delta_text = "".join(all_texts[start:batch_end]) + is_last = i == len(chunks) - 1 + + result = parser.parse_delta( + delta_text, + batch_ids, + request, + prompt_token_ids=[] if start == 0 else None, + finished=finished_on_last and is_last, + ) + results.append(result) + return results + + emitted_up_to = 0 + is_first = True + + for start in range(0, len(tokens), chunk_size): + batch_end = min(start + chunk_size, len(tokens)) + + if batch_end < len(tokens): + held_chars = 0 + safe_end = batch_end + while safe_end > emitted_up_to and held_chars < holdback_chars: + safe_end -= 1 + held_chars += len(all_texts[safe_end]) + else: + safe_end = batch_end + + if safe_end <= emitted_up_to: + continue + + batch_ids = all_ids[emitted_up_to:safe_end] + delta_text = "".join(all_texts[emitted_up_to:safe_end]) + emitted_up_to = safe_end + + is_last_chunk = batch_end >= len(tokens) + result = parser.parse_delta( + delta_text, + batch_ids, + request, + prompt_token_ids=[] if is_first else None, + finished=finished_on_last and is_last_chunk, + ) + results.append(result) + is_first = False + + if emitted_up_to < len(tokens): + batch_ids = all_ids[emitted_up_to:] + delta_text = "".join(all_texts[emitted_up_to:]) + result = parser.parse_delta( + delta_text, + batch_ids, + request, + prompt_token_ids=[] if is_first else None, + finished=finished_on_last, + ) + results.append(result) + + return results + + +def replay_with_text_holdback( + parser, + tokens: list[tuple[int, str]], + text_delay: int = 1, + tools: list[dict] | None = None, +) -> list[DeltaMessage | None]: + """Replay token-by-token with text arriving *text_delay* steps late. + + Simulates the production detokenizer holdback where token IDs arrive + immediately but decoded text is delayed. On the last token all + remaining held-back text is flushed, matching real server behavior:: + + step 0: ids=[tok0], text="" (held back) + step 1: ids=[tok1], text=tok0_text (tok0 released) + ... + step N-1: ids=[tokN-1], text=remaining_texts (flush all) + + This exercises the TokenIDScanner deferred-terminal path that + ``replay_streaming`` (which keeps text and IDs aligned) does not. + """ + results: list[DeltaMessage | None] = [] + request = _test_request(tools=tools) + + n = len(tokens) + held_texts: list[str] = [] + + for i in range(n): + token_id = tokens[i][0] + held_texts.append(tokens[i][1]) + + is_last = i == n - 1 + if is_last: + delta_text = "".join(held_texts) + held_texts.clear() + elif len(held_texts) > text_delay: + delta_text = held_texts.pop(0) + else: + delta_text = "" + + result = parser.parse_delta( + delta_text, + [token_id], + request, + prompt_token_ids=[] if i == 0 else None, + finished=is_last, + ) + results.append(result) + + return results + + +def accumulate_deltas( + deltas: Sequence[DeltaMessage | None], +) -> dict: + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + tool_calls_by_idx: dict[int, dict] = {} + + for delta in deltas: + if delta is None: + continue + if delta.reasoning: + reasoning_parts.append(delta.reasoning) + if delta.content: + content_parts.append(delta.content) + if delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.name: + existing = tool_calls_by_idx.get(tc.index) + if existing is None: + tool_calls_by_idx[tc.index] = { + "name": tc.function.name, + "_args_parts": [tc.function.arguments or ""], + } + else: + existing["_args_parts"].append(tc.function.arguments or "") + elif tc.function and tc.function.arguments: + existing = tool_calls_by_idx.get(tc.index) + if existing is not None: + existing["_args_parts"].append(tc.function.arguments) + + return { + "reasoning": "".join(reasoning_parts), + "content": "".join(content_parts), + "tool_calls": [ + {"name": tc["name"], "arguments": "".join(tc["_args_parts"])} + for tc in tool_calls_by_idx.values() + ], + } + + +def collect_output(results: list[DeltaMessage | None]) -> ParseOutput: + """Accumulate ``DeltaMessage`` results into a :class:`ParseOutput`.""" + result = accumulate_deltas(results) + return ParseOutput( + reasoning=result["reasoning"], + content=result["content"], + tool_calls=result["tool_calls"], + ) + + +def assert_parse_output(actual: ParseOutput, sample: Sample) -> None: + """Compare actual parse output against expected values from a sample.""" + if sample.expected_reasoning is not None: + assert actual.reasoning == sample.expected_reasoning, ( + f"Reasoning mismatch:\n" + f" expected: {sample.expected_reasoning!r}\n" + f" actual: {actual.reasoning!r}" + ) + + if sample.expected_content is not None: + assert actual.content == sample.expected_content, ( + f"Content mismatch:\n" + f" expected: {sample.expected_content!r}\n" + f" actual: {actual.content!r}" + ) + if sample.expected_tool_calls is not None: + assert len(actual.tool_calls) == len(sample.expected_tool_calls), ( + f"Tool call count mismatch: " + f"expected {len(sample.expected_tool_calls)}, " + f"got {len(actual.tool_calls)}" + ) + for i, (expected_tc, actual_tc) in enumerate( + zip(sample.expected_tool_calls, actual.tool_calls) + ): + assert actual_tc["name"] == expected_tc["name"], ( + f"Tool call {i} name mismatch: " + f"expected {expected_tc['name']!r}, " + f"got {actual_tc['name']!r}" + ) + if "arguments" in expected_tc: + expected_args = expected_tc["arguments"] + actual_args_str = actual_tc.get("arguments", "{}") + if isinstance(expected_args, dict): + try: + actual_args = json.loads(actual_args_str) + except json.JSONDecodeError as e: + raise AssertionError( + f"Tool call {i} arguments not valid JSON: " + f"{actual_args_str!r}" + ) from e + assert actual_args == expected_args, ( + f"Tool call {i} arguments mismatch:\n" + f" expected: {expected_args}\n" + f" actual: {actual_args}" + ) + + +def assert_no_terminal_leakage( + actual: ParseOutput, + terminals: list[str], + context: str = "", +) -> None: + """Assert that none of *terminals* appear in reasoning or content.""" + suffix = f" ({context})" if context else "" + for terminal in terminals: + assert terminal not in actual.reasoning, ( + f"{terminal!r} leaked into reasoning{suffix}" + ) + assert terminal not in actual.content, ( + f"{terminal!r} leaked into content{suffix}" + ) diff --git a/tests/parser/engine/streaming_helpers.py b/tests/parser/engine/streaming_helpers.py new file mode 100644 index 0000000000..48077ce8ed --- /dev/null +++ b/tests/parser/engine/streaming_helpers.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared streaming simulation helpers for parser engine tests.""" + +from __future__ import annotations + +from typing import Any + +from vllm.entrypoints.openai.engine.protocol import DeltaMessage + + +def _build_token_id_map(parser) -> dict[str, int]: + """Map special token text to token IDs from the parser's config.""" + token_id_map: dict[str, int] = {} + cfg = getattr(parser, "parser_engine_config", None) + vocab = getattr(parser, "vocab", None) + if cfg is not None and vocab is not None: + for text in (cfg.token_id_terminals or {}).values(): + tid = vocab.get(text) + if tid is not None: + token_id_map[text] = tid + return token_id_map + + +def simulate_tool_streaming( + parser, + request, + chunks: list[str], +) -> list[tuple[DeltaMessage | None, str]]: + """Feed text chunks through ``extract_tool_calls_streaming()``.""" + token_id_map = _build_token_id_map(parser) + + results: list[tuple[Any, str]] = [] + previous_text = "" + previous_token_ids: list[int] = [] + + for chunk in chunks: + current_text = previous_text + chunk + + delta_token_ids: list[int] = [ + tid for text, tid in token_id_map.items() if text in chunk + ] + + current_token_ids = previous_token_ids + delta_token_ids + + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=chunk, + previous_token_ids=tuple(previous_token_ids), + current_token_ids=tuple(current_token_ids), + delta_token_ids=tuple(delta_token_ids), + request=request, + ) + results.append((delta, current_text)) + previous_text = current_text + previous_token_ids = list(current_token_ids) + + return results + + +def collect_tool_arguments( + results: list[tuple[DeltaMessage | None, str]], +) -> str: + """Concatenate all streamed argument fragments.""" + args_text = "" + for delta, _ in results: + if delta and delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.arguments: + args_text += tc.function.arguments + return args_text + + +def collect_content( + results: list[tuple[DeltaMessage | None, str]], +) -> str: + """Concatenate all streamed content parts.""" + parts: list[str] = [] + for delta, _ in results: + if delta and delta.content: + parts.append(delta.content) + return "".join(parts) + + +def collect_function_name( + results: list[tuple[DeltaMessage | None, str]], +) -> str | None: + """Return first function name from deltas.""" + for delta, _ in results: + if delta and delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.name: + return tc.function.name + return None + + +def simulate_reasoning_streaming( + parser, + chunks: list[str], + delta_token_ids_per_chunk: list[tuple[int, ...]] | None = None, +) -> tuple[str, str]: + """Feed chunks through ``extract_reasoning_streaming()``. + + Returns ``(reasoning_text, content_text)`` tuple. + """ + token_id_map = ( + _build_token_id_map(parser) if delta_token_ids_per_chunk is None else {} + ) + + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + prev_text = "" + prev_ids: list[int] = [] + for i, chunk in enumerate(chunks): + cur_text = prev_text + chunk + if delta_token_ids_per_chunk is not None: + d_ids = delta_token_ids_per_chunk[i] + else: + d_ids = tuple(tid for text, tid in token_id_map.items() if text in chunk) + cur_ids = prev_ids + list(d_ids) + delta = parser.extract_reasoning_streaming( + previous_text=prev_text, + current_text=cur_text, + delta_text=chunk, + previous_token_ids=tuple(prev_ids), + current_token_ids=tuple(cur_ids), + delta_token_ids=d_ids, + ) + if delta: + if delta.reasoning: + reasoning_parts.append(delta.reasoning) + if delta.content: + content_parts.append(delta.content) + prev_text = cur_text + prev_ids = list(cur_ids) + return "".join(reasoning_parts), "".join(content_parts) diff --git a/tests/parser/engine/test_delegating_replay.py b/tests/parser/engine/test_delegating_replay.py new file mode 100644 index 0000000000..86ff3a1868 --- /dev/null +++ b/tests/parser/engine/test_delegating_replay.py @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Replay tests for DelegatingParser with engine adapters. + +Exercises DelegatingParser in engine-adapter mode to verify that delegated +routing produces correct output across chunk sizes. +See test_replay.py for tests that target engine parsers directly. +""" + +from __future__ import annotations + +from functools import lru_cache + +import pytest +from pydantic import TypeAdapter + +from tests.parser.engine.replay_harness import ( + assert_parse_output, + collect_output, + make_mock_tokenizer, + replay_streaming, +) +from tests.parser.engine.trace_builder import build_samples +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, +) +from vllm.parser.abstract_parser import Parser +from vllm.parser.parser_manager import ParserManager + +_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam]) + +_PAIRINGS: dict[str, tuple[str, str]] = { + "engine": ("qwen3_coder", "qwen3"), +} + +CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None] + + +@lru_cache +def _get_delegating_parser_cls(pairings: str) -> type[Parser]: + tool_name, reasoning_name = _PAIRINGS[pairings] + parser_cls = ParserManager.get_parser( + tool_parser_name=tool_name, + reasoning_parser_name=reasoning_name, + enable_auto_tools=True, + ) + assert parser_cls is not None + return parser_cls + + +_all_samples = build_samples("qwen3") + + +@pytest.mark.parametrize( + "pairings", + list(_PAIRINGS), + ids=lambda p: f"mode={p}", +) +@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}") +@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id) +def test_delegating_replay(sample, chunk_size, pairings): + parser_cls = _get_delegating_parser_cls(pairings=pairings) + + tokenizer = make_mock_tokenizer(sample) + validated_tools = ( + _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None + ) + parser = parser_cls( + tokenizer, + validated_tools, + chat_template_kwargs=sample.chat_template_kwargs, + ) + + deltas = replay_streaming( + parser, + sample.tokens, + chunk_size=chunk_size, + finished_on_last=True, + tools=sample.tools, + ) + output = collect_output(deltas) + assert_parse_output(output, sample) diff --git a/tests/parser/engine/test_engine.py b/tests/parser/engine/test_engine.py new file mode 100644 index 0000000000..0ea8afd8b9 --- /dev/null +++ b/tests/parser/engine/test_engine.py @@ -0,0 +1,846 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the streaming parser engine core pipeline.""" + +from unittest.mock import MagicMock + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from vllm.parser.engine.events import EventType, SemanticEvent +from vllm.parser.engine.incremental_lexer import ( + LexerShape, + TerminalDef, + terminals_from_literals, +) +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine + + +def _hermes_config() -> ParserEngineConfig: + """Simple Hermes-style config: JSON.""" + return ParserEngineConfig( + name="hermes_test", + terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + token_id_terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + +def _think_config() -> ParserEngineConfig: + """Simple think-tag reasoning config: ....""" + return ParserEngineConfig( + name="think_test", + terminals={ + "THINK_START": "", + "THINK_END": "", + }, + transitions={ + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + }, + ) + + +class TestNonStreaming: + def test_plain_text(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + events = engine.parse_complete("Hello, world!") + assert len(events) == 1 + assert events[0].type == EventType.TEXT_CHUNK + assert events[0].value == "Hello, world!" + + def test_single_tool_call(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + text = ( + '{"name": "get_weather",' + ' "arguments": {"city": "SF"}}' + "" + ) + events = engine.parse_complete(text) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START in types + assert EventType.TOOL_CALL_END in types + assert EventType.ARG_VALUE_CHUNK in types + + arg_text = "".join( + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK + ) + assert '"name": "get_weather"' in arg_text + assert '"city": "SF"' in arg_text + + def test_text_then_tool_call(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + text = 'Sure!{"name": "add"}' + events = engine.parse_complete(text) + + types = [e.type for e in events] + assert types[0] == EventType.TEXT_CHUNK + assert events[0].value == "Sure!" + assert EventType.TOOL_CALL_START in types + assert EventType.TOOL_CALL_END in types + + def test_multiple_tool_calls(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + text = ( + '{"name": "a"}{"name": "b"}' + ) + events = engine.parse_complete(text) + + starts = [e for e in events if e.type == EventType.TOOL_CALL_START] + ends = [e for e in events if e.type == EventType.TOOL_CALL_END] + assert len(starts) == 2 + assert len(ends) == 2 + assert starts[0].tool_index == 0 + assert starts[1].tool_index == 1 + + def test_reasoning(self): + engine = StreamingParserEngine(_think_config(), tokenizer=None) + text = "Let me think...The answer is 42." + events = engine.parse_complete(text) + + types = [e.type for e in events] + assert types[0] == EventType.REASONING_START + assert EventType.REASONING_CHUNK in types + assert EventType.REASONING_END in types + assert EventType.TEXT_CHUNK in types + + reasoning = "".join( + e.value for e in events if e.type == EventType.REASONING_CHUNK + ) + assert "Let me think..." in reasoning + + content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK) + assert "The answer is 42." in content + + +class TestStreaming: + @staticmethod + def _feed_chars( + engine: StreamingParserEngine, + text: str, + ) -> list[SemanticEvent]: + """Feed text one character at a time.""" + all_events = [] + for ch in text: + all_events.extend(engine.feed(ch, [])) + all_events.extend(engine.finish()) + return all_events + + @staticmethod + def _feed_chunks( + engine: StreamingParserEngine, + text: str, + chunk_size: int, + ) -> list[SemanticEvent]: + """Feed text in fixed-size chunks.""" + all_events = [] + for i in range(0, len(text), chunk_size): + chunk = text[i : i + chunk_size] + all_events.extend(engine.feed(chunk, [])) + all_events.extend(engine.finish()) + return all_events + + def test_char_by_char_tool_call(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + text = '{"name": "add", "arguments": {"a": 1}}' + events = self._feed_chars(engine, text) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START in types + assert EventType.TOOL_CALL_END in types + assert EventType.ARG_VALUE_CHUNK in types + + arg_text = "".join( + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK + ) + assert '"name": "add"' in arg_text + + @pytest.mark.parametrize( + "text", + [ + '{"name": "get", "arguments": {"x": "hello"}}', + '{"name": "f", "arguments": ' + '{"items": [1, [2, 3]], "obj": {"k": "v"}}}' + "", + ], + ids=["flat_args", "nested_arrays"], + ) + def test_chunk_sizes_produce_same_content(self, text): + """Different chunk sizes must produce identical concatenated content.""" + results = {} + for chunk_size in [1, 2, 3, 5, 7, len(text)]: + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + events = self._feed_chunks(engine, text, chunk_size) + arg_text = "".join( + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK + ) + results[chunk_size] = arg_text + + values = list(results.values()) + for v in values[1:]: + assert v == values[0], f"Mismatch: {results}" + + def test_prefix_buffering_prevents_premature_emit(self): + """Text like '", []) + starts = [e for e in events2 if e.type == EventType.TOOL_CALL_START] + assert len(starts) == 1 + + def test_prefix_buffering_flush_on_mismatch(self): + """Text like 'rest", []) + events2.extend(engine.finish()) + content = "".join(e.value for e in events2 if e.type == EventType.TEXT_CHUNK) + assert content == "rest" + + def test_reasoning_streaming(self): + engine = StreamingParserEngine(_think_config(), tokenizer=None) + events = self._feed_chars(engine, "hmmanswer") + + reasoning = "".join( + e.value for e in events if e.type == EventType.REASONING_CHUNK + ) + content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK) + assert "hmm" in reasoning + assert "answer" in content + + def test_text_between_tool_calls(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + text = ( + 'Hi{"name":"a"}' + 'mid{"name":"b"}end' + ) + events = self._feed_chunks(engine, text, 3) + + texts = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK) + assert "Hi" in texts + assert "mid" in texts + assert "end" in texts + + starts = [e for e in events if e.type == EventType.TOOL_CALL_START] + assert len(starts) == 2 + + def test_unmatched_close_brace_does_not_poison_depth(self): + """A stray } in malformed JSON must not kill streaming for + all subsequent content.""" + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + engine.feed("", []) + + malformed = '}{{"a": 1}}' + events = self._feed_chars(engine, malformed + "") + + arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK] + assert len(arg_chunks) > 1, ( + "Content after stray } should still stream incrementally" + ) + arg_text = "".join(arg_chunks) + assert '"a": 1' in arg_text + + def test_json_args_no_premature_close_brace(self): + """Closing braces of the top-level JSON shouldn't be streamed + until confirmed by the end tag.""" + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + + engine.feed("", []) + events = engine.feed('{"name": "f"}', []) + + arg_text = "".join( + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK + ) + assert "}" not in arg_text, "Top-level } should be held back" + + events2 = engine.feed("", []) + arg_text2 = "".join( + e.value for e in events2 if e.type == EventType.ARG_VALUE_CHUNK + ) + assert "}" in arg_text2, "} should flush on end tag" + + +_START_ID = 50 +_END_ID = 51 +_TOOL_START_ID = 60 +_TOOL_END_ID = 61 + + +def _make_think_tokenizer(): + tok = MagicMock() + tok.encode.return_value = [1, 2, 3] + tok.get_vocab.return_value = {"": _START_ID, "": _END_ID} + tok.decode.side_effect = lambda ids: { + _START_ID: "", + _END_ID: "", + }.get(ids[0], f"tok{ids[0]}") + return tok + + +def _make_hermes_tokenizer(): + """Tokenizer that resolves tool_call tags to special IDs.""" + _special = {_TOOL_START_ID: "", _TOOL_END_ID: ""} + tok = MagicMock() + tok.encode.return_value = [1, 2, 3] + tok.get_vocab.return_value = { + "": _TOOL_START_ID, + "": _TOOL_END_ID, + } + tok.decode.side_effect = lambda ids: "".join( + _special.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids + ) + return tok + + +class TestLexerBufferFlush: + """Lexer buffer must be flushed before PreLexedTerminal transitions.""" + + def test_buffered_prefix_emitted_in_current_state(self): + """Text buffered by the lexer (e.g. '<') must be emitted as + REASONING_CHUNK before THINK_END transitions to CONTENT.""" + engine = StreamingParserEngine(_think_config(), _make_think_tokenizer()) + + events = engine.feed("", [_START_ID]) + assert any(e.type == EventType.REASONING_START for e in events) + + events = engine.feed("reasoning text<", []) + reasoning_text = "".join( + e.value for e in events if e.type == EventType.REASONING_CHUNK + ) + assert "reasoning text" in reasoning_text + + events = engine.feed("", [_END_ID]) + event_types = [e.type for e in events] + if EventType.REASONING_CHUNK in event_types: + rc_idx = event_types.index(EventType.REASONING_CHUNK) + re_idx = event_types.index(EventType.REASONING_END) + assert rc_idx < re_idx, ( + "'<' must be emitted as REASONING_CHUNK before REASONING_END" + ) + flushed = events[rc_idx].value + assert "<" in flushed + + def test_empty_buffer_no_extra_events(self): + """When the lexer buffer is empty, flushing is a no-op.""" + engine = StreamingParserEngine(_think_config(), _make_think_tokenizer()) + + engine.feed("", [_START_ID]) + engine.feed("clean text", []) + + events = engine.feed("", [_END_ID]) + assert any(e.type == EventType.REASONING_END for e in events) + chunk_events = [e for e in events if e.type == EventType.REASONING_CHUNK] + assert all(e.value for e in chunk_events) + + +class TestTokenIdFiltering: + """When token IDs are available, lex-matched terminals that also + have token_id_terminal entries should be demoted to content.""" + + def test_lex_matched_terminal_demoted_after_token_ids_seen(self): + """After receiving token IDs, text that matches a token-ID + terminal should be treated as content, not trigger a transition.""" + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer()) + + # First feed with a non-special token ID to set _ever_had_token_ids + engine.feed("prefix ", [1]) + + # Now feed text containing as literal text + events = engine.feed( + "Use to invoke tools.", [2, 3, 4, 5] + ) + events.extend(engine.finish()) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START not in types + assert EventType.TEXT_CHUNK in types + + text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK) + assert "" in text + + def test_scanner_matched_terminal_bypasses_filter(self): + """PreLexedTerminals from the scanner bypass the filter and + still trigger state transitions.""" + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer()) + + events = engine.feed("", [_TOOL_START_ID]) + assert any(e.type == EventType.TOOL_CALL_START for e in events) + + events = engine.feed('{"name": "f"}', [2, 3]) + events.extend(engine.feed("", [_TOOL_END_ID])) + events.extend(engine.finish()) + assert any(e.type == EventType.TOOL_CALL_END for e in events) + + def test_no_filtering_without_token_ids(self): + """When no token IDs are ever provided (non-streaming), + text matching still triggers transitions.""" + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer()) + + events = engine.feed('{"name": "f"}', []) + events.extend(engine.finish()) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START in types + assert EventType.TOOL_CALL_END in types + + def test_mixed_text_then_real_tool_call(self): + """Text mentioning tool syntax followed by a real special-token + tool call.""" + engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer()) + + events1 = engine.feed("Mention in text. ", [1, 2, 3, 4]) + events2 = engine.feed("", [_TOOL_START_ID]) + events3 = engine.feed('{"name": "a"}', [5, 6]) + events4 = engine.feed("", [_TOOL_END_ID]) + events4.extend(engine.finish()) + + all_events = events1 + events2 + events3 + events4 + + content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK) + assert "" in content + + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1 + + +def _func_prefix_config() -> ParserEngineConfig: + """Config mixing token-ID terminals (TOOL_START/END) with + text-only terminals (FUNC_PREFIX) and fallback transitions.""" + return ParserEngineConfig( + name="func_prefix_test", + terminals={ + "TOOL_START": "", + "TOOL_END": "", + "FUNC_PREFIX": "", + "CLOSE_ANGLE": ">", + }, + token_id_terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.TOOL_CALL_START,), + ), + (ParserState.CONTENT, "FUNC_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + skip_in_token_id_mode=True, + ), + (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition( + ParserState.TOOL_NAME, + (), + ), + (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition( + ParserState.TOOL_ARGS, + (), + ), + (ParserState.TOOL_ARGS, "FUNC_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( + ParserState.CONTENT, + (), + ), + (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + skip_in_token_id_mode=True, + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_NAME: EventType.TOOL_NAME, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + +def _make_func_prefix_tokenizer(): + return make_mock_tokenizer( + { + "": _TOOL_START_ID, + "": _TOOL_END_ID, + } + ) + + +class TestTextOnlyFallbackFiltering: + """When token IDs are available, transitions marked + skip_in_token_id_mode should be skipped.""" + + def test_func_prefix_in_prose_demoted_in_strict_mode(self): + """ in prose should NOT trigger a tool call + when strict mode is active.""" + engine = StreamingParserEngine( + _func_prefix_config(), _make_func_prefix_tokenizer() + ) + engine.feed("prefix ", [1]) + + events = engine.feed("Use to check.", [2, 3, 4, 5]) + events.extend(engine.finish()) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START not in types + assert EventType.TEXT_CHUNK in types + text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK) + assert " FUNC_PREFIX (text) should + still parse a tool call normally in strict mode.""" + engine = StreamingParserEngine( + _func_prefix_config(), _make_func_prefix_tokenizer() + ) + + events1 = engine.feed("", [_TOOL_START_ID]) + assert any(e.type == EventType.TOOL_CALL_START for e in events1) + + events2 = engine.feed("", [2, 3]) + events3 = engine.feed("args", [4]) + events4 = engine.feed("", [5, 6]) + events4.extend(engine.feed("", [_TOOL_END_ID])) + events4.extend(engine.finish()) + + all_events = events1 + events2 + events3 + events4 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1 + assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1 + + def test_fallback_fires_without_token_ids(self): + """When no token IDs are provided, fallback transitions should + still fire normally.""" + engine = StreamingParserEngine( + _func_prefix_config(), _make_func_prefix_tokenizer() + ) + + events = engine.feed("args", []) + events.extend(engine.finish()) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START in types + assert EventType.TOOL_CALL_END in types + + def test_tool_between_fallback_blocked_in_strict_mode(self): + """The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be + blocked in strict mode.""" + engine = StreamingParserEngine( + _func_prefix_config(), _make_func_prefix_tokenizer() + ) + + engine.feed("", [_TOOL_START_ID]) + engine.feed("", [2, 3]) + engine.feed("args", [4]) + engine.feed("", [5, 6]) + engine.feed("", [_TOOL_END_ID]) + + events = engine.feed("more", [7, 8, 9]) + events.extend(engine.finish()) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START not in types + + +class TestNoUnusedTokenizerAttr: + """StreamingParserEngine no longer stores a redundant _tokenizer.""" + + def test_no_tokenizer_attribute(self): + config = ParserEngineConfig(name="test") + engine = StreamingParserEngine(config, tokenizer=None) + assert not hasattr(engine, "_tokenizer") + + +class TestArgsResetOnReentry: + """When leaving TOOL_ARGS and later re-entering (e.g. two tool + calls), the entering-TOOL_ARGS block resets args tracking. The + redundant reset on exit was removed.""" + + @staticmethod + def _multi_tool_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="multi_tool", + terminals={ + "TOOL_START": "", + "TOOL_END": "", + "TOOL_SEP": "", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_BETWEEN, "TOOL_SEP"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + def test_args_tracking_across_reentry(self): + engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None) + + events = engine.feed( + '{"city": "SF"}' + "" + '{"name": "bar"}', + [], + ) + + tool_starts = [e for e in events if e.type == EventType.TOOL_CALL_START] + tool_ends = [e for e in events if e.type == EventType.TOOL_CALL_END] + arg_chunks = [e for e in events if e.type == EventType.ARG_VALUE_CHUNK] + + assert len(tool_starts) == 2 + assert len(tool_ends) == 2 + assert tool_starts[0].tool_index == 0 + assert tool_starts[1].tool_index == 1 + + first_args = "".join(e.value for e in arg_chunks if e.tool_index == 0) + second_args = "".join(e.value for e in arg_chunks if e.tool_index == 1) + assert '"city"' in first_args + assert '"name"' in second_args + + def test_brace_depth_resets_on_reentry(self): + """Verify _args_brace_depth resets when re-entering TOOL_ARGS.""" + engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None) + engine.feed("", []) + assert engine.state == ParserState.TOOL_ARGS + assert engine._args_brace_depth == 0 + + engine.feed('{"a": 1}', []) + engine.feed("", []) + assert engine.state == ParserState.TOOL_BETWEEN + + engine.feed("", []) + assert engine.state == ParserState.TOOL_ARGS + assert engine._args_brace_depth == 0 + assert engine._args_in_string is False + assert engine._args_escape_next is False + + +class TestToolPreambleFinish: + """finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool + call was started (tool_index >= 0), but not when tool_index is -1.""" + + @staticmethod + def _preamble_with_tool_call_start_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="preamble_tcs", + terminals={"TOOL_START": ""}, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.TOOL_CALL_START,), + ), + }, + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK}, + ) + + @staticmethod + def _preamble_without_tool_call_start_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="preamble_no_tcs", + terminals={"TOOL_CALLS_START": ""}, + transitions={ + (ParserState.CONTENT, "TOOL_CALLS_START"): Transition( + ParserState.TOOL_PREAMBLE, + (), + ), + }, + content_events={ParserState.CONTENT: EventType.TEXT_CHUNK}, + ) + + def test_finish_emits_tool_call_end_with_tool_index(self): + config = self._preamble_with_tool_call_start_config() + engine = StreamingParserEngine(config, tokenizer=None) + + engine.feed("", []) + assert engine.state == ParserState.TOOL_PREAMBLE + assert engine.tool_index == 0 + + finish_events = engine.finish() + end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END] + assert len(end_events) == 1 + assert end_events[0].tool_index == 0 + + def test_finish_no_tool_call_end_without_tool_index(self): + config = self._preamble_without_tool_call_start_config() + engine = StreamingParserEngine(config, tokenizer=None) + + engine.feed("", []) + assert engine.state == ParserState.TOOL_PREAMBLE + assert engine.tool_index == -1 + + finish_events = engine.finish() + end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END] + assert len(end_events) == 0 + assert engine.state == ParserState.CONTENT + + +class TestRegexTerminalInfraRemoved: + """TerminalDef.priority, LexerShape.regex_terminals, and the regex + matching loop were removed.""" + + def test_terminal_def_no_priority(self): + import regex as re + + td = TerminalDef(name="X", pattern=re.compile("x")) + assert not hasattr(td, "priority") + + def test_lexer_shape_no_regex_terminals(self): + shape = LexerShape([]) + assert not hasattr(shape, "regex_terminals") + + def test_terminals_from_literals_still_works(self): + literals = {"TOOL_START": "", "TOOL_END": ""} + defs = terminals_from_literals(literals) + assert len(defs) == 2 + names = {d.name for d in defs} + assert names == {"TOOL_START", "TOOL_END"} + for d in defs: + assert d.is_literal + assert d.literal in ("", "") + + +class TestMultiCharTerminalInArgs: + """Regression: multi-char terminals falling through in TOOL_ARGS + must be fed char-by-char via _feed_args_text, not _feed_args_char.""" + + @staticmethod + def _newline_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="newline_test", + terminals={ + "TOOL_START": "", + "TOOL_END": "", + "NEWLINE": "\n", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + def test_newline_in_args_parsed_correctly(self): + engine = StreamingParserEngine(self._newline_config(), tokenizer=None) + text = '{"name": "f",\n"arguments": {"a": 1}}' + events = engine.parse_complete(text) + + arg_text = "".join( + e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK + ) + assert '"name": "f"' in arg_text + assert '"arguments"' in arg_text + + def test_newline_in_args_streaming(self): + engine = StreamingParserEngine(self._newline_config(), tokenizer=None) + all_events = TestStreaming._feed_chars( + engine, '{"name": "f",\n"a": 1}' + ) + + arg_text = "".join( + e.value for e in all_events if e.type == EventType.ARG_VALUE_CHUNK + ) + assert '"name": "f"' in arg_text + assert '"a": 1' in arg_text + + +class TestSkipToolParsing: + """When skip_tool_parsing is set, tool tags become content.""" + + def test_tool_tags_emitted_as_content(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + engine.skip_tool_parsing = True + + text = '{"name": "f"}' + events = engine.parse_complete(text) + + types = [e.type for e in events] + assert EventType.TOOL_CALL_START not in types + assert EventType.TOOL_CALL_END not in types + + content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK) + assert "" in content + assert "" in content + + def test_skip_tool_streaming(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + engine.skip_tool_parsing = True + + all_events = TestStreaming._feed_chars( + engine, '{"name": "f"}' + ) + + types = [e.type for e in all_events] + assert EventType.TOOL_CALL_START not in types + + content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK) + assert "" in content + + def test_reset_preserves_skip_tool_parsing(self): + engine = StreamingParserEngine(_hermes_config(), tokenizer=None) + engine.skip_tool_parsing = True + engine.reset() + assert engine.skip_tool_parsing is True diff --git a/tests/parser/engine/test_parser_engine.py b/tests/parser/engine/test_parser_engine.py new file mode 100644 index 0000000000..c2bcd91c53 --- /dev/null +++ b/tests/parser/engine/test_parser_engine.py @@ -0,0 +1,1356 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for :class:`ParserEngine` — the glue layer between +:class:`StreamingParserEngine` events and the serving layer's +DeltaMessage / ExtractedToolCallInformation protocol. +""" + +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import regex as re + +from tests.parser.engine.conftest import make_mock_tokenizer +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, + DeltaToolCall, + FunctionDefinition, +) +from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.engine.adapters import make_adapters +from vllm.parser.engine.events import EventType, SemanticEvent +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) + +# ── Shared test configs ────────────────────────────────────────────── + +_VOCAB: dict[str, int] = { + "": 200, + "": 201, + "": 202, + "": 203, +} + + +def _combined_config() -> ParserEngineConfig: + """Config with reasoning tags and tool-call tags.""" + return ParserEngineConfig( + name="combined_test", + terminals={ + "THINK_START": "", + "THINK_END": "", + "TOOL_START": "", + "TOOL_END": "", + }, + token_id_terminals={ + "THINK_START": "", + "THINK_END": "", + "TOOL_START": "", + "TOOL_END": "", + }, + transitions={ + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + initial_state=ParserState.REASONING, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.REASONING: EventType.REASONING_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + +def _hermes_config() -> ParserEngineConfig: + """Tool-call-only config (no reasoning).""" + return ParserEngineConfig( + name="hermes_test", + terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + token_id_terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + +def _make_engine( + config: ParserEngineConfig | None = None, + tools: list | None = None, +) -> ParserEngine: + tokenizer = make_mock_tokenizer(_VOCAB) + cfg = config or _combined_config() + return ParserEngine( + tokenizer, + tools=tools, + parser_engine_config=cfg, + ) + + +# ── TestEventsToDelta ──────────────────────────────────────────────── + + +class TestEventsToDelta: + """Unit tests for ParserEngine._events_to_delta().""" + + def test_text_chunk_produces_content(self): + engine = _make_engine() + delta = engine._events_to_delta( + [ + SemanticEvent(EventType.TEXT_CHUNK, "Hello world"), + ] + ) + assert delta is not None + assert delta.content == "Hello world" + assert not delta.tool_calls + + def test_reasoning_chunk_produces_reasoning(self): + engine = _make_engine() + delta = engine._events_to_delta( + [ + SemanticEvent(EventType.REASONING_CHUNK, "Let me think"), + ] + ) + assert delta is not None + assert delta.reasoning == "Let me think" + assert delta.content is None + + def test_empty_events_returns_none(self): + engine = _make_engine() + delta = engine._events_to_delta([]) + assert delta is None + + def test_tool_call_produces_tool_call_delta(self): + engine = _make_engine() + events = [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0), + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + '{"location": "NYC"}', + tool_index=0, + ), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + ] + delta = engine._events_to_delta(events) + assert delta is not None + assert len(delta.tool_calls) > 0 + names = [ + tc.function.name + for tc in delta.tool_calls + if tc.function and tc.function.name + ] + assert "get_weather" in names + + def test_reasoning_end_sets_flag(self): + engine = _make_engine() + assert engine._reasoning_ended is False + engine._events_to_delta([SemanticEvent(EventType.REASONING_END)]) + assert engine._reasoning_ended is True + + def test_mixed_content_and_reasoning(self): + engine = _make_engine() + events = [ + SemanticEvent(EventType.REASONING_CHUNK, "thinking..."), + SemanticEvent(EventType.REASONING_END), + SemanticEvent(EventType.TEXT_CHUNK, "answer"), + ] + delta = engine._events_to_delta(events) + assert delta is not None + assert delta.reasoning == "thinking..." + assert delta.content == "answer" + + @pytest.mark.parametrize( + "events,expected,excluded", + [ + ( + [SemanticEvent(EventType.TEXT_CHUNK, "Hello world")], + "content", + ["tool_calls", "reasoning"], + ), + ( + [SemanticEvent(EventType.REASONING_CHUNK, "Let me think")], + "reasoning", + ["tool_calls", "content"], + ), + ( + [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "fn", tool_index=0), + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"k":1}', tool_index=0), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + ], + "tool_calls", + ["content", "reasoning"], + ), + ], + ids=["content_only", "reasoning_only", "tool_call_only"], + ) + def test_delta_excludes_unset_fields(self, events, expected, excluded): + engine = _make_engine() + delta = engine._events_to_delta(events) + assert delta is not None + dumped = delta.model_dump(exclude_unset=True) + assert expected in dumped + for field in excluded: + assert field not in dumped + + def test_kimi_k2_tool_call_id_includes_func_name(self): + engine = _make_engine() + engine._stream_state.tool_call_id_type = "kimi_k2" + events = [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0), + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + '{"city": "NYC"}', + tool_index=0, + ), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + ] + delta = engine._events_to_delta(events) + assert delta is not None + assert len(delta.tool_calls) == 1 + assert delta.tool_calls[0].id == "functions.get_weather:0" + + def test_multiple_arg_chunks_same_batch_coalesced(self): + """Multiple events for the same tool in one batch must produce + at most one DeltaToolCall per index.""" + engine = _make_engine() + events = [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0), + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + '{"city": ', + tool_index=0, + ), + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + '"Tokyo"}', + tool_index=0, + ), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + ] + delta = engine._events_to_delta(events) + assert delta is not None + indices = [tc.index for tc in delta.tool_calls] + assert len(indices) == len(set(indices)), ( + f"Duplicate indices in tool_calls: {delta.tool_calls}" + ) + assert delta.tool_calls[0].function.name == "get_weather" + assert delta.tool_calls[0].id is not None + + +# ── TestCoalesceToolCallDeltas ────────────────────────────────────── + + +class TestCoalesceToolCallDeltas: + """Unit tests for ParserEngine._coalesce_tool_call_deltas().""" + + def test_no_duplicates_unchanged(self): + deltas = [ + DeltaToolCall( + index=0, + id="a", + type="function", + function=DeltaFunctionCall(name="f"), + ), + DeltaToolCall( + index=1, + function=DeltaFunctionCall(arguments="{}"), + ), + ] + result = ParserEngine._coalesce_tool_call_deltas(deltas) + assert len(result) == 2 + assert result[0].index == 0 + assert result[1].index == 1 + + def test_name_and_args_same_index_merged(self): + deltas = [ + DeltaToolCall( + index=0, + id="call_1", + type="function", + function=DeltaFunctionCall(name="get_weather"), + ), + DeltaToolCall( + index=0, + function=DeltaFunctionCall(arguments='{"city":'), + ), + DeltaToolCall( + index=0, + function=DeltaFunctionCall(arguments='"Tokyo"}'), + ), + ] + result = ParserEngine._coalesce_tool_call_deltas(deltas) + assert len(result) == 1 + assert result[0].index == 0 + assert result[0].id == "call_1" + assert result[0].type == "function" + assert result[0].function.name == "get_weather" + assert result[0].function.arguments == '{"city":"Tokyo"}' + + def test_empty_list(self): + assert ParserEngine._coalesce_tool_call_deltas([]) == [] + + def test_single_element(self): + tc = DeltaToolCall( + index=0, + function=DeltaFunctionCall(name="f"), + ) + result = ParserEngine._coalesce_tool_call_deltas([tc]) + assert result == [tc] + + def test_partial_duplicates(self): + deltas = [ + DeltaToolCall( + index=0, + id="a", + type="function", + function=DeltaFunctionCall(name="f1"), + ), + DeltaToolCall( + index=1, + id="b", + type="function", + function=DeltaFunctionCall(name="f2"), + ), + DeltaToolCall( + index=0, + function=DeltaFunctionCall(arguments='{"x":1}'), + ), + ] + result = ParserEngine._coalesce_tool_call_deltas(deltas) + assert len(result) == 2 + assert result[0].index == 0 + assert result[0].function.name == "f1" + assert result[0].function.arguments == '{"x":1}' + assert result[1].index == 1 + + def test_id_type_from_later_entry(self): + deltas = [ + DeltaToolCall( + index=0, + function=DeltaFunctionCall(arguments='{"a":1}'), + ), + DeltaToolCall( + index=0, + id="call_1", + type="function", + function=DeltaFunctionCall(name="f"), + ), + ] + result = ParserEngine._coalesce_tool_call_deltas(deltas) + assert len(result) == 1 + assert result[0].id == "call_1" + assert result[0].type == "function" + assert result[0].function.name == "f" + assert result[0].function.arguments == '{"a":1}' + + +# ── TestContentWhitespaceHandling ──────────────────────────────────── + + +class TestContentWhitespaceHandling: + """Unit tests for whitespace deferral / dropping in _events_to_delta.""" + + def test_whitespace_only_deferred_until_next_tick(self): + engine = _make_engine() + d1 = engine._events_to_delta( + [SemanticEvent(EventType.TEXT_CHUNK, " \n")], + ) + assert d1 is None + d2 = engine._events_to_delta( + [SemanticEvent(EventType.TEXT_CHUNK, "hello")], + ) + assert d2 is not None + assert d2.content == " \nhello" + + def test_whitespace_only_emitted_on_finished(self): + engine = _make_engine() + d = engine._events_to_delta( + [SemanticEvent(EventType.TEXT_CHUNK, " \n")], + finished=True, + ) + assert d is not None + assert d.content == " \n" + + def test_whitespace_dropped_before_tool_call(self): + engine = _make_engine() + engine._events_to_delta( + [ + SemanticEvent(EventType.TEXT_CHUNK, " \n"), + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + ] + ) + d = engine._events_to_delta( + [ + SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0), + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + '{"a":1}', + tool_index=0, + ), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + ] + ) + assert d is not None + assert d.content is None + assert d.tool_calls + + def test_real_content_before_tool_preserved(self): + engine = _make_engine() + d = engine._events_to_delta( + [ + SemanticEvent(EventType.TEXT_CHUNK, "prefix"), + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + ] + ) + assert d is not None + assert d.content == "prefix" + + def test_whitespace_after_nonws_content_preserved(self): + engine = _make_engine() + engine._events_to_delta( + [SemanticEvent(EventType.TEXT_CHUNK, "hello")], + ) + d = engine._events_to_delta( + [SemanticEvent(EventType.TEXT_CHUNK, " \n")], + ) + assert d is not None + assert d.content == " \n" + + def test_whitespace_after_nonws_not_dropped_with_tools(self): + engine = _make_engine() + engine._events_to_delta( + [SemanticEvent(EventType.TEXT_CHUNK, "hello")], + ) + d = engine._events_to_delta( + [ + SemanticEvent(EventType.TEXT_CHUNK, " \n"), + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + ] + ) + assert d is not None + assert d.content == " \n" + + +# ── TestPostToolContentDeferral ────────────────────────────────────── + + +class TestPostToolContentDeferral: + """Regression: content after TOOL_CALL_END in the same batch must not + produce a mixed DeltaMessage(content=..., tool_calls=...) — that causes + split_delta to reorder content before tool_calls, breaking the Responses + API state machine.""" + + def test_text_after_tool_end_deferred(self): + engine = _make_engine() + events = [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0), + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city":"NYC"}', tool_index=0), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + SemanticEvent(EventType.TEXT_CHUNK, "\nHere is the result"), + ] + delta = engine._events_to_delta(events) + assert delta is not None + assert delta.tool_calls + assert delta.content is None + + deferred = engine._events_to_delta([]) + assert deferred is not None + assert deferred.content == "\nHere is the result" + assert not deferred.tool_calls + + def test_text_after_tool_deferred_even_when_finished(self): + engine = _make_engine() + events = [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0), + SemanticEvent(EventType.ARG_VALUE_CHUNK, "{}", tool_index=0), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + SemanticEvent(EventType.TEXT_CHUNK, "done"), + ] + delta = engine._events_to_delta(events, finished=True) + assert delta is not None + assert delta.tool_calls + assert delta.content is None + + def test_text_before_tool_not_deferred(self): + engine = _make_engine() + engine._content_has_nonws = True + events = [ + SemanticEvent(EventType.TEXT_CHUNK, "hello"), + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0), + SemanticEvent(EventType.ARG_VALUE_CHUNK, "{}", tool_index=0), + SemanticEvent(EventType.TOOL_CALL_END, tool_index=0), + ] + delta = engine._events_to_delta(events) + assert delta is not None + assert delta.content == "hello" + assert delta.tool_calls + + def test_deferred_content_not_flushed_during_arg_continuation(self): + """Deferred content from batch N must not mix with arg-continuation + tool events in batch N+1 — that creates a DeltaMessage with both + content and nameless tool_calls, which crashes the Responses API + state machine (name=None → Pydantic ValidationError).""" + engine = _make_engine() + engine._content_has_nonws = True + + batch1 = [ + SemanticEvent(EventType.TOOL_CALL_START, tool_index=0), + SemanticEvent(EventType.TOOL_NAME, "get_weather", tool_index=0), + SemanticEvent(EventType.ARG_VALUE_CHUNK, '{"city":', tool_index=0), + SemanticEvent(EventType.TEXT_CHUNK, "\n"), + ] + delta1 = engine._events_to_delta(batch1) + assert delta1 is not None + assert delta1.tool_calls + assert delta1.content is None + + batch2 = [ + SemanticEvent(EventType.ARG_VALUE_CHUNK, '"NYC"}', tool_index=0), + ] + delta2 = engine._events_to_delta(batch2) + assert delta2 is not None + assert delta2.tool_calls + assert delta2.content is None + + flush = engine._events_to_delta([]) + assert flush is not None + assert flush.content == "\n" + assert not flush.tool_calls + + +# ── TestFixArgTypes ────────────────────────────────────────────────── + + +def _make_tool(name: str, properties: dict) -> ChatCompletionToolsParam: + return ChatCompletionToolsParam( + type="function", + function=FunctionDefinition( + name=name, + parameters={"type": "object", "properties": properties}, + ), + ) + + +class TestFixArgTypes: + """Tests for ParserEngine._fix_arg_types().""" + + def test_string_param_reverted_from_int(self): + tool = _make_tool("f", {"zipcode": {"type": "string"}}) + engine = _make_engine(tools=[tool]) + result = engine._fix_arg_types('{"zipcode": 12345}', "f") + assert '"zipcode": "12345"' in result + + def test_string_param_reverted_from_bool(self): + tool = _make_tool("f", {"flag": {"type": "string"}}) + engine = _make_engine(tools=[tool]) + result = engine._fix_arg_types('{"flag": true}', "f") + assert '"flag": "true"' in result + + def test_string_param_reverted_from_null(self): + tool = _make_tool("f", {"val": {"type": "string"}}) + engine = _make_engine(tools=[tool]) + result = engine._fix_arg_types('{"val": null}', "f") + assert '"val": "null"' in result + + def test_int_param_not_changed(self): + tool = _make_tool("f", {"count": {"type": "integer"}}) + engine = _make_engine(tools=[tool]) + result = engine._fix_arg_types('{"count": 42}', "f") + assert '"count": 42' in result + + def test_no_tools_returns_unchanged(self): + engine = _make_engine(tools=None) + original = '{"a": 1}' + assert engine._fix_arg_types(original, "f") == original + + def test_unknown_function_returns_unchanged(self): + tool = _make_tool("known", {"x": {"type": "string"}}) + engine = _make_engine(tools=[tool]) + original = '{"x": 1}' + assert engine._fix_arg_types(original, "unknown") == original + + def test_invalid_json_returns_unchanged(self): + tool = _make_tool("f", {"x": {"type": "string"}}) + engine = _make_engine(tools=[tool]) + original = "not json" + assert engine._fix_arg_types(original, "f") == original + + def test_string_value_not_touched(self): + tool = _make_tool("f", {"name": {"type": "string"}}) + engine = _make_engine(tools=[tool]) + original = '{"name": "Alice"}' + assert engine._fix_arg_types(original, "f") == original + + +# ── TestBuildExtractedResult ───────────────────────────────────────── + + +class TestBuildExtractedResult: + """Tests for ParserEngine._build_extracted_result().""" + + def test_no_tool_calls(self): + engine = _make_engine() + result = engine._build_extracted_result() + assert result.tools_called is False + assert result.tool_calls == [] + + def test_single_tool_call(self): + engine = _make_engine(_hermes_config()) + text = '{"name": "f", "arguments": {"a": 1}}' + events = engine._engine.feed(text, []) + events.extend(engine._engine.finish()) + delta = engine._events_to_delta(events) + result = engine._build_extracted_result(delta) + assert result.tools_called is True + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "f" + + def test_content_passthrough(self): + engine = _make_engine(_hermes_config()) + text = "Hello world" + events = engine._engine.feed(text, []) + events.extend(engine._engine.finish()) + delta = engine._events_to_delta(events, finished=True) + result = engine._build_extracted_result(delta) + assert result.tools_called is False + assert result.content == "Hello world" + + +# ── TestEngineBasedPath ────────────────────────────────────────────── + + +class TestEngineBasedPath: + """Tests for the _engine_based accumulation behavior in + DelegatingParser.parse_delta.""" + + def test_engine_based_true_when_both_parsers_engine(self): + r = SimpleNamespace(engine_based_streaming=True) + t = SimpleNamespace(engine_based_streaming=True) + engine_based = r.engine_based_streaming and t.engine_based_streaming + assert engine_based is True + + def test_engine_based_false_when_reasoning_parser_not_engine(self): + r = SimpleNamespace(engine_based_streaming=False) + t = SimpleNamespace(engine_based_streaming=True) + engine_based = r.engine_based_streaming and t.engine_based_streaming + assert engine_based is False + + def test_parse_delta_streaming(self, mock_request): + """Engine's parse_delta returns content from streaming events.""" + engine = _make_engine(_hermes_config()) + engine._streaming_initialized = True + result = engine.parse_delta( + "Hello", + [], + mock_request, + finished=False, + ) + assert result is not None + assert result.content == "Hello" + + def test_parse_delta_tool_call(self, mock_request): + """Engine's parse_delta handles tool calls in streaming.""" + engine = _make_engine(_hermes_config()) + engine._streaming_initialized = True + result = engine.parse_delta( + '{"name": "f", "arguments": {}}', + [], + mock_request, + finished=True, + ) + assert result is not None + assert len(result.tool_calls) > 0 + + +# ── TestParseTokenIdPassthrough ──────────────────────────────────── + + +class TestParseTokenIdPassthrough: + """parse() must forward model_output_token_ids to _single_pass_parse + so that token-ID-based strict terminal matching is active.""" + + def test_literal_tool_tag_in_content_preserved_with_token_ids(self, mock_request): + engine = _make_engine(_hermes_config()) + text = ( + "Use to call tools." + '{"name": "f", "arguments": {"a": 1}}' + ) + token_ids = [ + 65, + 66, + 67, + 68, + 69, + 70, + 71, # "Use to call tools." + 202, # real + 72, + 73, + 74, # '{"name": "f", ...}' + 203, # real + ] + + _, content, tool_calls = engine.parse( + text, mock_request, model_output_token_ids=token_ids + ) + + assert content is not None + assert "" in content + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "f" + + def test_parse_with_token_ids_basic(self, mock_request): + engine = _make_engine(_hermes_config()) + text = '{"name": "h", "arguments": {"x": 1}}' + token_ids = [202, 65, 66, 67, 203] + + _, content, tool_calls = engine.parse( + text, mock_request, model_output_token_ids=token_ids + ) + + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "h" + + def test_parse_without_token_ids_backward_compat(self, mock_request): + engine = _make_engine(_hermes_config()) + text = '{"name": "g", "arguments": {}}' + + _, content, tool_calls = engine.parse(text, mock_request) + + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "g" + + +# ── TestAdapterFinishOnStreamEnd ──────────────────────────────────── + + +class _CombinedTestEngine(ParserEngine): + def __init__(self, tokenizer, tools=None, **kwargs): + super().__init__( + tokenizer, tools, parser_engine_config=_combined_config(), **kwargs + ) + + +_CombinedReasoningAdapter, _CombinedToolAdapter = make_adapters(_CombinedTestEngine) + + +class _CombinedDelegating(DelegatingParser): + reasoning_parser_cls = _CombinedReasoningAdapter + tool_parser_cls = _CombinedToolAdapter + + +def _make_delegating_request(): + req = MagicMock(spec=ChatCompletionRequest) + req.tools = [] + req.tool_choice = "auto" + return req + + +class TestAdapterFinishOnStreamEnd: + """Engine adapters must flush buffered text when streaming ends. + + When a DelegatingParser wraps engine adapters, the underlying + StreamingParserEngine.finish() must be called on the last + parse_delta(finished=True) so that lexer-buffered text (terminal + prefixes) and scanner-deferred terminals are not silently lost. + """ + + def test_lexer_buffer_flushed_on_finished(self): + """Text buffered as a potential terminal prefix must be emitted + as content when the stream ends.""" + tokenizer = make_mock_tokenizer(_VOCAB) + parser = _CombinedDelegating(tokenizer) + request = _make_delegating_request() + + # Feed reasoning then content with a trailing '<' that looks like + # the start of a terminal ('' or ''). + parser.parse_delta("", [201], request, finished=False) + delta = parser.parse_delta("Hello world<", [], request, finished=True) + # The '<' must NOT be silently dropped. + assert delta is not None + assert delta.content is not None + assert "<" in delta.content, ( + "Trailing '<' lost: lexer buffer was not flushed on finish" + ) + + def test_args_buffer_flushed_on_finished(self): + """Pending arg buffer text must be emitted when stream ends + mid-tool-call (closing brace held back in buffer).""" + tokenizer = make_mock_tokenizer(_VOCAB) + parser = _CombinedDelegating(tokenizer) + request = _make_delegating_request() + + parser.parse_delta("", [201], request, finished=False) + parser.parse_delta("", [202], request, finished=False) + parser.parse_delta('{"name": "f"}', [], request, finished=False) + # The closing } is held back in args buffer, waiting for + # a TOOL_END terminal. Stream ends without one — finish() + # must flush the buffer. + delta = parser.parse_delta("", [], request, finished=True) + assert delta is not None, ( + "Engine finish should produce a delta with flushed args/end" + ) + + +# ── TestReasoningOnlyDelegatingParser ───────────────────────────── + + +class _ReasoningOnlyDelegating(DelegatingParser): + """DelegatingParser with reasoning adapter but NO tool adapter.""" + + reasoning_parser_cls = _CombinedReasoningAdapter + tool_parser_cls = None + + +class TestReasoningOnlyEndTokenLeak: + """When there is no tool parser, the content passthrough must not + re-emit the end-of-reasoning marker (e.g. ````) as content. + + Regression test for the scenario where ```` arrives as a + single-token delta: the engine correctly consumes it (emitting + REASONING_END with no content), but the content passthrough + fired because ``delta_message is None`` and reasoning had just ended. + """ + + def test_think_end_not_leaked_as_content(self): + tokenizer = make_mock_tokenizer(_VOCAB) + parser = _ReasoningOnlyDelegating(tokenizer) + request = _make_delegating_request() + + # Feed reasoning text. + d1 = parser.parse_delta( + "I am thinking", + [], + request, + finished=False, + ) + assert d1 is not None + assert d1.reasoning is not None + assert d1.content is None + + # Feed as a single-token delta. + d2 = parser.parse_delta( + "", + [201], + request, + finished=False, + ) + # The end-of-reasoning marker must NOT appear as content. + if d2 is not None: + assert d2.content is None, f" leaked as content: {d2.content!r}" + + # Feed content after reasoning. + d3 = parser.parse_delta( + "\n\nHello!", + [], + request, + finished=False, + ) + assert d3 is not None + assert d3.content is not None + assert "" not in d3.content + + def test_streaming_content_matches_non_streaming(self): + """Concatenated streaming content must match extract_reasoning.""" + tokenizer = make_mock_tokenizer(_VOCAB) + # No in input: the combined config starts in REASONING + # state, so all text before is reasoning. + full_text = "reasoning\n\nHello!" + + # Non-streaming extraction. + parser_ns = _ReasoningOnlyDelegating(tokenizer) + request = _make_delegating_request() + reasoning, content = parser_ns.extract_reasoning(full_text, request) + assert reasoning == "reasoning" + assert content == "\n\nHello!" + + # Streaming extraction — simulate per-token deltas. + parser_s = _ReasoningOnlyDelegating(tokenizer) + deltas = [ + ("reasoning", []), + ("", [201]), + ("\n\n", []), + ("Hello!", []), + ] + content_parts: list[str] = [] + for text, ids in deltas: + dm = parser_s.parse_delta(text, ids, request, finished=False) + if dm is not None and dm.content: + content_parts.append(dm.content) + dm = parser_s.parse_delta("", [], request, finished=True) + if dm is not None and dm.content: + content_parts.append(dm.content) + + streaming_content = "".join(content_parts) + assert streaming_content == content, ( + f"Streaming content {streaming_content!r} " + f"does not match non-streaming {content!r}" + ) + + def test_multi_token_delta_preserves_content_after_think_end(self): + """Content after in the same delta must not be lost.""" + tokenizer = make_mock_tokenizer(_VOCAB) + parser = _ReasoningOnlyDelegating(tokenizer) + request = _make_delegating_request() + + # Feed reasoning text. + d1 = parser.parse_delta( + "thinking", + [], + request, + finished=False, + ) + assert d1 is not None + assert d1.reasoning is not None + + # Feed and content in the same delta (e.g. speculative + # decoding accepting multiple tokens at once). Token IDs must + # cover all text so the scanner can split correctly. + # chr(10)='\n', chr(72)='H', chr(105)='i', chr(33)='!' + d2 = parser.parse_delta( + "\n\nHi!", + [201, 10, 10, 72, 105, 33], + request, + finished=False, + ) + assert d2 is not None, "Content after in multi-token delta was lost" + assert d2.content is not None, ( + "Content after in multi-token delta was nullified" + ) + assert "" not in d2.content + assert "Hi!" in d2.content + + +# ── TestToolAdapterForwardsKwargs ────────────────────────────────── + + +class TestToolAdapterForwardsKwargs: + """ParserEngineToolAdapter.__init__ must forward **kwargs to the + parser engine class so chat_template_kwargs reach model parsers.""" + + @pytest.mark.parametrize( + "enable_thinking,expected_state", + [ + (False, ParserState.CONTENT), + (True, ParserState.REASONING), + ], + ) + def test_kwargs_forwarded_to_parser_engine(self, enable_thinking, expected_state): + from vllm.parser.qwen3 import Qwen3Parser + + vocab = {"": 100, "": 101} + tokenizer = make_mock_tokenizer(vocab) + + _, ToolAdapter = make_adapters(Qwen3Parser) + adapter = ToolAdapter( + tokenizer, + tools=None, + chat_template_kwargs={"enable_thinking": enable_thinking}, + ) + engine = adapter._parser_engine + assert engine.parser_engine_config.initial_state == expected_state + + +# ── TestExtractContentIdsNoEmptyReturn ───────────────────────────── + + +class TestExtractContentIdsNoEmptyReturn: + """extract_content_ids must return input_ids (not []) when there is + no THINK_END token ID and _reasoning_ended is True.""" + + _NO_THINK_CONFIG = ParserEngineConfig(name="no_think_end", token_id_terminals={}) + + @pytest.mark.parametrize("input_ids", [[1, 2, 3], []]) + def test_returns_input_ids_without_think_end(self, input_ids): + engine = _make_engine(self._NO_THINK_CONFIG) + assert engine._reasoning_end_token_id is None + engine._reasoning_ended = True + assert engine.extract_content_ids(input_ids) == input_ids + + +# ── TestValuePostprocessorRemoved ────────────────────────────────── + + +class TestValuePostprocessorRemoved: + """ParserEngineConfig no longer has a value_postprocessor field.""" + + def test_no_value_postprocessor_field(self): + config = ParserEngineConfig(name="test") + assert not hasattr(config, "value_postprocessor") + + def test_constructor_rejects_value_postprocessor(self): + with pytest.raises(TypeError): + ParserEngineConfig( + name="test", + value_postprocessor=lambda x: x, # type: ignore[call-arg] + ) + + +# ── TestArgDeltaWithConverter ───────────────────────────────────── + + +_KV_RE = re.compile(r"(\w+)=(\S+)") + + +def _kv_converter(raw_args: str, partial: bool) -> str: + params: dict[str, str] = {} + for m in _KV_RE.finditer(raw_args): + params[m.group(1)] = m.group(2) + return json.dumps(params, ensure_ascii=False) + + +def _converter_config( + converter=_kv_converter, + name: str = "converter_test", +) -> ParserEngineConfig: + return ParserEngineConfig( + name=name, + terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + token_id_terminals={ + "TOOL_START": "", + "TOOL_END": "", + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_ARGS, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + arg_converter=converter, + stream_arg_deltas=True, + ) + + +def _collect_arg_deltas(deltas: list) -> str: + parts: list[str] = [] + for d in deltas: + if d is None: + continue + for tc in d.tool_calls or []: + if tc.function and tc.function.arguments: + parts.append(tc.function.arguments) + return "".join(parts) + + +def _run_streaming_tool(engine, name: str, chunks: list[str]) -> dict: + deltas = [] + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_CALL_START, tool_index=0)] + ) + ) + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_NAME, name, tool_index=0)] + ) + ) + for chunk in chunks: + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.ARG_VALUE_CHUNK, chunk, tool_index=0)] + ) + ) + deltas.append( + engine._events_to_delta([SemanticEvent(EventType.TOOL_CALL_END, tool_index=0)]) + ) + return json.loads(_collect_arg_deltas(deltas)) + + +class TestArgDeltaWithConverter: + """Exercise _compute_arg_delta with arg_converter + stream_arg_deltas. + + The startswith guard on line 814 of parser_engine.py validates that + converted JSON grows prefix-monotonically across streaming ticks. + These tests exercise that path with a synthetic config. + """ + + def test_streaming_arg_deltas_prefix_monotonic(self): + engine = _make_engine(_converter_config()) + deltas = [] + + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_CALL_START, tool_index=0)], + ) + ) + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0)], + ) + ) + deltas.append( + engine._events_to_delta( + [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + "a=hello ", + tool_index=0, + ) + ], + ) + ) + deltas.append( + engine._events_to_delta( + [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + "b=world ", + tool_index=0, + ) + ], + ) + ) + deltas.append( + engine._events_to_delta( + [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + "c=ok", + tool_index=0, + ) + ], + ) + ) + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_CALL_END, tool_index=0)], + ) + ) + + all_args = _collect_arg_deltas(deltas) + assert json.loads(all_args) == { + "a": "hello", + "b": "world", + "c": "ok", + } + + def test_streaming_arg_deltas_with_type_coercion(self): + tool = _make_tool( + "f", + { + "count": {"type": "integer"}, + "name": {"type": "string"}, + }, + ) + engine = _make_engine(_converter_config(), tools=[tool]) + deltas = [] + + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_CALL_START, tool_index=0)], + ) + ) + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_NAME, "f", tool_index=0)], + ) + ) + deltas.append( + engine._events_to_delta( + [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + "count=5 ", + tool_index=0, + ) + ], + ) + ) + deltas.append( + engine._events_to_delta( + [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + "name=test", + tool_index=0, + ) + ], + ) + ) + deltas.append( + engine._events_to_delta( + [SemanticEvent(EventType.TOOL_CALL_END, tool_index=0)], + ) + ) + + all_args = _collect_arg_deltas(deltas) + parsed = json.loads(all_args) + assert parsed == {"count": 5, "name": "test"} + assert isinstance(parsed["count"], int) + + +# ── TestSafeArgPrefix ──────────────────────────────────────────── + + +class TestSafeArgPrefix: + """Unit tests for ParserEngine._safe_arg_prefix.""" + + @pytest.mark.parametrize( + "json_str, expected", + [ + ('{"a": 1}', '{"a": '), + ('{"a": 1, "b": 2}', '{"a": 1, "b": '), + ('{"a": "hello", "b": "world"}', '{"a": "hello", "b": '), + ('{"obj": {"x": 1}, "b": 2}', '{"obj": {"x": 1}, "b": '), + ('{"url": "http://x:80", "b": 1}', '{"url": "http://x:80", "b": '), + ('{"a": 1', '{"a": '), + ("{}", ""), + ("{", ""), + ("", ""), + ('{"k":1}', '{"k":'), + ('{"k": 1, "v":2}', '{"k": 1, "v":'), + ], + ) + def test_safe_arg_prefix(self, json_str, expected): + assert ParserEngine._safe_arg_prefix(json_str) == expected + + +# ── Coercion instability regression tests ──────────────────────── + + +def _growing_kv_converter(raw_args: str, partial: bool) -> str: + """Converter that produces growing bare values (no delimiter).""" + params: dict[str, str] = {} + for part in raw_args.split(" "): + if "=" in part: + k, v = part.split("=", 1) + params[k] = v + return json.dumps(params, ensure_ascii=False) + + +class TestCoercionInstabilityRegression: + """Regression tests for _fix_arg_types coercion instability. + + These tests exercise scenarios where a trailing value's coercion + status changes between ticks (e.g. "4" coerces to int but "4e" + does not). Before the _safe_arg_prefix fix, these would violate + the startswith prefix invariant and permanently drop deltas. + """ + + def test_coercion_flip_does_not_corrupt_stream(self): + tool = _make_tool( + "f", + { + "count": {"type": "integer"}, + "flag": {"type": "string"}, + }, + ) + engine = _make_engine(_converter_config(), tools=[tool]) + parsed = _run_streaming_tool( + engine, + "f", + ["count=42 ", "flag=ok"], + ) + assert parsed == {"count": 42, "flag": "ok"} + assert isinstance(parsed["count"], int) + + def test_bool_partial_value_coercion_is_safe(self): + """Boolean value building char by char must not break prefix.""" + tool = _make_tool( + "f", + { + "name": {"type": "string"}, + "flag": {"type": "boolean"}, + }, + ) + cfg = _converter_config(_growing_kv_converter) + engine = _make_engine(cfg, tools=[tool]) + parsed = _run_streaming_tool( + engine, + "f", + ["name=hello ", "flag=t", "r", "u", "e"], + ) + assert parsed == {"name": "hello", "flag": True} + assert isinstance(parsed["flag"], bool) + + def test_int_partial_value_flip_is_safe(self): + """Integer that becomes non-coercible must not break prefix. + + A dummy first arg is needed so the name emission consumes the + first ARG_VALUE_CHUNK, ensuring _compute_arg_delta runs for the + chunk where val="4" coerces to int 4. On the next chunk val + grows to "4e" which is NOT a valid int, flipping the coercion. + """ + tool = _make_tool( + "f", + { + "dummy": {"type": "string"}, + "val": {"type": "integer"}, + "extra": {"type": "string"}, + }, + ) + cfg = _converter_config(_growing_kv_converter) + engine = _make_engine(cfg, tools=[tool]) + parsed = _run_streaming_tool( + engine, + "f", + ["dummy=x ", "val=4", "e ", "extra=ok"], + ) + assert parsed["dummy"] == "x" + assert parsed["val"] == "4e" + assert isinstance(parsed["val"], str) + assert parsed["extra"] == "ok" diff --git a/tests/parser/engine/test_qwen3.py b/tests/parser/engine/test_qwen3.py new file mode 100644 index 0000000000..7c2255ac7b --- /dev/null +++ b/tests/parser/engine/test_qwen3.py @@ -0,0 +1,1095 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based Qwen3 tool call parser. + +These validate that the engine-driven parser correctly handles +Qwen3 XML-style tool calls. +""" + +import json +from unittest.mock import MagicMock + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import ( + collect_content, + collect_function_name, + collect_tool_arguments, + simulate_tool_streaming, +) +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.qwen3 import ( + TOOL_CALL_END, + TOOL_CALL_START, + qwen3_config, +) + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer( + { + TOOL_CALL_START: 100, + TOOL_CALL_END: 101, + } + ) + + +@pytest.fixture +def parser(mock_tokenizer): + return ParserEngine( + mock_tokenizer, + parser_engine_config=qwen3_config(thinking=False), + ) + + +class TestNonStreaming: + def test_no_tool_calls(self, parser, mock_request): + result = parser.extract_tool_calls( + "This is a regular response without any tool calls.", + mock_request, + ) + assert result.tools_called is False + assert result.tool_calls == [] + assert result.content == ("This is a regular response without any tool calls.") + + def test_single_tool_call(self, parser, mock_request): + text = ( + "\n" + "\n" + "Tokyo\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "get_weather" + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"city": "Tokyo"} + + def test_parallel_tool_calls(self, parser, mock_request): + text = ( + "\n" + "\n" + "Tokyo\n" + "\n" + "" + "\n" + "\n" + "Asia/Tokyo\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 2 + assert result.tool_calls[0].function.name == "get_weather" + assert result.tool_calls[1].function.name == "get_time" + + args0 = json.loads(result.tool_calls[0].function.arguments) + assert args0 == {"city": "Tokyo"} + args1 = json.loads(result.tool_calls[1].function.arguments) + assert args1 == {"timezone": "Asia/Tokyo"} + + def test_various_data_types(self, parser, mock_request): + text = ( + "\n\n" + "hello\n" + "42\n" + "3.14\n" + "true\n" + "null\n" + '["a", "b", "c"]\n' + '{"nested": "value"}\n' + "\n" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + args = json.loads(result.tool_calls[0].function.arguments) + assert args["string_field"] == "hello" + assert args["int_field"] == "42" + assert args["float_field"] == "3.14" + assert args["bool_field"] == "true" + assert args["null_field"] == "null" + assert args["array_field"] == '["a", "b", "c"]' + assert args["object_field"] == '{"nested": "value"}' + + def test_empty_arguments(self, parser, mock_request): + text = "\n\n\n" + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert result.tool_calls[0].function.name == "refresh" + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {} + + def test_surrounding_text(self, parser, mock_request): + text = ( + "Let me check the weather for you.\n\n" + "\n\n" + "Tokyo\n" + "\n\n\n" + "I will get that information." + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert result.content is not None + assert "Let me check the weather" in result.content + assert result.tool_calls[0].function.name == "get_weather" + + def test_escaped_strings(self, parser, mock_request): + text = ( + "\n\n" + 'He said "hello"\n' + "C:\\Users\\file.txt\n" + "line1\nline2\n" + "\n" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + args = json.loads(result.tool_calls[0].function.arguments) + assert args["quoted"] == 'He said "hello"' + assert args["path"] == "C:\\Users\\file.txt" + assert args["newline"] == "line1\nline2" + + def test_multiple_parameters(self, parser, mock_request): + text = ( + "\n\n" + "vllm parsing\n" + "10\n" + "false\n" + "\n" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + args = json.loads(result.tool_calls[0].function.arguments) + assert args == { + "query": "vllm parsing", + "limit": "10", + "exact_match": "false", + } + + def test_multiline_param_values(self, parser, mock_request): + """Parameter values spanning multiple lines.""" + text = ( + "\n" + "\n" + "\n" + "ls -la /tmp\n" + "\n" + "\n" + "List files in /tmp directory\n" + "\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "Bash" + args = json.loads(result.tool_calls[0].function.arguments) + assert args["command"] == "ls -la /tmp" + assert args["description"] == "List files in /tmp directory" + + def test_multiline_two_tool_calls(self, parser, mock_request): + """Two tool calls with multi-line parameter values (bug report).""" + text = ( + "\n" + "\n" + "\n" + "find /workspace -name '*.py' | head -20\n" + "\n" + "\n" + "Find Python files\n" + "\n" + "\n" + "" + "\n" + "\n" + "\n" + "/workspace/main.py\n" + "\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 2 + assert result.tool_calls[0].function.name == "Bash" + assert result.tool_calls[1].function.name == "Read" + args0 = json.loads(result.tool_calls[0].function.arguments) + assert "find /workspace" in args0["command"] + assert "Find Python files" in args0["description"] + args1 = json.loads(result.tool_calls[1].function.arguments) + assert "/workspace/main.py" in args1["file_path"] + + def test_consecutive_tool_calls_without_tool_end(self, parser, mock_request): + text = ( + "\n" + "\n" + "Tokyo\n" + "\n" + "\n" + "\n" + "Paris\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 2 + args0 = json.loads(result.tool_calls[0].function.arguments) + assert args0 == {"city": "Tokyo"} + args1 = json.loads(result.tool_calls[1].function.arguments) + assert args1 == {"city": "Paris"} + + def test_nested_json_array_parameter(self, parser, mock_request): + text = ( + "\n" + "\n" + "" + '[{"question": "Pick a color",' + ' "multiSelect": false, "answer": null}]' + "\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + args = json.loads(result.tool_calls[0].function.arguments) + assert args == { + "questions": '[{"question": "Pick a color",' + ' "multiSelect": false, "answer": null}]', + } + + +class TestStreaming: + def test_basic_streaming(self, parser, mock_request): + chunks = [ + "\n", + "\n", + "Tokyo", + "\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "get_weather" + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert parsed == {"city": "Tokyo"} + + def test_streaming_multi_param(self, parser, mock_request): + chunks = [ + "\n", + "\n", + "Tokyo\n", + "celsius\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "get_weather" + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert parsed == {"city": "Tokyo", "unit": "celsius"} + + def test_streaming_args_arrive_incrementally(self, parser, mock_request): + """Arguments must stream as intermediate deltas, not batch at + tool-end.""" + chunks = [ + "\n", + "\n", + "Tokyo\n", + "celsius\n", + "5\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + arg_deltas: list[str] = [] + for delta, _ in results: + if delta and delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.arguments: + arg_deltas.append(tc.function.arguments) + + assert len(arg_deltas) > 1, ( + f"Expected arguments across multiple deltas, got {len(arg_deltas)}: " + f"{arg_deltas}" + ) + concatenated = "".join(arg_deltas) + parsed = json.loads(concatenated) + assert parsed == {"city": "Tokyo", "unit": "celsius", "days": "5"} + + def test_streaming_text_before_tool(self, parser, mock_request): + chunks = [ + "Let me check ", + "the weather. ", + "\n", + "\n", + "Tokyo\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + assert collect_content(results).strip().startswith("Let me check") + + def test_streaming_empty_args(self, parser, mock_request): + chunks = [ + "\n", + "\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "refresh" + + def test_streaming_split_parameter_tag(self, parser, mock_request): + """Parameter tag split across chunks.""" + chunks = [ + "\n", + "\n", + "Alice", + "\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "test" + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert parsed["name"] == "Alice" + + def test_streaming_numeric_values(self, parser, mock_request): + chunks = [ + "\n", + "\n", + "42\n", + "true\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + args_text = collect_tool_arguments(results) + if args_text: + parsed = json.loads(args_text) + assert parsed["count"] == "42" + assert parsed["active"] == "true" + + def test_streaming_parallel_calls(self, parser, mock_request): + chunks = [ + "\n", + "\n", + "Tokyo\n", + "\n", + "", + "\n", + "\n", + "JST\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + names = [] + for delta, _ in results: + if delta and delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.name: + names.append(tc.function.name) + + assert "get_weather" in names + assert "get_time" in names + + def test_streaming_value_split_across_chunks(self, parser, mock_request): + """Parameter value split across multiple chunks.""" + chunks = [ + "\n", + "\n", + "hello ", + "world", + " test\n", + "\n", + "", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert parsed["query"] == "hello world test" + + def test_streaming_split_tool_call_tag(self, parser, mock_request): + """ arrives as a single special token; the rest of + the content is split into fine-grained chunks.""" + chunks = [ + "\n", + "\n", + "1", + "\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "test" + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert parsed["x"] == "1" + + def test_char_by_char_streaming(self, mock_request): + """Feed text character-by-character to test lexer robustness. + + Uses a tokenizer without special token IDs because char-by-char + delivery only occurs when the tokenizer splits the tag across + multiple sub-word tokens (i.e., no dedicated special token). + """ + tokenizer = MagicMock() + tokenizer.encode.return_value = [1, 2, 3] + tokenizer.get_vocab.return_value = {} + tokenizer.decode.side_effect = lambda ids: "".join( + chr(i) if i < 128 else f"<{i}>" for i in ids + ) + no_tid_parser = ParserEngine( + tokenizer, parser_engine_config=qwen3_config(thinking=False) + ) + + full_text = ( + "\n" + "\n" + "hi\n" + "\n" + "" + ) + chunks = list(full_text) + results = simulate_tool_streaming(no_tid_parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "echo" + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert parsed == {"msg": "hi"} + + def test_streaming_multiline_param_values(self, parser, mock_request): + """Multi-line parameter values in streaming mode.""" + chunks = [ + "\n", + "\n", + "\n", + "ls -la /tmp\n", + "\n", + "\n", + "List files\n", + "\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser, mock_request, chunks) + + name = collect_function_name(results) + assert name == "Bash" + + args_text = collect_tool_arguments(results) + assert args_text + parsed = json.loads(args_text) + assert "ls -la /tmp" in parsed["command"] + assert "List files" in parsed["description"] + + def test_streaming_multiline_two_tool_calls(self, parser, mock_request): + """Two tool calls with multi-line values — matches bug report.""" + chunks = [ + "\n", + "\n", + "\n", + "find /workspace -name '*.py' | head -20\n", + "\n", + "\n", + "Find Python files\n", + "\n", + "\n", + "", + "\n", + "\n", + "\n", + "/workspace/main.py\n", + "\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser, mock_request, chunks) + + names = [] + for delta, _ in results: + if delta and delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.name: + names.append(tc.function.name) + + assert "Bash" in names + assert "Read" in names + + +class TestArgConverter: + """Direct tests for the Qwen3 arg_converter with multi-line values.""" + + def test_multiline_param_values(self): + from vllm.parser.qwen3 import ( + _qwen3_arg_converter, + ) + + raw = ( + "\n" + "ls -la /tmp\n" + "\n" + "\n" + "List files\n" + "\n" + ) + result = json.loads(_qwen3_arg_converter(raw, partial=False)) + assert result["command"] == "ls -la /tmp" + assert result["description"] == "List files" + + def test_two_multiline_params(self): + from vllm.parser.qwen3 import ( + _qwen3_arg_converter, + ) + + raw = ( + "\nfoo\nbar\n\n" + "\nbaz\nqux\n\n" + ) + result = json.loads(_qwen3_arg_converter(raw, partial=False)) + assert result["a"] == "foo\nbar" + assert result["b"] == "baz\nqux" + + def test_partial_multiline(self): + from vllm.parser.qwen3 import ( + _qwen3_arg_converter, + ) + + raw = "\nls -la\n\npartial value" + result = json.loads(_qwen3_arg_converter(raw, partial=True)) + assert result["command"] == "ls -la" + assert result["desc"] == "\npartial value" + + +class TestSchemaAwareTypeCoercion: + """Verify that _fix_arg_types corrects miscoerced values using the + tool schema.""" + + @pytest.fixture + def tools(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + return [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "TaskUpdate", + "parameters": { + "type": "object", + "properties": { + "taskId": {"type": "string"}, + "count": {"type": "integer"}, + "ratio": {"type": "number"}, + "flag": {"type": "string"}, + }, + }, + }, + ) + ] + + @pytest.fixture + def parser_with_tools(self, mock_tokenizer, tools): + return ParserEngine( + mock_tokenizer, + tools=tools, + parser_engine_config=qwen3_config(thinking=False), + ) + + def test_string_param_not_coerced_to_int(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "1\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + assert result.tools_called + args = json.loads(result.tool_calls[0].function.arguments) + assert args["taskId"] == "1" + assert isinstance(args["taskId"], str) + + def test_string_param_not_coerced_to_bool(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "true\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["flag"] == "true" + assert isinstance(args["flag"], str) + + def test_int_param_still_coerced(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "42\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["count"] == 42 + assert isinstance(args["count"], int) + + def test_no_tools_keeps_strings(self, parser, mock_request): + text = ( + "\n" + "\n" + "1\n" + "\n" + "" + ) + result = parser.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["taskId"] == "1" + assert isinstance(args["taskId"], str) + + def test_streaming_string_param_not_coerced(self, parser_with_tools, mock_request): + chunks = [ + "\n", + "\n", + "1\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) + args_str = collect_tool_arguments(results) + args = json.loads(args_str) + assert args["taskId"] == "1" + assert isinstance(args["taskId"], str) + + +class TestAnyOfTypeCoercion: + """Verify that _fix_arg_types handles union types (anyOf/oneOf).""" + + @pytest.fixture + def tools_with_anyof(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + return [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "set_config", + "parameters": { + "type": "object", + "properties": { + "port": { + "anyOf": [ + {"type": "string"}, + {"type": "null"}, + ], + }, + "count": {"type": "integer"}, + }, + }, + }, + ) + ] + + @pytest.fixture + def parser_with_anyof(self, mock_tokenizer, tools_with_anyof): + return ParserEngine( + mock_tokenizer, + tools=tools_with_anyof, + parser_engine_config=qwen3_config(thinking=False), + ) + + def test_anyof_string_param_not_coerced(self, parser_with_anyof, mock_request): + """A param with anyOf including 'string' must not be coerced + to integer.""" + text = ( + "\n" + "\n" + "8080\n" + "\n" + "" + ) + result = parser_with_anyof.extract_tool_calls(text, mock_request) + assert result.tools_called + args = json.loads(result.tool_calls[0].function.arguments) + assert args["port"] == "8080" + + +class TestSchemaCoercionBoolNumberNull: + """Verify that _fix_arg_types coerces string values to non-string + schema types using coerce_to_schema_type.""" + + @pytest.fixture + def tools(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + return [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "configure", + "parameters": { + "type": "object", + "properties": { + "enabled": {"type": "boolean"}, + "ratio": {"type": "number"}, + "count": {"type": "integer"}, + "value": {"type": ["integer", "null"]}, + "label": {"type": "string"}, + }, + }, + }, + ) + ] + + @pytest.fixture + def parser_with_tools(self, mock_tokenizer, tools): + return ParserEngine( + mock_tokenizer, + tools=tools, + parser_engine_config=qwen3_config(thinking=False), + ) + + def test_bool_param_coerced(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "true\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["enabled"] is True + assert isinstance(args["enabled"], bool) + + def test_number_param_whole_normalized(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "5.0\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["ratio"] == 5 + assert isinstance(args["ratio"], int) + + def test_number_param_fractional(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "3.14\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["ratio"] == pytest.approx(3.14) + assert isinstance(args["ratio"], float) + + def test_null_coerced_when_in_schema(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "null\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["value"] is None + + def test_null_stays_string_without_null_schema( + self, parser_with_tools, mock_request + ): + text = ( + "\n" + "\n" + "null\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["label"] == "null" + assert isinstance(args["label"], str) + + def test_streaming_bool_param_coerced(self, parser_with_tools, mock_request): + chunks = [ + "\n", + "\n", + "true\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) + args_str = collect_tool_arguments(results) + args = json.loads(args_str) + assert args["enabled"] is True + assert isinstance(args["enabled"], bool) + + def test_streaming_number_param_coerced(self, parser_with_tools, mock_request): + chunks = [ + "\n", + "\n", + "3.14\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) + args_str = collect_tool_arguments(results) + args = json.loads(args_str) + assert args["ratio"] == pytest.approx(3.14) + assert isinstance(args["ratio"], float) + + def test_streaming_matches_non_streaming_comprehensive( + self, parser_with_tools, mock_request + ): + text = ( + "\n" + "\n" + "true\n" + "5.0\n" + "42\n" + "null\n" + "hello\n" + "\n" + "" + ) + non_stream = parser_with_tools.extract_tool_calls(text, mock_request) + ns_args = json.loads(non_stream.tool_calls[0].function.arguments) + + chunks = [line + "\n" for line in text.split("\n") if line] + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) + s_args = json.loads(collect_tool_arguments(results)) + + assert s_args == ns_args + assert ns_args == { + "enabled": True, + "ratio": 5, + "count": 42, + "value": None, + "label": "hello", + } + + +class TestNestedSchemaCoercion: + """Verify that _fix_arg_types recurses into nested objects and arrays.""" + + @pytest.fixture + def tools(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + return [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "search", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "filters": { + "type": "object", + "properties": { + "language": {"type": "string"}, + "min_stars": {"type": "integer"}, + }, + }, + "tags": { + "type": "array", + "items": {"type": "string"}, + }, + "limits": { + "type": "array", + "items": {"type": "integer"}, + }, + "verbose": {"type": "boolean"}, + }, + }, + }, + ), + ChatCompletionToolsParam( + type="function", + function={ + "name": "AskUserQuestion", + "parameters": { + "type": "object", + "properties": { + "questions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "question": {"type": "string"}, + "multiSelect": { + "type": "boolean", + }, + "answer": { + "type": ["string", "null"], + }, + }, + }, + }, + }, + }, + }, + ), + ] + + @pytest.fixture + def parser_with_tools(self, mock_tokenizer, tools): + return ParserEngine( + mock_tokenizer, + tools=tools, + parser_engine_config=qwen3_config(thinking=False), + ) + + def test_nested_object_coerced(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + '{"language": "python",' + ' "min_stars": 100}\n' + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["filters"] == {"language": "python", "min_stars": 100} + assert isinstance(args["filters"]["min_stars"], int) + + def test_nested_array_items_coerced(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + "[10, 20, 30]\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["limits"] == [10, 20, 30] + assert all(isinstance(v, int) for v in args["limits"]) + + def test_nested_string_array_not_coerced(self, parser_with_tools, mock_request): + text = ( + "\n" + "\n" + '["ml", "42"]\n' + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["tags"] == ["ml", "42"] + assert all(isinstance(v, str) for v in args["tags"]) + + def test_array_of_objects_with_bool_and_null_coerced( + self, parser_with_tools, mock_request + ): + text = ( + "\n" + "\n" + "" + '[{"question": "Pick a color",' + ' "multiSelect": false, "answer": null}]' + "\n" + "\n" + "" + ) + result = parser_with_tools.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + questions = args["questions"] + assert isinstance(questions, list) + assert len(questions) == 1 + assert questions[0]["question"] == "Pick a color" + assert questions[0]["multiSelect"] is False + assert questions[0]["answer"] is None + + def test_streaming_array_of_objects_with_bool_and_null_coerced( + self, parser_with_tools, mock_request + ): + chunks = [ + "\n", + "\n", + '[{"question": "Pick a color",', + ' "multiSelect": false, "answer": null}]', + "\n", + "\n", + "", + ] + results = simulate_tool_streaming(parser_with_tools, mock_request, chunks) + args_str = collect_tool_arguments(results) + args = json.loads(args_str) + questions = args["questions"] + assert isinstance(questions, list) + assert len(questions) == 1 + assert questions[0]["question"] == "Pick a color" + assert questions[0]["multiSelect"] is False + assert questions[0]["answer"] is None diff --git a/tests/parser/engine/test_qwen3_reasoning.py b/tests/parser/engine/test_qwen3_reasoning.py new file mode 100644 index 0000000000..a46294d966 --- /dev/null +++ b/tests/parser/engine/test_qwen3_reasoning.py @@ -0,0 +1,549 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based Qwen3 reasoning parser. + +Validates that ``Qwen3Parser`` correctly handles +````/```` reasoning with Qwen3-specific extensions: +- ```` as implicit reasoning end (terminal + token ID) +- Stripping ```` from generated output (old template compat) +- No terminal text (````, ````) leaks into output +""" + +import dataclasses + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming +from vllm.parser.engine.parser_engine_config import ParserState +from vllm.parser.qwen3 import Qwen3Parser, qwen3_config + +_THINK_START_ID = 50 +_THINK_END_ID = 51 +_TOOL_CALL_ID = 60 +_TOOL_CALL_END_ID = 61 +_TEXT_ID = 100 + +_QWEN3_VOCAB = { + "": _THINK_START_ID, + "": _THINK_END_ID, + "": _TOOL_CALL_ID, + "": _TOOL_CALL_END_ID, +} + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_QWEN3_VOCAB) + + +@pytest.fixture +def parser(mock_tokenizer): + return Qwen3Parser(mock_tokenizer) + + +class TestNonStreaming: + def test_reasoning_then_content(self, parser): + text = "Let me analyze.The answer is 42." + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "Let me analyze." + assert content == "The answer is 42." + + def test_no_start_token_in_output(self, parser): + """Qwen3.5+ style: in prompt, only in output.""" + text = "Let me think about this.The answer is 42." + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "Let me think about this." + assert content == "The answer is 42." + + def test_reasoning_only(self, parser): + text = "Still thinking..." + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "Still thinking..." + assert content is None + + def test_no_end_tag_all_reasoning(self, parser): + """No means truncated output — everything is reasoning.""" + text = "Hello, no reasoning here." + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "Hello, no reasoning here." + assert content is None + + def test_multiline_reasoning(self, parser): + text = ( + "Step 1: parse.\nStep 2: compute.\nStep 3: output.Result: 7." + ) + reasoning, content = parser.extract_reasoning(text, None) + assert "Step 1" in reasoning + assert "Step 3" in reasoning + assert content == "Result: 7." + + def test_tool_call_implicit_end(self, parser): + """ without acts as implicit reasoning end.""" + text = ( + "I need to read the file.\n\n" + "\n\n" + "ls\n" + "\n" + ) + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "I need to read the file.\n\n" + assert "" not in reasoning + assert "" not in reasoning + + def test_tool_call_implicit_end_no_think(self, parser): + """ as implicit end, no in output.""" + text = ( + "I need to read the file.\n\n" + "\n\n" + "ls\n" + "\n" + ) + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "I need to read the file.\n\n" + assert "" not in reasoning + + def test_live_scenario_think_end_before_tool_call(self, parser): + """Real model output: immediately before . + + Regression test for the bug where and + leaked into reasoning content. + """ + text = ( + "The user wants to see what files are in the current directory" + " and their contents. Let me start by listing the directory." + "" + "/Users/test/demo" + "" + ) + reasoning, content = parser.extract_reasoning(text, None) + expected_reasoning = ( + "The user wants to see what files are in the current directory" + " and their contents. Let me start by listing the directory." + ) + assert reasoning == expected_reasoning + assert "" not in reasoning + assert "" not in reasoning + assert "" not in (reasoning or "") + assert "" not in (reasoning or "") + + def test_no_terminal_text_in_content(self, parser): + """Terminal text must never appear in content output.""" + text = "Reasoning here.Content here." + reasoning, content = parser.extract_reasoning(text, None) + assert "" not in (content or "") + assert "" not in (content or "") + + def test_duplicate_think_end_absorbed(self, parser): + """Duplicate in CONTENT state must not leak.""" + text = "Reasoning here.Content here.More content." + reasoning, content = parser.extract_reasoning(text, None) + assert reasoning == "Reasoning here." + assert content == "Content here.More content." + + +class TestIsReasoningEnd: + def test_think_end_token(self, parser): + assert parser.is_reasoning_end([_THINK_START_ID, 1, _THINK_END_ID]) + + def test_no_end_token(self, parser): + assert not parser.is_reasoning_end([_THINK_START_ID, 1, 2]) + + def test_start_after_end_means_not_ended(self, parser): + assert not parser.is_reasoning_end([_THINK_END_ID, _THINK_START_ID, 1]) + + def test_tool_call_as_implicit_end(self, parser): + """Unpaired is implicit reasoning end.""" + assert parser.is_reasoning_end([_THINK_START_ID, 1, _TOOL_CALL_ID]) + + def test_paired_tool_call_not_end(self, parser): + """Paired ... (from template) is NOT end.""" + assert not parser.is_reasoning_end( + [_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID] + ) + + def test_tool_call_after_think_end(self, parser): + """ after — already ended.""" + assert parser.is_reasoning_end( + [_THINK_START_ID, 1, _THINK_END_ID, _TOOL_CALL_ID] + ) + + def test_empty_ids(self, parser): + assert not parser.is_reasoning_end([]) + + +class TestStreaming: + def test_basic_streaming(self, parser): + reasoning, content = simulate_reasoning_streaming( + parser, + ["", "thinking", " hard", "", "done"], + [ + (_THINK_START_ID,), + (1,), + (2,), + (_THINK_END_ID,), + (3,), + ], + ) + assert reasoning == "thinking hard" + assert content == "done" + + def test_streaming_no_start_token(self, parser): + """Qwen3.5 style: no in output, just reasoning then .""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["reasoning ", "text", "", "content"], + [ + (1,), + (2,), + (_THINK_END_ID,), + (3,), + ], + ) + assert reasoning == "reasoning text" + assert content == "content" + + def test_streaming_start_token_stripped(self, parser): + """ in output (old template) should be stripped.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["reasoning", "", "content"], + [ + (_THINK_START_ID, 1), + (_THINK_END_ID,), + (2,), + ], + ) + assert reasoning == "reasoning" + assert content == "content" + + def test_streaming_tool_call_implicit_end(self, parser): + """ ends reasoning implicitly during streaming.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["I need to check.", "", "\n"], + [ + (1,), + (_TOOL_CALL_ID,), + (2,), + ], + ) + assert reasoning == "I need to check." + assert "" not in reasoning + assert "" not in reasoning + assert content is not None + + def test_streaming_content_after_think_end(self, parser): + """Content deltas after are routed as content.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["reasoning", "", "content1", " content2"], + [ + (1,), + (_THINK_END_ID,), + (2,), + (3,), + ], + ) + assert reasoning == "reasoning" + assert content == "content1 content2" + + def test_streaming_content_after_tool_call(self, parser): + """Content deltas after are routed as content.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["thinking", "", ""], + [ + (1,), + (_TOOL_CALL_ID,), + (2,), + ], + ) + assert reasoning == "thinking" + assert "" not in reasoning + assert content is not None + + def test_streaming_end_grouped_with_content(self, parser): + """ grouped with following content in one delta.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["reasoning", "the answer"], + [ + (1,), + (_THINK_END_ID, 2), + ], + ) + assert reasoning == "reasoning" + assert content == "the answer" + + def test_streaming_think_and_end_in_one_delta(self, parser): + """ and in the same delta.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["reasoning"], + [ + (_THINK_START_ID, 1, _THINK_END_ID), + ], + ) + assert reasoning == "reasoning" + assert content == "" + + def test_streaming_pure_content_no_think(self, parser): + """No think tokens at all — everything is reasoning (truncated).""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["hello ", "world"], + [ + (1,), + (2,), + ], + ) + assert reasoning == "hello world" + assert content == "" + + def test_streaming_think_end_and_tool_call_same_delta(self, parser): + """ and in the same delta — no leakage. + + Regression test: the old override split at without + stripping , causing to leak into reasoning. + """ + reasoning, content = simulate_reasoning_streaming( + parser, + [ + "Let me list the directory.", + "", + "", + "/tmp", + ], + [ + (1,), + (_THINK_END_ID, _TOOL_CALL_ID), + (2,), + (3,), + ], + ) + assert reasoning == "Let me list the directory." + assert "" not in reasoning + assert "" not in reasoning + assert "", "content"], + [ + (1,), + (_THINK_END_ID,), + (2,), + ], + ) + assert "" not in reasoning + assert "" not in content + assert "" not in reasoning + + def test_streaming_duplicate_think_end_absorbed(self, parser): + """Duplicate token in CONTENT state must not leak.""" + reasoning, content = simulate_reasoning_streaming( + parser, + ["reasoning", "", "content", "", "more"], + [ + (1,), + (_THINK_END_ID,), + (2,), + (_THINK_END_ID,), + (3,), + ], + ) + assert reasoning == "reasoning" + assert content == "contentmore" + + +class TestTrailingWhitespaceStripping: + """When strip_trailing_reasoning_whitespace is True, + trailing whitespace before must be stripped. + + Models often generate trailing newlines before , and these + accumulate across multi-turn conversations via a feedback loop. + """ + + @pytest.fixture + def parser_with_strip(self): + cfg = dataclasses.replace( + qwen3_config(), + strip_trailing_reasoning_whitespace=True, + ) + return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg) + + def test_non_streaming_trailing_newline(self, parser_with_strip): + text = "Reasoning here.\nContent." + reasoning, content = parser_with_strip.extract_reasoning(text, None) + assert reasoning == "Reasoning here." + assert content == "Content." + + def test_non_streaming_multiple_trailing_newlines(self, parser_with_strip): + text = "Reasoning here.\n\n\nContent." + reasoning, content = parser_with_strip.extract_reasoning(text, None) + assert reasoning == "Reasoning here." + assert content == "Content." + + def test_non_streaming_internal_newlines_preserved(self, parser_with_strip): + text = "Step 1.\n\nStep 2.\n\nStep 3.Answer." + reasoning, content = parser_with_strip.extract_reasoning(text, None) + assert reasoning == "Step 1.\n\nStep 2.\n\nStep 3." + assert content == "Answer." + + def test_non_streaming_only_newlines_becomes_none(self, parser_with_strip): + text = "\n\n\nContent." + reasoning, content = parser_with_strip.extract_reasoning(text, None) + assert reasoning is None + assert content == "Content." + + def test_streaming_trailing_newline_stripped(self, parser_with_strip): + reasoning, content = simulate_reasoning_streaming( + parser_with_strip, + ["thinking.\n", "", "done"], + [ + (1,), + (_THINK_END_ID,), + (2,), + ], + ) + assert reasoning == "thinking." + assert content == "done" + + def test_streaming_multiple_trailing_newlines_stripped(self, parser_with_strip): + reasoning, content = simulate_reasoning_streaming( + parser_with_strip, + ["thinking.\n", "\n", "\n", "", "done"], + [ + (1,), + (2,), + (3,), + (_THINK_END_ID,), + (4,), + ], + ) + assert reasoning == "thinking." + assert content == "done" + + def test_streaming_internal_newlines_preserved(self, parser_with_strip): + reasoning, content = simulate_reasoning_streaming( + parser_with_strip, + ["Step 1.\n", "\nStep 2.\n", "", "Answer"], + [ + (1,), + (2,), + (_THINK_END_ID,), + (3,), + ], + ) + assert reasoning == "Step 1.\n\nStep 2." + assert content == "Answer" + + def test_streaming_trailing_newlines_before_tool_call(self, parser_with_strip): + """Trailing newlines before implicit end are stripped.""" + reasoning, content = simulate_reasoning_streaming( + parser_with_strip, + ["I'll check.\n\n", "", ""], + [ + (1,), + (_TOOL_CALL_ID,), + (2,), + ], + ) + assert reasoning == "I'll check." + assert "" not in reasoning + + +class TestWhitespaceStrippingDisabled: + """When strip_trailing_reasoning_whitespace is False, + trailing whitespace in reasoning must be preserved.""" + + @pytest.fixture + def parser_no_strip(self): + cfg = dataclasses.replace( + qwen3_config(), + strip_trailing_reasoning_whitespace=False, + ) + return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg) + + def test_non_streaming_preserves_trailing_newline(self, parser_no_strip): + text = "Reasoning here.\nContent." + reasoning, content = parser_no_strip.extract_reasoning(text, None) + assert reasoning == "Reasoning here.\n" + assert content == "Content." + + def test_streaming_preserves_trailing_newlines(self, parser_no_strip): + reasoning, content = simulate_reasoning_streaming( + parser_no_strip, + ["thinking.\n", "\n", "", "done"], + [ + (1,), + (2,), + (_THINK_END_ID,), + (3,), + ], + ) + assert reasoning == "thinking.\n\n" + assert content == "done" + + +class TestThinkingDisabled: + """When ``enable_thinking=False``, the chat template pre-fills a closed + ``\\n\\n\\n\\n`` block. The model output starts in content + state, so the parser's initial state must be CONTENT — not REASONING. + """ + + def test_thinking_disabled_initial_state_is_content(self, mock_tokenizer): + p = Qwen3Parser( + mock_tokenizer, + chat_template_kwargs={"enable_thinking": False}, + ) + assert p.parser_engine_config.initial_state == ParserState.CONTENT + + def test_thinking_enabled_initial_state_is_reasoning(self, mock_tokenizer): + p = Qwen3Parser( + mock_tokenizer, + chat_template_kwargs={"enable_thinking": True}, + ) + assert p.parser_engine_config.initial_state == ParserState.REASONING + + def test_default_initial_state_is_reasoning(self, mock_tokenizer): + p = Qwen3Parser(mock_tokenizer) + assert p.parser_engine_config.initial_state == ParserState.REASONING + + def test_thinking_disabled_streaming_content_only(self, mock_tokenizer): + """Plain text with thinking disabled must stream as content, not + reasoning. Before the fix, the REASONING initial state caused all + output to be emitted as reasoning chunks.""" + p = Qwen3Parser( + mock_tokenizer, + chat_template_kwargs={"enable_thinking": False}, + ) + reasoning, content = simulate_reasoning_streaming( + p, + ["The answer", " is 42."], + [ + (_TEXT_ID,), + (_TEXT_ID,), + ], + ) + assert content == "The answer is 42." + assert reasoning == "" + + def test_thinking_disabled_non_streaming(self, mock_tokenizer): + p = Qwen3Parser( + mock_tokenizer, + chat_template_kwargs={"enable_thinking": False}, + ) + reasoning, content = p.extract_reasoning("The answer is 42.", None) + assert reasoning is None + assert content == "The answer is 42." diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py new file mode 100644 index 0000000000..7d257feb9d --- /dev/null +++ b/tests/parser/engine/test_replay.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters). + +Replays dynamically built token sequences at different chunk sizes and +holdback depths to verify chunk-size invariance and terminal-token hygiene. +""" + +from __future__ import annotations + +import pytest + +from tests.parser.engine.replay_harness import ( + _test_request, + assert_no_terminal_leakage, + assert_parse_output, + collect_output, + make_mock_tokenizer, + replay_streaming, +) +from tests.parser.engine.trace_builder import build_samples +from vllm.parser.abstract_parser import Parser +from vllm.parser.engine.registered_adapters import ( + Qwen3Parser, +) + +_ENGINE_PARSERS: dict[str, type[Parser]] = { + "qwen3_engine": Qwen3Parser, +} + +_qwen3_samples = build_samples("qwen3") + +_QWEN3_TERMINALS = [ + "", + "", + "", + "", + "", +] + +HOLDBACK_CONFIGS = [6, 12, 24] + + +@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}") +@pytest.mark.parametrize("chunk_size", [5, 10], ids=lambda c: f"chunk{c}") +@pytest.mark.parametrize("sample", _qwen3_samples, ids=lambda s: s.id) +class TestQwen3ReplayWithHoldback: + """Replay Qwen3 with simulated detokenizer holdback.""" + + def test_replay(self, sample, chunk_size, holdback): + tokenizer = make_mock_tokenizer(sample) + parser = Qwen3Parser(tokenizer, sample.tools) + deltas = replay_streaming( + parser, + sample.tokens, + chunk_size=chunk_size, + holdback_chars=holdback, + ) + output = collect_output(deltas) + + assert_parse_output(output, sample) + assert_no_terminal_leakage( + output, + _QWEN3_TERMINALS, + context=f"chunk_size={chunk_size}, holdback={holdback}", + ) + + +_TOOL_CALL_SAMPLES = [ + (Qwen3Parser, s) + for s in _qwen3_samples + if s.expected_tool_calls and s.expected_reasoning +] + + +def _suppressed_expectations(sample) -> tuple[str, str]: + """Compute expected (reasoning, content) when tools are suppressed. + + When an explicit reasoning-end delimiter (````, ````) + is present, reasoning ends there and the tool call block becomes content. + When reasoning ends implicitly (the tool-start token triggers both + REASONING_END and TOOL_CALL_START), reasoning still ends at the tool + start and the raw tool call block becomes content text — only the + structured tool parsing is suppressed, not the reasoning boundary. + """ + full_text = "".join(text for _, text in sample.tokens) + reasoning = sample.expected_reasoning + idx = full_text.find(reasoning) + if idx < 0: + return (full_text, "") + after_reasoning = full_text[idx + len(reasoning) :] + for delim in ("", ""): + pos = after_reasoning.find(delim) + if pos >= 0: + return (reasoning, after_reasoning[pos + len(delim) :]) + for delim in ("",): + pos = after_reasoning.find(delim) + if pos >= 0: + return (reasoning, after_reasoning[pos:]) + return (full_text, "") + + +_DUMMY_TOOLS = [ + { + "type": "function", + "function": {"name": "stub", "parameters": {"type": "object"}}, + } +] + + +@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}") +@pytest.mark.parametrize( + "parser_cls,sample", + _TOOL_CALL_SAMPLES, + ids=lambda v: v.id if hasattr(v, "id") else v.__name__, +) +class TestSkipToolParsingReplay: + """Replay with skip_tool_parsing=True (tool_choice='none'). + + Verifies that reasoning is extracted normally and the raw tool call + block appears as content text with no tool calls parsed. + """ + + def test_replay(self, parser_cls, sample, chunk_size): + tokenizer = make_mock_tokenizer(sample) + kwargs = {} + if sample.chat_template_kwargs: + kwargs["chat_template_kwargs"] = sample.chat_template_kwargs + parser = parser_cls(tokenizer, **kwargs) + + request = _test_request() + request.tool_choice = "none" + request.tools = _DUMMY_TOOLS + + all_ids = [tid for tid, _ in sample.tokens] + all_texts = [text for _, text in sample.tokens] + if chunk_size is None: + chunk_size = len(all_ids) + + results = [] + chunks = list(range(0, len(all_ids), chunk_size)) + for i, start in enumerate(chunks): + end = min(start + chunk_size, len(all_ids)) + is_last = i == len(chunks) - 1 + result = parser.parse_delta( + "".join(all_texts[start:end]), + all_ids[start:end], + request, + prompt_token_ids=[] if start == 0 else None, + finished=is_last, + ) + results.append(result) + + output = collect_output(results) + + expected_reasoning, expected_content = _suppressed_expectations(sample) + + assert output.reasoning == expected_reasoning, ( + f"Reasoning mismatch:\n" + f" expected: {expected_reasoning!r}\n" + f" actual: {output.reasoning!r}" + ) + assert output.tool_calls == [], ( + f"Expected no tool calls but got {output.tool_calls}" + ) + assert output.content == expected_content, ( + f"Content mismatch:\n" + f" expected: {expected_content!r}\n" + f" actual: {output.content!r}" + ) + + +class TestAdapterReferences: + """Verify make_adapters sets reasoning/tool parser class refs on parser engine + parser classes so the serving layer finds them and calls adjust_request.""" + + @pytest.mark.parametrize( + "parser_name", + list(_ENGINE_PARSERS.keys()), + ) + def test_adapter_cls_refs_set(self, parser_name): + parser_cls = _ENGINE_PARSERS[parser_name] + assert parser_cls.reasoning_parser_cls is not None, ( + f"{parser_name}: reasoning_parser_cls is None" + ) + assert parser_cls.tool_parser_cls is not None, ( + f"{parser_name}: tool_parser_cls is None" + ) diff --git a/tests/parser/engine/test_token_id_scanner.py b/tests/parser/engine/test_token_id_scanner.py new file mode 100644 index 0000000000..8284646ba1 --- /dev/null +++ b/tests/parser/engine/test_token_id_scanner.py @@ -0,0 +1,631 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for TokenIDScanner, focusing on hold-back text recovery. + +Uses gemma4_config for all end-to-end engine tests, covering +reasoning channels, tool calls, and combined flows.""" + +from unittest.mock import MagicMock + +import pytest + +from vllm.parser.engine.events import EventType +from vllm.parser.engine.token_id_scanner import ( + PreLexedTerminal, + TextChunk, + TokenIDScanner, +) + +CHANNEL_START = "<|channel>" +CHANNEL_END = "" +CHANNEL_START_ID = 100 +CHANNEL_END_ID = 101 +REGULAR_TOKEN_ID = 200 +TOOL_START = "" +TOOL_END = "" +TOOL_START_ID = 110 +TOOL_END_ID = 111 + + +@pytest.fixture +def tokenizer(): + tok = MagicMock() + tok.get_vocab.return_value = { + CHANNEL_START: CHANNEL_START_ID, + CHANNEL_END: CHANNEL_END_ID, + } + tok.decode.side_effect = lambda ids: { + CHANNEL_START_ID: CHANNEL_START, + CHANNEL_END_ID: CHANNEL_END, + REGULAR_TOKEN_ID: "regular", + }.get(ids[0], f"") + return tok + + +@pytest.fixture +def scanner(tokenizer): + return TokenIDScanner( + token_id_to_terminal={ + CHANNEL_START_ID: "THINK_START", + CHANNEL_END_ID: "THINK_END", + }, + tokenizer=tokenizer, + ) + + +class TestJoinDecodedTextReturnsStr: + """_join_decoded_text now returns str unconditionally (was + str | None when an isinstance guard made a branch unreachable).""" + + @pytest.fixture + def bare_scanner(self): + return TokenIDScanner({}, tokenizer=None, drop_token_ids=set()) + + def test_mixed_items(self, bare_scanner): + items = [ + TextChunk("hello "), + PreLexedTerminal("TOOL_START", 42, ""), + TextChunk(" world"), + ] + result = bare_scanner._join_decoded_text(items) + assert isinstance(result, str) + assert result == "hello world" + + def test_empty_list(self, bare_scanner): + result = bare_scanner._join_decoded_text([]) + assert isinstance(result, str) + assert result == "" + + def test_only_text_chunks(self, bare_scanner): + result = bare_scanner._join_decoded_text([TextChunk("abc"), TextChunk("def")]) + assert result == "abcdef" + + +class TestHoldbackTextRecovery: + def test_holdback_text_with_special_token_text_absent(self, scanner): + """delta_text has hold-back text but the special token's text is + NOT in delta_text (held back by the detokenizer). Terminal is + deferred until the text arrives in a subsequent delta.""" + result = scanner.scan( + delta_text="processed is appropriate.", + delta_token_ids=[CHANNEL_END_ID], + ) + + assert len(result) == 0 + + # Second scan: terminal text arrives (detokenizer flushes). + # Deferred terminal resolves with holdback text before it. + result2 = scanner.scan( + delta_text="Understood.", + delta_token_ids=[20, 21], + ) + pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)] + assert len(pre_lexed) == 1 + assert pre_lexed[0].terminal == "THINK_END" + texts = [r.text for r in result2 if isinstance(r, TextChunk)] + combined = "".join(texts) + assert "processed is appropriate." in combined + assert "Understood." in combined + + def test_holdback_text_with_special_token_text_present(self, scanner): + """delta_text includes hold-back text AND the special token text.""" + result = scanner.scan( + delta_text="holdback text", + delta_token_ids=[CHANNEL_END_ID], + ) + + assert len(result) == 2 + assert isinstance(result[0], TextChunk) + assert result[0].text == "holdback text" + assert isinstance(result[1], PreLexedTerminal) + assert result[1].terminal == "THINK_END" + + def test_no_holdback_text(self, scanner): + """delta_text is exactly the special token text — no hold-back.""" + result = scanner.scan( + delta_text="", + delta_token_ids=[CHANNEL_END_ID], + ) + + assert len(result) == 1 + assert isinstance(result[0], PreLexedTerminal) + assert result[0].terminal == "THINK_END" + + def test_empty_delta_text(self, scanner): + """delta_text is empty — terminal deferred until text arrives.""" + result = scanner.scan( + delta_text="", + delta_token_ids=[CHANNEL_END_ID], + ) + + assert len(result) == 0 + + flushed = scanner.flush_pending() + assert len(flushed) == 1 + assert isinstance(flushed[0], PreLexedTerminal) + assert flushed[0].terminal == "THINK_END" + + def test_empty_delta_text_drops_individual_decode_text(self, tokenizer): + """delta_text="" with multiple tokens including special: all + results deferred — individually-decoded TextChunks are unreliable + and PreLexedTerminals wait for text confirmation.""" + tool_start_id = 400 + tok_a = 201 + tok_b = 202 + tokenizer.decode.side_effect = lambda ids: { + tool_start_id: "<|tool_call>", + tok_a: "call:", + tok_b: "get_weather", + }.get(ids[0], "?") + + scanner = TokenIDScanner( + token_id_to_terminal={tool_start_id: "TOOL_START"}, + tokenizer=tokenizer, + ) + + result = scanner.scan( + delta_text="", + delta_token_ids=[tool_start_id, tok_a, tok_b], + ) + + assert len(result) == 0 + + flushed = scanner.flush_pending() + assert len(flushed) == 1 + assert isinstance(flushed[0], PreLexedTerminal) + assert flushed[0].terminal == "TOOL_START" + + def test_holdback_before_start_tag(self, scanner): + """Hold-back text before a reasoning start tag.""" + result = scanner.scan( + delta_text="prefix text<|channel>", + delta_token_ids=[CHANNEL_START_ID], + ) + + assert len(result) == 2 + assert isinstance(result[0], TextChunk) + assert result[0].text == "prefix text" + assert isinstance(result[1], PreLexedTerminal) + assert result[1].terminal == "THINK_START" + + def test_multi_token_batch_special_in_middle(self, scanner, tokenizer): + """Stream-interval > 1: batch has regular tokens + special token. + delta_text differs from individual decodes (context-dependent).""" + tok_a = 201 + tok_b = 202 + tokenizer.decode.side_effect = lambda ids: { + tok_a: "wordA", + tok_b: "wordB", + CHANNEL_END_ID: CHANNEL_END, + }.get(ids[0], "?") + + scanner_multi = TokenIDScanner( + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"}, + tokenizer=tokenizer, + ) + + result = scanner_multi.scan( + delta_text="holdback wordA wordB", + delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b], + ) + + texts = [r.text for r in result if isinstance(r, TextChunk)] + terminals = [r.terminal for r in result if isinstance(r, PreLexedTerminal)] + assert "THINK_END" in terminals + assert "holdback wordA" in "".join(texts) + + def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer): + """Stream-interval > 1: batch has regular + special token, but + delta_text doesn't contain the special token text at all + (held back by detokenizer along with trailing regular tokens). + Terminal is deferred until text arrives.""" + tok_a = 201 + tok_b = 202 + tokenizer.decode.side_effect = lambda ids: { + tok_a: "alpha", + tok_b: "beta", + CHANNEL_END_ID: CHANNEL_END, + }.get(ids[0], "?") + + scanner_multi = TokenIDScanner( + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"}, + tokenizer=tokenizer, + ) + + result = scanner_multi.scan( + delta_text="holdback alpha", + delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b], + ) + + assert len(result) == 0 + + # Next delta: terminal text arrives (detokenizer flushes). + # Deferred terminal resolves with holdback text before it. + result2 = scanner_multi.scan( + delta_text=" more text", + delta_token_ids=[300], + ) + pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)] + assert len(pre_lexed) == 1 + assert pre_lexed[0].terminal == "THINK_END" + text_chunks = [r for r in result2 if isinstance(r, TextChunk)] + combined = "".join(t.text for t in text_chunks) + assert "holdback alpha" in combined + assert "more text" in combined + + def test_holdback_with_content_after_special_token(self, tokenizer): + """delta_text has hold-back + special token + content after, + with corresponding token IDs for all parts.""" + tok_content = 210 + tokenizer.decode.side_effect = lambda ids: { + CHANNEL_END_ID: CHANNEL_END, + tok_content: "content start", + }.get(ids[0], "?") + + scanner = TokenIDScanner( + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"}, + tokenizer=tokenizer, + ) + + result = scanner.scan( + delta_text="reasoning end.content start", + delta_token_ids=[CHANNEL_END_ID, tok_content], + ) + + pre_lexed = [r for r in result if isinstance(r, PreLexedTerminal)] + assert len(pre_lexed) == 1 + assert pre_lexed[0].terminal == "THINK_END" + + text_chunks = [r for r in result if isinstance(r, TextChunk)] + combined = "".join(t.text for t in text_chunks) + assert "reasoning end." in combined + + +class TestDropTokens: + def test_drop_token_with_holdback(self, tokenizer): + """Drop tokens stripped from delta_text, hold-back text preserved. + Terminal is deferred when its text is absent from delta_text.""" + drop_id = 300 + tokenizer.decode.side_effect = lambda ids: { + CHANNEL_END_ID: CHANNEL_END, + drop_id: "", + }.get(ids[0], "?") + + scanner = TokenIDScanner( + token_id_to_terminal={CHANNEL_END_ID: "THINK_END"}, + tokenizer=tokenizer, + drop_token_ids={drop_id}, + ) + + result = scanner.scan( + delta_text="holdback", + delta_token_ids=[drop_id, CHANNEL_END_ID], + ) + + assert len(result) == 0 + + # Terminal text arrives in next delta; deferred terminal resolves. + result2 = scanner.scan( + delta_text="content", + delta_token_ids=[20], + ) + pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)] + assert len(pre_lexed) == 1 + assert pre_lexed[0].terminal == "THINK_END" + texts = [r.text for r in result2 if isinstance(r, TextChunk)] + combined = "".join(texts) + assert "holdback" in combined + assert "" not in combined + + assert len(scanner.flush_pending()) == 0 + + +class TestEndToEndReasoningHoldback: + """End-to-end tests through the full parser engine simulating + stream-interval > 1 and detokenizer hold-back, using + gemma4_config.""" + + def test_reasoning_content_not_truncated(self): + from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, + ) + from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine + + config = ParserEngineConfig( + name="test_channel", + initial_state=ParserState.CONTENT, + terminals={ + "THINK_START": CHANNEL_START, + "THINK_END": CHANNEL_END, + }, + token_id_terminals={ + "THINK_START": CHANNEL_START, + "THINK_END": CHANNEL_END, + }, + transitions={ + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + }, + ) + tok = MagicMock() + vocab = { + CHANNEL_START: CHANNEL_START_ID, + CHANNEL_END: CHANNEL_END_ID, + } + tok.get_vocab.return_value = vocab + tok.decode.side_effect = lambda ids: { + CHANNEL_START_ID: CHANNEL_START, + CHANNEL_END_ID: CHANNEL_END, + }.get(ids[0], f"tok{ids[0]}") + + engine = StreamingParserEngine(config, tok) + all_events = [] + + # Delta 1: channel start token (text includes start tag) + all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID])) + + # Delta 2: reasoning text (normal content, no special tokens) + all_events.extend( + engine.feed( + "thought\nThe request was received and ", + [10, 11, 12, 13, 14], + ) + ) + + # Delta 3: MORE reasoning text, the detokenizer held some back. + # Then channel end token arrives in token_ids, but its text + # is NOT in delta_text (held back by detokenizer). + # delta_text = previously held-back reasoning text only. + all_events.extend( + engine.feed( + "processed is appropriate.", + [CHANNEL_END_ID], + ) + ) + + # Delta 4: detokenizer flushes held-back channel end text + # plus new content tokens. + all_events.extend( + engine.feed( + "Understood.", + [20, 21], + ) + ) + + all_events.extend(engine.finish()) + + reasoning_text = "".join( + e.value for e in all_events if e.type == EventType.REASONING_CHUNK + ) + content_text = "".join( + e.value for e in all_events if e.type == EventType.TEXT_CHUNK + ) + + assert "processed is appropriate." in reasoning_text + assert "Understood." in content_text + + def test_backtick_content_not_truncated(self): + """Reproduces the hostname backtick truncation case.""" + from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, + ) + from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine + + config = ParserEngineConfig( + name="test_channel", + initial_state=ParserState.CONTENT, + terminals={ + "THINK_START": CHANNEL_START, + "THINK_END": CHANNEL_END, + }, + token_id_terminals={ + "THINK_START": CHANNEL_START, + "THINK_END": CHANNEL_END, + }, + transitions={ + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + }, + ) + tok = MagicMock() + vocab = { + CHANNEL_START: CHANNEL_START_ID, + CHANNEL_END: CHANNEL_END_ID, + } + tok.get_vocab.return_value = vocab + tok.decode.side_effect = lambda ids: { + CHANNEL_START_ID: CHANNEL_START, + CHANNEL_END_ID: CHANNEL_END, + }.get(ids[0], f"tok{ids[0]}") + + engine = StreamingParserEngine(config, tok) + all_events = [] + + all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID])) + all_events.extend( + engine.feed( + "thought\n1/10 completed. Next: ", + [10, 11, 12, 13], + ) + ) + + # Hold-back text includes backtick content; channel end text + # absent from delta_text. + all_events.extend( + engine.feed( + "`hostname`.\n", + [CHANNEL_END_ID], + ) + ) + + # Next delta flushes channel end + tool call start + all_events.extend( + engine.feed( + "tool output", + [20, 21], + ) + ) + + all_events.extend(engine.finish()) + + reasoning_text = "".join( + e.value for e in all_events if e.type == EventType.REASONING_CHUNK + ) + + assert "`hostname`." in reasoning_text + + +class TestRebuildFromAnchorsLiteralLookalike: + """When delta_text contains a literal mention of a special token's + text before the real special token, _rebuild_from_anchors must + anchor at the real occurrence, not the literal one.""" + + @pytest.fixture + def tool_scanner(self): + tok = MagicMock() + tok.get_vocab.return_value = { + TOOL_START: TOOL_START_ID, + TOOL_END: TOOL_END_ID, + } + tok.decode.side_effect = lambda ids: { + TOOL_START_ID: TOOL_START, + TOOL_END_ID: TOOL_END, + }.get(ids[0], f"t{ids[0]}") + return TokenIDScanner( + {TOOL_START_ID: "TOOL_START", TOOL_END_ID: "TOOL_END"}, + tok, + ) + + def test_literal_before_real_anchor(self, tool_scanner): + """Literal in prose followed by a real + special token — the scanner must split at the real one.""" + delta_text = 'Use like this: {"name":"f"}' + delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID] + items = tool_scanner.scan(delta_text, delta_token_ids) + + text_parts = [it.text for it in items if isinstance(it, TextChunk)] + terminals = [it for it in items if isinstance(it, PreLexedTerminal)] + + assert len(terminals) == 2 + assert terminals[0].terminal == "TOOL_START" + assert terminals[1].terminal == "TOOL_END" + + # The literal mention must appear in a text chunk, not be + # consumed by the TOOL_START anchor. + joined_text = "".join(text_parts) + assert "" in joined_text + assert '{"name":"f"}' in joined_text + + def test_multiple_tool_calls_with_literal_between(self, tool_scanner): + """Two real tool calls with a literal mention between them.""" + delta_text = ( + '{"name":"a"}' + " see syntax " + '{"name":"b"}' + ) + delta_token_ids = [ + TOOL_START_ID, + 1, + TOOL_END_ID, + 2, + 3, + 4, + TOOL_START_ID, + 5, + TOOL_END_ID, + ] + items = tool_scanner.scan(delta_text, delta_token_ids) + + terminals = [it for it in items if isinstance(it, PreLexedTerminal)] + assert len(terminals) == 4 + + text_parts = [it.text for it in items if isinstance(it, TextChunk)] + joined_text = "".join(text_parts) + # The literal mention between the two real calls must be in text + assert " syntax" in joined_text + + +class TestRebuildFromAnchorsCascadingDeferral: + """When a middle anchor's text is absent from delta_text, + only that anchor should be deferred — not subsequent ones + with valid positions.""" + + @pytest.fixture + def bare_scanner(self): + tok = MagicMock() + tok.decode.side_effect = lambda ids: f"t{ids[0]}" + return TokenIDScanner({}, tok) + + def test_middle_anchor_missing_does_not_cascade(self, bare_scanner): + a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START) + b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END) + c = PreLexedTerminal("TOOL_END", TOOL_END_ID, TOOL_END) + delta_text = f"prefix{TOOL_START}middle{TOOL_END}suffix" + results = [a, b, c] + + rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results) + + terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)] + texts = [r for r in rebuilt if isinstance(r, TextChunk)] + joined = "".join(t.text for t in texts) + + assert len(terminals) == 2 + assert terminals[0].terminal == "TOOL_START" + assert terminals[1].terminal == "TOOL_END" + assert "prefix" in joined + assert "middle" in joined + assert "suffix" in joined + assert len(bare_scanner._deferred_terminals) == 1 + assert bare_scanner._deferred_terminals[0].terminal == "THINK_END" + assert bare_scanner._deferred_post_text == "" + + def test_first_anchor_missing_rest_still_emitted(self, bare_scanner): + a = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END) + b = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START) + delta_text = f"text{TOOL_START}more" + results = [a, b] + + rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results) + + terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)] + assert len(terminals) == 1 + assert terminals[0].terminal == "TOOL_START" + assert len(bare_scanner._deferred_terminals) == 1 + assert bare_scanner._deferred_terminals[0].terminal == "THINK_END" + + def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner): + a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START) + b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END) + delta_text = f"text{TOOL_START}more" + results = [a, b] + + rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results) + + terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)] + assert len(terminals) == 1 + assert terminals[0].terminal == "TOOL_START" + texts = [r for r in rebuilt if isinstance(r, TextChunk)] + joined = "".join(t.text for t in texts) + assert "text" in joined + # "more" is deferred along with the missing terminal — + # it will be resolved in the next scan when the terminal + # text arrives. + assert bare_scanner._deferred_post_text == "more" + assert len(bare_scanner._deferred_terminals) == 1 + assert bare_scanner._deferred_terminals[0].terminal == "THINK_END" diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py new file mode 100644 index 0000000000..7c84a9134f --- /dev/null +++ b/tests/parser/engine/trace_builder.py @@ -0,0 +1,410 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""On-demand trace builder for parser engine testing and benchmarks. + +Generates token sequences programmatically from model-agnostic scenario +definitions. Each model format handler knows how to render scenarios +into the model's output format, tokenize them with correct special token +IDs, and compute expected parse outputs. + +Every generated sample is self-validated by replaying it through the +real parser before being returned. +""" + +from __future__ import annotations + +import functools +import json +from dataclasses import dataclass +from typing import Any + +from tests.parser.engine.replay_harness import ( + MockTokenizer, + Sample, + assert_parse_output, + collect_output, + replay_streaming, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, +) +from vllm.parser.engine.registered_adapters import ( + Qwen3Parser, +) + +# ── Data structures ────────────────────────────────────────────────── + + +@dataclass +class ToolCallSpec: + name: str + arguments: dict[str, Any] + + +@dataclass +class Scenario: + id: str + description: str + reasoning: str | None = None + content: str | None = None + tool_calls: list[ToolCallSpec] | None = None + + +# ── Scenarios ──────────────────────────────────────────────────────── + +_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"}) +_BASH_TOOL = ToolCallSpec( + "bash", {"command": "hostname", "description": "Get hostname"} +) +_WEATHER_TOOL = ToolCallSpec( + "get_weather", + {"city": "Dallas", "state": "TX", "unit": "fahrenheit"}, +) +_COMPLEX_TOOL = ToolCallSpec( + "search", + { + "query": "vllm parser", + "filters": {"language": "python", "min_stars": 100}, + "tags": ["ml", "inference"], + "limit": 10, + "verbose": True, + }, +) + +SCENARIOS: list[Scenario] = [ + Scenario( + id="think-then-tool", + description="Reasoning then single tool call", + reasoning="Let me check the file.", + tool_calls=[_READ_TOOL], + ), + Scenario( + id="think-then-parallel-tools", + description="Reasoning then two parallel tool calls", + reasoning="I need to run both commands.", + tool_calls=[_BASH_TOOL, _WEATHER_TOOL], + ), + Scenario( + id="think-then-content", + description="Reasoning then content response", + reasoning="Let me think about this carefully.", + content="The answer is 42.", + ), + Scenario( + id="content-only", + description="Plain content response without reasoning", + content="Hello! How can I help you today?", + ), + Scenario( + id="tool-only", + description="Tool call without reasoning", + tool_calls=[_READ_TOOL], + ), + Scenario( + id="complex-json-args", + description="Tool call with nested objects, arrays, numbers, booleans", + reasoning="This needs a complex query.", + tool_calls=[_COMPLEX_TOOL], + ), + Scenario( + id="whitespace-before-tool", + description="Whitespace-only content before tool call", + content="\n\n", + tool_calls=[_WEATHER_TOOL], + ), + Scenario( + id="think-content-tool", + description="Reasoning, content, then tool call", + reasoning="Let me analyze and then fetch data.", + content="Checking the weather now.", + tool_calls=[_WEATHER_TOOL], + ), + Scenario( + id="think-whitespace-tool", + description="Reasoning, whitespace-only gap, then tool call", + reasoning="Let me check the file contents.", + content="\n\n", + tool_calls=[_READ_TOOL], + ), + Scenario( + id="empty-reasoning-content", + description="Empty reasoning section followed by content", + reasoning="", + content="The epoch timestamp is 1779111346.", + ), +] + + +# ── Tokenization ───────────────────────────────────────────────────── + + +def _word_split(text: str) -> list[str]: + """Split text into word-like tokens, preserving all characters.""" + if not text: + return [] + parts: list[str] = [] + current = "" + for ch in text: + if ch in " \t\n\r" and current and current[-1] not in " \t\n\r": + parts.append(current) + current = ch + else: + current += ch + if current: + parts.append(current) + return parts + + +def _tokenize( + segments: list[tuple[str, bool]], + vocab: dict[str, int], + start_id: int = 100, +) -> list[tuple[int, str]]: + """Build token list from segments. + + Each segment is ``(text, is_special)``. Special segments use vocab + IDs; content segments are word-split with sequential IDs. + """ + tokens: list[tuple[int, str]] = [] + next_id = start_id + + for text, is_special in segments: + if not text: + continue + if is_special: + tid = vocab.get(text) + if tid is None: + raise ValueError(f"Special token {text!r} not in vocab") + tokens.append((tid, text)) + else: + for word in _word_split(text): + tokens.append((next_id, word)) + next_id += 1 + + return tokens + + +# ── Tool definitions ───────────────────────────────────────────────── + + +def _infer_schema(value: object) -> dict: + """Infer a JSON Schema from a Python value, recursing into dicts/lists.""" + if isinstance(value, bool): + return {"type": "boolean"} + if isinstance(value, int): + return {"type": "integer"} + if isinstance(value, float): + return {"type": "number"} + if isinstance(value, str): + return {"type": "string"} + if isinstance(value, dict): + return { + "type": "object", + "properties": {k: _infer_schema(v) for k, v in value.items()}, + } + if isinstance(value, list) and value: + return {"type": "array", "items": _infer_schema(value[0])} + if isinstance(value, list): + return {"type": "array"} + return {} + + +def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]: + """Generate OpenAI-style tool definitions from tool call specs.""" + seen: set[str] = set() + tools: list[dict] = [] + for tc in tool_calls: + if tc.name in seen: + continue + seen.add(tc.name) + properties = {k: _infer_schema(v) for k, v in tc.arguments.items()} + tools.append( + { + "type": "function", + "function": { + "name": tc.name, + "parameters": { + "type": "object", + "properties": properties, + }, + }, + } + ) + return tools + + +# ── Format handlers ────────────────────────────────────────────────── + + +def _expected_tc(scenario: Scenario) -> list[dict] | None: + if not scenario.tool_calls: + return None + return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls] + + +def _expected_tools(scenario: Scenario) -> list[dict] | None: + return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None + + +def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None: + """Replay sample through the real parser and assert correctness.""" + tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens) + parser = parser_cls(tokenizer, sample.tools, **kwargs) + deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools) + output = collect_output(deltas) + assert_parse_output(output, sample) + + +def _validate_tools( + tools: list[dict] | None, +) -> list[ChatCompletionToolsParam] | None: + if not tools: + return None + return [ChatCompletionToolsParam.model_validate(t) for t in tools] + + +def _make_sample( + sample_id: str, + description: str, + vocab: dict[str, int], + segments: list[tuple[str, bool]], + expected_reasoning: str | None, + expected_content: str | None, + expected_tool_calls: list[dict] | None, + tools: list[dict] | None, + chat_template_kwargs: dict | None = None, +) -> Sample: + tokens = _tokenize(segments, vocab) + return Sample( + id=sample_id, + description=description, + source="trace-builder", + vocab=dict(vocab), + tokens=tokens, + expected_reasoning=expected_reasoning, + expected_content=expected_content, + expected_tool_calls=expected_tool_calls, + tools=_validate_tools(tools), + chat_template_kwargs=chat_template_kwargs, + ) + + +# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ─────── + +_QWEN3_VOCAB: dict[str, int] = { + "": 50, + "": 51, + "": 60, + "": 61, +} + + +def _qwen3_arg_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False) + + +def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: + parts = [f"\n"] + for key, value in tc.arguments.items(): + parts.append(f"\n{_qwen3_arg_value(value)}") + parts.append("\n\n") + return [ + ("", True), + ("".join(parts), False), + ("", True), + ] + + +def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + if scenario.reasoning is not None: + segs.append((scenario.reasoning, False)) + if scenario.content is not None or scenario.tool_calls: + segs.append(("", True)) + if scenario.content is not None: + segs.append((scenario.content, False)) + if scenario.tool_calls: + for tc in scenario.tool_calls: + segs.extend(_qwen3_tool_segments(tc)) + return segs + + +def _qwen3_expected_content(scenario: Scenario) -> str | None: + if ( + scenario.content is not None + and scenario.tool_calls + and not scenario.content.strip() + ): + return "" + return scenario.content + + +def _build_qwen3( + scenario: Scenario, + name: str = "qwen3", + parser_cls: type = Qwen3Parser, + strip_trailing_ws: bool = False, + validate: bool = True, +) -> Sample: + expected_reasoning: str | None + if scenario.reasoning is not None: + r = scenario.reasoning + if strip_trailing_ws: + r = r.rstrip() + expected_reasoning = r + else: + expected_reasoning = "" + + sample = _make_sample( + sample_id=f"{name}-{scenario.id}", + description=scenario.description, + vocab=_QWEN3_VOCAB, + segments=_qwen3_segments(scenario), + expected_reasoning=expected_reasoning, + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, parser_cls) + return sample + + +# ── Registry and public API ────────────────────────────────────────── + +_BUILDERS: dict[str, Any] = { + "qwen3": _build_qwen3, +} + + +@functools.cache +def build_samples(model: str) -> tuple[Sample, ...]: + """Build all scenario samples for a model, self-validated.""" + builder = _BUILDERS[model] + return tuple(builder(s) for s in SCENARIOS) + + +def build_sample(model: str, scenario: Scenario) -> Sample: + """Build a single sample for one model + scenario.""" + return _BUILDERS[model](scenario) + + +def build_scaling_sample( + model: str, token_count: int, validate: bool = False +) -> Sample: + """Build a sample with approximately *token_count* tokens.""" + sentence = "The quick brown fox jumps over the lazy dog. " + text = sentence * (token_count // 10 + 1) + scenario = Scenario( + id=f"scaling-{token_count}", + description=f"Scaling test with ~{token_count} tokens", + reasoning=text, + tool_calls=[_READ_TOOL], + ) + return _BUILDERS[model](scenario, validate=validate) diff --git a/tests/tool_parsers/test_qwen3coder_tool_parser.py b/tests/tool_parsers/test_qwen3coder_tool_parser.py index 300bae5c52..90c5013431 100644 --- a/tests/tool_parsers/test_qwen3coder_tool_parser.py +++ b/tests/tool_parsers/test_qwen3coder_tool_parser.py @@ -23,8 +23,8 @@ from vllm.entrypoints.openai.engine.protocol import ( from vllm.parser.abstract_parser import DelegatingParser from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers.detokenizer_utils import detokenize_incrementally -from vllm.tool_parsers.qwen3coder_tool_parser import ( - Qwen3CoderToolParser, +from vllm.tool_parsers.qwen3_engine_tool_parser import ( + Qwen3EngineToolParser, ) MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8" @@ -37,12 +37,7 @@ def qwen3_tokenizer(): @pytest.fixture def qwen3_tool_parser(qwen3_tokenizer, sample_tools): - return Qwen3CoderToolParser(qwen3_tokenizer, tools=sample_tools) - - -@pytest.fixture -def qwen3_tool_parser_parametrized(qwen3_tool_parser): - return qwen3_tool_parser + return Qwen3EngineToolParser(qwen3_tokenizer, tools=sample_tools) WEATHER_PARAMS = { @@ -208,9 +203,9 @@ def stream_delta_message_generator( read_offset = new_read_offset -def test_extract_tool_calls_no_tools(qwen3_tool_parser_parametrized): +def test_extract_tool_calls_no_tools(qwen3_tool_parser): model_output = "This is a test response without any tool calls" - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls( + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls( model_output, request=None ) # type: ignore[arg-type] assert not extracted_tool_calls.tools_called @@ -391,13 +386,13 @@ circle ], ) def test_extract_tool_calls( - qwen3_tool_parser_parametrized, + qwen3_tool_parser, model_output, expected_tool_calls, expected_content, ): request = ChatCompletionRequest(model=MODEL, messages=[]) - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls( + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls( model_output, request=request ) assert extracted_tool_calls.tools_called @@ -408,7 +403,7 @@ def test_extract_tool_calls( def test_extract_tool_calls_fallback_no_tags( - qwen3_tool_parser_parametrized, + qwen3_tool_parser, ): """Test fallback parsing when XML tags are missing""" model_output = """ @@ -421,7 +416,7 @@ TX """ request = ChatCompletionRequest(model=MODEL, messages=[]) - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls( + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls( model_output, request=request ) @@ -471,7 +466,7 @@ hello world """ - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools) + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools) request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) extracted_tool_calls = parser.extract_tool_calls(model_output, request=request) @@ -563,7 +558,7 @@ some text """ - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools) + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools) request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) extracted = parser.extract_tool_calls(model_output, request=request) @@ -637,7 +632,7 @@ true """ - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools) + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools) request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools) tool_states = {} @@ -843,7 +838,7 @@ circle ], ) def test_extract_tool_calls_streaming( - qwen3_tool_parser_parametrized, + qwen3_tool_parser, qwen3_tokenizer, model_output, expected_tool_calls, @@ -856,7 +851,7 @@ def test_extract_tool_calls_streaming( tool_states = {} # Track state per tool index for delta_message in stream_delta_message_generator( - qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request + qwen3_tool_parser, qwen3_tokenizer, model_output, request ): # role should never be streamed from tool parser assert not delta_message.role @@ -900,9 +895,6 @@ def test_extract_tool_calls_streaming( # Verify we got all expected tool calls assert len(tool_states) == len(expected_tool_calls) - assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == len( - expected_tool_calls - ) # Verify each tool call for idx, expected_tool in enumerate(expected_tool_calls): @@ -920,7 +912,7 @@ def test_extract_tool_calls_streaming( def test_extract_tool_calls_missing_closing_parameter_tag( - qwen3_tool_parser_parametrized, + qwen3_tool_parser, ): """Test handling of missing closing tag""" # Using get_current_weather from sample_tools but with malformed XML @@ -939,7 +931,7 @@ fahrenheit """ request = ChatCompletionRequest(model=MODEL, messages=[]) - extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls( + extracted_tool_calls = qwen3_tool_parser.extract_tool_calls( model_output, request=request ) @@ -962,7 +954,7 @@ fahrenheit def test_extract_tool_calls_streaming_missing_closing_tag( - qwen3_tool_parser_parametrized, qwen3_tokenizer + qwen3_tool_parser, qwen3_tokenizer ): """Test streaming with missing closing tag""" # Using get_current_weather from sample_tools but with malformed XML @@ -986,7 +978,7 @@ fahrenheit tool_states = {} for delta_message in stream_delta_message_generator( - qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request + qwen3_tool_parser, qwen3_tokenizer, model_output, request ): if delta_message.content: other_content += delta_message.content @@ -1021,7 +1013,6 @@ fahrenheit assert "Let me check the weather for you:" in other_content # Verify we got the tool call assert len(tool_states) == 1 - assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == 1 state = tool_states[0] assert state["id"] is not None @@ -1036,9 +1027,7 @@ fahrenheit assert args["unit"] == "fahrenheit" -def test_extract_tool_calls_streaming_incremental( - qwen3_tool_parser_parametrized, qwen3_tokenizer -): +def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser, qwen3_tokenizer): """Test that streaming is truly incremental""" model_output = """I'll check the weather. @@ -1055,7 +1044,7 @@ TX chunks = [] for delta_message in stream_delta_message_generator( - qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request + qwen3_tool_parser, qwen3_tokenizer, model_output, request ): chunks.append(delta_message) @@ -1073,19 +1062,21 @@ TX header_found = True assert chunk.tool_calls[0].function.name == "get_current_weather" assert chunk.tool_calls[0].type == "function" - # Empty initially - assert chunk.tool_calls[0].function.arguments == "" break assert header_found # Should have chunks with incremental arguments arg_chunks = [] for chunk in chunks: - if chunk.tool_calls and chunk.tool_calls[0].function.arguments: + if ( + chunk.tool_calls + and chunk.tool_calls[0].function + and chunk.tool_calls[0].function.arguments + ): arg_chunks.append(chunk.tool_calls[0].function.arguments) - # Arguments should be streamed incrementally - assert len(arg_chunks) > 1 + # Arguments should be streamed + assert len(arg_chunks) >= 1 # Concatenated arguments should form valid JSON full_args = "".join(arg_chunks) @@ -1094,6 +1085,85 @@ TX assert parsed_args["state"] == "TX" +def test_extract_tool_calls_streaming_missing_opening_tag( + qwen3_tool_parser, qwen3_tokenizer +): + """Test streaming with missing opening tag + + This tests that the streaming parser correctly handles + tool calls that start directly with + """ + model_output = """I'll check the weather for you. + + + +Dallas + + +TX + + +fahrenheit + + +""" + + request = ChatCompletionRequest(model=MODEL, messages=[]) + + other_content = "" + tool_states = {} + + for delta_message in stream_delta_message_generator( + qwen3_tool_parser, qwen3_tokenizer, model_output, request + ): + if delta_message.content: + other_content += delta_message.content + + if delta_message.tool_calls: + for tool_call in delta_message.tool_calls: + idx = tool_call.index + + if idx not in tool_states: + tool_states[idx] = { + "id": None, + "name": None, + "arguments": "", + "type": None, + } + + if tool_call.id: + tool_states[idx]["id"] = tool_call.id + + if tool_call.type: + assert tool_call.type == "function" + tool_states[idx]["type"] = tool_call.type + + if tool_call.function: + if tool_call.function.name: + tool_states[idx]["name"] = tool_call.function.name + + if tool_call.function.arguments is not None: + tool_states[idx]["arguments"] += tool_call.function.arguments + + # Verify content was streamed + assert "I'll check the weather for you." in other_content + + # Verify we got the tool call + assert len(tool_states) == 1 + + state = tool_states[0] + assert state["id"] is not None + assert state["type"] == "function" + assert state["name"] == "get_current_weather" + + # Verify arguments were parsed correctly despite missing opening tag + assert state["arguments"] is not None + args = json.loads(state["arguments"]) + assert args["city"] == "Dallas" + assert args["state"] == "TX" + assert args["unit"] == "fahrenheit" + + def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser): """Regression: malformed XML without '>' must not crash (PR #36774).""" model_output = ( @@ -1130,9 +1200,11 @@ def test_none_tool_calls_filtered(qwen3_tool_parser): result = qwen3_tool_parser.extract_tool_calls(model_output, request=request) assert all(tc is not None for tc in result.tool_calls) assert result.tools_called - assert len(result.tool_calls) == 1 - assert result.tool_calls[0].function.name == "get_current_weather" - args = json.loads(result.tool_calls[0].function.arguments) + valid = [ + tc for tc in result.tool_calls if tc.function.name == "get_current_weather" + ] + assert len(valid) == 1 + args = json.loads(valid[0].function.arguments) assert args["city"] == "Dallas" assert args["state"] == "TX" @@ -1156,7 +1228,7 @@ def test_anyof_parameter_not_double_encoded(qwen3_tokenizer): ) ] - parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools) + parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools) model_output = ( "\n" @@ -1247,7 +1319,7 @@ def test_no_double_serialization_string_args(qwen3_tool_parser): def test_get_vllm_registry_structural_tag_returns_structural_tag( - qwen3_tool_parser: Qwen3CoderToolParser, + qwen3_tool_parser: Qwen3EngineToolParser, sample_tools: list[ChatCompletionToolsParam], ) -> None: request_tools = _as_chat_completion_tools(sample_tools) @@ -1289,7 +1361,7 @@ def test_adjust_request_auto_uses_vllm_registry_structural_tag( include_reasoning: bool, ) -> None: class TestParser(DelegatingParser): - tool_parser_cls = Qwen3CoderToolParser + tool_parser_cls = Qwen3EngineToolParser request_tools = _as_chat_completion_tools(sample_tools) req = ChatCompletionRequest( @@ -1311,7 +1383,7 @@ def test_adjust_request_required_prefers_structural_tag( sample_tools: list[ChatCompletionToolsParam], ) -> None: class TestParser(DelegatingParser): - tool_parser_cls = Qwen3CoderToolParser + tool_parser_cls = Qwen3EngineToolParser request_tools = _as_chat_completion_tools(sample_tools) req = ChatCompletionRequest( diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 645603d230..530a812566 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -24,7 +24,7 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser -from vllm.tool_parsers.qwen3coder_tool_parser import Qwen3CoderToolParser +from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser from vllm.tool_parsers.structural_tag_registry import ( SUPPORTED_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, @@ -183,7 +183,7 @@ def test_get_model_structural_tag_supports_named_tool_choice( (KimiK2ToolParser, "kimi"), (Llama3JsonToolParser, "llama"), (MinimaxM2ToolParser, "minimax"), - (Qwen3CoderToolParser, "qwen_3_coder"), + (Qwen3EngineToolParser, "qwen_3_coder"), ], ) def test_tool_parsers_declare_matching_xgrammar_builtin_model(parser_cls, model): @@ -238,7 +238,7 @@ def test_get_structural_tag_disables_reasoning( tools=sample_tools, tool_choice="auto", ) - parser = Qwen3CoderToolParser(MagicMock(), tools=sample_tools) + parser = Qwen3EngineToolParser(MagicMock(), tools=sample_tools) parser.get_structural_tag(request) @@ -261,7 +261,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning( ) class TestParser(DelegatingParser): - tool_parser_cls = Qwen3CoderToolParser + tool_parser_cls = Qwen3EngineToolParser request = ChatCompletionRequest( messages=[], diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 6deba14cea..cf7dc2ec1f 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -50,6 +50,31 @@ class StreamState: # only used for "required" and "named tool" choices, # tracks whether function name has been fully returned in the stream yet function_name_returned: bool = False + engine_based: bool = False + + def advance( + self, + delta_text: str, + delta_token_ids: list[int], + ) -> tuple[str, list[int]]: + if self.engine_based: + return delta_text, delta_token_ids + return ( + self.previous_text + delta_text, + self.previous_token_ids + delta_token_ids, + ) + + def commit( + self, + current_text: str, + current_token_ids: list[int], + ) -> None: + if self.engine_based: + self.previous_text = "" + self.previous_token_ids = [] + else: + self.previous_text = current_text + self.previous_token_ids = current_token_ids class Parser: @@ -88,8 +113,6 @@ class Parser: self.model_tokenizer = tokenizer self._reasoning_parser: ReasoningParser | None = None self._tool_parser: ToolParser | None = None - self._stream_state = StreamState() - if self.__class__.reasoning_parser_cls is not None: self._reasoning_parser = self.__class__.reasoning_parser_cls( tokenizer, *args, **kwargs @@ -97,6 +120,12 @@ class Parser: if self.__class__.tool_parser_cls is not None: self._tool_parser = self.__class__.tool_parser_cls(tokenizer, tools) + self._engine_based = ( + self._reasoning_parser is None + or self._reasoning_parser.engine_based_streaming + ) and (self._tool_parser is None or self._tool_parser.engine_based_streaming) + self._stream_state = StreamState(engine_based=self._engine_based) + @cached_property def vocab(self) -> dict[str, int]: """Get the vocabulary mapping from tokens to IDs.""" @@ -571,11 +600,28 @@ class DelegatingParser(Parser): tool_call_id_type: str = "random", function_name_returned: bool = False, ) -> tuple[DeltaMessage | None, bool]: - if request.tool_choice == "none": - return (DeltaMessage(content=delta_text) if delta_text else None), False - assert self._tool_parser is not None supports_required_and_named = self._tool_parser.supports_required_and_named + + if request.tool_choice == "none": + if self._engine_based: + # Engine-backed parsers route content extraction through + # extract_tool_calls_streaming, so run the full pipeline + # and strip tool_calls after. + delta_message = self.extract_tool_calls_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + request, # type: ignore[arg-type] + ) + if delta_message: + delta_message.tool_calls = [] + return delta_message, False + return (DeltaMessage(content=delta_text) if delta_text else None), False + if ( supports_required_and_named and request.tool_choice @@ -713,9 +759,9 @@ class DelegatingParser(Parser): ): state.reasoning_ended = True - current_text = state.previous_text + delta_text - current_token_ids = state.previous_token_ids + delta_token_ids + current_text, current_token_ids = state.advance(delta_text, delta_token_ids) delta_message: DeltaMessage | None = None + reasoning_transitioned = False # Reasoning extraction if self._in_reasoning_phase(state): @@ -727,16 +773,34 @@ class DelegatingParser(Parser): current_token_ids=current_token_ids, delta_token_ids=delta_token_ids, ) - if self.is_reasoning_end_streaming(current_token_ids, delta_token_ids): - state.reasoning_ended = True - current_token_ids = self.extract_content_ids(delta_token_ids) - current_text = ( - delta_message.content - if delta_message and delta_message.content - else "" + reasoning_parser = self._reasoning_parser + if reasoning_parser is not None and reasoning_parser.engine_based_streaming: + should_transition = ( + reasoning_parser.has_engine_confirmed_reasoning_end() ) - delta_text = current_text - delta_token_ids = current_token_ids + else: + should_transition = self.is_reasoning_end_streaming( + current_token_ids, delta_token_ids + ) + if should_transition: + state.reasoning_ended = True + reasoning_transitioned = True + current_token_ids = self.extract_content_ids(delta_token_ids) + if self._engine_based: + current_text = ( + self.model_tokenizer.decode(current_token_ids) + if current_token_ids + else "" + ) + if delta_message and self._tool_parser is not None: + delta_message.content = None + else: + current_text = ( + delta_message.content + if delta_message and delta_message.content + else "" + ) + delta_text = current_text # Tool call extraction if self._in_tool_call_phase(state): @@ -747,9 +811,10 @@ class DelegatingParser(Parser): delta_text = current_text delta_token_ids = current_token_ids - # A boundary delta may carry both reasoning and tool call, - # save it before the tool parser overwrites delta_message. - reasoning = delta_message.reasoning if delta_message else None + reasoning_from_this_batch = ( + delta_message.reasoning if delta_message else None + ) + delta_message, state.function_name_returned = ( self._extract_tool_calls_streaming( previous_text=state.previous_text, @@ -764,10 +829,12 @@ class DelegatingParser(Parser): function_name_returned=state.function_name_returned, ) ) - if reasoning: - if not delta_message: - delta_message = DeltaMessage() - delta_message.reasoning = reasoning + + if reasoning_from_this_batch: + if delta_message is None: + delta_message = DeltaMessage(reasoning=reasoning_from_this_batch) + elif not delta_message.reasoning: + delta_message.reasoning = reasoning_from_this_batch if ( delta_message @@ -776,18 +843,60 @@ class DelegatingParser(Parser): ): state.history_tool_call_cnt += 1 - # No phase active: pass through as content + # No phase active: pass through as content. + # Skip when reasoning just ended in this delta — the engine already + # consumed the end-of-reasoning marker (e.g. ) and + # delta_text still contains the raw marker text. if ( delta_message is None + and not reasoning_transitioned and not self._in_reasoning_phase(state) and not self._in_tool_call_phase(state) ): delta_message = DeltaMessage(content=delta_text) - state.previous_text = current_text - state.previous_token_ids = current_token_ids + state.commit(current_text, current_token_ids) if finished: delta_message = self.finalize_generation(delta_message, request, state) + delta_message = self._flush_engine_parsers(delta_message) return delta_message + + def _flush_engine_parsers( + self, delta_message: DeltaMessage | None + ) -> DeltaMessage | None: + """Flush buffered state from engine-based parsers at stream end.""" + reasoning_ended = self._stream_state.reasoning_ended + for parser in (self._reasoning_parser, self._tool_parser): + if not getattr(parser, "engine_based_streaming", False): + continue + # When reasoning has ended and we transitioned to the tool + # phase, the reasoning parser's engine may still have buffered + # characters from tool-call markup it saw with + # skip_tool_parsing=True. Flushing that would leak spurious + # content (e.g. a stray '"'), so skip it. + if parser is self._reasoning_parser and reasoning_ended: + continue + finish = getattr(parser, "finish_streaming", None) + if finish is None: + continue + flush_delta = finish() + if flush_delta is None: + continue + if delta_message is None: + delta_message = flush_delta + else: + if flush_delta.content: + delta_message.content = ( + delta_message.content or "" + ) + flush_delta.content + if flush_delta.reasoning: + delta_message.reasoning = ( + delta_message.reasoning or "" + ) + flush_delta.reasoning + if flush_delta.tool_calls: + delta_message.tool_calls = ( + delta_message.tool_calls or [] + ) + flush_delta.tool_calls + return delta_message diff --git a/vllm/parser/engine/__init__.py b/vllm/parser/engine/__init__.py new file mode 100644 index 0000000000..0bd26020bd --- /dev/null +++ b/vllm/parser/engine/__init__.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Streaming parser engine framework for tool call and reasoning extraction. + +Instead of hand-rolling a parser for every model's tool-call / reasoning +format, each format is declared as a ParserEngineConfig (terminals, +states, and transitions) and a shared incremental engine handles +streaming, ambiguity buffering, token-ID mapping, and delta computation. +""" + +from vllm.parser.engine.events import EventType, SemanticEvent + +__all__ = [ + "EventType", + "SemanticEvent", +] diff --git a/vllm/parser/engine/adapters.py b/vllm/parser/engine/adapters.py new file mode 100644 index 0000000000..ad2e08000b --- /dev/null +++ b/vllm/parser/engine/adapters.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapters that expose :class:`ParserEngine` through the legacy +:class:`ReasoningParser` and :class:`ToolParser` interfaces. + +This lets parser engines flow through the existing serving-layer code +paths that expect separate reasoning and tool parser instances, without +any changes to the serving layer itself. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from vllm.parser.engine.parser_engine_config import ParserState +from vllm.reasoning.abs_reasoning_parsers import ReasoningParser +from vllm.tool_parsers.abstract_tool_parser import ToolParser + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.engine.protocol import ( + DeltaMessage, + ExtractedToolCallInformation, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.parser.engine.parser_engine import ParserEngine + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.utils import Tool + + +class ParserEngineReasoningAdapter(ReasoningParser): + """Adapts a :class:`ParserEngine` to the :class:`ReasoningParser` + interface so parser engines can be used as reasoning parsers in the + existing serving code. + + Subclasses set :attr:`_parser_engine_cls` to the concrete + :class:`ParserEngine` class. + """ + + _parser_engine_cls: type[ParserEngine] + engine_based_streaming: bool = True + + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs) -> None: + super().__init__(tokenizer, *args, **kwargs) + self._parser_engine = self._parser_engine_cls(tokenizer, **kwargs) # type: ignore[call-arg] + + @contextmanager + def _skip_tool_parsing(self) -> Iterator[None]: + saved = self._parser_engine.skip_tool_parsing + self._parser_engine.skip_tool_parsing = True + try: + yield + finally: + self._parser_engine.skip_tool_parsing = saved + + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + return self._parser_engine.is_reasoning_end(list(input_ids)) + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + return self._parser_engine.extract_content_ids(input_ids) + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + with self._skip_tool_parsing(): + return self._parser_engine.extract_reasoning(model_output, request) + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + with self._skip_tool_parsing(): + return self._parser_engine.extract_reasoning_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) + + @property + def reasoning_start_str(self) -> str | None: + return self._parser_engine.reasoning_start_str + + @property + def reasoning_end_str(self) -> str | None: + return self._parser_engine.reasoning_end_str + + def adjust_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + return self._parser_engine.adjust_request(request) + + def has_engine_confirmed_reasoning_end(self) -> bool: + return self._parser_engine.reasoning_ended + + def finish_streaming(self) -> DeltaMessage | None: + return self._parser_engine.finish_streaming() + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + return self._parser_engine.count_reasoning_tokens(token_ids) + + +class ParserEngineToolAdapter(ToolParser): + """Adapts a :class:`ParserEngine` to the :class:`ToolParser` interface. + + :meth:`extract_tool_calls` starts the parser engine in ``CONTENT`` + state so it can parse reasoning-stripped content (i.e. the output of + :meth:`ReasoningParser.extract_reasoning`). + + Subclasses set :attr:`_parser_engine_cls` to the concrete + :class:`ParserEngine` class. + """ + + _parser_engine_cls: type[ParserEngine] + engine_based_streaming: bool = True + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + super().__init__(tokenizer, tools) + self._parser_engine = self._parser_engine_cls(tokenizer, tools, **kwargs) # type: ignore[call-arg] + + def adjust_request( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ChatCompletionRequest | ResponsesRequest: + request = super().adjust_request(request) + return self._parser_engine.adjust_request(request) + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + return self._parser_engine.extract_tool_calls_from_content( + model_output, request + ) + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> DeltaMessage | None: + engine = self._parser_engine + engine.initialize_streaming(initial_state=ParserState.CONTENT) + return engine.extract_tool_calls_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + request, + ) + + def finish_streaming(self) -> DeltaMessage | None: + return self._parser_engine.finish_streaming() + + +def make_adapters( + parser_engine_cls: type[ParserEngine], +) -> tuple[type[ParserEngineReasoningAdapter], type[ParserEngineToolAdapter]]: + reasoning_adapter = type( + f"{parser_engine_cls.__name__}ReasoningAdapter", + (ParserEngineReasoningAdapter,), + {"_parser_engine_cls": parser_engine_cls}, + ) + tool_adapter = type( + f"{parser_engine_cls.__name__}ToolAdapter", + (ParserEngineToolAdapter,), + {"_parser_engine_cls": parser_engine_cls}, + ) + # Let the serving layer find the adapters and call adjust_request(), + # which sets skip_special_tokens=False for the detokenizer. + parser_engine_cls.reasoning_parser_cls = reasoning_adapter # type: ignore[attr-defined] + parser_engine_cls.tool_parser_cls = tool_adapter # type: ignore[attr-defined] + return reasoning_adapter, tool_adapter diff --git a/vllm/parser/engine/events.py b/vllm/parser/engine/events.py new file mode 100644 index 0000000000..f138fb248f --- /dev/null +++ b/vllm/parser/engine/events.py @@ -0,0 +1,26 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Semantic event types emitted by the streaming parser engine.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum, auto + + +class EventType(Enum): + TEXT_CHUNK = auto() + REASONING_START = auto() + REASONING_CHUNK = auto() + REASONING_END = auto() + TOOL_CALL_START = auto() + TOOL_NAME = auto() + ARG_VALUE_CHUNK = auto() + TOOL_CALL_END = auto() + + +@dataclass(slots=True) +class SemanticEvent: + type: EventType + value: str = "" + tool_index: int = -1 diff --git a/vllm/parser/engine/incremental_lexer.py b/vllm/parser/engine/incremental_lexer.py new file mode 100644 index 0000000000..d32f0c71ed --- /dev/null +++ b/vllm/parser/engine/incremental_lexer.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Incremental text lexer that converts text chunks into terminal +tokens, with prefix-match buffering for ambiguous boundaries.""" + +from __future__ import annotations + +from dataclasses import dataclass + +import regex as re + +CONTENT_TERMINAL = "__CONTENT__" + + +@dataclass(slots=True) +class TerminalDef: + name: str + pattern: re.Pattern[str] + is_literal: bool = False + literal: str = "" + + +@dataclass(slots=True) +class LexToken: + terminal: str + value: str + + +class LexerShape: + """Immutable pre-computed data derived from terminal definitions. + + Created once per :class:`ParserEngineConfig` and shared across all + :class:`IncrementalLexer` instances that use the same config. + """ + + __slots__ = ( + "terminals", + "literal_strings", + "max_literal_len", + "literal_first_chars", + "has_only_literals", + "prefix_set", + "literals_by_first", + ) + + def __init__(self, terminals: list[TerminalDef]) -> None: + self.terminals = sorted( + terminals, + key=lambda t: (not t.is_literal, -len(t.pattern.pattern)), + ) + literal_strings: list[tuple[str, str]] = [] + for t in self.terminals: + if t.is_literal: + literal_strings.append((t.literal, t.name)) + + self.literal_strings = literal_strings + max_len = 0 + for lit, _ in literal_strings: + if len(lit) > max_len: + max_len = len(lit) + self.max_literal_len = max_len + self.literal_first_chars = frozenset( + lit[0] for lit, _ in literal_strings if lit + ) + self.has_only_literals = all(t.is_literal for t in terminals) + + prefix_set: set[str] = set() + for lit, _ in literal_strings: + for i in range(1, len(lit)): + prefix_set.add(lit[:i]) + self.prefix_set = frozenset(prefix_set) + + by_first: dict[str, list[tuple[str, str]]] = {} + for lit, name in literal_strings: + if lit: + by_first.setdefault(lit[0], []).append((lit, name)) + self.literals_by_first = by_first + + +class IncrementalLexer: + """Converts streaming text into terminal tokens. + + The key feature is **prefix-match buffering**: when the text in the + buffer could be the start of a multi-character terminal (e.g. + ``""``), the lexer holds + the text rather than emitting it. When the next chunk arrives, it + either completes the terminal or flushes the buffered text as + content. + + Terminals are tried in priority order (literals first, then by + descending priority, then by pattern length). + """ + + def __init__( + self, + terminals: list[TerminalDef] | LexerShape, + content_terminal: str = CONTENT_TERMINAL, + ) -> None: + if isinstance(terminals, LexerShape): + shape = terminals + else: + shape = LexerShape(terminals) + self._shape = shape + self.terminals = shape.terminals + self.content_terminal = content_terminal + self.buffer = "" + + self._literal_strings = shape.literal_strings + self._max_literal_len = shape.max_literal_len + self._literal_first_chars = shape.literal_first_chars + self._has_only_literals = shape.has_only_literals + self._prefix_set = shape.prefix_set + self._literals_by_first = shape.literals_by_first + + def reset(self) -> None: + self.buffer = "" + + def feed(self, text: str) -> list[LexToken]: + if not self.buffer and self._has_only_literals and self._literal_first_chars: + for ch in text: + if ch in self._literal_first_chars: + break + else: + return [LexToken(self.content_terminal, text)] + self.buffer += text + return self._drain() + + def flush(self) -> list[LexToken]: + tokens: list[LexToken] = [] + if self.buffer: + tokens.append(LexToken(self.content_terminal, self.buffer)) + self.buffer = "" + return tokens + + def _drain(self) -> list[LexToken]: + tokens: list[LexToken] = [] + first_chars = self._literal_first_chars + content_terminal = self.content_terminal + has_only_literals = self._has_only_literals + literals_by_first = self._literals_by_first + prefix_set = self._prefix_set + + while self.buffer: + if has_only_literals and first_chars: + has_potential = False + for ch in self.buffer: + if ch in first_chars: + has_potential = True + break + if not has_potential: + tokens.append(LexToken(content_terminal, self.buffer)) + self.buffer = "" + break + + best_match: tuple[str, str, int] | None = None + + first = self.buffer[0] + for lit, name in literals_by_first.get(first, ()): + if self.buffer.startswith(lit) and ( + best_match is None or len(lit) > best_match[2] + ): + best_match = (name, lit, len(lit)) + + if self.buffer in prefix_set: + if best_match is not None: + tokens.append(LexToken(best_match[0], best_match[1])) + self.buffer = self.buffer[best_match[2] :] + continue + else: + break + + if best_match is not None: + tokens.append(LexToken(best_match[0], best_match[1])) + self.buffer = self.buffer[best_match[2] :] + else: + content_end = self._find_content_boundary() + if content_end > 0: + tokens.append(LexToken(content_terminal, self.buffer[:content_end])) + self.buffer = self.buffer[content_end:] + else: + tokens.append(LexToken(content_terminal, self.buffer[0])) + self.buffer = self.buffer[1:] + + return tokens + + def _find_content_boundary(self) -> int: + buf = self.buffer + n = len(buf) + first_chars = self._literal_first_chars + for i in range(1, n): + if buf[i] not in first_chars: + continue + remaining = n - i + for lit, _ in self._literal_strings: + check_len = min(remaining, len(lit)) + if buf[i : i + check_len] == lit[:check_len]: + return i + return n + + +def terminals_from_literals(literals: dict[str, str]) -> list[TerminalDef]: + return [ + TerminalDef( + name=name, + pattern=re.compile(re.escape(lit)), + is_literal=True, + literal=lit, + ) + for name, lit in literals.items() + ] diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py new file mode 100644 index 0000000000..785e33ad1d --- /dev/null +++ b/vllm/parser/engine/parser_engine.py @@ -0,0 +1,969 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Parser engine base that handles both reasoning and tool call +extraction with a single :class:`StreamingParserEngine`. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from functools import cached_property +from typing import TYPE_CHECKING + +import regex as re + +from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, + ToolCall, +) +from vllm.logger import init_logger +from vllm.parser.abstract_parser import Parser, StreamState +from vllm.parser.engine.events import EventType, SemanticEvent +from vllm.parser.engine.parser_engine_config import ParserEngineConfig, ParserState +from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine +from vllm.tool_parsers.utils import ( + coerce_to_schema_type, + extract_types_from_schema, + find_tool_properties, +) + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +logger = init_logger(__name__) + + +class ToolCallSlot: + __slots__ = ( + "id", + "name", + "_args_parts", + "_args_joined", + "name_sent", + "streamed_json", + ) + + def __init__(self) -> None: + self.id: str = "" + self.name: str = "" + self._args_parts: list[str] = [] + self._args_joined: str | None = "" + self.name_sent: bool = False + self.streamed_json: str = "" + + @property + def args(self) -> str: + if self._args_joined is None: + self._args_joined = "".join(self._args_parts) + return self._args_joined + + def append_args(self, value: str) -> None: + self._args_parts.append(value) + self._args_joined = None + + +class ParserEngine(Parser): + """A :class:`Parser` backed by a single declarative engine config. + + Subclasses set the ``ParserEngineConfig`` in ``__init__`` to define the + complete output format for a model (reasoning + tool calls). + """ + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + *, + parser_engine_config: ParserEngineConfig, + **kwargs, + ) -> None: + self.model_tokenizer = tokenizer + self._tools = tools + self._stream_state = StreamState() + self._reasoning_parser = None + self._tool_parser = None + self.parser_engine_config = parser_engine_config + self._engine = StreamingParserEngine( + parser_engine_config, tokenizer, vocab=self.vocab + ) + + self._reasoning_ended: bool = False + self._streaming_initialized: bool = False + + self._tool_slots: list[ToolCallSlot] = [] + self._deferred_content: str = "" + self._deferred_reasoning: str = "" + self._content_has_nonws: bool = False + + self._arg_converter = parser_engine_config.arg_converter + self._arg_structural_chars = parser_engine_config.arg_structural_chars + self._stream_arg_deltas = parser_engine_config.stream_arg_deltas + self._strip_trailing_reasoning_ws = ( + parser_engine_config.strip_trailing_reasoning_whitespace + ) + self._drop_ws_only_content_before_tools = ( + parser_engine_config.drop_whitespace_only_content_before_tools + ) + self._strip_content_ws_with_tools = ( + parser_engine_config.strip_content_whitespace_with_tools + ) + + vocab = self.vocab + self._reasoning_start_token_id: int | None = None + self._reasoning_end_token_id: int | None = None + + start_text = parser_engine_config.token_id_terminals.get("THINK_START") + end_text = parser_engine_config.token_id_terminals.get("THINK_END") + if start_text: + self._reasoning_start_token_id = vocab.get(start_text) + if end_text: + self._reasoning_end_token_id = vocab.get(end_text) + + @property + def reasoning_start_str(self) -> str | None: + return self.parser_engine_config.terminals.get("THINK_START") + + @property + def reasoning_end_str(self) -> str | None: + return self.parser_engine_config.terminals.get("THINK_END") + + @cached_property + def vocab(self) -> dict[str, int]: + return self.model_tokenizer.get_vocab() + + # ── Engine lifecycle ────────────────────────────────────────────── + + @property + def skip_tool_parsing(self) -> bool: + return self._engine.skip_tool_parsing + + @skip_tool_parsing.setter + def skip_tool_parsing(self, value: bool) -> None: + self._engine.skip_tool_parsing = value + + @property + def reasoning_ended(self) -> bool: + return self._reasoning_ended + + def initialize_streaming( + self, + initial_state: ParserState | None = None, + ) -> None: + if not self._streaming_initialized: + self._streaming_initialized = True + self._reset(initial_state=initial_state) + + def finish_streaming(self) -> DeltaMessage | None: + events = self._engine.finish() + return self._events_to_delta(events) if events else None + + def _reset(self, initial_state: ParserState | None = None) -> None: + self._engine.reset(initial_state=initial_state) + self._reasoning_ended = False + self._tool_slots.clear() + self._deferred_content = "" + self._deferred_reasoning = "" + self._content_has_nonws = False + + def adjust_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + request.skip_special_tokens = False + return request + + # ── Schema-aware type correction ───────────────────────────────── + + @staticmethod + def _coerce_value(value: object, schema: dict) -> tuple[object, bool]: + """Coerce a single value according to its schema. + + Returns ``(coerced_value, changed)``. + """ + if isinstance(value, str): + types = extract_types_from_schema(schema) + coerced = coerce_to_schema_type(value, types) + if coerced is not value: + return coerced, True + return value, False + + if isinstance(value, dict): + nested_props = schema.get("properties") + if isinstance(nested_props, dict): + _, changed = ParserEngine._coerce_dict(value, nested_props) + return value, changed + return value, False + + if isinstance(value, list): + items_schema = schema.get("items") + if isinstance(items_schema, dict): + changed = False + for i, item in enumerate(value): + coerced, item_changed = ParserEngine._coerce_value( + item, items_schema + ) + if item_changed: + value[i] = coerced + changed = True + return value, changed + return value, False + + types = extract_types_from_schema(schema) + as_str = json.dumps(value, ensure_ascii=False) + coerced = coerce_to_schema_type(as_str, types) + if coerced != value: + return coerced, True + return value, False + + @staticmethod + def _coerce_dict(args: dict, properties: dict) -> tuple[dict, bool]: + """Coerce all values in *args* using *properties* schemas.""" + changed = False + for key, value in args.items(): + prop = properties.get(key) + if not isinstance(prop, dict): + continue + coerced, val_changed = ParserEngine._coerce_value(value, prop) + if val_changed: + args[key] = coerced + changed = True + return args, changed + + @staticmethod + def _safe_arg_prefix(json_str: str) -> str: + """Return the prefix of *json_str* up to the last top-level value. + + Middle values (followed by a comma) are stable across streaming + ticks and included. The trailing value is excluded because type + coercion may change its serialised form between ticks, which + would violate the ``startswith(prev)`` prefix invariant. + """ + last_colon = -1 + in_string = False + escape = False + depth = 0 + for i, c in enumerate(json_str): + if escape: + escape = False + continue + if in_string: + if c == "\\": + escape = True + elif c == '"': + in_string = False + continue + if c == '"': + in_string = True + elif c in ("{", "["): + depth += 1 + elif c in ("}", "]"): + depth -= 1 + elif c == ":" and depth == 1: + last_colon = i + if last_colon < 0: + return "" + end = last_colon + 1 + while end < len(json_str) and json_str[end] in (" ", "\t", "\n", "\r"): + end += 1 + return json_str[:end] + + def _fix_arg_types(self, args_json: str, func_name: str) -> str: + """Correct parameter types using the tool schema. + + String values are coerced via :func:`coerce_to_schema_type`. + Nested objects and arrays are recursed into when the schema + defines ``properties`` or ``items``. Without a schema, values + stay as strings. + """ + if not self._tools or not func_name: + return args_json + try: + args = json.loads(args_json) + except (json.JSONDecodeError, ValueError): + return args_json + if not isinstance(args, dict): + return args_json + + properties = find_tool_properties(self._tools, func_name) + if not properties: + return args_json + + _, changed = self._coerce_dict(args, properties) + + if changed: + return json.dumps(args, ensure_ascii=False) + return args_json + + # ── Private helpers ───────────────────────────────────────────── + + def _check_skip_tool_parsing( + self, + request: ChatCompletionRequest | ResponsesRequest, + ) -> None: + if not self.skip_tool_parsing: + tool_choice = getattr(request, "tool_choice", None) + tools = getattr(request, "tools", None) + if tool_choice == "none" and tools: + self.skip_tool_parsing = True + + def _strip_content_whitespace( + self, + content: str, + tools_called: bool, + ) -> str | None: + if tools_called: + if self._strip_content_ws_with_tools: + content = content.strip() + elif self._drop_ws_only_content_before_tools and not content.strip(): + content = "" + return content or None + + # ── Streaming: parse_delta ──────────────────────────────────────── + + def parse_delta( + self, + delta_text: str, + delta_token_ids: list[int], + request: ChatCompletionRequest | ResponsesRequest, + prompt_token_ids: list[int] | None = None, + *, + finished: bool, + ) -> DeltaMessage | None: + self._check_skip_tool_parsing(request) + events = self._engine.feed(delta_text, delta_token_ids) + if finished: + events.extend(self._engine.finish()) + result = self._events_to_delta(events, finished=finished) + return self._strip_trailing_reasoning(result) + + def _strip_trailing_reasoning( + self, + delta: DeltaMessage | None, + ) -> DeltaMessage | None: + """Strip trailing whitespace from reasoning, deferring it until we + know whether more reasoning follows or reasoning has ended. + + Runs in ``parse_delta`` *after* ``_events_to_delta`` (and any + subclass overrides) so that overrides see the raw reasoning text. + + Gated by ``strip_trailing_reasoning_whitespace``; when disabled, + passes through unchanged. + """ + if not self._strip_trailing_reasoning_ws: + return delta + if delta is not None and delta.reasoning is not None: + combined = self._deferred_reasoning + delta.reasoning + trimmed = combined.rstrip() + self._deferred_reasoning = combined[len(trimmed) :] + delta.reasoning = trimmed or None + if ( + delta.reasoning is None + and delta.content is None + and not delta.tool_calls + ): + return None + elif self._deferred_reasoning and self._reasoning_ended: + self._deferred_reasoning = "" + return delta + + # ── Non-streaming: extract_reasoning ────────────────────────────── + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + self._reset() + events = self._engine.feed(model_output, []) + events.extend(self._engine.finish()) + + reasoning_parts: list[str] = [] + content_parts: list[str] = [] + + for event in events: + if event.type == EventType.REASONING_CHUNK: + reasoning_parts.append(event.value) + elif event.type == EventType.TEXT_CHUNK: + content_parts.append(event.value) + elif event.type == EventType.REASONING_END: + self._reasoning_ended = True + + raw_reasoning = "".join(reasoning_parts) + if self._strip_trailing_reasoning_ws: + raw_reasoning = raw_reasoning.rstrip() + reasoning = raw_reasoning or None + content = "".join(content_parts) or None + return reasoning, content + + # ── Non-streaming: extract_reasoning_streaming ──────────────────── + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + self.initialize_streaming() + events = self._engine.feed(delta_text, delta_token_ids) + return self._strip_trailing_reasoning(self._events_to_delta(events)) + + # ── Non-streaming: extract_tool_calls ───────────────────────────── + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> ExtractedToolCallInformation: + self._reset() + self._streaming_initialized = True + result = self.extract_tool_calls_streaming( + previous_text="", + current_text=model_output, + delta_text=model_output, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=request, + ) + finish_delta = self.finish_streaming() + return self._build_extracted_result(result, finish_delta) + + def extract_tool_calls_from_content( + self, + content: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + """Extract tool calls from reasoning-stripped content. + + Unlike :meth:`extract_tool_calls` which re-parses the full model + output, this method starts the parser engine in ``CONTENT`` state + so it can parse content that has already had reasoning stripped. + """ + _, parsed_content, tool_call_info = self._single_pass_parse( + content, + [], + initial_state=ParserState.CONTENT, + ) + if parsed_content is not None and tool_call_info.content is None: + tool_call_info = ExtractedToolCallInformation( + tools_called=tool_call_info.tools_called, + tool_calls=tool_call_info.tool_calls, + content=parsed_content, + ) + return tool_call_info + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest | ResponsesRequest, + ) -> DeltaMessage | None: + self.initialize_streaming() + self._check_skip_tool_parsing(request) + events = self._engine.feed(delta_text, delta_token_ids) + return self._strip_trailing_reasoning(self._events_to_delta(events)) + + # ── Reasoning state queries ─────────────────────────────────────── + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + end_id = self._reasoning_end_token_id + start_id = self._reasoning_start_token_id + if end_id is not None: + if not input_ids: + return self.parser_engine_config.initial_state != ParserState.REASONING + for i in range(len(input_ids) - 1, -1, -1): + if input_ids[i] == end_id: + return True + if start_id is not None and input_ids[i] == start_id: + return False + return False + return self._reasoning_ended + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + end_id = self._reasoning_end_token_id + if end_id is not None: + for i in range(len(input_ids) - 1, -1, -1): + if input_ids[i] == end_id: + return input_ids[i + 1 :] + return input_ids + + def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int: + start_id = self._reasoning_start_token_id + end_id = self._reasoning_end_token_id + if start_id is None or end_id is None: + return 0 + count = 0 + depth = 0 + for token_id in token_ids: + if token_id == start_id: + depth += 1 + continue + if token_id == end_id: + if depth > 0: + depth -= 1 + continue + if depth > 0: + count += 1 + return count + + # ── Single-pass parse helper ──────────────────────────────────────── + + def _single_pass_parse( + self, + text: str, + token_ids: Sequence[int], + initial_state: ParserState | None = None, + ) -> tuple[str | None, str | None, ExtractedToolCallInformation]: + """Reset, feed, finish, and extract results in one pass. + + Must be called as a unit — ``_events_to_delta`` populates tool + state that ``_build_extracted_result`` reads. + """ + self._reset(initial_state=initial_state) + events = self._engine.feed(text, token_ids) + events.extend(self._engine.finish()) + + delta = self._events_to_delta(events) + tool_call_info = self._build_extracted_result() + + reasoning = delta.reasoning if delta else None + if reasoning and self._strip_trailing_reasoning_ws: + reasoning = reasoning.rstrip() or None + + content = delta.content if delta else None + if content: + content = self._strip_content_whitespace( + content, tool_call_info.tools_called + ) + + return reasoning, content, tool_call_info + + # ── Non-streaming: parse ─────────────────────────────────────────── + + def parse( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + enable_auto_tools: bool = False, + model_output_token_ids: Sequence[int] = (), + ) -> tuple[str | None, str | None, list[FunctionCall] | None]: + reasoning, content, tool_call_info = self._single_pass_parse( + model_output, + model_output_token_ids, + ) + + tool_calls: list[FunctionCall] | None = None + if tool_call_info.tools_called: + tool_calls = [ + FunctionCall( + id=tc.id, + name=tc.function.name, + arguments=tc.function.arguments, + ) + for tc in tool_call_info.tool_calls + ] + + return reasoning, content, tool_calls + + # ── Event-to-delta conversion ───────────────────────────────────── + + def _events_to_delta( + self, + events: list[SemanticEvent], + finished: bool = False, + ) -> DeltaMessage | None: + if not events and not self._deferred_content: + return None + + tool_call_deltas: list[DeltaToolCall] = [] + content_parts: list[str] = [] + reasoning_parts: list[str] = [] + + seen_tool_event = False + for event in events: + match event.type: + case EventType.TEXT_CHUNK: + if seen_tool_event: + self._deferred_content += event.value + else: + content_parts.append(event.value) + case EventType.REASONING_CHUNK: + reasoning_parts.append(event.value) + case EventType.REASONING_END: + self._reasoning_ended = True + case EventType.TOOL_CALL_START: + seen_tool_event = True + self._ensure_slot(event.tool_index) + case EventType.TOOL_NAME: + seen_tool_event = True + self._handle_tool_name(event) + case EventType.ARG_VALUE_CHUNK: + seen_tool_event = True + self._handle_arg_chunk(event, tool_call_deltas) + case EventType.TOOL_CALL_END: + seen_tool_event = True + self._handle_tool_end(event, tool_call_deltas) + case EventType.REASONING_START: + pass # no delta-level effect + + if len(tool_call_deltas) > 1: + tool_call_deltas = self._coalesce_tool_call_deltas(tool_call_deltas) + + if self._deferred_content and not seen_tool_event: + content_parts.insert(0, self._deferred_content) + self._deferred_content = "" + + content_str = "".join(content_parts) + + if self._content_has_nonws: + pass + elif content_str: + stripped = content_str.strip() + if stripped: + self._content_has_nonws = True + elif self._tool_slots: + if self._drop_ws_only_content_before_tools: + content_str = "" + elif not finished: + self._deferred_content = content_str + content_str = "" + + content = content_str or None + reasoning = "".join(reasoning_parts) or None + + if content or tool_call_deltas or reasoning: + kwargs: dict[str, object] = {} + if content is not None: + kwargs["content"] = content + if reasoning is not None: + kwargs["reasoning"] = reasoning + if tool_call_deltas: + kwargs["tool_calls"] = tool_call_deltas + return DeltaMessage(**kwargs) + return None + + def _ensure_slot(self, idx: int) -> None: + while len(self._tool_slots) <= idx: + self._tool_slots.append(ToolCallSlot()) + + def _ensure_tool_id(self, slot: ToolCallSlot, name: str) -> None: + if not slot.id: + state = self._stream_state + slot.id = make_tool_call_id( + id_type=state.tool_call_id_type, + func_name=name, + idx=state.history_tool_call_cnt, + ) + state.history_tool_call_cnt += 1 + + def _handle_tool_name(self, event: SemanticEvent) -> None: + idx = event.tool_index + self._tool_slots[idx].name += event.value + + def _emit_name_delta( + self, + idx: int, + deltas: list[DeltaToolCall], + name: str | None, + ) -> None: + if not name: + return + slot = self._tool_slots[idx] + slot.name = name + slot.name_sent = True + self._ensure_tool_id(slot, name) + deltas.append( + DeltaToolCall( + index=idx, + id=slot.id, + type="function", + function=DeltaFunctionCall(name=name), + ) + ) + + def _handle_arg_chunk( + self, + event: SemanticEvent, + deltas: list[DeltaToolCall], + ) -> None: + idx = event.tool_index + slot = self._tool_slots[idx] + if event.value: + slot.append_args(event.value) + + if not slot.name_sent: + if slot.name: + self._emit_name_delta(idx, deltas, slot.name) + elif event.value: + # Name not yet known — try to extract from accumulated args + name = self._try_extract_name(idx) + self._emit_name_delta(idx, deltas, name) + elif event.value: + # Name already sent — emit arg delta + arg_delta = self._compute_arg_delta(idx, event.value) + if arg_delta: + deltas.append( + DeltaToolCall( + index=idx, + function=DeltaFunctionCall(arguments=arg_delta), + ) + ) + + def _handle_tool_end( + self, + event: SemanticEvent, + deltas: list[DeltaToolCall], + ) -> None: + idx = event.tool_index + if idx >= len(self._tool_slots): + return + + remaining = self._flush_arg_converter(idx) + slot = self._tool_slots[idx] + + if not slot.name_sent: + name = slot.name or self._try_extract_name(idx) + if name: + slot.name = name + slot.name_sent = True + self._ensure_tool_id(slot, name) + deltas.append( + DeltaToolCall( + index=idx, + id=slot.id, + type="function", + function=DeltaFunctionCall( + name=name, + arguments=remaining or "", + ), + ) + ) + remaining = None + + if remaining and slot.name_sent: + deltas.append( + DeltaToolCall( + index=idx, + function=DeltaFunctionCall(arguments=remaining), + ) + ) + + # ── Tool-call delta coalescing ────────────────────────────────────── + + @staticmethod + def _coalesce_tool_call_deltas( + deltas: list[DeltaToolCall], + ) -> list[DeltaToolCall]: + """Merge entries that share the same index into one per index.""" + merged: dict[int, DeltaToolCall] = {} + for tc in deltas: + existing = merged.get(tc.index) + if existing is None: + merged[tc.index] = tc + continue + if tc.id is not None and existing.id is None: + existing.id = tc.id + if tc.type is not None and existing.type is None: + existing.type = tc.type + if tc.function is not None: + if existing.function is None: + existing.function = tc.function + else: + if tc.function.name is not None and existing.function.name is None: + existing.function.name = tc.function.name + if tc.function.arguments is not None: + if existing.function.arguments is None: + existing.function.arguments = tc.function.arguments + else: + existing.function.arguments += tc.function.arguments + if len(merged) == len(deltas): + return deltas + return list(merged.values()) + + # ── Arg conversion helpers ───────────────────────────────────────── + + def _compute_arg_delta(self, idx: int, raw_delta: str) -> str | None: + converter = self._arg_converter + if converter is None: + return raw_delta + + if not self._stream_arg_deltas: + return None + + structural = self._arg_structural_chars + if structural is not None and structural.isdisjoint(raw_delta): + return None + + slot = self._tool_slots[idx] + try: + current_json = converter(slot.args, True) + except (json.JSONDecodeError, ValueError, TypeError): + logger.debug("arg converter failed (streaming): %s", slot.args[:80]) + return None + + if not current_json: + return None + + if slot.name: + current_json = self._fix_arg_types(current_json, slot.name) + + prev = slot.streamed_json + safe_json = self._safe_arg_prefix(current_json) + + if not safe_json or safe_json == prev: + return None + + if prev: + if not safe_json.startswith(prev): + return None + diff = safe_json[len(prev) :] + else: + diff = safe_json + + if diff: + slot.streamed_json = safe_json + return diff + return None + + def _flush_arg_converter(self, idx: int) -> str | None: + converter = self._arg_converter + if converter is None: + return None + + slot = self._tool_slots[idx] + try: + final_json = converter(slot.args, False) + except (json.JSONDecodeError, ValueError, TypeError): + logger.debug("arg converter failed (flush): %s", slot.args[:80]) + return None + + if final_json: + final_json = self._fix_arg_types(final_json, slot.name) + + prev = slot.streamed_json + if final_json and len(final_json) > len(prev): + if prev and not final_json.startswith(prev): + return None + diff = final_json[len(prev) :] + slot.streamed_json = final_json + return diff + return None + + _NAME_RE = re.compile(r'"name"\s*:\s*"([^"]*)"') + + def _try_extract_name(self, idx: int) -> str | None: + m = self._NAME_RE.search(self._tool_slots[idx].args) + if m: + name = m.group(1) + if name: + return name + return None + + # ── Build ExtractedToolCallInformation ───────────────────────────── + + def _build_extracted_result( + self, + *deltas: DeltaMessage | None, + ) -> ExtractedToolCallInformation: + content_parts: list[str] = [] + for delta in deltas: + if delta is not None and delta.content: + content_parts.append(delta.content) + + tool_calls: list[ToolCall] = [] + for idx, slot in enumerate(self._tool_slots): + if not slot.name and not slot.args: + continue + + name = slot.name.strip() + raw_body = slot.args + + if not name and raw_body.strip(): + name, args_json = self._extract_name_and_args(raw_body) + elif raw_body.strip(): + converter = self._arg_converter + if converter is not None: + try: + args_json = converter(raw_body, False) + except (json.JSONDecodeError, ValueError, TypeError): + logger.debug( + "arg converter failed (extract): %s", raw_body[:80] + ) + args_json = self._extract_args_json(raw_body, name) + else: + args_json = self._extract_args_json(raw_body, name) + else: + args_json = "{}" + + if name: + self._ensure_tool_id(slot, name) + args_json = self._fix_arg_types(args_json, name) + tool_calls.append( + ToolCall( + id=slot.id, + function=FunctionCall(name=name, arguments=args_json), + ) + ) + + content_str = "".join(content_parts) + content = self._strip_content_whitespace(content_str, len(tool_calls) > 0) + + return ExtractedToolCallInformation( + tools_called=len(tool_calls) > 0, + tool_calls=tool_calls, + content=content, + ) + + @staticmethod + def _extract_args_value(parsed: dict) -> str | None: + for key in ("arguments", "parameters"): + if key in parsed: + val = parsed[key] + if isinstance(val, str): + return val + return json.dumps(val, ensure_ascii=False) + return None + + def _extract_name_and_args( + self, + raw_body: str, + ) -> tuple[str, str]: + raw_body = raw_body.strip() + try: + parsed = json.loads(raw_body) + except json.JSONDecodeError: + return "", raw_body + + if not isinstance(parsed, dict): + return "", raw_body + + name = parsed.get("name", "") + args = self._extract_args_value(parsed) + if args is not None: + return name, args + + without_name = {k: v for k, v in parsed.items() if k != "name"} + return name, json.dumps(without_name, ensure_ascii=False) + + def _extract_args_json(self, raw_args: str, func_name: str) -> str: + if not raw_args.strip(): + return "{}" + _, args = self._extract_name_and_args(raw_args) + return args diff --git a/vllm/parser/engine/parser_engine_config.py b/vllm/parser/engine/parser_engine_config.py new file mode 100644 index 0000000000..20b4fa096d --- /dev/null +++ b/vllm/parser/engine/parser_engine_config.py @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Declarative configuration for model tool-call and reasoning formats. + +Each model format is described by a :class:`ParserEngineConfig` that specifies: + +* **terminals** – literal strings or regex patterns that delimit the format + (e.g. ````, ````). +* **token_id_terminals** – terminals that should be matched by token ID + rather than (or in addition to) text. +* **transitions** – a state machine mapping + ``(state, terminal) → (new_state, events_to_emit)`` that drives semantic + event generation during streaming. +* **content_events** – what :class:`EventType` to emit for plain content + (non-terminal text) in each state. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from enum import Enum, auto +from functools import cached_property + +from vllm.parser.engine.events import EventType + +STRUCTURAL_DROP_TOKENS: frozenset[str] = frozenset( + { + "", + "", + "", + "", + "", + } +) + + +class ParserState(Enum): + CONTENT = auto() + REASONING = auto() + TOOL_PREAMBLE = auto() + TOOL_NAME = auto() + TOOL_ARGS = auto() + TOOL_BETWEEN = auto() + + +@dataclass(frozen=True, slots=True) +class Transition: + next_state: ParserState + events: tuple[EventType, ...] = field(default_factory=tuple) + skip_in_token_id_mode: bool = False + + +@dataclass(frozen=True) +class ParserEngineConfig: + """Declarative description of a model's tool-call / reasoning format. + + The engine feeds terminals from the incremental lexer into the + transition table and emits the corresponding semantic events. + Content tokens (text between terminals) are classified by the + current state via ``content_events``. + """ + + name: str + + terminals: dict[str, str] = field(default_factory=dict) + + token_id_terminals: dict[str, str] = field(default_factory=dict) + + transitions: dict[tuple[ParserState, str], Transition] = field( + default_factory=dict, + ) + + content_events: dict[ParserState, EventType] = field( + default_factory=lambda: { + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.REASONING: EventType.REASONING_CHUNK, + ParserState.TOOL_NAME: EventType.TOOL_NAME, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + ) + + initial_state: ParserState = ParserState.CONTENT + + arg_converter: Callable[[str, bool], str] | None = None + + stream_arg_deltas: bool = True + + tool_args_json: bool = True + + arg_structural_chars: frozenset[str] | None = None + + # Prevents trailing-whitespace accumulation across multi-turn conversations. + strip_trailing_reasoning_whitespace: bool = True + + # Drop content that is entirely whitespace when tool calls follow. + drop_whitespace_only_content_before_tools: bool = True + + # .strip() content text when tool calls are present. + strip_content_whitespace_with_tools: bool = True + + drop_tokens: frozenset[str] = field(default_factory=frozenset) + + @cached_property + def terminal_defs(self): + from vllm.parser.engine.incremental_lexer import terminals_from_literals + + return terminals_from_literals(self.terminals) + + @cached_property + def lexer_shape(self): + from vllm.parser.engine.incremental_lexer import LexerShape + + return LexerShape(self.terminal_defs) diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py new file mode 100644 index 0000000000..302344efe3 --- /dev/null +++ b/vllm/parser/engine/registered_adapters.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Concrete adapter classes for each registered parser engine. + +These are created via :func:`make_adapters` and exposed as module-level +names so that :class:`ReasoningParserManager` and +:class:`ToolParserManager` can load them lazily. +""" + +from vllm.parser.engine.adapters import make_adapters +from vllm.parser.qwen3 import Qwen3Parser + +( + Qwen3ParserReasoningAdapter, + Qwen3ParserToolAdapter, +) = make_adapters(Qwen3Parser) diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py new file mode 100644 index 0000000000..aced616806 --- /dev/null +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -0,0 +1,408 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Streaming parser engine that orchestrates token ID scanning, +incremental lexing, and state-machine-driven semantic event emission.""" + +from __future__ import annotations + +from collections.abc import Sequence + +from vllm.parser.engine.events import EventType, SemanticEvent +from vllm.parser.engine.incremental_lexer import ( + CONTENT_TERMINAL, + IncrementalLexer, + LexToken, +) +from vllm.parser.engine.parser_engine_config import ( + STRUCTURAL_DROP_TOKENS, + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.parser.engine.token_id_scanner import ( + LexerInput, + PreLexedTerminal, + TextChunk, + TokenIDScanner, +) + + +class StreamingParserEngine: + """Consumes ``(delta_text, delta_token_ids)`` pairs and produces a + stream of :class:`SemanticEvent` instances. + + This is the main entry point for streaming parsing. + Create one per request (it is stateful). + + The pipeline is:: + + delta_text + delta_token_ids + → TokenIDScanner (special token pre-lexing) + → IncrementalLexer (text → terminal tokens with prefix buffering) + → State Machine (terminal → semantic events) + → list[SemanticEvent] + + Usage:: + + engine = StreamingParserEngine(config, tokenizer) + for each streaming delta: + events = engine.feed(delta_text, delta_token_ids) + # convert events to DeltaMessage + """ + + def __init__( + self, + config: ParserEngineConfig, + tokenizer, + initial_state: ParserState | None = None, + vocab: dict[str, int] | None = None, + ) -> None: + self.config = config + + resolved_token_ids: dict[int, str] = {} + drop_token_ids: set[int] = set() + if tokenizer is not None: + if vocab is None: + vocab = tokenizer.get_vocab() + if config.token_id_terminals: + for terminal_name, token_text in config.token_id_terminals.items(): + tid = vocab.get(token_text) + if tid is not None: + resolved_token_ids[tid] = terminal_name + all_drop = config.drop_tokens | STRUCTURAL_DROP_TOKENS + for token_text in all_drop: + tid = vocab.get(token_text) + if tid is not None: + drop_token_ids.add(tid) + for attr in ("eos_token_id", "bos_token_id", "pad_token_id"): + tid = getattr(tokenizer, attr, None) + if tid is not None: + drop_token_ids.add(tid) + + self._resolved_token_ids = resolved_token_ids + self._drop_token_ids = drop_token_ids + + self._scanner = TokenIDScanner( + resolved_token_ids, + tokenizer, + drop_token_ids, + ) + + self._token_id_terminal_names: frozenset[str] = frozenset( + resolved_token_ids.values() + ) + + self._lexer = IncrementalLexer( + config.lexer_shape, content_terminal=CONTENT_TERMINAL + ) + + self._tool_terminals: frozenset[str] = frozenset( + terminal + for (state, terminal), tr in config.transitions.items() + if tr.next_state in self._TOOL_STATES or state in self._TOOL_STATES + ) + + self.skip_tool_parsing = False + self.reset(initial_state=initial_state) + + def _reset_args_state(self) -> None: + self._args_buffer: str = "" + self._args_safe_end: int = 0 + self._args_brace_depth: int = 0 + self._args_in_string: bool = False + self._args_escape_next: bool = False + + def reset(self, initial_state: ParserState | None = None) -> None: + """Reset mutable state for reuse across requests. + + Preserves cached immutable structures (compiled terminals, + resolved token IDs, lexer shape, token text cache) to avoid + redundant initialization work. + """ + self.state = ( + initial_state if initial_state is not None else self.config.initial_state + ) + self.tool_index = -1 + self._ever_had_token_ids = False + # DO NOT reset skip_tool_parsing here — callers set it before + # calling methods that trigger reset() (e.g. extract_reasoning), + # and clearing it silently breaks non-streaming tool-call-as- + # implicit-reasoning-end (content returns None). + self._scanner.reset() + self._lexer.reset() + self._reset_args_state() + + def feed( + self, + delta_text: str, + delta_token_ids: Sequence[int], + ) -> list[SemanticEvent]: + if delta_token_ids: + self._ever_had_token_ids = True + + # Fast path: skip scanner and lexer when the delta is plain + # content with no special tokens and no terminal-starting chars. + if ( + delta_text + and not self._lexer.buffer + and not self._scanner._deferred_terminals + and self._lexer._literal_first_chars.isdisjoint(delta_text) + ): + has_special = False + for tid in delta_token_ids: + if tid in self._resolved_token_ids or tid in self._drop_token_ids: + has_special = True + break + if not has_special: + return self._emit_for_state(delta_text) + + scanner_items = self._scanner.scan(delta_text, delta_token_ids) + + if len(scanner_items) == 1 and isinstance(scanner_items[0], TextChunk): + lex_tokens = self._lexer.feed(scanner_items[0].text) + if len(lex_tokens) == 1 and lex_tokens[0].terminal == CONTENT_TERMINAL: + text = lex_tokens[0].value + return self._emit_for_state(text) + return self._process_lex_tokens(lex_tokens) + + return self._process_scanner_items(scanner_items) + + def _process_scanner_items( + self, items: Sequence[LexerInput] + ) -> list[SemanticEvent]: + events: list[SemanticEvent] = [] + for item in items: + if isinstance(item, PreLexedTerminal): + events.extend(self._process_lex_tokens(self._lexer.flush())) + events.extend(self._on_terminal(item.terminal, item.text)) + elif isinstance(item, TextChunk): + events.extend(self._process_lex_tokens(self._lexer.feed(item.text))) + return events + + def finish(self) -> list[SemanticEvent]: + events = self._process_scanner_items(self._scanner.flush_pending()) + + events.extend(self._process_lex_tokens(self._lexer.flush())) + + if self._args_buffer: + events.append( + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + value=self._args_buffer, + tool_index=self.tool_index, + ) + ) + self._args_buffer = "" + self._args_safe_end = 0 + + if self.state in ( + ParserState.TOOL_PREAMBLE, + ParserState.TOOL_ARGS, + ParserState.TOOL_NAME, + ParserState.TOOL_BETWEEN, + ): + if self.tool_index >= 0: + events.append( + SemanticEvent( + EventType.TOOL_CALL_END, + tool_index=self.tool_index, + ) + ) + self.state = ParserState.CONTENT + elif self.state == ParserState.REASONING: + events.append( + SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index) + ) + self.state = ParserState.CONTENT + + return events + + def parse_complete(self, text: str) -> list[SemanticEvent]: + token_ids: list[int] = [] + events = self.feed(text, token_ids) + events.extend(self.finish()) + return events + + def _process_lex_tokens(self, tokens: list[LexToken]) -> list[SemanticEvent]: + events: list[SemanticEvent] = [] + strict = self._token_id_terminal_names if self._ever_had_token_ids else None + for tok in tokens: + if tok.terminal == CONTENT_TERMINAL or (strict and tok.terminal in strict): + events.extend(self._on_content(tok.value)) + else: + events.extend(self._on_terminal(tok.terminal, tok.value)) + return events + + _TOOL_STATES = frozenset( + { + ParserState.TOOL_PREAMBLE, + ParserState.TOOL_NAME, + ParserState.TOOL_ARGS, + ParserState.TOOL_BETWEEN, + } + ) + + def _on_terminal(self, terminal: str, value: str) -> list[SemanticEvent]: + key = (self.state, terminal) + transition = self.config.transitions.get(key) + + if transition is None: + return self._emit_for_state(value) + + if self.skip_tool_parsing and terminal in self._tool_terminals: + if EventType.REASONING_END in transition.events: + self.state = ParserState.CONTENT + return [ + SemanticEvent( + EventType.REASONING_END, + value=value, + tool_index=self.tool_index, + ), + SemanticEvent( + EventType.TEXT_CHUNK, + value=value, + tool_index=self.tool_index, + ), + ] + content_type = self.config.content_events.get(self.state) + if content_type is not None: + return [ + SemanticEvent(content_type, value=value, tool_index=self.tool_index) + ] + return [] + + if transition.skip_in_token_id_mode and self._ever_had_token_ids: + return self._emit_for_state(value) + + return self._apply_transition(transition, value) + + def _emit_for_state(self, text: str) -> list[SemanticEvent]: + if self.state == ParserState.TOOL_ARGS: + if self.config.tool_args_json: + return self._feed_args_text(text) + return [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + value=text, + tool_index=self.tool_index, + ) + ] + content_type = self.config.content_events.get(self.state) + if content_type is not None: + return [SemanticEvent(content_type, value=text, tool_index=self.tool_index)] + return [] + + def _on_content(self, text: str) -> list[SemanticEvent]: + if not text: + return [] + return self._emit_for_state(text) + + def _apply_transition( + self, + transition: Transition, + value: str, + ) -> list[SemanticEvent]: + events: list[SemanticEvent] = [] + + if ( + self.state == ParserState.TOOL_ARGS + and transition.next_state != ParserState.TOOL_ARGS + and self._args_buffer + ): + events.append( + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + value=self._args_buffer, + tool_index=self.tool_index, + ) + ) + self._args_buffer = "" + + self.state = transition.next_state + + for event_type in transition.events: + if event_type == EventType.TOOL_CALL_START: + self.tool_index += 1 + events.append( + SemanticEvent( + event_type, + value=value, + tool_index=self.tool_index, + ) + ) + + if self.state == ParserState.TOOL_ARGS: + self._args_brace_depth = 0 + self._args_in_string = False + self._args_escape_next = False + self._args_safe_end = 0 + + return events + + def _feed_args_text(self, text: str) -> list[SemanticEvent]: + """Feed text into the JSON argument streaming buffer. + + Streams argument characters incrementally while holding back + closing braces/brackets that might change as more input arrives. + """ + events: list[SemanticEvent] = [] + for ch in text: + result = self._feed_args_char(ch) + events.extend(result) + return events + + def _feed_args_char(self, ch: str) -> list[SemanticEvent]: + self._args_buffer += ch + + if self._args_escape_next: + self._args_escape_next = False + self._args_safe_end = len(self._args_buffer) + return self._flush_safe_args() + + if self._args_in_string: + if ch == "\\": + self._args_escape_next = True + elif ch == '"': + self._args_in_string = False + self._args_safe_end = len(self._args_buffer) + return self._flush_safe_args() + + if ch == '"': + self._args_in_string = True + self._args_safe_end = len(self._args_buffer) + return self._flush_safe_args() + + if ch in ("{", "["): + self._args_brace_depth += 1 + self._args_safe_end = len(self._args_buffer) + return self._flush_safe_args() + + if ch in ("}", "]"): + if self._args_brace_depth > 0: + self._args_brace_depth -= 1 + if self._args_brace_depth == 0: + return [] + self._args_safe_end = len(self._args_buffer) + return self._flush_safe_args() + + self._args_safe_end = len(self._args_buffer) + return self._flush_safe_args() + + def _flush_safe_args(self) -> list[SemanticEvent]: + """Emit buffered argument characters up to the safe-end watermark. + + Top-level closing braces are held back (safe_end not advanced) + until confirmed safe by a subsequent character or finish(). + """ + if self._args_safe_end == 0: + return [] + to_emit = self._args_buffer[: self._args_safe_end] + self._args_buffer = self._args_buffer[self._args_safe_end :] + self._args_safe_end = 0 + return [ + SemanticEvent( + EventType.ARG_VALUE_CHUNK, + value=to_emit, + tool_index=self.tool_index, + ) + ] diff --git a/vllm/parser/engine/token_id_scanner.py b/vllm/parser/engine/token_id_scanner.py new file mode 100644 index 0000000000..d9569de89a --- /dev/null +++ b/vllm/parser/engine/token_id_scanner.py @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Scan delta token IDs for special tokens and split the stream into +pre-lexed terminals and plain text chunks.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + + +@dataclass(slots=True) +class TextChunk: + text: str + + +@dataclass(slots=True) +class PreLexedTerminal: + terminal: str + token_id: int + text: str + + +LexerInput = TextChunk | PreLexedTerminal + + +class TokenIDScanner: + """Maps special token IDs in the delta to terminals. + + Before text-based lexing happens, the scanner checks each token ID + in the delta against a mapping of ``{token_id: terminal_name}``. + Matched tokens are emitted as :class:`PreLexedTerminal` items; + everything else is grouped into :class:`TextChunk` items for the + incremental lexer to process. + + When a terminal's text is not yet in ``delta_text`` (held back by + the detokenizer), the terminal is deferred until the text arrives + in a subsequent delta. + """ + + def __init__( + self, + token_id_to_terminal: dict[int, str], + tokenizer, + drop_token_ids: set[int] | None = None, + ) -> None: + self.token_id_to_terminal = token_id_to_terminal + self.tokenizer = tokenizer + self._token_text_cache: dict[int, str] = {} + self._drop_token_ids = drop_token_ids or set() + self._deferred_terminals: list[PreLexedTerminal] = [] + self._deferred_post_text: str = "" + + def reset(self) -> None: + """Clear mutable state for reuse. Preserves the token text cache.""" + self._deferred_terminals.clear() + self._deferred_post_text = "" + + def _decode_token(self, token_id: int) -> str: + if token_id not in self._token_text_cache: + self._token_text_cache[token_id] = self.tokenizer.decode([token_id]) + return self._token_text_cache[token_id] + + _EMPTY: tuple[LexerInput, ...] = () + + def scan( + self, + delta_text: str, + delta_token_ids: Sequence[int], + ) -> Sequence[LexerInput]: + prefix_items: list[LexerInput] = [] + effective_text = delta_text + + if self._deferred_terminals: + prefix_items, effective_text = self._resolve_deferred(delta_text) + + if not self.token_id_to_terminal and not self._drop_token_ids: + if effective_text: + prefix_items.append(TextChunk(effective_text)) + return prefix_items + + has_special = False + has_drop = False + token_id_to_terminal = self.token_id_to_terminal + drop_token_ids = self._drop_token_ids + for tid in delta_token_ids: + if tid in token_id_to_terminal: + has_special = True + if tid in drop_token_ids: + has_drop = True + + if not has_special and not has_drop: + if effective_text: + if not prefix_items: + return [TextChunk(effective_text)] + prefix_items.append(TextChunk(effective_text)) + return prefix_items or self._EMPTY + + token_texts = [self._decode_token(tid) for tid in delta_token_ids] + + results: list[LexerInput] = [] + text_accum: list[str] = [] + + for idx, tid in enumerate(delta_token_ids): + if tid in self._drop_token_ids: + continue + terminal = self.token_id_to_terminal.get(tid) + if terminal is not None: + if text_accum: + joined = "".join(text_accum) + if joined: + results.append(TextChunk(joined)) + text_accum.clear() + results.append(PreLexedTerminal(terminal, tid, token_texts[idx])) + else: + text_accum.append(token_texts[idx]) + + if text_accum: + joined = "".join(text_accum) + if joined: + results.append(TextChunk(joined)) + + if effective_text: + if has_drop: + clean_delta = effective_text + for idx, tid in enumerate(delta_token_ids): + if tid in self._drop_token_ids: + dropped = token_texts[idx] + pos = clean_delta.find(dropped) + if pos >= 0: + clean_delta = ( + clean_delta[:pos] + clean_delta[pos + len(dropped) :] + ) + if clean_delta: + if results: + results = self._recover_holdback_text(clean_delta, results) + else: + results = [TextChunk(clean_delta)] + else: + results = self._recover_holdback_text(effective_text, results) + else: + # No detokenizer text to validate against — individually-decoded + # TextChunks are unreliable (context-dependent decoding). + # Defer PreLexedTerminals so the state machine doesn't + # transition before the preceding text has arrived. The + # deferred terminals will be resolved against the actual + # delta_text in a subsequent scan() or flushed by finish(). + for r in results: + if isinstance(r, PreLexedTerminal): + self._deferred_terminals.append(r) + results = [] + + return prefix_items + results + + def flush_pending(self) -> list[LexerInput]: + if not self._deferred_terminals and not self._deferred_post_text: + return [] + results: list[LexerInput] = [] + if self._deferred_post_text: + results.append(TextChunk(self._deferred_post_text)) + self._deferred_post_text = "" + results.extend(self._deferred_terminals) + self._deferred_terminals.clear() + return results + + def _resolve_deferred( + self, + delta_text: str, + ) -> tuple[list[LexerInput], str]: + """Resolve deferred terminals against new delta_text. + + When a previous ``scan()`` deferred a terminal (its text hadn't + arrived yet), the next delta's text should contain that terminal's + text. Split delta_text at the terminal boundary: text before + belongs to the previous parser state, the terminal triggers the + state transition, and text after belongs to the new state. + + Returns ``(prefix_items, remaining_text)`` where prefix_items + are the resolved deferred terminals (with any preceding text) + and remaining_text is the unconsumed portion of delta_text that + should be scanned with the current delta's token IDs. + """ + deferred = self._deferred_terminals + self._deferred_terminals = [] + + results: list[LexerInput] = [] + remaining = delta_text + + if self._deferred_post_text: + remaining = self._deferred_post_text + remaining + self._deferred_post_text = "" + + # Duplicate-text deferred terminals resolve left-to-right via + # find(); correct when each terminal text appears once in sequence. + for terminal in deferred: + pos = remaining.find(terminal.text) + if pos > 0: + results.append(TextChunk(remaining[:pos])) + results.append(terminal) + remaining = remaining[pos + len(terminal.text) :] + elif pos == 0: + results.append(terminal) + remaining = remaining[len(terminal.text) :] + else: + # Accumulate text until terminal text arrives — + # only the terminal provides a reliable split point. + if remaining: + self._deferred_post_text += remaining + remaining = "" + self._deferred_terminals.append(terminal) + + return results, remaining + + def _recover_holdback_text( + self, + delta_text: str, + results: list[LexerInput], + ) -> list[LexerInput]: + """Recover detokenizer hold-back text not in delta_token_ids. + + The detokenizer may flush previously held-back text in + ``delta_text`` that has no corresponding token ID in + ``delta_token_ids``. This hold-back text always appears as a + prefix of ``delta_text``. + """ + if not results: + return [TextChunk(delta_text)] + + reconstructed = self._join_decoded_text(results) + + if not reconstructed: + return [TextChunk(delta_text)] + results + + pos = delta_text.find(reconstructed) + if pos > 0: + return [TextChunk(delta_text[:pos])] + results + if pos == 0: + return results + + # Fallback: SentencePiece context-dependent decoding mismatch. + # Rebuild from delta_text using PreLexedTerminals as split anchors. + return self._rebuild_from_anchors(delta_text, results) + + def _join_decoded_text(self, results: list[LexerInput]) -> str: + """Join TextChunk and PreLexedTerminal text into one string.""" + parts: list[str] = [] + for item in results: + if isinstance(item, (TextChunk, PreLexedTerminal)): + parts.append(item.text) + return "".join(parts) + + def _rebuild_from_anchors( + self, + delta_text: str, + results: list[LexerInput], + ) -> list[LexerInput]: + """Rebuild results from delta_text using terminals as anchors. + + When context-dependent decoding creates a mismatch between + individually-decoded tokens and delta_text, use + PreLexedTerminals as split points and reallocate text from + delta_text. If a terminal's text is not found in delta_text, + it is deferred to the next scan() call. + + Anchors are resolved right-to-left with ``rfind`` so that each + anchor binds to the *rightmost* available occurrence of its + text. This prevents earlier literal lookalikes (e.g. a user + mentioning ```` in prose) from stealing the position + of a real special-token anchor that appears later. + + If the same anchor text appears multiple times as real special + tokens (not prose), the rightmost-first binding could misalign. + In practice this doesn't happen: each special token ID maps to + a distinct PreLexedTerminal, and duplicates in prose are resolved + by the token-ID filtering layer above. + """ + anchors = [item for item in results if isinstance(item, PreLexedTerminal)] + if not anchors: + return [TextChunk(delta_text)] + + # Resolve positions right-to-left: each anchor gets the + # rightmost occurrence that is still before the next anchor. + positions: list[int] = [-1] * len(anchors) + search_end = len(delta_text) + for i in range(len(anchors) - 1, -1, -1): + pos = delta_text.rfind(anchors[i].text, 0, search_end) + if pos >= 0: + positions[i] = pos + search_end = pos + + # Build results left-to-right using the resolved positions. + new_results: list[LexerInput] = [] + consumed = 0 + for i, anchor in enumerate(anchors): + pos = positions[i] + if pos >= consumed: + if pos > consumed: + new_results.append(TextChunk(delta_text[consumed:pos])) + new_results.append(anchor) + consumed = pos + len(anchor.text) + else: + has_later_valid = any(p >= 0 for p in positions[i + 1 :]) + if not has_later_valid and consumed < len(delta_text): + self._deferred_post_text += delta_text[consumed:] + consumed = len(delta_text) + self._deferred_terminals.append(anchor) + if consumed < len(delta_text): + new_results.append(TextChunk(delta_text[consumed:])) + return new_results diff --git a/vllm/parser/qwen3.py b/vllm/parser/qwen3.py new file mode 100644 index 0000000000..4b03ee34a2 --- /dev/null +++ b/vllm/parser/qwen3.py @@ -0,0 +1,218 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Qwen3 parser for tool calls and reasoning. + +Qwen3 XML tool call format:: + + + + value + + + +The argument body consists of ``VALUE`` tags. +The ``_qwen3_arg_converter`` parses these into a JSON object. +""" + +from __future__ import annotations + +import functools +import json +from typing import TYPE_CHECKING + +import regex as re + +from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) + +if TYPE_CHECKING: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +TOOL_CALL_START = "" +TOOL_CALL_END = "" +FUNC_PREFIX = "]*)>" + r"(.*?)" + r"(?:<\s*/\s*parameter\s*>|(?=<\s*parameter\s*=))", + re.DOTALL, +) +_PARTIAL_PARAM_RE = re.compile(r"<\s*parameter\s*=\s*([^>]+)>([^<]*)$", re.DOTALL) + + +def _qwen3_arg_converter(raw_args: str, partial: bool) -> str: + params: dict[str, object] = {} + + for match in _PARAM_RE.finditer(raw_args): + name = match.group(1) + value = match.group(2) + params[name] = value.strip() + + if partial: + remaining = _PARAM_RE.sub("", raw_args) + m = _PARTIAL_PARAM_RE.search(remaining) + if m: + name = m.group(1) + value = m.group(2) + if name: + params[name] = value + + return json.dumps(params, ensure_ascii=False) + + +@functools.cache +def qwen3_config(thinking: bool = True) -> ParserEngineConfig: + return ParserEngineConfig( + name="qwen3", + initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + terminals={ + # Reasoning terminals + "THINK_START": "", + "THINK_END": "", + # Tool call terminals + "TOOL_START": TOOL_CALL_START, + "TOOL_END": TOOL_CALL_END, + "FUNC_PREFIX": FUNC_PREFIX, + "FUNC_END": FUNC_END, + "CLOSE_ANGLE": ">", + }, + token_id_terminals={ + "THINK_START": "", + "THINK_END": "", + "TOOL_START": TOOL_CALL_START, + "TOOL_END": TOOL_CALL_END, + }, + transitions={ + # -- Reasoning transitions -- + (ParserState.REASONING, "THINK_START"): Transition( + ParserState.REASONING, + (), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + # Absorb duplicate — model may emit it after + # already transitioning to CONTENT; drop it silently. + (ParserState.CONTENT, "THINK_END"): Transition( + ParserState.CONTENT, + (), + ), + # Tool call directly from reasoning (implicit end) + (ParserState.REASONING, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.REASONING_END, EventType.TOOL_CALL_START), + ), + # -- Tool call transitions -- + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.TOOL_CALL_START,), + ), + # Fallback: + (ParserState.CONTENT, "FUNC_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition( + ParserState.TOOL_NAME, + (), + ), + (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition( + ParserState.TOOL_ARGS, + (), + ), + # Malformed: while still in TOOL_NAME (no closing >) + (ParserState.TOOL_NAME, "FUNC_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_ARGS, "FUNC_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( + ParserState.CONTENT, + (), + ), + # Consecutive tool call without closing + (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + }, + arg_converter=_qwen3_arg_converter, + stream_arg_deltas=True, + strip_trailing_reasoning_whitespace=False, + tool_args_json=False, + ) + + +class Qwen3Parser(ParserEngine): + """Qwen3 parser: ````/```` reasoning + + ```` XML tool calls in a single engine. + + - ```` as implicit reasoning end + - Unpaired ```` token ID detection for ``is_reasoning_end`` + """ + + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + self.thinking_enabled = chat_kwargs.get("enable_thinking", True) + kwargs.setdefault( + "parser_engine_config", + qwen3_config(thinking=self.thinking_enabled), + ) + super().__init__( + tokenizer, + tools, + **kwargs, + ) + vocab = self.vocab + self._tool_call_token_id: int | None = vocab.get("") + self._tool_call_end_token_id: int | None = vocab.get("") + + def extract_reasoning( + self, + model_output: str, + request: ChatCompletionRequest | ResponsesRequest, + ) -> tuple[str | None, str | None]: + if not self.thinking_enabled: + return None, model_output + return super().extract_reasoning(model_output, request) + + def is_reasoning_end(self, input_ids: list[int]) -> bool: + if super().is_reasoning_end(input_ids): + return True + tool_call_id = self._tool_call_token_id + tool_call_end_id = self._tool_call_end_token_id + if tool_call_id is not None: + for i in range(len(input_ids) - 1, -1, -1): + if input_ids[i] == tool_call_id: + if tool_call_end_id is not None and any( + input_ids[j] == tool_call_end_id + for j in range(i + 1, len(input_ids)) + ): + continue + return True + return False diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index cd51f10650..5d301b8201 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -13,8 +13,8 @@ Register a lazy module mapping. Example: ReasoningParserManager.register_lazy_module( name="qwen3", - module_path="vllm.reasoning.qwen3_reasoning_parser", - class_name="Qwen3ReasoningParser", + module_path="vllm.reasoning.qwen3_engine_reasoning_parser", + class_name="Qwen3ParserReasoningAdapter", ) """ @@ -81,8 +81,8 @@ _REASONING_PARSERS_TO_REGISTER = { "KimiK2ReasoningParser", ), "mimo": ( - "qwen3_reasoning_parser", - "Qwen3ReasoningParser", + "qwen3_engine_reasoning_parser", + "Qwen3ParserReasoningAdapter", ), "minimax_m2": ( "minimax_m2_reasoning_parser", @@ -105,8 +105,8 @@ _REASONING_PARSERS_TO_REGISTER = { "Olmo3ReasoningParser", ), "qwen3": ( - "qwen3_reasoning_parser", - "Qwen3ReasoningParser", + "qwen3_engine_reasoning_parser", + "Qwen3ParserReasoningAdapter", ), "seed_oss": ( "seedoss_reasoning_parser", diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py index 74b3e62abc..4e519f6aeb 100644 --- a/vllm/reasoning/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -31,6 +31,8 @@ class ReasoningParser: It is used to extract reasoning content from the model output. """ + engine_based_streaming: bool = False + def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs): self.model_tokenizer = tokenizer # Optional vLLM ModelConfig from the server. Use get (not pop) so composite @@ -57,6 +59,17 @@ class ReasoningParser: """ return None + def has_engine_confirmed_reasoning_end(self) -> bool: + """Whether the engine has confirmed the reasoning end transition. + + Engine-based parsers may defer terminal processing when the + detokenizer holds back text. This method returns the engine's + *processed* state, not a raw token-ID check. + + Only called for parsers with ``engine_based_streaming = True``. + """ + return False + @abstractmethod def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: """ @@ -285,8 +298,8 @@ class ReasoningParserManager: Example: ReasoningParserManager.register_lazy_module( name="qwen3", - module_path="vllm.reasoning.parsers.qwen3_reasoning_parser", - class_name="Qwen3ReasoningParser", + module_path="vllm.reasoning.qwen3_engine_reasoning_parser", + class_name="Qwen3ParserReasoningAdapter", ) """ cls.lazy_parsers[name] = (module_path, class_name) diff --git a/vllm/reasoning/qwen3_engine_reasoning_parser.py b/vllm/reasoning/qwen3_engine_reasoning_parser.py new file mode 100644 index 0000000000..64e71f9f08 --- /dev/null +++ b/vllm/reasoning/qwen3_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import Qwen3ParserReasoningAdapter + +__all__ = ["Qwen3ParserReasoningAdapter"] diff --git a/vllm/reasoning/qwen3_reasoning_parser.py b/vllm/reasoning/qwen3_reasoning_parser.py deleted file mode 100644 index e38b0de3d8..0000000000 --- a/vllm/reasoning/qwen3_reasoning_parser.py +++ /dev/null @@ -1,231 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from collections.abc import Iterable, Sequence -from typing import TYPE_CHECKING - -from vllm.entrypoints.openai.engine.protocol import DeltaMessage -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser - -if TYPE_CHECKING: - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest - from vllm.tokenizers import TokenizerLike - - -class Qwen3ReasoningParser(BaseThinkingReasoningParser): - """ - Reasoning parser for the Qwen3/Qwen3.5 model family. - - The Qwen3 model family uses ... tokens to denote reasoning - text. Starting with Qwen3.5, the chat template places in the - prompt so only appears in the generated output. The model - provides a strict switch to disable reasoning output via the - 'enable_thinking=False' parameter. - - When thinking is disabled, the template places \\n\\n\\n\\n - in the prompt. The serving layer detects this via prompt_is_reasoning_end - and routes deltas as content without calling the streaming parser. - - NOTE: Models up to the 2507 release (e.g., Qwen/Qwen3-235B-A22B-Instruct-2507) - use an older chat template where the model generates itself. - This parser handles both styles: if appears in the generated output - it is stripped before extraction (non-streaming) or skipped (streaming). - - NOTE: Qwen3.5 models may emit inside the thinking block - without closing first. is treated as an implicit - end of reasoning, matching the approach in KimiK2ReasoningParser. - """ - - def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs): - super().__init__(tokenizer, *args, **kwargs) - - chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} - # Qwen3 defaults to thinking enabled; only treat output as - # pure content when the user explicitly disables it. - self.thinking_enabled = chat_kwargs.get("enable_thinking", True) - - self._tool_call_tag = "" - self._tool_call_token_id = self.vocab.get(self._tool_call_tag) - self._tool_call_end_tag = "" - self._tool_call_end_token_id = self.vocab.get(self._tool_call_end_tag) - - @property - def start_token(self) -> str: - """The token that starts reasoning content.""" - return "" - - @property - def end_token(self) -> str: - """The token that ends reasoning content.""" - return "" - - def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: - start_token_id = self.start_token_id - end_token_id = self.end_token_id - tool_call_token_id = self._tool_call_token_id - tool_call_end_token_id = self._tool_call_end_token_id - - for i in range(len(input_ids) - 1, -1, -1): - token_id = input_ids[i] - if token_id == start_token_id: - # Found before or - return False - if token_id == end_token_id: - return True - if tool_call_token_id is not None and token_id == tool_call_token_id: - # Only treat as implicit reasoning end if this - # is NOT followed by . Paired occurrences are - # template examples in the prompt, not model output. - if tool_call_end_token_id is not None and any( - input_ids[j] == tool_call_end_token_id - for j in range(i + 1, len(input_ids)) - ): - continue - return True - return False - - def is_reasoning_end_streaming( - self, input_ids: Sequence[int], delta_ids: Iterable[int] - ) -> bool: - if super().is_reasoning_end_streaming(input_ids, delta_ids): - return True - if self._tool_call_token_id is not None: - return self._tool_call_token_id in delta_ids - return False - - def extract_content_ids(self, input_ids: list[int]) -> list[int]: - """ - Extract content token ids from the input_ids. - """ - result = super().extract_content_ids(input_ids) - if result: - return result - # Fall back: content starts at (implicit reasoning end). - if ( - self._tool_call_token_id is not None - and self._tool_call_token_id in input_ids - ): - tool_call_index = ( - len(input_ids) - 1 - input_ids[::-1].index(self._tool_call_token_id) - ) - return input_ids[tool_call_index:] - return [] - - def extract_reasoning( - self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" - ) -> tuple[str | None, str | None]: - """ - Extract reasoning content from the model output. - - The token is placed in the prompt by the chat template, - so typically only appears in the generated output. - If is present (e.g. from a different template), it is - stripped before extraction. - - When thinking is explicitly disabled and no appears, - returns (None, model_output) — all output is content. - Otherwise (thinking enabled, default), a missing means - the output was truncated and everything is reasoning: - returns (model_output, None). - - Returns: - tuple[Optional[str], Optional[str]]: reasoning content and content - """ - - # Strip if present in the generated output. - model_output_parts = model_output.partition(self.start_token) - model_output = ( - model_output_parts[2] if model_output_parts[1] else model_output_parts[0] - ) - - if self.end_token in model_output: - reasoning, _, content = model_output.partition(self.end_token) - return reasoning, content or None - - if not self.thinking_enabled: - # Thinking explicitly disabled — treat everything as content. - return None, model_output - - # No — check for implicit reasoning end via . - tool_call_index = model_output.find(self._tool_call_tag) - if tool_call_index != -1: - reasoning = model_output[:tool_call_index] - content = model_output[tool_call_index:] - return reasoning or None, content or None - # Thinking enabled but no : output was truncated. - # Everything generated so far is reasoning. - return model_output, None - - def extract_reasoning_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> DeltaMessage | None: - """ - Extract reasoning content from a streaming delta. - - Since is placed in the prompt by the chat template, all - generated tokens before are reasoning and tokens after - are content. - - NOTE: When thinking is disabled, no think tokens appear in the - generated output. The serving layer detects this via - prompt_is_reasoning_end and routes deltas as content without - calling this method. - """ - # Strip from delta if present (old template / edge case - # where the model generates itself). - if self.start_token_id in delta_token_ids: - start_idx = delta_text.find(self.start_token) - if start_idx >= 0: - delta_text = delta_text[start_idx + len(self.start_token) :] - - if self.end_token_id in delta_token_ids: - # End token in this delta: split reasoning from content. - end_index = delta_text.find(self.end_token) - if end_index >= 0: - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - if not reasoning and not content: - return None - return DeltaMessage( - reasoning=reasoning if reasoning else None, - content=content if content else None, - ) - # end_token_id in IDs but not in text (already stripped) - return None - - # Implicit reasoning end via . - if ( - self._tool_call_token_id is not None - and self._tool_call_token_id in delta_token_ids - ): - tool_index = delta_text.find(self._tool_call_tag) - if tool_index >= 0: - reasoning = delta_text[:tool_index] - content = delta_text[tool_index:] - return DeltaMessage( - reasoning=reasoning if reasoning else None, - content=content if content else None, - ) - - # No end token in this delta. - if not delta_text: - # Nothing left after stripping start token. - return None - elif self.end_token_id in previous_token_ids: - # End token already passed: everything is content now. - return DeltaMessage(content=delta_text) - elif ( - self._tool_call_token_id is not None - and self._tool_call_token_id in previous_token_ids - ): - return DeltaMessage(content=delta_text) - else: - # No end token yet: still in reasoning phase. - return DeltaMessage(reasoning=delta_text) diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 6d122b4695..a6a931d5b2 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -119,8 +119,8 @@ _TOOL_PARSERS_TO_REGISTER = { "LongcatFlashToolParser", ), "mimo": ( - "qwen3coder_tool_parser", - "Qwen3CoderToolParser", + "qwen3_engine_tool_parser", + "Qwen3EngineToolParser", ), "minimax_m2": ( "minimax_m2_tool_parser", @@ -155,12 +155,12 @@ _TOOL_PARSERS_TO_REGISTER = { "PythonicToolParser", ), "qwen3_coder": ( - "qwen3coder_tool_parser", - "Qwen3CoderToolParser", + "qwen3_engine_tool_parser", + "Qwen3EngineToolParser", ), "qwen3_xml": ( - "qwen3coder_tool_parser", - "Qwen3CoderToolParser", + "qwen3_engine_tool_parser", + "Qwen3EngineToolParser", ), "seed_oss": ( "seed_oss_tool_parser", diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 3609bcbf45..a1c4cf1ffa 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -60,6 +60,7 @@ class ToolParser: # xgrammar builtin structural tag model key. Subclasses set this when # their parsed tool-call syntax matches a builtin xgrammar format. structural_tag_model: str | None = None + engine_based_streaming: bool = False def __init_subclass__(cls, **kwargs: Any) -> None: super().__init_subclass__(**kwargs) diff --git a/vllm/tool_parsers/qwen3_engine_tool_parser.py b/vllm/tool_parsers/qwen3_engine_tool_parser.py new file mode 100644 index 0000000000..2263a40b36 --- /dev/null +++ b/vllm/tool_parsers/qwen3_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import Qwen3ParserToolAdapter + + +class Qwen3EngineToolParser(Qwen3ParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = "qwen_3_coder" diff --git a/vllm/tool_parsers/qwen3coder_tool_parser.py b/vllm/tool_parsers/qwen3coder_tool_parser.py deleted file mode 100644 index f9d777af1e..0000000000 --- a/vllm/tool_parsers/qwen3coder_tool_parser.py +++ /dev/null @@ -1,586 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import json -import uuid -from collections.abc import Sequence -from typing import Any - -import regex as re - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - coerce_to_schema_type, - extract_types_from_schema, - find_tool_properties, -) - -logger = init_logger(__name__) - - -class Qwen3CoderToolParser(ToolParser): - structural_tag_model = "qwen_3_coder" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.current_tool_name_sent: bool = False - self.prev_tool_call_arr: list[dict] = [] - # Override base class type - we use string IDs for tool calls - self.current_tool_id: str | None = None # type: ignore - self.streamed_args_for_tool: list[str] = [] - - # Sentinel tokens for streaming mode - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - self.tool_call_prefix: str = "(.*?)", re.DOTALL - ) - self.tool_call_regex = re.compile( - r"(.*?)|(.*?)$", re.DOTALL - ) - self.tool_call_function_regex = re.compile( - r"||(?=)|$)", - re.DOTALL, - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) - self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) - - if self.tool_call_start_token_id is None or self.tool_call_end_token_id is None: - raise RuntimeError( - "Qwen3 XML Tool parser could not locate tool call start/end " - "tokens in the tokenizer!" - ) - - logger.debug( - "vLLM Successfully import tool parser %s !", self.__class__.__name__ - ) - - def _generate_tool_call_id(self) -> str: - """Generate a unique tool call ID.""" - return f"call_{uuid.uuid4().hex[:24]}" - - def _reset_streaming_state(self): - """Reset all streaming state.""" - self.current_tool_index = 0 - self.is_tool_call_started = False - self.header_sent = False - self.current_tool_id = None - self.current_function_name = None - self.current_param_name = None - self.current_param_value = "" - self.param_count = 0 - self.in_param = False - self.in_function = False - self.accumulated_text = "" - self.json_started = False - self.json_closed = False - # Store accumulated parameters for type conversion - self.accumulated_params = {} - self.streaming_request = None - - def _convert_param_value( - self, param_value: str, param_name: str, param_config: dict, func_name: str - ) -> Any: - """Convert parameter value based on its type in the schema.""" - if not isinstance(param_value, str): - return param_value - param_schema = param_config.get(param_name, {}) - param_types = extract_types_from_schema(param_schema) - return coerce_to_schema_type(param_value, param_types) - - def _parse_xml_function_call(self, function_call_str: str) -> ToolCall | None: - # Extract function name - end_index = function_call_str.find(">") - # If there's no ">" character, this is not a valid xml function call - if end_index == -1: - return None - function_name = function_call_str[:end_index] - param_config = find_tool_properties(self.tools, function_name) - parameters = function_call_str[end_index + 1 :] - param_dict = {} - for match_text in self.tool_call_parameter_regex.findall(parameters): - idx = match_text.index(">") - param_name = match_text[:idx] - param_value = str(match_text[idx + 1 :]) - # Remove prefix and trailing \n - if param_value.startswith("\n"): - param_value = param_value[1:] - if param_value.endswith("\n"): - param_value = param_value[:-1] - - param_dict[param_name] = self._convert_param_value( - param_value, param_name, param_config, function_name - ) - return ToolCall( - type="function", - function=FunctionCall( - name=function_name, arguments=json.dumps(param_dict, ensure_ascii=False) - ), - ) - - def _get_function_calls(self, model_output: str) -> list[str]: - # Find all tool calls - matched_ranges = self.tool_call_regex.findall(model_output) - raw_tool_calls = [ - match[0] if match[0] else match[1] for match in matched_ranges - ] - - # Back-off strategy if no tool_call tags found - if len(raw_tool_calls) == 0: - raw_tool_calls = [model_output] - - raw_function_calls = [] - for tool_call in raw_tool_calls: - raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call)) - - function_calls = [ - match[0] if match[0] else match[1] for match in raw_function_calls - ] - return function_calls - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # Quick check to avoid unnecessary processing - if self.tool_call_prefix not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - function_calls = self._get_function_calls(model_output) - if len(function_calls) == 0: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - tool_calls = [ - self._parse_xml_function_call(function_call_str) - for function_call_str in function_calls - ] - # Populate prev_tool_call_arr for serving layer to set finish_reason - self.prev_tool_call_arr.clear() # Clear previous calls - for tool_call in tool_calls: - if tool_call: - self.prev_tool_call_arr.append( - { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments, - } - ) - - # Extract content before tool calls - content_index = model_output.find(self.tool_call_start_token) - idx = model_output.find(self.tool_call_prefix) - content_index = content_index if content_index >= 0 else idx - content = model_output[:content_index] # .rstrip() - valid_tool_calls = [tc for tc in tool_calls if tc is not None] - return ExtractedToolCallInformation( - tools_called=(len(valid_tool_calls) > 0), - tool_calls=valid_tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # Store request for type conversion - if not previous_text: - self._reset_streaming_state() - self.streaming_request = request - - # If no delta text, return None unless it's an EOS token after tools - if not delta_text: - # Check if this is an EOS token after all tool calls are complete - # Check for tool calls in text even if is_tool_call_started - # is False (might have been reset after processing all tools) - if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids: - # Count complete tool calls - complete_calls = len( - self.tool_call_complete_regex.findall(current_text) - ) - - # If we have completed tool calls and populated - # prev_tool_call_arr - if complete_calls > 0 and len(self.prev_tool_call_arr) > 0: - # Check if all tool calls are closed - open_calls = current_text.count( - self.tool_call_start_token - ) - current_text.count(self.tool_call_end_token) - if open_calls == 0: - # Return empty delta for finish_reason processing - return DeltaMessage(content="") - elif not self.is_tool_call_started and current_text: - # This is a regular content response that's now complete - return DeltaMessage(content="") - return None - - # Update accumulated text - self.accumulated_text = current_text - - # Check if we need to advance to next tool - if self.json_closed and not self.in_function: - # Check if this tool call has ended - tool_ends = current_text.count(self.tool_call_end_token) - if tool_ends > self.current_tool_index: - # This tool has ended, advance to next - self.current_tool_index += 1 - self.header_sent = False - self.param_count = 0 - self.json_started = False - self.json_closed = False - self.accumulated_params = {} - - # Check if there are more tool calls - tool_starts = current_text.count(self.tool_call_start_token) - if self.current_tool_index >= tool_starts: - # No more tool calls - self.is_tool_call_started = False - # Continue processing next tool - return None - - # Handle normal content before tool calls - if not self.is_tool_call_started: - # Check if tool call is starting - if ( - self.tool_call_start_token_id in delta_token_ids - or self.tool_call_start_token in delta_text - ): - self.is_tool_call_started = True - # Return any content before the tool call - if self.tool_call_start_token in delta_text: - content_before = delta_text[ - : delta_text.index(self.tool_call_start_token) - ] - if content_before: - return DeltaMessage(content=content_before) - return None - else: - # Check if we're between tool calls - skip whitespace - if ( - current_text.rstrip().endswith(self.tool_call_end_token) - and delta_text.strip() == "" - ): - # We just ended a tool call, skip whitespace - return None - # Normal content, no tool call - return DeltaMessage(content=delta_text) - - # Check if we're between tool calls (waiting for next one) - # Count tool calls we've seen vs processed - tool_starts_count = current_text.count(self.tool_call_start_token) - if self.current_tool_index >= tool_starts_count: - # We're past all tool calls, shouldn't be here - return None - - # We're in a tool call, find the current tool call portion - # Need to find the correct tool call based on current_tool_index - tool_start_positions: list[int] = [] - idx = 0 - while True: - idx = current_text.find(self.tool_call_start_token, idx) - if idx == -1: - break - tool_start_positions.append(idx) - idx += len(self.tool_call_start_token) - - if self.current_tool_index >= len(tool_start_positions): - # No more tool calls to process yet - return None - - tool_start_idx = tool_start_positions[self.current_tool_index] - # Find where this tool call ends (or current position if not ended yet) - tool_end_idx = current_text.find(self.tool_call_end_token, tool_start_idx) - if tool_end_idx == -1: - tool_text = current_text[tool_start_idx:] - else: - tool_text = current_text[ - tool_start_idx : tool_end_idx + len(self.tool_call_end_token) - ] - - # Looking for function header - if not self.header_sent: - if self.tool_call_prefix in tool_text: - func_start = tool_text.find(self.tool_call_prefix) + len( - self.tool_call_prefix - ) - func_end = tool_text.find(">", func_start) - - if func_end != -1: - # Found complete function name - self.current_function_name = tool_text[func_start:func_end] - self.current_tool_id = self._generate_tool_call_id() - self.header_sent = True - self.in_function = True - - # Always append — each tool call is a separate - # invocation even if the function name is the same - # (e.g. two consecutive "read" calls). - self.prev_tool_call_arr.append( - { - "name": self.current_function_name, - "arguments": "{}", - } - ) - - # Initialize streamed args tracking for this tool. - # The serving layer reads streamed_args_for_tool to - # compute remaining arguments at stream end. Without - # this, IndexError occurs when the serving layer - # accesses streamed_args_for_tool[index]. - self.streamed_args_for_tool.append("") - - # Send header with function info - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - id=self.current_tool_id, - function=DeltaFunctionCall( - name=self.current_function_name, arguments="" - ), - type="function", - ) - ] - ) - return None - - # We've sent header, now handle function body - if self.in_function: - # Always send opening brace first, regardless of whether - # parameter_prefix is in the current delta. With speculative - # decoding, a single delta may contain both the opening brace - # and parameter data; skipping "{" here would desync - # json_started from what was actually streamed. - if not self.json_started: - self.json_started = True - self.streamed_args_for_tool[self.current_tool_index] += "{" - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall(arguments="{"), - ) - ] - ) - - # Find all parameter start positions in current tool_text - param_starts = [] - search_idx = 0 - while True: - search_idx = tool_text.find(self.parameter_prefix, search_idx) - if search_idx == -1: - break - param_starts.append(search_idx) - search_idx += len(self.parameter_prefix) - - # Process ALL complete params in a loop (spec decode fix). - # With speculative decoding a single delta can deliver - # multiple complete parameters at once. The old single-pass - # code would process one and ``return None`` if the next was - # incomplete — skipping any already-complete params that - # preceded it. Using a loop with ``break`` instead ensures - # we emit every complete parameter before yielding control. - json_fragments = [] - while not self.in_param and self.param_count < len(param_starts): - param_idx = param_starts[self.param_count] - param_start = param_idx + len(self.parameter_prefix) - remaining = tool_text[param_start:] - - if ">" not in remaining: - break - - name_end = remaining.find(">") - current_param_name = remaining[:name_end] - - value_start = param_start + name_end + 1 - value_text = tool_text[value_start:] - if value_text.startswith("\n"): - value_text = value_text[1:] - - param_end_idx = value_text.find(self.parameter_end_token) - if param_end_idx == -1: - next_param_idx = value_text.find(self.parameter_prefix) - func_end_idx = value_text.find(self.function_end_token) - - if next_param_idx != -1 and ( - func_end_idx == -1 or next_param_idx < func_end_idx - ): - param_end_idx = next_param_idx - elif func_end_idx != -1: - param_end_idx = func_end_idx - else: - # Fallback for malformed XML where - # is missing. Use as a delimiter - # if present in the value so we don't include - # the closing tag as part of the param value. - tool_end_in_value = value_text.find(self.tool_call_end_token) - if tool_end_in_value != -1: - param_end_idx = tool_end_in_value - else: - # Parameter incomplete — break so we still - # emit any fragments accumulated by earlier - # loop iterations. - break - - if param_end_idx == -1: - break - - param_value = value_text[:param_end_idx] - if param_value.endswith("\n"): - param_value = param_value[:-1] - - self.current_param_name = current_param_name - self.accumulated_params[current_param_name] = param_value - - param_config = find_tool_properties( - self.tools, self.current_function_name or "" - ) - - converted_value = self._convert_param_value( - param_value, - current_param_name, - param_config, - self.current_function_name or "", - ) - - serialized_value = json.dumps(converted_value, ensure_ascii=False) - - if self.param_count == 0: - json_fragment = f'"{current_param_name}": {serialized_value}' - else: - json_fragment = f', "{current_param_name}": {serialized_value}' - - self.param_count += 1 - json_fragments.append(json_fragment) - - if json_fragments: - combined = "".join(json_fragments) - - if self.current_tool_index < len(self.streamed_args_for_tool): - self.streamed_args_for_tool[self.current_tool_index] += combined - else: - logger.warning( - "streamed_args_for_tool out of sync: index=%d len=%d", - self.current_tool_index, - len(self.streamed_args_for_tool), - ) - - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall(arguments=combined), - ) - ] - ) - - # Check for function end AFTER processing parameters. - # This ordering is critical: with speculative decoding a - # burst can deliver the final parameter value together with - # . If the close check ran first it would emit - # "}" and set in_function=False before the parameter loop - # ever ran, causing the parameter to be silently dropped. - if not self.json_closed and self.function_end_token in tool_text: - self.json_closed = True - - func_start = tool_text.find(self.tool_call_prefix) + len( - self.tool_call_prefix - ) - func_content_end = tool_text.find(self.function_end_token, func_start) - if func_content_end != -1: - func_content = tool_text[func_start:func_content_end] - try: - parsed_tool = self._parse_xml_function_call( - func_content, - ) - if parsed_tool and self.current_tool_index < len( - self.prev_tool_call_arr - ): - self.prev_tool_call_arr[self.current_tool_index][ - "arguments" - ] = parsed_tool.function.arguments - except Exception: - logger.debug( - "Failed to parse tool call during streaming: %s", - tool_text, - exc_info=True, - ) - - if self.current_tool_index < len(self.streamed_args_for_tool): - self.streamed_args_for_tool[self.current_tool_index] += "}" - else: - logger.warning( - "streamed_args_for_tool out of sync: index=%d len=%d", - self.current_tool_index, - len(self.streamed_args_for_tool), - ) - - result = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall(arguments="}"), - ) - ] - ) - - self.in_function = False - self.json_closed = True - self.accumulated_params = {} - - return result - - return None