[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-06-15 11:59:05 +08:00
committed by GitHub
co-authored by Flora Feng
parent e3e3cd5458
commit c4a3f9d137
33 changed files with 8494 additions and 904 deletions
View File
+40
View File
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
@pytest.fixture()
def should_do_global_cleanup_after_test() -> bool:
return False
def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
"""Create a mock tokenizer with the given special-token vocabulary.
The returned mock supports get_vocab(), encode(), and decode().
decode() maps known token IDs back to their text and falls back to
chr(id) for ASCII IDs or ``<id>`` for others.
"""
id_to_text = {v: k for k, v in vocab.items()}
tokenizer = MagicMock()
tokenizer.encode.return_value = [1, 2, 3]
tokenizer.get_vocab.return_value = dict(vocab)
tokenizer.decode.side_effect = lambda ids: "".join(
id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
)
return tokenizer
@pytest.fixture
def mock_request():
req = MagicMock(spec=ChatCompletionRequest)
req.tools = []
req.tool_choice = "auto"
return req
+377
View File
@@ -0,0 +1,377 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Data-driven replay harness for parser engine testing.
Replays token sequences through parsers at different chunk sizes to
verify chunk-size invariance: the same token sequence must produce
identical output regardless of how tokens are batched.
"""
from __future__ import annotations
import json
from collections.abc import Sequence
from dataclasses import dataclass, field
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
@dataclass
class Sample:
"""One test sample loaded from a JSONL file."""
id: str
description: str
source: str
vocab: dict[str, int]
tokens: list[tuple[int, str]]
expected_reasoning: str | None
expected_content: str | None
expected_tool_calls: list[dict] | None
tools: list[dict] | None = None
chat_template_kwargs: dict | None = None
@dataclass
class ParseOutput:
"""Accumulated parse output from replaying a token stream."""
reasoning: str = ""
content: str = ""
tool_calls: list[dict] = field(default_factory=list)
class MockTokenizer:
"""Lightweight tokenizer mock that avoids unittest.mock overhead.
Used by ``benchmarks/benchmark_parsers.py`` in tight timing loops,
so hot-path methods (``decode``, ``get_vocab``) must be cheap.
MagicMock's call-recording machinery added ~40% overhead to small-
sample benchmarks, inflating the per-token cost of the parser engine.
"""
__slots__ = (
"_vocab",
"_token_ids",
"_token_decode_map",
"_special_ids",
"eos_token_id",
"bos_token_id",
"pad_token_id",
)
def __init__(
self,
vocab: dict[str, int],
tokens: list[tuple[int, str]],
) -> None:
self._vocab = vocab
self._token_ids = [tid for tid, _ in tokens]
self._token_decode_map = {tid: text for tid, text in tokens}
self._special_ids = set(vocab.values())
self.eos_token_id = None
self.bos_token_id = None
self.pad_token_id = None
def set_vocab(self, vocab: dict[str, int]) -> None:
self._vocab = vocab
def get_vocab(self) -> dict[str, int]:
return self._vocab
def encode(self, text: str, **kwargs) -> list[int]:
return self._token_ids
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
parts: list[str] = []
for tid in ids:
if skip_special_tokens and tid in self._special_ids:
continue
text = self._token_decode_map.get(tid, f"?{tid}?")
parts.append(text)
return "".join(parts)
def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
"""Build a mock tokenizer from a sample's vocab and token data."""
return MockTokenizer(
vocab=dict(sample.vocab),
tokens=sample.tokens,
)
def _test_request(
tools: list[dict] | None = None,
) -> ChatCompletionRequest:
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "test"}],
tools=tools,
)
def replay_streaming(
parser,
tokens: list[tuple[int, str]],
chunk_size: int | None = None,
holdback_chars: int = 0,
finished_on_last: bool = False,
tools: list[dict] | None = None,
) -> list[DeltaMessage | None]:
"""Feed tokens through ``parser.parse_delta()`` at a given chunk size.
Args:
parser: A :class:`Parser` instance with ``parse_delta()`` method.
tokens: List of ``(token_id, decoded_text)`` pairs.
chunk_size: Number of tokens per batch. ``None`` means all at once.
holdback_chars: Simulate detokenizer holdback by holding back
this many characters of decoded text between batches.
finished_on_last: When True, pass ``finished=True`` on the last
``parse_delta()`` call, matching real server behavior.
tools: Optional tool definitions to include on the request,
matching the serving layer where tools set
``tool_choice`` to ``"auto"``.
Returns:
List of ``DeltaMessage`` results from each ``parse_delta()`` call.
"""
if chunk_size is None:
chunk_size = len(tokens)
results: list[DeltaMessage | None] = []
all_ids = [tid for tid, _ in tokens]
all_texts = [text for _, text in tokens]
request = _test_request(tools=tools)
if holdback_chars <= 0:
chunks = list(range(0, len(tokens), chunk_size))
for i, start in enumerate(chunks):
batch_end = min(start + chunk_size, len(tokens))
batch_ids = all_ids[start:batch_end]
delta_text = "".join(all_texts[start:batch_end])
is_last = i == len(chunks) - 1
result = parser.parse_delta(
delta_text,
batch_ids,
request,
prompt_token_ids=[] if start == 0 else None,
finished=finished_on_last and is_last,
)
results.append(result)
return results
emitted_up_to = 0
is_first = True
for start in range(0, len(tokens), chunk_size):
batch_end = min(start + chunk_size, len(tokens))
if batch_end < len(tokens):
held_chars = 0
safe_end = batch_end
while safe_end > emitted_up_to and held_chars < holdback_chars:
safe_end -= 1
held_chars += len(all_texts[safe_end])
else:
safe_end = batch_end
if safe_end <= emitted_up_to:
continue
batch_ids = all_ids[emitted_up_to:safe_end]
delta_text = "".join(all_texts[emitted_up_to:safe_end])
emitted_up_to = safe_end
is_last_chunk = batch_end >= len(tokens)
result = parser.parse_delta(
delta_text,
batch_ids,
request,
prompt_token_ids=[] if is_first else None,
finished=finished_on_last and is_last_chunk,
)
results.append(result)
is_first = False
if emitted_up_to < len(tokens):
batch_ids = all_ids[emitted_up_to:]
delta_text = "".join(all_texts[emitted_up_to:])
result = parser.parse_delta(
delta_text,
batch_ids,
request,
prompt_token_ids=[] if is_first else None,
finished=finished_on_last,
)
results.append(result)
return results
def replay_with_text_holdback(
parser,
tokens: list[tuple[int, str]],
text_delay: int = 1,
tools: list[dict] | None = None,
) -> list[DeltaMessage | None]:
"""Replay token-by-token with text arriving *text_delay* steps late.
Simulates the production detokenizer holdback where token IDs arrive
immediately but decoded text is delayed. On the last token all
remaining held-back text is flushed, matching real server behavior::
step 0: ids=[tok0], text="" (held back)
step 1: ids=[tok1], text=tok0_text (tok0 released)
...
step N-1: ids=[tokN-1], text=remaining_texts (flush all)
This exercises the TokenIDScanner deferred-terminal path that
``replay_streaming`` (which keeps text and IDs aligned) does not.
"""
results: list[DeltaMessage | None] = []
request = _test_request(tools=tools)
n = len(tokens)
held_texts: list[str] = []
for i in range(n):
token_id = tokens[i][0]
held_texts.append(tokens[i][1])
is_last = i == n - 1
if is_last:
delta_text = "".join(held_texts)
held_texts.clear()
elif len(held_texts) > text_delay:
delta_text = held_texts.pop(0)
else:
delta_text = ""
result = parser.parse_delta(
delta_text,
[token_id],
request,
prompt_token_ids=[] if i == 0 else None,
finished=is_last,
)
results.append(result)
return results
def accumulate_deltas(
deltas: Sequence[DeltaMessage | None],
) -> dict:
reasoning_parts: list[str] = []
content_parts: list[str] = []
tool_calls_by_idx: dict[int, dict] = {}
for delta in deltas:
if delta is None:
continue
if delta.reasoning:
reasoning_parts.append(delta.reasoning)
if delta.content:
content_parts.append(delta.content)
if delta.tool_calls:
for tc in delta.tool_calls:
if tc.function and tc.function.name:
existing = tool_calls_by_idx.get(tc.index)
if existing is None:
tool_calls_by_idx[tc.index] = {
"name": tc.function.name,
"_args_parts": [tc.function.arguments or ""],
}
else:
existing["_args_parts"].append(tc.function.arguments or "")
elif tc.function and tc.function.arguments:
existing = tool_calls_by_idx.get(tc.index)
if existing is not None:
existing["_args_parts"].append(tc.function.arguments)
return {
"reasoning": "".join(reasoning_parts),
"content": "".join(content_parts),
"tool_calls": [
{"name": tc["name"], "arguments": "".join(tc["_args_parts"])}
for tc in tool_calls_by_idx.values()
],
}
def collect_output(results: list[DeltaMessage | None]) -> ParseOutput:
"""Accumulate ``DeltaMessage`` results into a :class:`ParseOutput`."""
result = accumulate_deltas(results)
return ParseOutput(
reasoning=result["reasoning"],
content=result["content"],
tool_calls=result["tool_calls"],
)
def assert_parse_output(actual: ParseOutput, sample: Sample) -> None:
"""Compare actual parse output against expected values from a sample."""
if sample.expected_reasoning is not None:
assert actual.reasoning == sample.expected_reasoning, (
f"Reasoning mismatch:\n"
f" expected: {sample.expected_reasoning!r}\n"
f" actual: {actual.reasoning!r}"
)
if sample.expected_content is not None:
assert actual.content == sample.expected_content, (
f"Content mismatch:\n"
f" expected: {sample.expected_content!r}\n"
f" actual: {actual.content!r}"
)
if sample.expected_tool_calls is not None:
assert len(actual.tool_calls) == len(sample.expected_tool_calls), (
f"Tool call count mismatch: "
f"expected {len(sample.expected_tool_calls)}, "
f"got {len(actual.tool_calls)}"
)
for i, (expected_tc, actual_tc) in enumerate(
zip(sample.expected_tool_calls, actual.tool_calls)
):
assert actual_tc["name"] == expected_tc["name"], (
f"Tool call {i} name mismatch: "
f"expected {expected_tc['name']!r}, "
f"got {actual_tc['name']!r}"
)
if "arguments" in expected_tc:
expected_args = expected_tc["arguments"]
actual_args_str = actual_tc.get("arguments", "{}")
if isinstance(expected_args, dict):
try:
actual_args = json.loads(actual_args_str)
except json.JSONDecodeError as e:
raise AssertionError(
f"Tool call {i} arguments not valid JSON: "
f"{actual_args_str!r}"
) from e
assert actual_args == expected_args, (
f"Tool call {i} arguments mismatch:\n"
f" expected: {expected_args}\n"
f" actual: {actual_args}"
)
def assert_no_terminal_leakage(
actual: ParseOutput,
terminals: list[str],
context: str = "",
) -> None:
"""Assert that none of *terminals* appear in reasoning or content."""
suffix = f" ({context})" if context else ""
for terminal in terminals:
assert terminal not in actual.reasoning, (
f"{terminal!r} leaked into reasoning{suffix}"
)
assert terminal not in actual.content, (
f"{terminal!r} leaked into content{suffix}"
)
+137
View File
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared streaming simulation helpers for parser engine tests."""
from __future__ import annotations
from typing import Any
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
def _build_token_id_map(parser) -> dict[str, int]:
"""Map special token text to token IDs from the parser's config."""
token_id_map: dict[str, int] = {}
cfg = getattr(parser, "parser_engine_config", None)
vocab = getattr(parser, "vocab", None)
if cfg is not None and vocab is not None:
for text in (cfg.token_id_terminals or {}).values():
tid = vocab.get(text)
if tid is not None:
token_id_map[text] = tid
return token_id_map
def simulate_tool_streaming(
parser,
request,
chunks: list[str],
) -> list[tuple[DeltaMessage | None, str]]:
"""Feed text chunks through ``extract_tool_calls_streaming()``."""
token_id_map = _build_token_id_map(parser)
results: list[tuple[Any, str]] = []
previous_text = ""
previous_token_ids: list[int] = []
for chunk in chunks:
current_text = previous_text + chunk
delta_token_ids: list[int] = [
tid for text, tid in token_id_map.items() if text in chunk
]
current_token_ids = previous_token_ids + delta_token_ids
delta = parser.extract_tool_calls_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=chunk,
previous_token_ids=tuple(previous_token_ids),
current_token_ids=tuple(current_token_ids),
delta_token_ids=tuple(delta_token_ids),
request=request,
)
results.append((delta, current_text))
previous_text = current_text
previous_token_ids = list(current_token_ids)
return results
def collect_tool_arguments(
results: list[tuple[DeltaMessage | None, str]],
) -> str:
"""Concatenate all streamed argument fragments."""
args_text = ""
for delta, _ in results:
if delta and delta.tool_calls:
for tc in delta.tool_calls:
if tc.function and tc.function.arguments:
args_text += tc.function.arguments
return args_text
def collect_content(
results: list[tuple[DeltaMessage | None, str]],
) -> str:
"""Concatenate all streamed content parts."""
parts: list[str] = []
for delta, _ in results:
if delta and delta.content:
parts.append(delta.content)
return "".join(parts)
def collect_function_name(
results: list[tuple[DeltaMessage | None, str]],
) -> str | None:
"""Return first function name from deltas."""
for delta, _ in results:
if delta and delta.tool_calls:
for tc in delta.tool_calls:
if tc.function and tc.function.name:
return tc.function.name
return None
def simulate_reasoning_streaming(
parser,
chunks: list[str],
delta_token_ids_per_chunk: list[tuple[int, ...]] | None = None,
) -> tuple[str, str]:
"""Feed chunks through ``extract_reasoning_streaming()``.
Returns ``(reasoning_text, content_text)`` tuple.
"""
token_id_map = (
_build_token_id_map(parser) if delta_token_ids_per_chunk is None else {}
)
reasoning_parts: list[str] = []
content_parts: list[str] = []
prev_text = ""
prev_ids: list[int] = []
for i, chunk in enumerate(chunks):
cur_text = prev_text + chunk
if delta_token_ids_per_chunk is not None:
d_ids = delta_token_ids_per_chunk[i]
else:
d_ids = tuple(tid for text, tid in token_id_map.items() if text in chunk)
cur_ids = prev_ids + list(d_ids)
delta = parser.extract_reasoning_streaming(
previous_text=prev_text,
current_text=cur_text,
delta_text=chunk,
previous_token_ids=tuple(prev_ids),
current_token_ids=tuple(cur_ids),
delta_token_ids=d_ids,
)
if delta:
if delta.reasoning:
reasoning_parts.append(delta.reasoning)
if delta.content:
content_parts.append(delta.content)
prev_text = cur_text
prev_ids = list(cur_ids)
return "".join(reasoning_parts), "".join(content_parts)
@@ -0,0 +1,82 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Replay tests for DelegatingParser with engine adapters.
Exercises DelegatingParser in engine-adapter mode to verify that delegated
routing produces correct output across chunk sizes.
See test_replay.py for tests that target engine parsers directly.
"""
from __future__ import annotations
from functools import lru_cache
import pytest
from pydantic import TypeAdapter
from tests.parser.engine.replay_harness import (
assert_parse_output,
collect_output,
make_mock_tokenizer,
replay_streaming,
)
from tests.parser.engine.trace_builder import build_samples
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.abstract_parser import Parser
from vllm.parser.parser_manager import ParserManager
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
_PAIRINGS: dict[str, tuple[str, str]] = {
"engine": ("qwen3_coder", "qwen3"),
}
CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
@lru_cache
def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
tool_name, reasoning_name = _PAIRINGS[pairings]
parser_cls = ParserManager.get_parser(
tool_parser_name=tool_name,
reasoning_parser_name=reasoning_name,
enable_auto_tools=True,
)
assert parser_cls is not None
return parser_cls
_all_samples = build_samples("qwen3")
@pytest.mark.parametrize(
"pairings",
list(_PAIRINGS),
ids=lambda p: f"mode={p}",
)
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
def test_delegating_replay(sample, chunk_size, pairings):
parser_cls = _get_delegating_parser_cls(pairings=pairings)
tokenizer = make_mock_tokenizer(sample)
validated_tools = (
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
)
parser = parser_cls(
tokenizer,
validated_tools,
chat_template_kwargs=sample.chat_template_kwargs,
)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=chunk_size,
finished_on_last=True,
tools=sample.tools,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
+846
View File
@@ -0,0 +1,846 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the streaming parser engine core pipeline."""
from unittest.mock import MagicMock
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from vllm.parser.engine.events import EventType, SemanticEvent
from vllm.parser.engine.incremental_lexer import (
LexerShape,
TerminalDef,
terminals_from_literals,
)
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
def _hermes_config() -> ParserEngineConfig:
"""Simple Hermes-style config: <tool_call>JSON</tool_call>."""
return ParserEngineConfig(
name="hermes_test",
terminals={
"TOOL_START": "<tool_call>",
"TOOL_END": "</tool_call>",
},
token_id_terminals={
"TOOL_START": "<tool_call>",
"TOOL_END": "</tool_call>",
},
transitions={
(ParserState.CONTENT, "TOOL_START"): Transition(
ParserState.TOOL_ARGS,
(EventType.TOOL_CALL_START,),
),
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
ParserState.CONTENT,
(EventType.TOOL_CALL_END,),
),
},
content_events={
ParserState.CONTENT: EventType.TEXT_CHUNK,
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
},
)
def _think_config() -> ParserEngineConfig:
"""Simple think-tag reasoning config: <think>...</think>."""
return ParserEngineConfig(
name="think_test",
terminals={
"THINK_START": "<think>",
"THINK_END": "</think>",
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
class TestNonStreaming:
def test_plain_text(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
events = engine.parse_complete("Hello, world!")
assert len(events) == 1
assert events[0].type == EventType.TEXT_CHUNK
assert events[0].value == "Hello, world!"
def test_single_tool_call(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
text = (
'<tool_call>{"name": "get_weather",'
' "arguments": {"city": "SF"}}'
"</tool_call>"
)
events = engine.parse_complete(text)
types = [e.type for e in events]
assert EventType.TOOL_CALL_START in types
assert EventType.TOOL_CALL_END in types
assert EventType.ARG_VALUE_CHUNK in types
arg_text = "".join(
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
)
assert '"name": "get_weather"' in arg_text
assert '"city": "SF"' in arg_text
def test_text_then_tool_call(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
text = 'Sure!<tool_call>{"name": "add"}</tool_call>'
events = engine.parse_complete(text)
types = [e.type for e in events]
assert types[0] == EventType.TEXT_CHUNK
assert events[0].value == "Sure!"
assert EventType.TOOL_CALL_START in types
assert EventType.TOOL_CALL_END in types
def test_multiple_tool_calls(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
text = (
'<tool_call>{"name": "a"}</tool_call><tool_call>{"name": "b"}</tool_call>'
)
events = engine.parse_complete(text)
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
assert len(starts) == 2
assert len(ends) == 2
assert starts[0].tool_index == 0
assert starts[1].tool_index == 1
def test_reasoning(self):
engine = StreamingParserEngine(_think_config(), tokenizer=None)
text = "<think>Let me think...</think>The answer is 42."
events = engine.parse_complete(text)
types = [e.type for e in events]
assert types[0] == EventType.REASONING_START
assert EventType.REASONING_CHUNK in types
assert EventType.REASONING_END in types
assert EventType.TEXT_CHUNK in types
reasoning = "".join(
e.value for e in events if e.type == EventType.REASONING_CHUNK
)
assert "Let me think..." in reasoning
content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
assert "The answer is 42." in content
class TestStreaming:
@staticmethod
def _feed_chars(
engine: StreamingParserEngine,
text: str,
) -> list[SemanticEvent]:
"""Feed text one character at a time."""
all_events = []
for ch in text:
all_events.extend(engine.feed(ch, []))
all_events.extend(engine.finish())
return all_events
@staticmethod
def _feed_chunks(
engine: StreamingParserEngine,
text: str,
chunk_size: int,
) -> list[SemanticEvent]:
"""Feed text in fixed-size chunks."""
all_events = []
for i in range(0, len(text), chunk_size):
chunk = text[i : i + chunk_size]
all_events.extend(engine.feed(chunk, []))
all_events.extend(engine.finish())
return all_events
def test_char_by_char_tool_call(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
text = '<tool_call>{"name": "add", "arguments": {"a": 1}}</tool_call>'
events = self._feed_chars(engine, text)
types = [e.type for e in events]
assert EventType.TOOL_CALL_START in types
assert EventType.TOOL_CALL_END in types
assert EventType.ARG_VALUE_CHUNK in types
arg_text = "".join(
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
)
assert '"name": "add"' in arg_text
@pytest.mark.parametrize(
"text",
[
'<tool_call>{"name": "get", "arguments": {"x": "hello"}}</tool_call>',
'<tool_call>{"name": "f", "arguments": '
'{"items": [1, [2, 3]], "obj": {"k": "v"}}}'
"</tool_call>",
],
ids=["flat_args", "nested_arrays"],
)
def test_chunk_sizes_produce_same_content(self, text):
"""Different chunk sizes must produce identical concatenated content."""
results = {}
for chunk_size in [1, 2, 3, 5, 7, len(text)]:
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
events = self._feed_chunks(engine, text, chunk_size)
arg_text = "".join(
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
)
results[chunk_size] = arg_text
values = list(results.values())
for v in values[1:]:
assert v == values[0], f"Mismatch: {results}"
def test_prefix_buffering_prevents_premature_emit(self):
"""Text like '<tool_' should be buffered, not emitted as content."""
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
events1 = engine.feed("<tool_", [])
content_events = [e for e in events1 if e.type == EventType.TEXT_CHUNK]
assert len(content_events) == 0, "Should buffer partial tag"
events2 = engine.feed("call>", [])
starts = [e for e in events2 if e.type == EventType.TOOL_CALL_START]
assert len(starts) == 1
def test_prefix_buffering_flush_on_mismatch(self):
"""Text like '<tool_box' should eventually flush as content."""
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
events1 = engine.feed("<tool_", [])
assert len([e for e in events1 if e.type == EventType.TEXT_CHUNK]) == 0
events2 = engine.feed("box>rest", [])
events2.extend(engine.finish())
content = "".join(e.value for e in events2 if e.type == EventType.TEXT_CHUNK)
assert content == "<tool_box>rest"
def test_reasoning_streaming(self):
engine = StreamingParserEngine(_think_config(), tokenizer=None)
events = self._feed_chars(engine, "<think>hmm</think>answer")
reasoning = "".join(
e.value for e in events if e.type == EventType.REASONING_CHUNK
)
content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
assert "hmm" in reasoning
assert "answer" in content
def test_text_between_tool_calls(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
text = (
'Hi<tool_call>{"name":"a"}</tool_call>'
'mid<tool_call>{"name":"b"}</tool_call>end'
)
events = self._feed_chunks(engine, text, 3)
texts = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
assert "Hi" in texts
assert "mid" in texts
assert "end" in texts
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
assert len(starts) == 2
def test_unmatched_close_brace_does_not_poison_depth(self):
"""A stray } in malformed JSON must not kill streaming for
all subsequent content."""
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.feed("<tool_call>", [])
malformed = '}{{"a": 1}}'
events = self._feed_chars(engine, malformed + "</tool_call>")
arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
assert len(arg_chunks) > 1, (
"Content after stray } should still stream incrementally"
)
arg_text = "".join(arg_chunks)
assert '"a": 1' in arg_text
def test_json_args_no_premature_close_brace(self):
"""Closing braces of the top-level JSON shouldn't be streamed
until confirmed by the end tag."""
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.feed("<tool_call>", [])
events = engine.feed('{"name": "f"}', [])
arg_text = "".join(
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
)
assert "}" not in arg_text, "Top-level } should be held back"
events2 = engine.feed("</tool_call>", [])
arg_text2 = "".join(
e.value for e in events2 if e.type == EventType.ARG_VALUE_CHUNK
)
assert "}" in arg_text2, "} should flush on end tag"
_START_ID = 50
_END_ID = 51
_TOOL_START_ID = 60
_TOOL_END_ID = 61
def _make_think_tokenizer():
tok = MagicMock()
tok.encode.return_value = [1, 2, 3]
tok.get_vocab.return_value = {"<think>": _START_ID, "</think>": _END_ID}
tok.decode.side_effect = lambda ids: {
_START_ID: "<think>",
_END_ID: "</think>",
}.get(ids[0], f"tok{ids[0]}")
return tok
def _make_hermes_tokenizer():
"""Tokenizer that resolves tool_call tags to special IDs."""
_special = {_TOOL_START_ID: "<tool_call>", _TOOL_END_ID: "</tool_call>"}
tok = MagicMock()
tok.encode.return_value = [1, 2, 3]
tok.get_vocab.return_value = {
"<tool_call>": _TOOL_START_ID,
"</tool_call>": _TOOL_END_ID,
}
tok.decode.side_effect = lambda ids: "".join(
_special.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
)
return tok
class TestLexerBufferFlush:
"""Lexer buffer must be flushed before PreLexedTerminal transitions."""
def test_buffered_prefix_emitted_in_current_state(self):
"""Text buffered by the lexer (e.g. '<') must be emitted as
REASONING_CHUNK before THINK_END transitions to CONTENT."""
engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
events = engine.feed("<think>", [_START_ID])
assert any(e.type == EventType.REASONING_START for e in events)
events = engine.feed("reasoning text<", [])
reasoning_text = "".join(
e.value for e in events if e.type == EventType.REASONING_CHUNK
)
assert "reasoning text" in reasoning_text
events = engine.feed("</think>", [_END_ID])
event_types = [e.type for e in events]
if EventType.REASONING_CHUNK in event_types:
rc_idx = event_types.index(EventType.REASONING_CHUNK)
re_idx = event_types.index(EventType.REASONING_END)
assert rc_idx < re_idx, (
"'<' must be emitted as REASONING_CHUNK before REASONING_END"
)
flushed = events[rc_idx].value
assert "<" in flushed
def test_empty_buffer_no_extra_events(self):
"""When the lexer buffer is empty, flushing is a no-op."""
engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
engine.feed("<think>", [_START_ID])
engine.feed("clean text", [])
events = engine.feed("</think>", [_END_ID])
assert any(e.type == EventType.REASONING_END for e in events)
chunk_events = [e for e in events if e.type == EventType.REASONING_CHUNK]
assert all(e.value for e in chunk_events)
class TestTokenIdFiltering:
"""When token IDs are available, lex-matched terminals that also
have token_id_terminal entries should be demoted to content."""
def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
"""After receiving token IDs, text that matches a token-ID
terminal should be treated as content, not trigger a transition."""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
# First feed with a non-special token ID to set _ever_had_token_ids
engine.feed("prefix ", [1])
# Now feed text containing <tool_call> as literal text
events = engine.feed(
"Use <tool_call> to invoke tools.</tool_call>", [2, 3, 4, 5]
)
events.extend(engine.finish())
types = [e.type for e in events]
assert EventType.TOOL_CALL_START not in types
assert EventType.TEXT_CHUNK in types
text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
assert "<tool_call>" in text
def test_scanner_matched_terminal_bypasses_filter(self):
"""PreLexedTerminals from the scanner bypass the filter and
still trigger state transitions."""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
events = engine.feed("<tool_call>", [_TOOL_START_ID])
assert any(e.type == EventType.TOOL_CALL_START for e in events)
events = engine.feed('{"name": "f"}', [2, 3])
events.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
events.extend(engine.finish())
assert any(e.type == EventType.TOOL_CALL_END for e in events)
def test_no_filtering_without_token_ids(self):
"""When no token IDs are ever provided (non-streaming),
text matching still triggers transitions."""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
events.extend(engine.finish())
types = [e.type for e in events]
assert EventType.TOOL_CALL_START in types
assert EventType.TOOL_CALL_END in types
def test_mixed_text_then_real_tool_call(self):
"""Text mentioning tool syntax followed by a real special-token
tool call."""
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
events2 = engine.feed("<tool_call>", [_TOOL_START_ID])
events3 = engine.feed('{"name": "a"}', [5, 6])
events4 = engine.feed("</tool_call>", [_TOOL_END_ID])
events4.extend(engine.finish())
all_events = events1 + events2 + events3 + events4
content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
assert "<tool_call>" in content
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
def _func_prefix_config() -> ParserEngineConfig:
"""Config mixing token-ID terminals (TOOL_START/END) with
text-only terminals (FUNC_PREFIX) and fallback transitions."""
return ParserEngineConfig(
name="func_prefix_test",
terminals={
"TOOL_START": "<tool_call>",
"TOOL_END": "</tool_call>",
"FUNC_PREFIX": "<function=",
"FUNC_END": "</function>",
"CLOSE_ANGLE": ">",
},
token_id_terminals={
"TOOL_START": "<tool_call>",
"TOOL_END": "</tool_call>",
},
transitions={
(ParserState.CONTENT, "TOOL_START"): Transition(
ParserState.TOOL_PREAMBLE,
(EventType.TOOL_CALL_START,),
),
(ParserState.CONTENT, "FUNC_PREFIX"): Transition(
ParserState.TOOL_NAME,
(EventType.TOOL_CALL_START,),
skip_in_token_id_mode=True,
),
(ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition(
ParserState.TOOL_NAME,
(),
),
(ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition(
ParserState.TOOL_ARGS,
(),
),
(ParserState.TOOL_ARGS, "FUNC_END"): Transition(
ParserState.TOOL_BETWEEN,
(EventType.TOOL_CALL_END,),
),
(ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
ParserState.CONTENT,
(),
),
(ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
ParserState.TOOL_PREAMBLE,
(EventType.TOOL_CALL_START,),
),
(ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition(
ParserState.TOOL_NAME,
(EventType.TOOL_CALL_START,),
skip_in_token_id_mode=True,
),
},
content_events={
ParserState.CONTENT: EventType.TEXT_CHUNK,
ParserState.TOOL_NAME: EventType.TOOL_NAME,
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
},
)
def _make_func_prefix_tokenizer():
return make_mock_tokenizer(
{
"<tool_call>": _TOOL_START_ID,
"</tool_call>": _TOOL_END_ID,
}
)
class TestTextOnlyFallbackFiltering:
"""When token IDs are available, transitions marked
skip_in_token_id_mode should be skipped."""
def test_func_prefix_in_prose_demoted_in_strict_mode(self):
"""<function=get_time> in prose should NOT trigger a tool call
when strict mode is active."""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
engine.feed("prefix ", [1])
events = engine.feed("Use <function=get_time> to check.", [2, 3, 4, 5])
events.extend(engine.finish())
types = [e.type for e in events]
assert EventType.TOOL_CALL_START not in types
assert EventType.TEXT_CHUNK in types
text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
assert "<function=" in text
def test_normal_flow_after_tool_start_still_works(self):
"""TOOL_START (special token) -> FUNC_PREFIX (text) should
still parse a tool call normally in strict mode."""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
events1 = engine.feed("<tool_call>", [_TOOL_START_ID])
assert any(e.type == EventType.TOOL_CALL_START for e in events1)
events2 = engine.feed("<function=get_weather>", [2, 3])
events3 = engine.feed("args", [4])
events4 = engine.feed("</function>", [5, 6])
events4.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
events4.extend(engine.finish())
all_events = events1 + events2 + events3 + events4
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
def test_fallback_fires_without_token_ids(self):
"""When no token IDs are provided, fallback transitions should
still fire normally."""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
events = engine.feed("<function=get_time>args</function>", [])
events.extend(engine.finish())
types = [e.type for e in events]
assert EventType.TOOL_CALL_START in types
assert EventType.TOOL_CALL_END in types
def test_tool_between_fallback_blocked_in_strict_mode(self):
"""The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
blocked in strict mode."""
engine = StreamingParserEngine(
_func_prefix_config(), _make_func_prefix_tokenizer()
)
engine.feed("<tool_call>", [_TOOL_START_ID])
engine.feed("<function=a>", [2, 3])
engine.feed("args", [4])
engine.feed("</function>", [5, 6])
engine.feed("</tool_call>", [_TOOL_END_ID])
events = engine.feed("<function=b>more</function>", [7, 8, 9])
events.extend(engine.finish())
types = [e.type for e in events]
assert EventType.TOOL_CALL_START not in types
class TestNoUnusedTokenizerAttr:
"""StreamingParserEngine no longer stores a redundant _tokenizer."""
def test_no_tokenizer_attribute(self):
config = ParserEngineConfig(name="test")
engine = StreamingParserEngine(config, tokenizer=None)
assert not hasattr(engine, "_tokenizer")
class TestArgsResetOnReentry:
"""When leaving TOOL_ARGS and later re-entering (e.g. two tool
calls), the entering-TOOL_ARGS block resets args tracking. The
redundant reset on exit was removed."""
@staticmethod
def _multi_tool_config() -> ParserEngineConfig:
return ParserEngineConfig(
name="multi_tool",
terminals={
"TOOL_START": "<tool_call>",
"TOOL_END": "</tool_call>",
"TOOL_SEP": "<tool_sep>",
},
transitions={
(ParserState.CONTENT, "TOOL_START"): Transition(
ParserState.TOOL_ARGS,
(EventType.TOOL_CALL_START,),
),
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
ParserState.TOOL_BETWEEN,
(EventType.TOOL_CALL_END,),
),
(ParserState.TOOL_BETWEEN, "TOOL_SEP"): Transition(
ParserState.TOOL_ARGS,
(EventType.TOOL_CALL_START,),
),
},
content_events={
ParserState.CONTENT: EventType.TEXT_CHUNK,
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
},
)
def test_args_tracking_across_reentry(self):
engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
events = engine.feed(
'<tool_call>{"city": "SF"}</tool_call>'
"<tool_sep>"
'{"name": "bar"}</tool_call>',
[],
)
tool_starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
tool_ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
arg_chunks = [e for e in events if e.type == EventType.ARG_VALUE_CHUNK]
assert len(tool_starts) == 2
assert len(tool_ends) == 2
assert tool_starts[0].tool_index == 0
assert tool_starts[1].tool_index == 1
first_args = "".join(e.value for e in arg_chunks if e.tool_index == 0)
second_args = "".join(e.value for e in arg_chunks if e.tool_index == 1)
assert '"city"' in first_args
assert '"name"' in second_args
def test_brace_depth_resets_on_reentry(self):
"""Verify _args_brace_depth resets when re-entering TOOL_ARGS."""
engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
engine.feed("<tool_call>", [])
assert engine.state == ParserState.TOOL_ARGS
assert engine._args_brace_depth == 0
engine.feed('{"a": 1}', [])
engine.feed("</tool_call>", [])
assert engine.state == ParserState.TOOL_BETWEEN
engine.feed("<tool_sep>", [])
assert engine.state == ParserState.TOOL_ARGS
assert engine._args_brace_depth == 0
assert engine._args_in_string is False
assert engine._args_escape_next is False
class TestToolPreambleFinish:
"""finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
call was started (tool_index >= 0), but not when tool_index is -1."""
@staticmethod
def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
return ParserEngineConfig(
name="preamble_tcs",
terminals={"TOOL_START": "<tool_call>"},
transitions={
(ParserState.CONTENT, "TOOL_START"): Transition(
ParserState.TOOL_PREAMBLE,
(EventType.TOOL_CALL_START,),
),
},
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
)
@staticmethod
def _preamble_without_tool_call_start_config() -> ParserEngineConfig:
return ParserEngineConfig(
name="preamble_no_tcs",
terminals={"TOOL_CALLS_START": "<tool_calls>"},
transitions={
(ParserState.CONTENT, "TOOL_CALLS_START"): Transition(
ParserState.TOOL_PREAMBLE,
(),
),
},
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
)
def test_finish_emits_tool_call_end_with_tool_index(self):
config = self._preamble_with_tool_call_start_config()
engine = StreamingParserEngine(config, tokenizer=None)
engine.feed("<tool_call>", [])
assert engine.state == ParserState.TOOL_PREAMBLE
assert engine.tool_index == 0
finish_events = engine.finish()
end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
assert len(end_events) == 1
assert end_events[0].tool_index == 0
def test_finish_no_tool_call_end_without_tool_index(self):
config = self._preamble_without_tool_call_start_config()
engine = StreamingParserEngine(config, tokenizer=None)
engine.feed("<tool_calls>", [])
assert engine.state == ParserState.TOOL_PREAMBLE
assert engine.tool_index == -1
finish_events = engine.finish()
end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
assert len(end_events) == 0
assert engine.state == ParserState.CONTENT
class TestRegexTerminalInfraRemoved:
"""TerminalDef.priority, LexerShape.regex_terminals, and the regex
matching loop were removed."""
def test_terminal_def_no_priority(self):
import regex as re
td = TerminalDef(name="X", pattern=re.compile("x"))
assert not hasattr(td, "priority")
def test_lexer_shape_no_regex_terminals(self):
shape = LexerShape([])
assert not hasattr(shape, "regex_terminals")
def test_terminals_from_literals_still_works(self):
literals = {"TOOL_START": "<tool_call>", "TOOL_END": "</tool_call>"}
defs = terminals_from_literals(literals)
assert len(defs) == 2
names = {d.name for d in defs}
assert names == {"TOOL_START", "TOOL_END"}
for d in defs:
assert d.is_literal
assert d.literal in ("<tool_call>", "</tool_call>")
class TestMultiCharTerminalInArgs:
"""Regression: multi-char terminals falling through in TOOL_ARGS
must be fed char-by-char via _feed_args_text, not _feed_args_char."""
@staticmethod
def _newline_config() -> ParserEngineConfig:
return ParserEngineConfig(
name="newline_test",
terminals={
"TOOL_START": "<tool_call>",
"TOOL_END": "</tool_call>",
"NEWLINE": "\n",
},
transitions={
(ParserState.CONTENT, "TOOL_START"): Transition(
ParserState.TOOL_ARGS,
(EventType.TOOL_CALL_START,),
),
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
ParserState.CONTENT,
(EventType.TOOL_CALL_END,),
),
},
content_events={
ParserState.CONTENT: EventType.TEXT_CHUNK,
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
},
)
def test_newline_in_args_parsed_correctly(self):
engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
text = '<tool_call>{"name": "f",\n"arguments": {"a": 1}}</tool_call>'
events = engine.parse_complete(text)
arg_text = "".join(
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
)
assert '"name": "f"' in arg_text
assert '"arguments"' in arg_text
def test_newline_in_args_streaming(self):
engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
all_events = TestStreaming._feed_chars(
engine, '<tool_call>{"name": "f",\n"a": 1}</tool_call>'
)
arg_text = "".join(
e.value for e in all_events if e.type == EventType.ARG_VALUE_CHUNK
)
assert '"name": "f"' in arg_text
assert '"a": 1' in arg_text
class TestSkipToolParsing:
"""When skip_tool_parsing is set, tool tags become content."""
def test_tool_tags_emitted_as_content(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.skip_tool_parsing = True
text = '<tool_call>{"name": "f"}</tool_call>'
events = engine.parse_complete(text)
types = [e.type for e in events]
assert EventType.TOOL_CALL_START not in types
assert EventType.TOOL_CALL_END not in types
content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
assert "<tool_call>" in content
assert "</tool_call>" in content
def test_skip_tool_streaming(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.skip_tool_parsing = True
all_events = TestStreaming._feed_chars(
engine, '<tool_call>{"name": "f"}</tool_call>'
)
types = [e.type for e in all_events]
assert EventType.TOOL_CALL_START not in types
content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
assert "<tool_call>" in content
def test_reset_preserves_skip_tool_parsing(self):
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
engine.skip_tool_parsing = True
engine.reset()
assert engine.skip_tool_parsing is True
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+549
View File
@@ -0,0 +1,549 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the engine-based Qwen3 reasoning parser.
Validates that ``Qwen3Parser`` correctly handles
``<think>``/``</think>`` reasoning with Qwen3-specific extensions:
- ``<tool_call>`` as implicit reasoning end (terminal + token ID)
- Stripping ``<think>`` from generated output (old template compat)
- No terminal text (``</think>``, ``<tool_call>``) leaks into output
"""
import dataclasses
import pytest
from tests.parser.engine.conftest import make_mock_tokenizer
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
from vllm.parser.engine.parser_engine_config import ParserState
from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
_THINK_START_ID = 50
_THINK_END_ID = 51
_TOOL_CALL_ID = 60
_TOOL_CALL_END_ID = 61
_TEXT_ID = 100
_QWEN3_VOCAB = {
"<think>": _THINK_START_ID,
"</think>": _THINK_END_ID,
"<tool_call>": _TOOL_CALL_ID,
"</tool_call>": _TOOL_CALL_END_ID,
}
@pytest.fixture
def mock_tokenizer():
return make_mock_tokenizer(_QWEN3_VOCAB)
@pytest.fixture
def parser(mock_tokenizer):
return Qwen3Parser(mock_tokenizer)
class TestNonStreaming:
def test_reasoning_then_content(self, parser):
text = "<think>Let me analyze.</think>The answer is 42."
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "Let me analyze."
assert content == "The answer is 42."
def test_no_start_token_in_output(self, parser):
"""Qwen3.5+ style: <think> in prompt, only </think> in output."""
text = "Let me think about this.</think>The answer is 42."
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "Let me think about this."
assert content == "The answer is 42."
def test_reasoning_only(self, parser):
text = "<think>Still thinking...</think>"
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "Still thinking..."
assert content is None
def test_no_end_tag_all_reasoning(self, parser):
"""No </think> means truncated output — everything is reasoning."""
text = "Hello, no reasoning here."
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "Hello, no reasoning here."
assert content is None
def test_multiline_reasoning(self, parser):
text = (
"<think>Step 1: parse.\nStep 2: compute.\nStep 3: output.</think>Result: 7."
)
reasoning, content = parser.extract_reasoning(text, None)
assert "Step 1" in reasoning
assert "Step 3" in reasoning
assert content == "Result: 7."
def test_tool_call_implicit_end(self, parser):
"""<tool_call> without </think> acts as implicit reasoning end."""
text = (
"<think>I need to read the file.\n\n"
"<tool_call>\n<function=bash>\n"
"<parameter=cmd>ls</parameter>\n"
"</function>\n</tool_call>"
)
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "I need to read the file.\n\n"
assert "</think>" not in reasoning
assert "<tool_call>" not in reasoning
def test_tool_call_implicit_end_no_think(self, parser):
"""<tool_call> as implicit end, no <think> in output."""
text = (
"I need to read the file.\n\n"
"<tool_call>\n<function=bash>\n"
"<parameter=cmd>ls</parameter>\n"
"</function>\n</tool_call>"
)
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "I need to read the file.\n\n"
assert "<tool_call>" not in reasoning
def test_live_scenario_think_end_before_tool_call(self, parser):
"""Real model output: </think> immediately before <tool_call>.
Regression test for the bug where </think> and <parameter=...>
leaked into reasoning content.
"""
text = (
"The user wants to see what files are in the current directory"
" and their contents. Let me start by listing the directory."
"</think><tool_call><function=read>"
"<parameter=filePath>/Users/test/demo</parameter>"
"</function></tool_call>"
)
reasoning, content = parser.extract_reasoning(text, None)
expected_reasoning = (
"The user wants to see what files are in the current directory"
" and their contents. Let me start by listing the directory."
)
assert reasoning == expected_reasoning
assert "</think>" not in reasoning
assert "<tool_call>" not in reasoning
assert "<parameter=" not in reasoning
def test_no_terminal_text_in_reasoning(self, parser):
"""Terminal text must never appear in reasoning output."""
text = "Reasoning here.</think>Content here."
reasoning, content = parser.extract_reasoning(text, None)
assert "</think>" not in (reasoning or "")
assert "<think>" not in (reasoning or "")
def test_no_terminal_text_in_content(self, parser):
"""Terminal text must never appear in content output."""
text = "Reasoning here.</think>Content here."
reasoning, content = parser.extract_reasoning(text, None)
assert "</think>" not in (content or "")
assert "<think>" not in (content or "")
def test_duplicate_think_end_absorbed(self, parser):
"""Duplicate </think> in CONTENT state must not leak."""
text = "Reasoning here.</think>Content here.</think>More content."
reasoning, content = parser.extract_reasoning(text, None)
assert reasoning == "Reasoning here."
assert content == "Content here.More content."
class TestIsReasoningEnd:
def test_think_end_token(self, parser):
assert parser.is_reasoning_end([_THINK_START_ID, 1, _THINK_END_ID])
def test_no_end_token(self, parser):
assert not parser.is_reasoning_end([_THINK_START_ID, 1, 2])
def test_start_after_end_means_not_ended(self, parser):
assert not parser.is_reasoning_end([_THINK_END_ID, _THINK_START_ID, 1])
def test_tool_call_as_implicit_end(self, parser):
"""Unpaired <tool_call> is implicit reasoning end."""
assert parser.is_reasoning_end([_THINK_START_ID, 1, _TOOL_CALL_ID])
def test_paired_tool_call_not_end(self, parser):
"""Paired <tool_call>...</tool_call> (from template) is NOT end."""
assert not parser.is_reasoning_end(
[_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
)
def test_tool_call_after_think_end(self, parser):
"""<tool_call> after </think> — already ended."""
assert parser.is_reasoning_end(
[_THINK_START_ID, 1, _THINK_END_ID, _TOOL_CALL_ID]
)
def test_empty_ids(self, parser):
assert not parser.is_reasoning_end([])
class TestStreaming:
def test_basic_streaming(self, parser):
reasoning, content = simulate_reasoning_streaming(
parser,
["<think>", "thinking", " hard", "</think>", "done"],
[
(_THINK_START_ID,),
(1,),
(2,),
(_THINK_END_ID,),
(3,),
],
)
assert reasoning == "thinking hard"
assert content == "done"
def test_streaming_no_start_token(self, parser):
"""Qwen3.5 style: no <think> in output, just reasoning then </think>."""
reasoning, content = simulate_reasoning_streaming(
parser,
["reasoning ", "text", "</think>", "content"],
[
(1,),
(2,),
(_THINK_END_ID,),
(3,),
],
)
assert reasoning == "reasoning text"
assert content == "content"
def test_streaming_start_token_stripped(self, parser):
"""<think> in output (old template) should be stripped."""
reasoning, content = simulate_reasoning_streaming(
parser,
["<think>reasoning", "</think>", "content"],
[
(_THINK_START_ID, 1),
(_THINK_END_ID,),
(2,),
],
)
assert reasoning == "reasoning"
assert content == "content"
def test_streaming_tool_call_implicit_end(self, parser):
"""<tool_call> ends reasoning implicitly during streaming."""
reasoning, content = simulate_reasoning_streaming(
parser,
["I need to check.", "<tool_call>", "\n<function=test>"],
[
(1,),
(_TOOL_CALL_ID,),
(2,),
],
)
assert reasoning == "I need to check."
assert "<tool_call>" not in reasoning
assert "</think>" not in reasoning
assert content is not None
def test_streaming_content_after_think_end(self, parser):
"""Content deltas after </think> are routed as content."""
reasoning, content = simulate_reasoning_streaming(
parser,
["reasoning", "</think>", "content1", " content2"],
[
(1,),
(_THINK_END_ID,),
(2,),
(3,),
],
)
assert reasoning == "reasoning"
assert content == "content1 content2"
def test_streaming_content_after_tool_call(self, parser):
"""Content deltas after <tool_call> are routed as content."""
reasoning, content = simulate_reasoning_streaming(
parser,
["thinking", "<tool_call>", "<function=f>"],
[
(1,),
(_TOOL_CALL_ID,),
(2,),
],
)
assert reasoning == "thinking"
assert "<tool_call>" not in reasoning
assert content is not None
def test_streaming_end_grouped_with_content(self, parser):
"""</think> grouped with following content in one delta."""
reasoning, content = simulate_reasoning_streaming(
parser,
["reasoning", "</think>the answer"],
[
(1,),
(_THINK_END_ID, 2),
],
)
assert reasoning == "reasoning"
assert content == "the answer"
def test_streaming_think_and_end_in_one_delta(self, parser):
"""<think> and </think> in the same delta."""
reasoning, content = simulate_reasoning_streaming(
parser,
["<think>reasoning</think>"],
[
(_THINK_START_ID, 1, _THINK_END_ID),
],
)
assert reasoning == "reasoning"
assert content == ""
def test_streaming_pure_content_no_think(self, parser):
"""No think tokens at all — everything is reasoning (truncated)."""
reasoning, content = simulate_reasoning_streaming(
parser,
["hello ", "world"],
[
(1,),
(2,),
],
)
assert reasoning == "hello world"
assert content == ""
def test_streaming_think_end_and_tool_call_same_delta(self, parser):
"""</think> and <tool_call> in the same delta — no leakage.
Regression test: the old override split at <tool_call> without
stripping </think>, causing </think> to leak into reasoning.
"""
reasoning, content = simulate_reasoning_streaming(
parser,
[
"Let me list the directory.",
"</think><tool_call>",
"<function=read>",
"<parameter=filePath>/tmp</parameter>",
],
[
(1,),
(_THINK_END_ID, _TOOL_CALL_ID),
(2,),
(3,),
],
)
assert reasoning == "Let me list the directory."
assert "</think>" not in reasoning
assert "<tool_call>" not in reasoning
assert "<parameter=" not in reasoning
assert content is not None
def test_streaming_no_terminal_text_leaks(self, parser):
"""Terminal text must never appear in reasoning or content."""
reasoning, content = simulate_reasoning_streaming(
parser,
["reasoning", "</think>", "content"],
[
(1,),
(_THINK_END_ID,),
(2,),
],
)
assert "</think>" not in reasoning
assert "</think>" not in content
assert "<think>" not in reasoning
def test_streaming_duplicate_think_end_absorbed(self, parser):
"""Duplicate </think> token in CONTENT state must not leak."""
reasoning, content = simulate_reasoning_streaming(
parser,
["reasoning", "</think>", "content", "</think>", "more"],
[
(1,),
(_THINK_END_ID,),
(2,),
(_THINK_END_ID,),
(3,),
],
)
assert reasoning == "reasoning"
assert content == "contentmore"
class TestTrailingWhitespaceStripping:
"""When strip_trailing_reasoning_whitespace is True,
trailing whitespace before </think> must be stripped.
Models often generate trailing newlines before </think>, and these
accumulate across multi-turn conversations via a feedback loop.
"""
@pytest.fixture
def parser_with_strip(self):
cfg = dataclasses.replace(
qwen3_config(),
strip_trailing_reasoning_whitespace=True,
)
return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
def test_non_streaming_trailing_newline(self, parser_with_strip):
text = "Reasoning here.\n</think>Content."
reasoning, content = parser_with_strip.extract_reasoning(text, None)
assert reasoning == "Reasoning here."
assert content == "Content."
def test_non_streaming_multiple_trailing_newlines(self, parser_with_strip):
text = "Reasoning here.\n\n\n</think>Content."
reasoning, content = parser_with_strip.extract_reasoning(text, None)
assert reasoning == "Reasoning here."
assert content == "Content."
def test_non_streaming_internal_newlines_preserved(self, parser_with_strip):
text = "Step 1.\n\nStep 2.\n\nStep 3.</think>Answer."
reasoning, content = parser_with_strip.extract_reasoning(text, None)
assert reasoning == "Step 1.\n\nStep 2.\n\nStep 3."
assert content == "Answer."
def test_non_streaming_only_newlines_becomes_none(self, parser_with_strip):
text = "\n\n\n</think>Content."
reasoning, content = parser_with_strip.extract_reasoning(text, None)
assert reasoning is None
assert content == "Content."
def test_streaming_trailing_newline_stripped(self, parser_with_strip):
reasoning, content = simulate_reasoning_streaming(
parser_with_strip,
["thinking.\n", "</think>", "done"],
[
(1,),
(_THINK_END_ID,),
(2,),
],
)
assert reasoning == "thinking."
assert content == "done"
def test_streaming_multiple_trailing_newlines_stripped(self, parser_with_strip):
reasoning, content = simulate_reasoning_streaming(
parser_with_strip,
["thinking.\n", "\n", "\n", "</think>", "done"],
[
(1,),
(2,),
(3,),
(_THINK_END_ID,),
(4,),
],
)
assert reasoning == "thinking."
assert content == "done"
def test_streaming_internal_newlines_preserved(self, parser_with_strip):
reasoning, content = simulate_reasoning_streaming(
parser_with_strip,
["Step 1.\n", "\nStep 2.\n", "</think>", "Answer"],
[
(1,),
(2,),
(_THINK_END_ID,),
(3,),
],
)
assert reasoning == "Step 1.\n\nStep 2."
assert content == "Answer"
def test_streaming_trailing_newlines_before_tool_call(self, parser_with_strip):
"""Trailing newlines before implicit <tool_call> end are stripped."""
reasoning, content = simulate_reasoning_streaming(
parser_with_strip,
["I'll check.\n\n", "<tool_call>", "<function=test>"],
[
(1,),
(_TOOL_CALL_ID,),
(2,),
],
)
assert reasoning == "I'll check."
assert "<tool_call>" not in reasoning
class TestWhitespaceStrippingDisabled:
"""When strip_trailing_reasoning_whitespace is False,
trailing whitespace in reasoning must be preserved."""
@pytest.fixture
def parser_no_strip(self):
cfg = dataclasses.replace(
qwen3_config(),
strip_trailing_reasoning_whitespace=False,
)
return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
def test_non_streaming_preserves_trailing_newline(self, parser_no_strip):
text = "Reasoning here.\n</think>Content."
reasoning, content = parser_no_strip.extract_reasoning(text, None)
assert reasoning == "Reasoning here.\n"
assert content == "Content."
def test_streaming_preserves_trailing_newlines(self, parser_no_strip):
reasoning, content = simulate_reasoning_streaming(
parser_no_strip,
["thinking.\n", "\n", "</think>", "done"],
[
(1,),
(2,),
(_THINK_END_ID,),
(3,),
],
)
assert reasoning == "thinking.\n\n"
assert content == "done"
class TestThinkingDisabled:
"""When ``enable_thinking=False``, the chat template pre-fills a closed
``<think>\\n\\n</think>\\n\\n`` block. The model output starts in content
state, so the parser's initial state must be CONTENT — not REASONING.
"""
def test_thinking_disabled_initial_state_is_content(self, mock_tokenizer):
p = Qwen3Parser(
mock_tokenizer,
chat_template_kwargs={"enable_thinking": False},
)
assert p.parser_engine_config.initial_state == ParserState.CONTENT
def test_thinking_enabled_initial_state_is_reasoning(self, mock_tokenizer):
p = Qwen3Parser(
mock_tokenizer,
chat_template_kwargs={"enable_thinking": True},
)
assert p.parser_engine_config.initial_state == ParserState.REASONING
def test_default_initial_state_is_reasoning(self, mock_tokenizer):
p = Qwen3Parser(mock_tokenizer)
assert p.parser_engine_config.initial_state == ParserState.REASONING
def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
"""Plain text with thinking disabled must stream as content, not
reasoning. Before the fix, the REASONING initial state caused all
output to be emitted as reasoning chunks."""
p = Qwen3Parser(
mock_tokenizer,
chat_template_kwargs={"enable_thinking": False},
)
reasoning, content = simulate_reasoning_streaming(
p,
["The answer", " is 42."],
[
(_TEXT_ID,),
(_TEXT_ID,),
],
)
assert content == "The answer is 42."
assert reasoning == ""
def test_thinking_disabled_non_streaming(self, mock_tokenizer):
p = Qwen3Parser(
mock_tokenizer,
chat_template_kwargs={"enable_thinking": False},
)
reasoning, content = p.extract_reasoning("The answer is 42.", None)
assert reasoning is None
assert content == "The answer is 42."
+189
View File
@@ -0,0 +1,189 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
Replays dynamically built token sequences at different chunk sizes and
holdback depths to verify chunk-size invariance and terminal-token hygiene.
"""
from __future__ import annotations
import pytest
from tests.parser.engine.replay_harness import (
_test_request,
assert_no_terminal_leakage,
assert_parse_output,
collect_output,
make_mock_tokenizer,
replay_streaming,
)
from tests.parser.engine.trace_builder import build_samples
from vllm.parser.abstract_parser import Parser
from vllm.parser.engine.registered_adapters import (
Qwen3Parser,
)
_ENGINE_PARSERS: dict[str, type[Parser]] = {
"qwen3_engine": Qwen3Parser,
}
_qwen3_samples = build_samples("qwen3")
_QWEN3_TERMINALS = [
"<think>",
"</think>",
"<tool_call>",
"</tool_call>",
"<function=",
"</function>",
]
HOLDBACK_CONFIGS = [6, 12, 24]
@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
@pytest.mark.parametrize("chunk_size", [5, 10], ids=lambda c: f"chunk{c}")
@pytest.mark.parametrize("sample", _qwen3_samples, ids=lambda s: s.id)
class TestQwen3ReplayWithHoldback:
"""Replay Qwen3 with simulated detokenizer holdback."""
def test_replay(self, sample, chunk_size, holdback):
tokenizer = make_mock_tokenizer(sample)
parser = Qwen3Parser(tokenizer, sample.tools)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=chunk_size,
holdback_chars=holdback,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
assert_no_terminal_leakage(
output,
_QWEN3_TERMINALS,
context=f"chunk_size={chunk_size}, holdback={holdback}",
)
_TOOL_CALL_SAMPLES = [
(Qwen3Parser, s)
for s in _qwen3_samples
if s.expected_tool_calls and s.expected_reasoning
]
def _suppressed_expectations(sample) -> tuple[str, str]:
"""Compute expected (reasoning, content) when tools are suppressed.
When an explicit reasoning-end delimiter (``</think>``, ``<channel|>``)
is present, reasoning ends there and the tool call block becomes content.
When reasoning ends implicitly (the tool-start token triggers both
REASONING_END and TOOL_CALL_START), reasoning still ends at the tool
start and the raw tool call block becomes content text — only the
structured tool parsing is suppressed, not the reasoning boundary.
"""
full_text = "".join(text for _, text in sample.tokens)
reasoning = sample.expected_reasoning
idx = full_text.find(reasoning)
if idx < 0:
return (full_text, "")
after_reasoning = full_text[idx + len(reasoning) :]
for delim in ("</think>", "<channel|>"):
pos = after_reasoning.find(delim)
if pos >= 0:
return (reasoning, after_reasoning[pos + len(delim) :])
for delim in ("<tool_call>",):
pos = after_reasoning.find(delim)
if pos >= 0:
return (reasoning, after_reasoning[pos:])
return (full_text, "")
_DUMMY_TOOLS = [
{
"type": "function",
"function": {"name": "stub", "parameters": {"type": "object"}},
}
]
@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
@pytest.mark.parametrize(
"parser_cls,sample",
_TOOL_CALL_SAMPLES,
ids=lambda v: v.id if hasattr(v, "id") else v.__name__,
)
class TestSkipToolParsingReplay:
"""Replay with skip_tool_parsing=True (tool_choice='none').
Verifies that reasoning is extracted normally and the raw tool call
block appears as content text with no tool calls parsed.
"""
def test_replay(self, parser_cls, sample, chunk_size):
tokenizer = make_mock_tokenizer(sample)
kwargs = {}
if sample.chat_template_kwargs:
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
parser = parser_cls(tokenizer, **kwargs)
request = _test_request()
request.tool_choice = "none"
request.tools = _DUMMY_TOOLS
all_ids = [tid for tid, _ in sample.tokens]
all_texts = [text for _, text in sample.tokens]
if chunk_size is None:
chunk_size = len(all_ids)
results = []
chunks = list(range(0, len(all_ids), chunk_size))
for i, start in enumerate(chunks):
end = min(start + chunk_size, len(all_ids))
is_last = i == len(chunks) - 1
result = parser.parse_delta(
"".join(all_texts[start:end]),
all_ids[start:end],
request,
prompt_token_ids=[] if start == 0 else None,
finished=is_last,
)
results.append(result)
output = collect_output(results)
expected_reasoning, expected_content = _suppressed_expectations(sample)
assert output.reasoning == expected_reasoning, (
f"Reasoning mismatch:\n"
f" expected: {expected_reasoning!r}\n"
f" actual: {output.reasoning!r}"
)
assert output.tool_calls == [], (
f"Expected no tool calls but got {output.tool_calls}"
)
assert output.content == expected_content, (
f"Content mismatch:\n"
f" expected: {expected_content!r}\n"
f" actual: {output.content!r}"
)
class TestAdapterReferences:
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
parser classes so the serving layer finds them and calls adjust_request."""
@pytest.mark.parametrize(
"parser_name",
list(_ENGINE_PARSERS.keys()),
)
def test_adapter_cls_refs_set(self, parser_name):
parser_cls = _ENGINE_PARSERS[parser_name]
assert parser_cls.reasoning_parser_cls is not None, (
f"{parser_name}: reasoning_parser_cls is None"
)
assert parser_cls.tool_parser_cls is not None, (
f"{parser_name}: tool_parser_cls is None"
)
@@ -0,0 +1,631 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for TokenIDScanner, focusing on hold-back text recovery.
Uses gemma4_config for all end-to-end engine tests, covering
reasoning channels, tool calls, and combined flows."""
from unittest.mock import MagicMock
import pytest
from vllm.parser.engine.events import EventType
from vllm.parser.engine.token_id_scanner import (
PreLexedTerminal,
TextChunk,
TokenIDScanner,
)
CHANNEL_START = "<|channel>"
CHANNEL_END = "<channel|>"
CHANNEL_START_ID = 100
CHANNEL_END_ID = 101
REGULAR_TOKEN_ID = 200
TOOL_START = "<tool_call>"
TOOL_END = "</tool_call>"
TOOL_START_ID = 110
TOOL_END_ID = 111
@pytest.fixture
def tokenizer():
tok = MagicMock()
tok.get_vocab.return_value = {
CHANNEL_START: CHANNEL_START_ID,
CHANNEL_END: CHANNEL_END_ID,
}
tok.decode.side_effect = lambda ids: {
CHANNEL_START_ID: CHANNEL_START,
CHANNEL_END_ID: CHANNEL_END,
REGULAR_TOKEN_ID: "regular",
}.get(ids[0], f"<unk:{ids[0]}>")
return tok
@pytest.fixture
def scanner(tokenizer):
return TokenIDScanner(
token_id_to_terminal={
CHANNEL_START_ID: "THINK_START",
CHANNEL_END_ID: "THINK_END",
},
tokenizer=tokenizer,
)
class TestJoinDecodedTextReturnsStr:
"""_join_decoded_text now returns str unconditionally (was
str | None when an isinstance guard made a branch unreachable)."""
@pytest.fixture
def bare_scanner(self):
return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
def test_mixed_items(self, bare_scanner):
items = [
TextChunk("hello "),
PreLexedTerminal("TOOL_START", 42, "<tool_call>"),
TextChunk(" world"),
]
result = bare_scanner._join_decoded_text(items)
assert isinstance(result, str)
assert result == "hello <tool_call> world"
def test_empty_list(self, bare_scanner):
result = bare_scanner._join_decoded_text([])
assert isinstance(result, str)
assert result == ""
def test_only_text_chunks(self, bare_scanner):
result = bare_scanner._join_decoded_text([TextChunk("abc"), TextChunk("def")])
assert result == "abcdef"
class TestHoldbackTextRecovery:
def test_holdback_text_with_special_token_text_absent(self, scanner):
"""delta_text has hold-back text but the special token's text is
NOT in delta_text (held back by the detokenizer). Terminal is
deferred until the text arrives in a subsequent delta."""
result = scanner.scan(
delta_text="processed is appropriate.",
delta_token_ids=[CHANNEL_END_ID],
)
assert len(result) == 0
# Second scan: terminal text arrives (detokenizer flushes).
# Deferred terminal resolves with holdback text before it.
result2 = scanner.scan(
delta_text="<channel|>Understood.",
delta_token_ids=[20, 21],
)
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
assert len(pre_lexed) == 1
assert pre_lexed[0].terminal == "THINK_END"
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
combined = "".join(texts)
assert "processed is appropriate." in combined
assert "Understood." in combined
def test_holdback_text_with_special_token_text_present(self, scanner):
"""delta_text includes hold-back text AND the special token text."""
result = scanner.scan(
delta_text="holdback text<channel|>",
delta_token_ids=[CHANNEL_END_ID],
)
assert len(result) == 2
assert isinstance(result[0], TextChunk)
assert result[0].text == "holdback text"
assert isinstance(result[1], PreLexedTerminal)
assert result[1].terminal == "THINK_END"
def test_no_holdback_text(self, scanner):
"""delta_text is exactly the special token text — no hold-back."""
result = scanner.scan(
delta_text="<channel|>",
delta_token_ids=[CHANNEL_END_ID],
)
assert len(result) == 1
assert isinstance(result[0], PreLexedTerminal)
assert result[0].terminal == "THINK_END"
def test_empty_delta_text(self, scanner):
"""delta_text is empty — terminal deferred until text arrives."""
result = scanner.scan(
delta_text="",
delta_token_ids=[CHANNEL_END_ID],
)
assert len(result) == 0
flushed = scanner.flush_pending()
assert len(flushed) == 1
assert isinstance(flushed[0], PreLexedTerminal)
assert flushed[0].terminal == "THINK_END"
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
"""delta_text="" with multiple tokens including special: all
results deferred — individually-decoded TextChunks are unreliable
and PreLexedTerminals wait for text confirmation."""
tool_start_id = 400
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
tool_start_id: "<|tool_call>",
tok_a: "call:",
tok_b: "get_weather",
}.get(ids[0], "?")
scanner = TokenIDScanner(
token_id_to_terminal={tool_start_id: "TOOL_START"},
tokenizer=tokenizer,
)
result = scanner.scan(
delta_text="",
delta_token_ids=[tool_start_id, tok_a, tok_b],
)
assert len(result) == 0
flushed = scanner.flush_pending()
assert len(flushed) == 1
assert isinstance(flushed[0], PreLexedTerminal)
assert flushed[0].terminal == "TOOL_START"
def test_holdback_before_start_tag(self, scanner):
"""Hold-back text before a reasoning start tag."""
result = scanner.scan(
delta_text="prefix text<|channel>",
delta_token_ids=[CHANNEL_START_ID],
)
assert len(result) == 2
assert isinstance(result[0], TextChunk)
assert result[0].text == "prefix text"
assert isinstance(result[1], PreLexedTerminal)
assert result[1].terminal == "THINK_START"
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
"""Stream-interval > 1: batch has regular tokens + special token.
delta_text differs from individual decodes (context-dependent)."""
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
tok_a: "wordA",
tok_b: "wordB",
CHANNEL_END_ID: CHANNEL_END,
}.get(ids[0], "?")
scanner_multi = TokenIDScanner(
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
tokenizer=tokenizer,
)
result = scanner_multi.scan(
delta_text="holdback wordA<channel|> wordB",
delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
)
texts = [r.text for r in result if isinstance(r, TextChunk)]
terminals = [r.terminal for r in result if isinstance(r, PreLexedTerminal)]
assert "THINK_END" in terminals
assert "holdback wordA" in "".join(texts)
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
"""Stream-interval > 1: batch has regular + special token, but
delta_text doesn't contain the special token text at all
(held back by detokenizer along with trailing regular tokens).
Terminal is deferred until text arrives."""
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
tok_a: "alpha",
tok_b: "beta",
CHANNEL_END_ID: CHANNEL_END,
}.get(ids[0], "?")
scanner_multi = TokenIDScanner(
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
tokenizer=tokenizer,
)
result = scanner_multi.scan(
delta_text="holdback alpha",
delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
)
assert len(result) == 0
# Next delta: terminal text arrives (detokenizer flushes).
# Deferred terminal resolves with holdback text before it.
result2 = scanner_multi.scan(
delta_text="<channel|> more text",
delta_token_ids=[300],
)
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
assert len(pre_lexed) == 1
assert pre_lexed[0].terminal == "THINK_END"
text_chunks = [r for r in result2 if isinstance(r, TextChunk)]
combined = "".join(t.text for t in text_chunks)
assert "holdback alpha" in combined
assert "more text" in combined
def test_holdback_with_content_after_special_token(self, tokenizer):
"""delta_text has hold-back + special token + content after,
with corresponding token IDs for all parts."""
tok_content = 210
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
tok_content: "content start",
}.get(ids[0], "?")
scanner = TokenIDScanner(
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
tokenizer=tokenizer,
)
result = scanner.scan(
delta_text="reasoning end.<channel|>content start",
delta_token_ids=[CHANNEL_END_ID, tok_content],
)
pre_lexed = [r for r in result if isinstance(r, PreLexedTerminal)]
assert len(pre_lexed) == 1
assert pre_lexed[0].terminal == "THINK_END"
text_chunks = [r for r in result if isinstance(r, TextChunk)]
combined = "".join(t.text for t in text_chunks)
assert "reasoning end." in combined
class TestDropTokens:
def test_drop_token_with_holdback(self, tokenizer):
"""Drop tokens stripped from delta_text, hold-back text preserved.
Terminal is deferred when its text is absent from delta_text."""
drop_id = 300
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
drop_id: "<eos>",
}.get(ids[0], "?")
scanner = TokenIDScanner(
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
tokenizer=tokenizer,
drop_token_ids={drop_id},
)
result = scanner.scan(
delta_text="holdback<eos>",
delta_token_ids=[drop_id, CHANNEL_END_ID],
)
assert len(result) == 0
# Terminal text arrives in next delta; deferred terminal resolves.
result2 = scanner.scan(
delta_text="<channel|>content",
delta_token_ids=[20],
)
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
assert len(pre_lexed) == 1
assert pre_lexed[0].terminal == "THINK_END"
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
combined = "".join(texts)
assert "holdback" in combined
assert "<eos>" not in combined
assert len(scanner.flush_pending()) == 0
class TestEndToEndReasoningHoldback:
"""End-to-end tests through the full parser engine simulating
stream-interval > 1 and detokenizer hold-back, using
gemma4_config."""
def test_reasoning_content_not_truncated(self):
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
config = ParserEngineConfig(
name="test_channel",
initial_state=ParserState.CONTENT,
terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
token_id_terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
tok = MagicMock()
vocab = {
CHANNEL_START: CHANNEL_START_ID,
CHANNEL_END: CHANNEL_END_ID,
}
tok.get_vocab.return_value = vocab
tok.decode.side_effect = lambda ids: {
CHANNEL_START_ID: CHANNEL_START,
CHANNEL_END_ID: CHANNEL_END,
}.get(ids[0], f"tok{ids[0]}")
engine = StreamingParserEngine(config, tok)
all_events = []
# Delta 1: channel start token (text includes start tag)
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
# Delta 2: reasoning text (normal content, no special tokens)
all_events.extend(
engine.feed(
"thought\nThe request was received and ",
[10, 11, 12, 13, 14],
)
)
# Delta 3: MORE reasoning text, the detokenizer held some back.
# Then channel end token arrives in token_ids, but its text
# is NOT in delta_text (held back by detokenizer).
# delta_text = previously held-back reasoning text only.
all_events.extend(
engine.feed(
"processed is appropriate.",
[CHANNEL_END_ID],
)
)
# Delta 4: detokenizer flushes held-back channel end text
# plus new content tokens.
all_events.extend(
engine.feed(
"<channel|>Understood.",
[20, 21],
)
)
all_events.extend(engine.finish())
reasoning_text = "".join(
e.value for e in all_events if e.type == EventType.REASONING_CHUNK
)
content_text = "".join(
e.value for e in all_events if e.type == EventType.TEXT_CHUNK
)
assert "processed is appropriate." in reasoning_text
assert "Understood." in content_text
def test_backtick_content_not_truncated(self):
"""Reproduces the hostname backtick truncation case."""
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
config = ParserEngineConfig(
name="test_channel",
initial_state=ParserState.CONTENT,
terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
token_id_terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
tok = MagicMock()
vocab = {
CHANNEL_START: CHANNEL_START_ID,
CHANNEL_END: CHANNEL_END_ID,
}
tok.get_vocab.return_value = vocab
tok.decode.side_effect = lambda ids: {
CHANNEL_START_ID: CHANNEL_START,
CHANNEL_END_ID: CHANNEL_END,
}.get(ids[0], f"tok{ids[0]}")
engine = StreamingParserEngine(config, tok)
all_events = []
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
all_events.extend(
engine.feed(
"thought\n1/10 completed. Next: ",
[10, 11, 12, 13],
)
)
# Hold-back text includes backtick content; channel end text
# absent from delta_text.
all_events.extend(
engine.feed(
"`hostname`.\n",
[CHANNEL_END_ID],
)
)
# Next delta flushes channel end + tool call start
all_events.extend(
engine.feed(
"<channel|>tool output",
[20, 21],
)
)
all_events.extend(engine.finish())
reasoning_text = "".join(
e.value for e in all_events if e.type == EventType.REASONING_CHUNK
)
assert "`hostname`." in reasoning_text
class TestRebuildFromAnchorsLiteralLookalike:
"""When delta_text contains a literal mention of a special token's
text before the real special token, _rebuild_from_anchors must
anchor at the real occurrence, not the literal one."""
@pytest.fixture
def tool_scanner(self):
tok = MagicMock()
tok.get_vocab.return_value = {
TOOL_START: TOOL_START_ID,
TOOL_END: TOOL_END_ID,
}
tok.decode.side_effect = lambda ids: {
TOOL_START_ID: TOOL_START,
TOOL_END_ID: TOOL_END,
}.get(ids[0], f"t{ids[0]}")
return TokenIDScanner(
{TOOL_START_ID: "TOOL_START", TOOL_END_ID: "TOOL_END"},
tok,
)
def test_literal_before_real_anchor(self, tool_scanner):
"""Literal <tool_call> in prose followed by a real <tool_call>
special token — the scanner must split at the real one."""
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
items = tool_scanner.scan(delta_text, delta_token_ids)
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
assert len(terminals) == 2
assert terminals[0].terminal == "TOOL_START"
assert terminals[1].terminal == "TOOL_END"
# The literal mention must appear in a text chunk, not be
# consumed by the TOOL_START anchor.
joined_text = "".join(text_parts)
assert "<tool_call>" in joined_text
assert '{"name":"f"}' in joined_text
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
"""Two real tool calls with a literal mention between them."""
delta_text = (
'<tool_call>{"name":"a"}</tool_call>'
" see <tool_call> syntax "
'<tool_call>{"name":"b"}</tool_call>'
)
delta_token_ids = [
TOOL_START_ID,
1,
TOOL_END_ID,
2,
3,
4,
TOOL_START_ID,
5,
TOOL_END_ID,
]
items = tool_scanner.scan(delta_text, delta_token_ids)
terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
assert len(terminals) == 4
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
joined_text = "".join(text_parts)
# The literal mention between the two real calls must be in text
assert "<tool_call> syntax" in joined_text
class TestRebuildFromAnchorsCascadingDeferral:
"""When a middle anchor's text is absent from delta_text,
only that anchor should be deferred — not subsequent ones
with valid positions."""
@pytest.fixture
def bare_scanner(self):
tok = MagicMock()
tok.decode.side_effect = lambda ids: f"t{ids[0]}"
return TokenIDScanner({}, tok)
def test_middle_anchor_missing_does_not_cascade(self, bare_scanner):
a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
c = PreLexedTerminal("TOOL_END", TOOL_END_ID, TOOL_END)
delta_text = f"prefix{TOOL_START}middle{TOOL_END}suffix"
results = [a, b, c]
rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
joined = "".join(t.text for t in texts)
assert len(terminals) == 2
assert terminals[0].terminal == "TOOL_START"
assert terminals[1].terminal == "TOOL_END"
assert "prefix" in joined
assert "middle" in joined
assert "suffix" in joined
assert len(bare_scanner._deferred_terminals) == 1
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
assert bare_scanner._deferred_post_text == ""
def test_first_anchor_missing_rest_still_emitted(self, bare_scanner):
a = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
b = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
delta_text = f"text{TOOL_START}more"
results = [a, b]
rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
assert len(terminals) == 1
assert terminals[0].terminal == "TOOL_START"
assert len(bare_scanner._deferred_terminals) == 1
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner):
a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
delta_text = f"text{TOOL_START}more"
results = [a, b]
rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
assert len(terminals) == 1
assert terminals[0].terminal == "TOOL_START"
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
joined = "".join(t.text for t in texts)
assert "text" in joined
# "more" is deferred along with the missing terminal —
# it will be resolved in the next scan when the terminal
# text arrives.
assert bare_scanner._deferred_post_text == "more"
assert len(bare_scanner._deferred_terminals) == 1
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
+410
View File
@@ -0,0 +1,410 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""On-demand trace builder for parser engine testing and benchmarks.
Generates token sequences programmatically from model-agnostic scenario
definitions. Each model format handler knows how to render scenarios
into the model's output format, tokenize them with correct special token
IDs, and compute expected parse outputs.
Every generated sample is self-validated by replaying it through the
real parser before being returned.
"""
from __future__ import annotations
import functools
import json
from dataclasses import dataclass
from typing import Any
from tests.parser.engine.replay_harness import (
MockTokenizer,
Sample,
assert_parse_output,
collect_output,
replay_streaming,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.engine.registered_adapters import (
Qwen3Parser,
)
# ── Data structures ──────────────────────────────────────────────────
@dataclass
class ToolCallSpec:
name: str
arguments: dict[str, Any]
@dataclass
class Scenario:
id: str
description: str
reasoning: str | None = None
content: str | None = None
tool_calls: list[ToolCallSpec] | None = None
# ── Scenarios ────────────────────────────────────────────────────────
_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"})
_BASH_TOOL = ToolCallSpec(
"bash", {"command": "hostname", "description": "Get hostname"}
)
_WEATHER_TOOL = ToolCallSpec(
"get_weather",
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"},
)
_COMPLEX_TOOL = ToolCallSpec(
"search",
{
"query": "vllm parser",
"filters": {"language": "python", "min_stars": 100},
"tags": ["ml", "inference"],
"limit": 10,
"verbose": True,
},
)
SCENARIOS: list[Scenario] = [
Scenario(
id="think-then-tool",
description="Reasoning then single tool call",
reasoning="Let me check the file.",
tool_calls=[_READ_TOOL],
),
Scenario(
id="think-then-parallel-tools",
description="Reasoning then two parallel tool calls",
reasoning="I need to run both commands.",
tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
),
Scenario(
id="think-then-content",
description="Reasoning then content response",
reasoning="Let me think about this carefully.",
content="The answer is 42.",
),
Scenario(
id="content-only",
description="Plain content response without reasoning",
content="Hello! How can I help you today?",
),
Scenario(
id="tool-only",
description="Tool call without reasoning",
tool_calls=[_READ_TOOL],
),
Scenario(
id="complex-json-args",
description="Tool call with nested objects, arrays, numbers, booleans",
reasoning="This needs a complex query.",
tool_calls=[_COMPLEX_TOOL],
),
Scenario(
id="whitespace-before-tool",
description="Whitespace-only content before tool call",
content="\n\n",
tool_calls=[_WEATHER_TOOL],
),
Scenario(
id="think-content-tool",
description="Reasoning, content, then tool call",
reasoning="Let me analyze and then fetch data.",
content="Checking the weather now.",
tool_calls=[_WEATHER_TOOL],
),
Scenario(
id="think-whitespace-tool",
description="Reasoning, whitespace-only gap, then tool call",
reasoning="Let me check the file contents.",
content="\n\n",
tool_calls=[_READ_TOOL],
),
Scenario(
id="empty-reasoning-content",
description="Empty reasoning section followed by content",
reasoning="",
content="The epoch timestamp is 1779111346.",
),
]
# ── Tokenization ─────────────────────────────────────────────────────
def _word_split(text: str) -> list[str]:
"""Split text into word-like tokens, preserving all characters."""
if not text:
return []
parts: list[str] = []
current = ""
for ch in text:
if ch in " \t\n\r" and current and current[-1] not in " \t\n\r":
parts.append(current)
current = ch
else:
current += ch
if current:
parts.append(current)
return parts
def _tokenize(
segments: list[tuple[str, bool]],
vocab: dict[str, int],
start_id: int = 100,
) -> list[tuple[int, str]]:
"""Build token list from segments.
Each segment is ``(text, is_special)``. Special segments use vocab
IDs; content segments are word-split with sequential IDs.
"""
tokens: list[tuple[int, str]] = []
next_id = start_id
for text, is_special in segments:
if not text:
continue
if is_special:
tid = vocab.get(text)
if tid is None:
raise ValueError(f"Special token {text!r} not in vocab")
tokens.append((tid, text))
else:
for word in _word_split(text):
tokens.append((next_id, word))
next_id += 1
return tokens
# ── Tool definitions ─────────────────────────────────────────────────
def _infer_schema(value: object) -> dict:
"""Infer a JSON Schema from a Python value, recursing into dicts/lists."""
if isinstance(value, bool):
return {"type": "boolean"}
if isinstance(value, int):
return {"type": "integer"}
if isinstance(value, float):
return {"type": "number"}
if isinstance(value, str):
return {"type": "string"}
if isinstance(value, dict):
return {
"type": "object",
"properties": {k: _infer_schema(v) for k, v in value.items()},
}
if isinstance(value, list) and value:
return {"type": "array", "items": _infer_schema(value[0])}
if isinstance(value, list):
return {"type": "array"}
return {}
def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]:
"""Generate OpenAI-style tool definitions from tool call specs."""
seen: set[str] = set()
tools: list[dict] = []
for tc in tool_calls:
if tc.name in seen:
continue
seen.add(tc.name)
properties = {k: _infer_schema(v) for k, v in tc.arguments.items()}
tools.append(
{
"type": "function",
"function": {
"name": tc.name,
"parameters": {
"type": "object",
"properties": properties,
},
},
}
)
return tools
# ── Format handlers ──────────────────────────────────────────────────
def _expected_tc(scenario: Scenario) -> list[dict] | None:
if not scenario.tool_calls:
return None
return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls]
def _expected_tools(scenario: Scenario) -> list[dict] | None:
return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None
def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
"""Replay sample through the real parser and assert correctness."""
tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
parser = parser_cls(tokenizer, sample.tools, **kwargs)
deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
output = collect_output(deltas)
assert_parse_output(output, sample)
def _validate_tools(
tools: list[dict] | None,
) -> list[ChatCompletionToolsParam] | None:
if not tools:
return None
return [ChatCompletionToolsParam.model_validate(t) for t in tools]
def _make_sample(
sample_id: str,
description: str,
vocab: dict[str, int],
segments: list[tuple[str, bool]],
expected_reasoning: str | None,
expected_content: str | None,
expected_tool_calls: list[dict] | None,
tools: list[dict] | None,
chat_template_kwargs: dict | None = None,
) -> Sample:
tokens = _tokenize(segments, vocab)
return Sample(
id=sample_id,
description=description,
source="trace-builder",
vocab=dict(vocab),
tokens=tokens,
expected_reasoning=expected_reasoning,
expected_content=expected_content,
expected_tool_calls=expected_tool_calls,
tools=_validate_tools(tools),
chat_template_kwargs=chat_template_kwargs,
)
# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
_QWEN3_VOCAB: dict[str, int] = {
"<think>": 50,
"</think>": 51,
"<tool_call>": 60,
"</tool_call>": 61,
}
def _qwen3_arg_value(value: Any) -> str:
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, (int, float)):
return str(value)
if isinstance(value, str):
return value
return json.dumps(value, ensure_ascii=False)
def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
parts = [f"\n<function={tc.name}>"]
for key, value in tc.arguments.items():
parts.append(f"\n<parameter={key}>{_qwen3_arg_value(value)}</parameter>")
parts.append("\n</function>\n")
return [
("<tool_call>", True),
("".join(parts), False),
("</tool_call>", True),
]
def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if scenario.reasoning is not None:
segs.append((scenario.reasoning, False))
if scenario.content is not None or scenario.tool_calls:
segs.append(("</think>", True))
if scenario.content is not None:
segs.append((scenario.content, False))
if scenario.tool_calls:
for tc in scenario.tool_calls:
segs.extend(_qwen3_tool_segments(tc))
return segs
def _qwen3_expected_content(scenario: Scenario) -> str | None:
if (
scenario.content is not None
and scenario.tool_calls
and not scenario.content.strip()
):
return ""
return scenario.content
def _build_qwen3(
scenario: Scenario,
name: str = "qwen3",
parser_cls: type = Qwen3Parser,
strip_trailing_ws: bool = False,
validate: bool = True,
) -> Sample:
expected_reasoning: str | None
if scenario.reasoning is not None:
r = scenario.reasoning
if strip_trailing_ws:
r = r.rstrip()
expected_reasoning = r
else:
expected_reasoning = ""
sample = _make_sample(
sample_id=f"{name}-{scenario.id}",
description=scenario.description,
vocab=_QWEN3_VOCAB,
segments=_qwen3_segments(scenario),
expected_reasoning=expected_reasoning,
expected_content=_qwen3_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
)
if validate:
_validate_sample(sample, parser_cls)
return sample
# ── Registry and public API ──────────────────────────────────────────
_BUILDERS: dict[str, Any] = {
"qwen3": _build_qwen3,
}
@functools.cache
def build_samples(model: str) -> tuple[Sample, ...]:
"""Build all scenario samples for a model, self-validated."""
builder = _BUILDERS[model]
return tuple(builder(s) for s in SCENARIOS)
def build_sample(model: str, scenario: Scenario) -> Sample:
"""Build a single sample for one model + scenario."""
return _BUILDERS[model](scenario)
def build_scaling_sample(
model: str, token_count: int, validate: bool = False
) -> Sample:
"""Build a sample with approximately *token_count* tokens."""
sentence = "The quick brown fox jumps over the lazy dog. "
text = sentence * (token_count // 10 + 1)
scenario = Scenario(
id=f"scaling-{token_count}",
description=f"Scaling test with ~{token_count} tokens",
reasoning=text,
tool_calls=[_READ_TOOL],
)
return _BUILDERS[model](scenario, validate=validate)