[Frontend] Add Streaming Parser Engine and new Qwen3 Parser (#45413)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
e3e3cd5458
commit
c4a3f9d137
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def should_do_global_cleanup_after_test() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
|
||||
"""Create a mock tokenizer with the given special-token vocabulary.
|
||||
|
||||
The returned mock supports get_vocab(), encode(), and decode().
|
||||
decode() maps known token IDs back to their text and falls back to
|
||||
chr(id) for ASCII IDs or ``<id>`` for others.
|
||||
"""
|
||||
id_to_text = {v: k for k, v in vocab.items()}
|
||||
tokenizer = MagicMock()
|
||||
tokenizer.encode.return_value = [1, 2, 3]
|
||||
tokenizer.get_vocab.return_value = dict(vocab)
|
||||
tokenizer.decode.side_effect = lambda ids: "".join(
|
||||
id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
|
||||
)
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request():
|
||||
req = MagicMock(spec=ChatCompletionRequest)
|
||||
req.tools = []
|
||||
req.tool_choice = "auto"
|
||||
return req
|
||||
@@ -0,0 +1,377 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Data-driven replay harness for parser engine testing.
|
||||
|
||||
Replays token sequences through parsers at different chunk sizes to
|
||||
verify chunk-size invariance: the same token sequence must produce
|
||||
identical output regardless of how tokens are batched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
|
||||
|
||||
@dataclass
|
||||
class Sample:
|
||||
"""One test sample loaded from a JSONL file."""
|
||||
|
||||
id: str
|
||||
description: str
|
||||
source: str
|
||||
vocab: dict[str, int]
|
||||
tokens: list[tuple[int, str]]
|
||||
expected_reasoning: str | None
|
||||
expected_content: str | None
|
||||
expected_tool_calls: list[dict] | None
|
||||
tools: list[dict] | None = None
|
||||
chat_template_kwargs: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParseOutput:
|
||||
"""Accumulated parse output from replaying a token stream."""
|
||||
|
||||
reasoning: str = ""
|
||||
content: str = ""
|
||||
tool_calls: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
class MockTokenizer:
|
||||
"""Lightweight tokenizer mock that avoids unittest.mock overhead.
|
||||
|
||||
Used by ``benchmarks/benchmark_parsers.py`` in tight timing loops,
|
||||
so hot-path methods (``decode``, ``get_vocab``) must be cheap.
|
||||
MagicMock's call-recording machinery added ~40% overhead to small-
|
||||
sample benchmarks, inflating the per-token cost of the parser engine.
|
||||
"""
|
||||
|
||||
__slots__ = (
|
||||
"_vocab",
|
||||
"_token_ids",
|
||||
"_token_decode_map",
|
||||
"_special_ids",
|
||||
"eos_token_id",
|
||||
"bos_token_id",
|
||||
"pad_token_id",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vocab: dict[str, int],
|
||||
tokens: list[tuple[int, str]],
|
||||
) -> None:
|
||||
self._vocab = vocab
|
||||
self._token_ids = [tid for tid, _ in tokens]
|
||||
self._token_decode_map = {tid: text for tid, text in tokens}
|
||||
self._special_ids = set(vocab.values())
|
||||
self.eos_token_id = None
|
||||
self.bos_token_id = None
|
||||
self.pad_token_id = None
|
||||
|
||||
def set_vocab(self, vocab: dict[str, int]) -> None:
|
||||
self._vocab = vocab
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return self._vocab
|
||||
|
||||
def encode(self, text: str, **kwargs) -> list[int]:
|
||||
return self._token_ids
|
||||
|
||||
def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
|
||||
parts: list[str] = []
|
||||
for tid in ids:
|
||||
if skip_special_tokens and tid in self._special_ids:
|
||||
continue
|
||||
text = self._token_decode_map.get(tid, f"?{tid}?")
|
||||
parts.append(text)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
|
||||
"""Build a mock tokenizer from a sample's vocab and token data."""
|
||||
return MockTokenizer(
|
||||
vocab=dict(sample.vocab),
|
||||
tokens=sample.tokens,
|
||||
)
|
||||
|
||||
|
||||
def _test_request(
|
||||
tools: list[dict] | None = None,
|
||||
) -> ChatCompletionRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
|
||||
def replay_streaming(
|
||||
parser,
|
||||
tokens: list[tuple[int, str]],
|
||||
chunk_size: int | None = None,
|
||||
holdback_chars: int = 0,
|
||||
finished_on_last: bool = False,
|
||||
tools: list[dict] | None = None,
|
||||
) -> list[DeltaMessage | None]:
|
||||
"""Feed tokens through ``parser.parse_delta()`` at a given chunk size.
|
||||
|
||||
Args:
|
||||
parser: A :class:`Parser` instance with ``parse_delta()`` method.
|
||||
tokens: List of ``(token_id, decoded_text)`` pairs.
|
||||
chunk_size: Number of tokens per batch. ``None`` means all at once.
|
||||
holdback_chars: Simulate detokenizer holdback by holding back
|
||||
this many characters of decoded text between batches.
|
||||
finished_on_last: When True, pass ``finished=True`` on the last
|
||||
``parse_delta()`` call, matching real server behavior.
|
||||
tools: Optional tool definitions to include on the request,
|
||||
matching the serving layer where tools set
|
||||
``tool_choice`` to ``"auto"``.
|
||||
|
||||
Returns:
|
||||
List of ``DeltaMessage`` results from each ``parse_delta()`` call.
|
||||
"""
|
||||
if chunk_size is None:
|
||||
chunk_size = len(tokens)
|
||||
|
||||
results: list[DeltaMessage | None] = []
|
||||
all_ids = [tid for tid, _ in tokens]
|
||||
all_texts = [text for _, text in tokens]
|
||||
|
||||
request = _test_request(tools=tools)
|
||||
|
||||
if holdback_chars <= 0:
|
||||
chunks = list(range(0, len(tokens), chunk_size))
|
||||
for i, start in enumerate(chunks):
|
||||
batch_end = min(start + chunk_size, len(tokens))
|
||||
batch_ids = all_ids[start:batch_end]
|
||||
delta_text = "".join(all_texts[start:batch_end])
|
||||
is_last = i == len(chunks) - 1
|
||||
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
batch_ids,
|
||||
request,
|
||||
prompt_token_ids=[] if start == 0 else None,
|
||||
finished=finished_on_last and is_last,
|
||||
)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
emitted_up_to = 0
|
||||
is_first = True
|
||||
|
||||
for start in range(0, len(tokens), chunk_size):
|
||||
batch_end = min(start + chunk_size, len(tokens))
|
||||
|
||||
if batch_end < len(tokens):
|
||||
held_chars = 0
|
||||
safe_end = batch_end
|
||||
while safe_end > emitted_up_to and held_chars < holdback_chars:
|
||||
safe_end -= 1
|
||||
held_chars += len(all_texts[safe_end])
|
||||
else:
|
||||
safe_end = batch_end
|
||||
|
||||
if safe_end <= emitted_up_to:
|
||||
continue
|
||||
|
||||
batch_ids = all_ids[emitted_up_to:safe_end]
|
||||
delta_text = "".join(all_texts[emitted_up_to:safe_end])
|
||||
emitted_up_to = safe_end
|
||||
|
||||
is_last_chunk = batch_end >= len(tokens)
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
batch_ids,
|
||||
request,
|
||||
prompt_token_ids=[] if is_first else None,
|
||||
finished=finished_on_last and is_last_chunk,
|
||||
)
|
||||
results.append(result)
|
||||
is_first = False
|
||||
|
||||
if emitted_up_to < len(tokens):
|
||||
batch_ids = all_ids[emitted_up_to:]
|
||||
delta_text = "".join(all_texts[emitted_up_to:])
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
batch_ids,
|
||||
request,
|
||||
prompt_token_ids=[] if is_first else None,
|
||||
finished=finished_on_last,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def replay_with_text_holdback(
|
||||
parser,
|
||||
tokens: list[tuple[int, str]],
|
||||
text_delay: int = 1,
|
||||
tools: list[dict] | None = None,
|
||||
) -> list[DeltaMessage | None]:
|
||||
"""Replay token-by-token with text arriving *text_delay* steps late.
|
||||
|
||||
Simulates the production detokenizer holdback where token IDs arrive
|
||||
immediately but decoded text is delayed. On the last token all
|
||||
remaining held-back text is flushed, matching real server behavior::
|
||||
|
||||
step 0: ids=[tok0], text="" (held back)
|
||||
step 1: ids=[tok1], text=tok0_text (tok0 released)
|
||||
...
|
||||
step N-1: ids=[tokN-1], text=remaining_texts (flush all)
|
||||
|
||||
This exercises the TokenIDScanner deferred-terminal path that
|
||||
``replay_streaming`` (which keeps text and IDs aligned) does not.
|
||||
"""
|
||||
results: list[DeltaMessage | None] = []
|
||||
request = _test_request(tools=tools)
|
||||
|
||||
n = len(tokens)
|
||||
held_texts: list[str] = []
|
||||
|
||||
for i in range(n):
|
||||
token_id = tokens[i][0]
|
||||
held_texts.append(tokens[i][1])
|
||||
|
||||
is_last = i == n - 1
|
||||
if is_last:
|
||||
delta_text = "".join(held_texts)
|
||||
held_texts.clear()
|
||||
elif len(held_texts) > text_delay:
|
||||
delta_text = held_texts.pop(0)
|
||||
else:
|
||||
delta_text = ""
|
||||
|
||||
result = parser.parse_delta(
|
||||
delta_text,
|
||||
[token_id],
|
||||
request,
|
||||
prompt_token_ids=[] if i == 0 else None,
|
||||
finished=is_last,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def accumulate_deltas(
|
||||
deltas: Sequence[DeltaMessage | None],
|
||||
) -> dict:
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
tool_calls_by_idx: dict[int, dict] = {}
|
||||
|
||||
for delta in deltas:
|
||||
if delta is None:
|
||||
continue
|
||||
if delta.reasoning:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
if delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
if tc.function and tc.function.name:
|
||||
existing = tool_calls_by_idx.get(tc.index)
|
||||
if existing is None:
|
||||
tool_calls_by_idx[tc.index] = {
|
||||
"name": tc.function.name,
|
||||
"_args_parts": [tc.function.arguments or ""],
|
||||
}
|
||||
else:
|
||||
existing["_args_parts"].append(tc.function.arguments or "")
|
||||
elif tc.function and tc.function.arguments:
|
||||
existing = tool_calls_by_idx.get(tc.index)
|
||||
if existing is not None:
|
||||
existing["_args_parts"].append(tc.function.arguments)
|
||||
|
||||
return {
|
||||
"reasoning": "".join(reasoning_parts),
|
||||
"content": "".join(content_parts),
|
||||
"tool_calls": [
|
||||
{"name": tc["name"], "arguments": "".join(tc["_args_parts"])}
|
||||
for tc in tool_calls_by_idx.values()
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def collect_output(results: list[DeltaMessage | None]) -> ParseOutput:
|
||||
"""Accumulate ``DeltaMessage`` results into a :class:`ParseOutput`."""
|
||||
result = accumulate_deltas(results)
|
||||
return ParseOutput(
|
||||
reasoning=result["reasoning"],
|
||||
content=result["content"],
|
||||
tool_calls=result["tool_calls"],
|
||||
)
|
||||
|
||||
|
||||
def assert_parse_output(actual: ParseOutput, sample: Sample) -> None:
|
||||
"""Compare actual parse output against expected values from a sample."""
|
||||
if sample.expected_reasoning is not None:
|
||||
assert actual.reasoning == sample.expected_reasoning, (
|
||||
f"Reasoning mismatch:\n"
|
||||
f" expected: {sample.expected_reasoning!r}\n"
|
||||
f" actual: {actual.reasoning!r}"
|
||||
)
|
||||
|
||||
if sample.expected_content is not None:
|
||||
assert actual.content == sample.expected_content, (
|
||||
f"Content mismatch:\n"
|
||||
f" expected: {sample.expected_content!r}\n"
|
||||
f" actual: {actual.content!r}"
|
||||
)
|
||||
if sample.expected_tool_calls is not None:
|
||||
assert len(actual.tool_calls) == len(sample.expected_tool_calls), (
|
||||
f"Tool call count mismatch: "
|
||||
f"expected {len(sample.expected_tool_calls)}, "
|
||||
f"got {len(actual.tool_calls)}"
|
||||
)
|
||||
for i, (expected_tc, actual_tc) in enumerate(
|
||||
zip(sample.expected_tool_calls, actual.tool_calls)
|
||||
):
|
||||
assert actual_tc["name"] == expected_tc["name"], (
|
||||
f"Tool call {i} name mismatch: "
|
||||
f"expected {expected_tc['name']!r}, "
|
||||
f"got {actual_tc['name']!r}"
|
||||
)
|
||||
if "arguments" in expected_tc:
|
||||
expected_args = expected_tc["arguments"]
|
||||
actual_args_str = actual_tc.get("arguments", "{}")
|
||||
if isinstance(expected_args, dict):
|
||||
try:
|
||||
actual_args = json.loads(actual_args_str)
|
||||
except json.JSONDecodeError as e:
|
||||
raise AssertionError(
|
||||
f"Tool call {i} arguments not valid JSON: "
|
||||
f"{actual_args_str!r}"
|
||||
) from e
|
||||
assert actual_args == expected_args, (
|
||||
f"Tool call {i} arguments mismatch:\n"
|
||||
f" expected: {expected_args}\n"
|
||||
f" actual: {actual_args}"
|
||||
)
|
||||
|
||||
|
||||
def assert_no_terminal_leakage(
|
||||
actual: ParseOutput,
|
||||
terminals: list[str],
|
||||
context: str = "",
|
||||
) -> None:
|
||||
"""Assert that none of *terminals* appear in reasoning or content."""
|
||||
suffix = f" ({context})" if context else ""
|
||||
for terminal in terminals:
|
||||
assert terminal not in actual.reasoning, (
|
||||
f"{terminal!r} leaked into reasoning{suffix}"
|
||||
)
|
||||
assert terminal not in actual.content, (
|
||||
f"{terminal!r} leaked into content{suffix}"
|
||||
)
|
||||
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared streaming simulation helpers for parser engine tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
|
||||
|
||||
|
||||
def _build_token_id_map(parser) -> dict[str, int]:
|
||||
"""Map special token text to token IDs from the parser's config."""
|
||||
token_id_map: dict[str, int] = {}
|
||||
cfg = getattr(parser, "parser_engine_config", None)
|
||||
vocab = getattr(parser, "vocab", None)
|
||||
if cfg is not None and vocab is not None:
|
||||
for text in (cfg.token_id_terminals or {}).values():
|
||||
tid = vocab.get(text)
|
||||
if tid is not None:
|
||||
token_id_map[text] = tid
|
||||
return token_id_map
|
||||
|
||||
|
||||
def simulate_tool_streaming(
|
||||
parser,
|
||||
request,
|
||||
chunks: list[str],
|
||||
) -> list[tuple[DeltaMessage | None, str]]:
|
||||
"""Feed text chunks through ``extract_tool_calls_streaming()``."""
|
||||
token_id_map = _build_token_id_map(parser)
|
||||
|
||||
results: list[tuple[Any, str]] = []
|
||||
previous_text = ""
|
||||
previous_token_ids: list[int] = []
|
||||
|
||||
for chunk in chunks:
|
||||
current_text = previous_text + chunk
|
||||
|
||||
delta_token_ids: list[int] = [
|
||||
tid for text, tid in token_id_map.items() if text in chunk
|
||||
]
|
||||
|
||||
current_token_ids = previous_token_ids + delta_token_ids
|
||||
|
||||
delta = parser.extract_tool_calls_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=tuple(previous_token_ids),
|
||||
current_token_ids=tuple(current_token_ids),
|
||||
delta_token_ids=tuple(delta_token_ids),
|
||||
request=request,
|
||||
)
|
||||
results.append((delta, current_text))
|
||||
previous_text = current_text
|
||||
previous_token_ids = list(current_token_ids)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def collect_tool_arguments(
|
||||
results: list[tuple[DeltaMessage | None, str]],
|
||||
) -> str:
|
||||
"""Concatenate all streamed argument fragments."""
|
||||
args_text = ""
|
||||
for delta, _ in results:
|
||||
if delta and delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
if tc.function and tc.function.arguments:
|
||||
args_text += tc.function.arguments
|
||||
return args_text
|
||||
|
||||
|
||||
def collect_content(
|
||||
results: list[tuple[DeltaMessage | None, str]],
|
||||
) -> str:
|
||||
"""Concatenate all streamed content parts."""
|
||||
parts: list[str] = []
|
||||
for delta, _ in results:
|
||||
if delta and delta.content:
|
||||
parts.append(delta.content)
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def collect_function_name(
|
||||
results: list[tuple[DeltaMessage | None, str]],
|
||||
) -> str | None:
|
||||
"""Return first function name from deltas."""
|
||||
for delta, _ in results:
|
||||
if delta and delta.tool_calls:
|
||||
for tc in delta.tool_calls:
|
||||
if tc.function and tc.function.name:
|
||||
return tc.function.name
|
||||
return None
|
||||
|
||||
|
||||
def simulate_reasoning_streaming(
|
||||
parser,
|
||||
chunks: list[str],
|
||||
delta_token_ids_per_chunk: list[tuple[int, ...]] | None = None,
|
||||
) -> tuple[str, str]:
|
||||
"""Feed chunks through ``extract_reasoning_streaming()``.
|
||||
|
||||
Returns ``(reasoning_text, content_text)`` tuple.
|
||||
"""
|
||||
token_id_map = (
|
||||
_build_token_id_map(parser) if delta_token_ids_per_chunk is None else {}
|
||||
)
|
||||
|
||||
reasoning_parts: list[str] = []
|
||||
content_parts: list[str] = []
|
||||
prev_text = ""
|
||||
prev_ids: list[int] = []
|
||||
for i, chunk in enumerate(chunks):
|
||||
cur_text = prev_text + chunk
|
||||
if delta_token_ids_per_chunk is not None:
|
||||
d_ids = delta_token_ids_per_chunk[i]
|
||||
else:
|
||||
d_ids = tuple(tid for text, tid in token_id_map.items() if text in chunk)
|
||||
cur_ids = prev_ids + list(d_ids)
|
||||
delta = parser.extract_reasoning_streaming(
|
||||
previous_text=prev_text,
|
||||
current_text=cur_text,
|
||||
delta_text=chunk,
|
||||
previous_token_ids=tuple(prev_ids),
|
||||
current_token_ids=tuple(cur_ids),
|
||||
delta_token_ids=d_ids,
|
||||
)
|
||||
if delta:
|
||||
if delta.reasoning:
|
||||
reasoning_parts.append(delta.reasoning)
|
||||
if delta.content:
|
||||
content_parts.append(delta.content)
|
||||
prev_text = cur_text
|
||||
prev_ids = list(cur_ids)
|
||||
return "".join(reasoning_parts), "".join(content_parts)
|
||||
@@ -0,0 +1,82 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Replay tests for DelegatingParser with engine adapters.
|
||||
|
||||
Exercises DelegatingParser in engine-adapter mode to verify that delegated
|
||||
routing produces correct output across chunk sizes.
|
||||
See test_replay.py for tests that target engine parsers directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import lru_cache
|
||||
|
||||
import pytest
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
replay_streaming,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import build_samples
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
|
||||
|
||||
_PAIRINGS: dict[str, tuple[str, str]] = {
|
||||
"engine": ("qwen3_coder", "qwen3"),
|
||||
}
|
||||
|
||||
CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
|
||||
|
||||
|
||||
@lru_cache
|
||||
def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
|
||||
tool_name, reasoning_name = _PAIRINGS[pairings]
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name=tool_name,
|
||||
reasoning_parser_name=reasoning_name,
|
||||
enable_auto_tools=True,
|
||||
)
|
||||
assert parser_cls is not None
|
||||
return parser_cls
|
||||
|
||||
|
||||
_all_samples = build_samples("qwen3")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pairings",
|
||||
list(_PAIRINGS),
|
||||
ids=lambda p: f"mode={p}",
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
|
||||
@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
|
||||
def test_delegating_replay(sample, chunk_size, pairings):
|
||||
parser_cls = _get_delegating_parser_cls(pairings=pairings)
|
||||
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
validated_tools = (
|
||||
_TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
|
||||
)
|
||||
parser = parser_cls(
|
||||
tokenizer,
|
||||
validated_tools,
|
||||
chat_template_kwargs=sample.chat_template_kwargs,
|
||||
)
|
||||
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=True,
|
||||
tools=sample.tools,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
assert_parse_output(output, sample)
|
||||
@@ -0,0 +1,846 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the streaming parser engine core pipeline."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from vllm.parser.engine.events import EventType, SemanticEvent
|
||||
from vllm.parser.engine.incremental_lexer import (
|
||||
LexerShape,
|
||||
TerminalDef,
|
||||
terminals_from_literals,
|
||||
)
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
|
||||
def _hermes_config() -> ParserEngineConfig:
|
||||
"""Simple Hermes-style config: <tool_call>JSON</tool_call>."""
|
||||
return ParserEngineConfig(
|
||||
name="hermes_test",
|
||||
terminals={
|
||||
"TOOL_START": "<tool_call>",
|
||||
"TOOL_END": "</tool_call>",
|
||||
},
|
||||
token_id_terminals={
|
||||
"TOOL_START": "<tool_call>",
|
||||
"TOOL_END": "</tool_call>",
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _think_config() -> ParserEngineConfig:
|
||||
"""Simple think-tag reasoning config: <think>...</think>."""
|
||||
return ParserEngineConfig(
|
||||
name="think_test",
|
||||
terminals={
|
||||
"THINK_START": "<think>",
|
||||
"THINK_END": "</think>",
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_plain_text(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
events = engine.parse_complete("Hello, world!")
|
||||
assert len(events) == 1
|
||||
assert events[0].type == EventType.TEXT_CHUNK
|
||||
assert events[0].value == "Hello, world!"
|
||||
|
||||
def test_single_tool_call(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
text = (
|
||||
'<tool_call>{"name": "get_weather",'
|
||||
' "arguments": {"city": "SF"}}'
|
||||
"</tool_call>"
|
||||
)
|
||||
events = engine.parse_complete(text)
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START in types
|
||||
assert EventType.TOOL_CALL_END in types
|
||||
assert EventType.ARG_VALUE_CHUNK in types
|
||||
|
||||
arg_text = "".join(
|
||||
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
assert '"name": "get_weather"' in arg_text
|
||||
assert '"city": "SF"' in arg_text
|
||||
|
||||
def test_text_then_tool_call(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
text = 'Sure!<tool_call>{"name": "add"}</tool_call>'
|
||||
events = engine.parse_complete(text)
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert types[0] == EventType.TEXT_CHUNK
|
||||
assert events[0].value == "Sure!"
|
||||
assert EventType.TOOL_CALL_START in types
|
||||
assert EventType.TOOL_CALL_END in types
|
||||
|
||||
def test_multiple_tool_calls(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
text = (
|
||||
'<tool_call>{"name": "a"}</tool_call><tool_call>{"name": "b"}</tool_call>'
|
||||
)
|
||||
events = engine.parse_complete(text)
|
||||
|
||||
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
|
||||
ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
|
||||
assert len(starts) == 2
|
||||
assert len(ends) == 2
|
||||
assert starts[0].tool_index == 0
|
||||
assert starts[1].tool_index == 1
|
||||
|
||||
def test_reasoning(self):
|
||||
engine = StreamingParserEngine(_think_config(), tokenizer=None)
|
||||
text = "<think>Let me think...</think>The answer is 42."
|
||||
events = engine.parse_complete(text)
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert types[0] == EventType.REASONING_START
|
||||
assert EventType.REASONING_CHUNK in types
|
||||
assert EventType.REASONING_END in types
|
||||
assert EventType.TEXT_CHUNK in types
|
||||
|
||||
reasoning = "".join(
|
||||
e.value for e in events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
assert "Let me think..." in reasoning
|
||||
|
||||
content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "The answer is 42." in content
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
@staticmethod
|
||||
def _feed_chars(
|
||||
engine: StreamingParserEngine,
|
||||
text: str,
|
||||
) -> list[SemanticEvent]:
|
||||
"""Feed text one character at a time."""
|
||||
all_events = []
|
||||
for ch in text:
|
||||
all_events.extend(engine.feed(ch, []))
|
||||
all_events.extend(engine.finish())
|
||||
return all_events
|
||||
|
||||
@staticmethod
|
||||
def _feed_chunks(
|
||||
engine: StreamingParserEngine,
|
||||
text: str,
|
||||
chunk_size: int,
|
||||
) -> list[SemanticEvent]:
|
||||
"""Feed text in fixed-size chunks."""
|
||||
all_events = []
|
||||
for i in range(0, len(text), chunk_size):
|
||||
chunk = text[i : i + chunk_size]
|
||||
all_events.extend(engine.feed(chunk, []))
|
||||
all_events.extend(engine.finish())
|
||||
return all_events
|
||||
|
||||
def test_char_by_char_tool_call(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
text = '<tool_call>{"name": "add", "arguments": {"a": 1}}</tool_call>'
|
||||
events = self._feed_chars(engine, text)
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START in types
|
||||
assert EventType.TOOL_CALL_END in types
|
||||
assert EventType.ARG_VALUE_CHUNK in types
|
||||
|
||||
arg_text = "".join(
|
||||
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
assert '"name": "add"' in arg_text
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
'<tool_call>{"name": "get", "arguments": {"x": "hello"}}</tool_call>',
|
||||
'<tool_call>{"name": "f", "arguments": '
|
||||
'{"items": [1, [2, 3]], "obj": {"k": "v"}}}'
|
||||
"</tool_call>",
|
||||
],
|
||||
ids=["flat_args", "nested_arrays"],
|
||||
)
|
||||
def test_chunk_sizes_produce_same_content(self, text):
|
||||
"""Different chunk sizes must produce identical concatenated content."""
|
||||
results = {}
|
||||
for chunk_size in [1, 2, 3, 5, 7, len(text)]:
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
events = self._feed_chunks(engine, text, chunk_size)
|
||||
arg_text = "".join(
|
||||
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
results[chunk_size] = arg_text
|
||||
|
||||
values = list(results.values())
|
||||
for v in values[1:]:
|
||||
assert v == values[0], f"Mismatch: {results}"
|
||||
|
||||
def test_prefix_buffering_prevents_premature_emit(self):
|
||||
"""Text like '<tool_' should be buffered, not emitted as content."""
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
|
||||
events1 = engine.feed("<tool_", [])
|
||||
content_events = [e for e in events1 if e.type == EventType.TEXT_CHUNK]
|
||||
assert len(content_events) == 0, "Should buffer partial tag"
|
||||
|
||||
events2 = engine.feed("call>", [])
|
||||
starts = [e for e in events2 if e.type == EventType.TOOL_CALL_START]
|
||||
assert len(starts) == 1
|
||||
|
||||
def test_prefix_buffering_flush_on_mismatch(self):
|
||||
"""Text like '<tool_box' should eventually flush as content."""
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
|
||||
events1 = engine.feed("<tool_", [])
|
||||
assert len([e for e in events1 if e.type == EventType.TEXT_CHUNK]) == 0
|
||||
|
||||
events2 = engine.feed("box>rest", [])
|
||||
events2.extend(engine.finish())
|
||||
content = "".join(e.value for e in events2 if e.type == EventType.TEXT_CHUNK)
|
||||
assert content == "<tool_box>rest"
|
||||
|
||||
def test_reasoning_streaming(self):
|
||||
engine = StreamingParserEngine(_think_config(), tokenizer=None)
|
||||
events = self._feed_chars(engine, "<think>hmm</think>answer")
|
||||
|
||||
reasoning = "".join(
|
||||
e.value for e in events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "hmm" in reasoning
|
||||
assert "answer" in content
|
||||
|
||||
def test_text_between_tool_calls(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
text = (
|
||||
'Hi<tool_call>{"name":"a"}</tool_call>'
|
||||
'mid<tool_call>{"name":"b"}</tool_call>end'
|
||||
)
|
||||
events = self._feed_chunks(engine, text, 3)
|
||||
|
||||
texts = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "Hi" in texts
|
||||
assert "mid" in texts
|
||||
assert "end" in texts
|
||||
|
||||
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
|
||||
assert len(starts) == 2
|
||||
|
||||
def test_unmatched_close_brace_does_not_poison_depth(self):
|
||||
"""A stray } in malformed JSON must not kill streaming for
|
||||
all subsequent content."""
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
engine.feed("<tool_call>", [])
|
||||
|
||||
malformed = '}{{"a": 1}}'
|
||||
events = self._feed_chars(engine, malformed + "</tool_call>")
|
||||
|
||||
arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
|
||||
assert len(arg_chunks) > 1, (
|
||||
"Content after stray } should still stream incrementally"
|
||||
)
|
||||
arg_text = "".join(arg_chunks)
|
||||
assert '"a": 1' in arg_text
|
||||
|
||||
def test_json_args_no_premature_close_brace(self):
|
||||
"""Closing braces of the top-level JSON shouldn't be streamed
|
||||
until confirmed by the end tag."""
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
|
||||
engine.feed("<tool_call>", [])
|
||||
events = engine.feed('{"name": "f"}', [])
|
||||
|
||||
arg_text = "".join(
|
||||
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
assert "}" not in arg_text, "Top-level } should be held back"
|
||||
|
||||
events2 = engine.feed("</tool_call>", [])
|
||||
arg_text2 = "".join(
|
||||
e.value for e in events2 if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
assert "}" in arg_text2, "} should flush on end tag"
|
||||
|
||||
|
||||
_START_ID = 50
|
||||
_END_ID = 51
|
||||
_TOOL_START_ID = 60
|
||||
_TOOL_END_ID = 61
|
||||
|
||||
|
||||
def _make_think_tokenizer():
|
||||
tok = MagicMock()
|
||||
tok.encode.return_value = [1, 2, 3]
|
||||
tok.get_vocab.return_value = {"<think>": _START_ID, "</think>": _END_ID}
|
||||
tok.decode.side_effect = lambda ids: {
|
||||
_START_ID: "<think>",
|
||||
_END_ID: "</think>",
|
||||
}.get(ids[0], f"tok{ids[0]}")
|
||||
return tok
|
||||
|
||||
|
||||
def _make_hermes_tokenizer():
|
||||
"""Tokenizer that resolves tool_call tags to special IDs."""
|
||||
_special = {_TOOL_START_ID: "<tool_call>", _TOOL_END_ID: "</tool_call>"}
|
||||
tok = MagicMock()
|
||||
tok.encode.return_value = [1, 2, 3]
|
||||
tok.get_vocab.return_value = {
|
||||
"<tool_call>": _TOOL_START_ID,
|
||||
"</tool_call>": _TOOL_END_ID,
|
||||
}
|
||||
tok.decode.side_effect = lambda ids: "".join(
|
||||
_special.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
|
||||
)
|
||||
return tok
|
||||
|
||||
|
||||
class TestLexerBufferFlush:
|
||||
"""Lexer buffer must be flushed before PreLexedTerminal transitions."""
|
||||
|
||||
def test_buffered_prefix_emitted_in_current_state(self):
|
||||
"""Text buffered by the lexer (e.g. '<') must be emitted as
|
||||
REASONING_CHUNK before THINK_END transitions to CONTENT."""
|
||||
engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
|
||||
|
||||
events = engine.feed("<think>", [_START_ID])
|
||||
assert any(e.type == EventType.REASONING_START for e in events)
|
||||
|
||||
events = engine.feed("reasoning text<", [])
|
||||
reasoning_text = "".join(
|
||||
e.value for e in events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
assert "reasoning text" in reasoning_text
|
||||
|
||||
events = engine.feed("</think>", [_END_ID])
|
||||
event_types = [e.type for e in events]
|
||||
if EventType.REASONING_CHUNK in event_types:
|
||||
rc_idx = event_types.index(EventType.REASONING_CHUNK)
|
||||
re_idx = event_types.index(EventType.REASONING_END)
|
||||
assert rc_idx < re_idx, (
|
||||
"'<' must be emitted as REASONING_CHUNK before REASONING_END"
|
||||
)
|
||||
flushed = events[rc_idx].value
|
||||
assert "<" in flushed
|
||||
|
||||
def test_empty_buffer_no_extra_events(self):
|
||||
"""When the lexer buffer is empty, flushing is a no-op."""
|
||||
engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
|
||||
|
||||
engine.feed("<think>", [_START_ID])
|
||||
engine.feed("clean text", [])
|
||||
|
||||
events = engine.feed("</think>", [_END_ID])
|
||||
assert any(e.type == EventType.REASONING_END for e in events)
|
||||
chunk_events = [e for e in events if e.type == EventType.REASONING_CHUNK]
|
||||
assert all(e.value for e in chunk_events)
|
||||
|
||||
|
||||
class TestTokenIdFiltering:
|
||||
"""When token IDs are available, lex-matched terminals that also
|
||||
have token_id_terminal entries should be demoted to content."""
|
||||
|
||||
def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
|
||||
"""After receiving token IDs, text that matches a token-ID
|
||||
terminal should be treated as content, not trigger a transition."""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
# First feed with a non-special token ID to set _ever_had_token_ids
|
||||
engine.feed("prefix ", [1])
|
||||
|
||||
# Now feed text containing <tool_call> as literal text
|
||||
events = engine.feed(
|
||||
"Use <tool_call> to invoke tools.</tool_call>", [2, 3, 4, 5]
|
||||
)
|
||||
events.extend(engine.finish())
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START not in types
|
||||
assert EventType.TEXT_CHUNK in types
|
||||
|
||||
text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "<tool_call>" in text
|
||||
|
||||
def test_scanner_matched_terminal_bypasses_filter(self):
|
||||
"""PreLexedTerminals from the scanner bypass the filter and
|
||||
still trigger state transitions."""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
events = engine.feed("<tool_call>", [_TOOL_START_ID])
|
||||
assert any(e.type == EventType.TOOL_CALL_START for e in events)
|
||||
|
||||
events = engine.feed('{"name": "f"}', [2, 3])
|
||||
events.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
|
||||
events.extend(engine.finish())
|
||||
assert any(e.type == EventType.TOOL_CALL_END for e in events)
|
||||
|
||||
def test_no_filtering_without_token_ids(self):
|
||||
"""When no token IDs are ever provided (non-streaming),
|
||||
text matching still triggers transitions."""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
|
||||
events.extend(engine.finish())
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START in types
|
||||
assert EventType.TOOL_CALL_END in types
|
||||
|
||||
def test_mixed_text_then_real_tool_call(self):
|
||||
"""Text mentioning tool syntax followed by a real special-token
|
||||
tool call."""
|
||||
engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
|
||||
|
||||
events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
|
||||
events2 = engine.feed("<tool_call>", [_TOOL_START_ID])
|
||||
events3 = engine.feed('{"name": "a"}', [5, 6])
|
||||
events4 = engine.feed("</tool_call>", [_TOOL_END_ID])
|
||||
events4.extend(engine.finish())
|
||||
|
||||
all_events = events1 + events2 + events3 + events4
|
||||
|
||||
content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "<tool_call>" in content
|
||||
|
||||
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
|
||||
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
|
||||
|
||||
|
||||
def _func_prefix_config() -> ParserEngineConfig:
|
||||
"""Config mixing token-ID terminals (TOOL_START/END) with
|
||||
text-only terminals (FUNC_PREFIX) and fallback transitions."""
|
||||
return ParserEngineConfig(
|
||||
name="func_prefix_test",
|
||||
terminals={
|
||||
"TOOL_START": "<tool_call>",
|
||||
"TOOL_END": "</tool_call>",
|
||||
"FUNC_PREFIX": "<function=",
|
||||
"FUNC_END": "</function>",
|
||||
"CLOSE_ANGLE": ">",
|
||||
},
|
||||
token_id_terminals={
|
||||
"TOOL_START": "<tool_call>",
|
||||
"TOOL_END": "</tool_call>",
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.CONTENT, "FUNC_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
skip_in_token_id_mode=True,
|
||||
),
|
||||
(ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "FUNC_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition(
|
||||
ParserState.TOOL_NAME,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
skip_in_token_id_mode=True,
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.TOOL_NAME: EventType.TOOL_NAME,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _make_func_prefix_tokenizer():
|
||||
return make_mock_tokenizer(
|
||||
{
|
||||
"<tool_call>": _TOOL_START_ID,
|
||||
"</tool_call>": _TOOL_END_ID,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class TestTextOnlyFallbackFiltering:
|
||||
"""When token IDs are available, transitions marked
|
||||
skip_in_token_id_mode should be skipped."""
|
||||
|
||||
def test_func_prefix_in_prose_demoted_in_strict_mode(self):
|
||||
"""<function=get_time> in prose should NOT trigger a tool call
|
||||
when strict mode is active."""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
engine.feed("prefix ", [1])
|
||||
|
||||
events = engine.feed("Use <function=get_time> to check.", [2, 3, 4, 5])
|
||||
events.extend(engine.finish())
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START not in types
|
||||
assert EventType.TEXT_CHUNK in types
|
||||
text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "<function=" in text
|
||||
|
||||
def test_normal_flow_after_tool_start_still_works(self):
|
||||
"""TOOL_START (special token) -> FUNC_PREFIX (text) should
|
||||
still parse a tool call normally in strict mode."""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
|
||||
events1 = engine.feed("<tool_call>", [_TOOL_START_ID])
|
||||
assert any(e.type == EventType.TOOL_CALL_START for e in events1)
|
||||
|
||||
events2 = engine.feed("<function=get_weather>", [2, 3])
|
||||
events3 = engine.feed("args", [4])
|
||||
events4 = engine.feed("</function>", [5, 6])
|
||||
events4.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
|
||||
events4.extend(engine.finish())
|
||||
|
||||
all_events = events1 + events2 + events3 + events4
|
||||
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
|
||||
assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
|
||||
|
||||
def test_fallback_fires_without_token_ids(self):
|
||||
"""When no token IDs are provided, fallback transitions should
|
||||
still fire normally."""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
|
||||
events = engine.feed("<function=get_time>args</function>", [])
|
||||
events.extend(engine.finish())
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START in types
|
||||
assert EventType.TOOL_CALL_END in types
|
||||
|
||||
def test_tool_between_fallback_blocked_in_strict_mode(self):
|
||||
"""The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
|
||||
blocked in strict mode."""
|
||||
engine = StreamingParserEngine(
|
||||
_func_prefix_config(), _make_func_prefix_tokenizer()
|
||||
)
|
||||
|
||||
engine.feed("<tool_call>", [_TOOL_START_ID])
|
||||
engine.feed("<function=a>", [2, 3])
|
||||
engine.feed("args", [4])
|
||||
engine.feed("</function>", [5, 6])
|
||||
engine.feed("</tool_call>", [_TOOL_END_ID])
|
||||
|
||||
events = engine.feed("<function=b>more</function>", [7, 8, 9])
|
||||
events.extend(engine.finish())
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START not in types
|
||||
|
||||
|
||||
class TestNoUnusedTokenizerAttr:
|
||||
"""StreamingParserEngine no longer stores a redundant _tokenizer."""
|
||||
|
||||
def test_no_tokenizer_attribute(self):
|
||||
config = ParserEngineConfig(name="test")
|
||||
engine = StreamingParserEngine(config, tokenizer=None)
|
||||
assert not hasattr(engine, "_tokenizer")
|
||||
|
||||
|
||||
class TestArgsResetOnReentry:
|
||||
"""When leaving TOOL_ARGS and later re-entering (e.g. two tool
|
||||
calls), the entering-TOOL_ARGS block resets args tracking. The
|
||||
redundant reset on exit was removed."""
|
||||
|
||||
@staticmethod
|
||||
def _multi_tool_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="multi_tool",
|
||||
terminals={
|
||||
"TOOL_START": "<tool_call>",
|
||||
"TOOL_END": "</tool_call>",
|
||||
"TOOL_SEP": "<tool_sep>",
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.TOOL_BETWEEN,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
(ParserState.TOOL_BETWEEN, "TOOL_SEP"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
)
|
||||
|
||||
def test_args_tracking_across_reentry(self):
|
||||
engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
|
||||
|
||||
events = engine.feed(
|
||||
'<tool_call>{"city": "SF"}</tool_call>'
|
||||
"<tool_sep>"
|
||||
'{"name": "bar"}</tool_call>',
|
||||
[],
|
||||
)
|
||||
|
||||
tool_starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
|
||||
tool_ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
|
||||
arg_chunks = [e for e in events if e.type == EventType.ARG_VALUE_CHUNK]
|
||||
|
||||
assert len(tool_starts) == 2
|
||||
assert len(tool_ends) == 2
|
||||
assert tool_starts[0].tool_index == 0
|
||||
assert tool_starts[1].tool_index == 1
|
||||
|
||||
first_args = "".join(e.value for e in arg_chunks if e.tool_index == 0)
|
||||
second_args = "".join(e.value for e in arg_chunks if e.tool_index == 1)
|
||||
assert '"city"' in first_args
|
||||
assert '"name"' in second_args
|
||||
|
||||
def test_brace_depth_resets_on_reentry(self):
|
||||
"""Verify _args_brace_depth resets when re-entering TOOL_ARGS."""
|
||||
engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
|
||||
engine.feed("<tool_call>", [])
|
||||
assert engine.state == ParserState.TOOL_ARGS
|
||||
assert engine._args_brace_depth == 0
|
||||
|
||||
engine.feed('{"a": 1}', [])
|
||||
engine.feed("</tool_call>", [])
|
||||
assert engine.state == ParserState.TOOL_BETWEEN
|
||||
|
||||
engine.feed("<tool_sep>", [])
|
||||
assert engine.state == ParserState.TOOL_ARGS
|
||||
assert engine._args_brace_depth == 0
|
||||
assert engine._args_in_string is False
|
||||
assert engine._args_escape_next is False
|
||||
|
||||
|
||||
class TestToolPreambleFinish:
|
||||
"""finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
|
||||
call was started (tool_index >= 0), but not when tool_index is -1."""
|
||||
|
||||
@staticmethod
|
||||
def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="preamble_tcs",
|
||||
terminals={"TOOL_START": "<tool_call>"},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
},
|
||||
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _preamble_without_tool_call_start_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="preamble_no_tcs",
|
||||
terminals={"TOOL_CALLS_START": "<tool_calls>"},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_CALLS_START"): Transition(
|
||||
ParserState.TOOL_PREAMBLE,
|
||||
(),
|
||||
),
|
||||
},
|
||||
content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
|
||||
)
|
||||
|
||||
def test_finish_emits_tool_call_end_with_tool_index(self):
|
||||
config = self._preamble_with_tool_call_start_config()
|
||||
engine = StreamingParserEngine(config, tokenizer=None)
|
||||
|
||||
engine.feed("<tool_call>", [])
|
||||
assert engine.state == ParserState.TOOL_PREAMBLE
|
||||
assert engine.tool_index == 0
|
||||
|
||||
finish_events = engine.finish()
|
||||
end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
|
||||
assert len(end_events) == 1
|
||||
assert end_events[0].tool_index == 0
|
||||
|
||||
def test_finish_no_tool_call_end_without_tool_index(self):
|
||||
config = self._preamble_without_tool_call_start_config()
|
||||
engine = StreamingParserEngine(config, tokenizer=None)
|
||||
|
||||
engine.feed("<tool_calls>", [])
|
||||
assert engine.state == ParserState.TOOL_PREAMBLE
|
||||
assert engine.tool_index == -1
|
||||
|
||||
finish_events = engine.finish()
|
||||
end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
|
||||
assert len(end_events) == 0
|
||||
assert engine.state == ParserState.CONTENT
|
||||
|
||||
|
||||
class TestRegexTerminalInfraRemoved:
|
||||
"""TerminalDef.priority, LexerShape.regex_terminals, and the regex
|
||||
matching loop were removed."""
|
||||
|
||||
def test_terminal_def_no_priority(self):
|
||||
import regex as re
|
||||
|
||||
td = TerminalDef(name="X", pattern=re.compile("x"))
|
||||
assert not hasattr(td, "priority")
|
||||
|
||||
def test_lexer_shape_no_regex_terminals(self):
|
||||
shape = LexerShape([])
|
||||
assert not hasattr(shape, "regex_terminals")
|
||||
|
||||
def test_terminals_from_literals_still_works(self):
|
||||
literals = {"TOOL_START": "<tool_call>", "TOOL_END": "</tool_call>"}
|
||||
defs = terminals_from_literals(literals)
|
||||
assert len(defs) == 2
|
||||
names = {d.name for d in defs}
|
||||
assert names == {"TOOL_START", "TOOL_END"}
|
||||
for d in defs:
|
||||
assert d.is_literal
|
||||
assert d.literal in ("<tool_call>", "</tool_call>")
|
||||
|
||||
|
||||
class TestMultiCharTerminalInArgs:
|
||||
"""Regression: multi-char terminals falling through in TOOL_ARGS
|
||||
must be fed char-by-char via _feed_args_text, not _feed_args_char."""
|
||||
|
||||
@staticmethod
|
||||
def _newline_config() -> ParserEngineConfig:
|
||||
return ParserEngineConfig(
|
||||
name="newline_test",
|
||||
terminals={
|
||||
"TOOL_START": "<tool_call>",
|
||||
"TOOL_END": "</tool_call>",
|
||||
"NEWLINE": "\n",
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "TOOL_START"): Transition(
|
||||
ParserState.TOOL_ARGS,
|
||||
(EventType.TOOL_CALL_START,),
|
||||
),
|
||||
(ParserState.TOOL_ARGS, "TOOL_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.TOOL_CALL_END,),
|
||||
),
|
||||
},
|
||||
content_events={
|
||||
ParserState.CONTENT: EventType.TEXT_CHUNK,
|
||||
ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
|
||||
},
|
||||
)
|
||||
|
||||
def test_newline_in_args_parsed_correctly(self):
|
||||
engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
|
||||
text = '<tool_call>{"name": "f",\n"arguments": {"a": 1}}</tool_call>'
|
||||
events = engine.parse_complete(text)
|
||||
|
||||
arg_text = "".join(
|
||||
e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
assert '"name": "f"' in arg_text
|
||||
assert '"arguments"' in arg_text
|
||||
|
||||
def test_newline_in_args_streaming(self):
|
||||
engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
|
||||
all_events = TestStreaming._feed_chars(
|
||||
engine, '<tool_call>{"name": "f",\n"a": 1}</tool_call>'
|
||||
)
|
||||
|
||||
arg_text = "".join(
|
||||
e.value for e in all_events if e.type == EventType.ARG_VALUE_CHUNK
|
||||
)
|
||||
assert '"name": "f"' in arg_text
|
||||
assert '"a": 1' in arg_text
|
||||
|
||||
|
||||
class TestSkipToolParsing:
|
||||
"""When skip_tool_parsing is set, tool tags become content."""
|
||||
|
||||
def test_tool_tags_emitted_as_content(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
engine.skip_tool_parsing = True
|
||||
|
||||
text = '<tool_call>{"name": "f"}</tool_call>'
|
||||
events = engine.parse_complete(text)
|
||||
|
||||
types = [e.type for e in events]
|
||||
assert EventType.TOOL_CALL_START not in types
|
||||
assert EventType.TOOL_CALL_END not in types
|
||||
|
||||
content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "<tool_call>" in content
|
||||
assert "</tool_call>" in content
|
||||
|
||||
def test_skip_tool_streaming(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
engine.skip_tool_parsing = True
|
||||
|
||||
all_events = TestStreaming._feed_chars(
|
||||
engine, '<tool_call>{"name": "f"}</tool_call>'
|
||||
)
|
||||
|
||||
types = [e.type for e in all_events]
|
||||
assert EventType.TOOL_CALL_START not in types
|
||||
|
||||
content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
|
||||
assert "<tool_call>" in content
|
||||
|
||||
def test_reset_preserves_skip_tool_parsing(self):
|
||||
engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
|
||||
engine.skip_tool_parsing = True
|
||||
engine.reset()
|
||||
assert engine.skip_tool_parsing is True
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,549 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the engine-based Qwen3 reasoning parser.
|
||||
|
||||
Validates that ``Qwen3Parser`` correctly handles
|
||||
``<think>``/``</think>`` reasoning with Qwen3-specific extensions:
|
||||
- ``<tool_call>`` as implicit reasoning end (terminal + token ID)
|
||||
- Stripping ``<think>`` from generated output (old template compat)
|
||||
- No terminal text (``</think>``, ``<tool_call>``) leaks into output
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.conftest import make_mock_tokenizer
|
||||
from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
|
||||
from vllm.parser.engine.parser_engine_config import ParserState
|
||||
from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
|
||||
|
||||
_THINK_START_ID = 50
|
||||
_THINK_END_ID = 51
|
||||
_TOOL_CALL_ID = 60
|
||||
_TOOL_CALL_END_ID = 61
|
||||
_TEXT_ID = 100
|
||||
|
||||
_QWEN3_VOCAB = {
|
||||
"<think>": _THINK_START_ID,
|
||||
"</think>": _THINK_END_ID,
|
||||
"<tool_call>": _TOOL_CALL_ID,
|
||||
"</tool_call>": _TOOL_CALL_END_ID,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_tokenizer():
|
||||
return make_mock_tokenizer(_QWEN3_VOCAB)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def parser(mock_tokenizer):
|
||||
return Qwen3Parser(mock_tokenizer)
|
||||
|
||||
|
||||
class TestNonStreaming:
|
||||
def test_reasoning_then_content(self, parser):
|
||||
text = "<think>Let me analyze.</think>The answer is 42."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "Let me analyze."
|
||||
assert content == "The answer is 42."
|
||||
|
||||
def test_no_start_token_in_output(self, parser):
|
||||
"""Qwen3.5+ style: <think> in prompt, only </think> in output."""
|
||||
text = "Let me think about this.</think>The answer is 42."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "Let me think about this."
|
||||
assert content == "The answer is 42."
|
||||
|
||||
def test_reasoning_only(self, parser):
|
||||
text = "<think>Still thinking...</think>"
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "Still thinking..."
|
||||
assert content is None
|
||||
|
||||
def test_no_end_tag_all_reasoning(self, parser):
|
||||
"""No </think> means truncated output — everything is reasoning."""
|
||||
text = "Hello, no reasoning here."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "Hello, no reasoning here."
|
||||
assert content is None
|
||||
|
||||
def test_multiline_reasoning(self, parser):
|
||||
text = (
|
||||
"<think>Step 1: parse.\nStep 2: compute.\nStep 3: output.</think>Result: 7."
|
||||
)
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert "Step 1" in reasoning
|
||||
assert "Step 3" in reasoning
|
||||
assert content == "Result: 7."
|
||||
|
||||
def test_tool_call_implicit_end(self, parser):
|
||||
"""<tool_call> without </think> acts as implicit reasoning end."""
|
||||
text = (
|
||||
"<think>I need to read the file.\n\n"
|
||||
"<tool_call>\n<function=bash>\n"
|
||||
"<parameter=cmd>ls</parameter>\n"
|
||||
"</function>\n</tool_call>"
|
||||
)
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "I need to read the file.\n\n"
|
||||
assert "</think>" not in reasoning
|
||||
assert "<tool_call>" not in reasoning
|
||||
|
||||
def test_tool_call_implicit_end_no_think(self, parser):
|
||||
"""<tool_call> as implicit end, no <think> in output."""
|
||||
text = (
|
||||
"I need to read the file.\n\n"
|
||||
"<tool_call>\n<function=bash>\n"
|
||||
"<parameter=cmd>ls</parameter>\n"
|
||||
"</function>\n</tool_call>"
|
||||
)
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "I need to read the file.\n\n"
|
||||
assert "<tool_call>" not in reasoning
|
||||
|
||||
def test_live_scenario_think_end_before_tool_call(self, parser):
|
||||
"""Real model output: </think> immediately before <tool_call>.
|
||||
|
||||
Regression test for the bug where </think> and <parameter=...>
|
||||
leaked into reasoning content.
|
||||
"""
|
||||
text = (
|
||||
"The user wants to see what files are in the current directory"
|
||||
" and their contents. Let me start by listing the directory."
|
||||
"</think><tool_call><function=read>"
|
||||
"<parameter=filePath>/Users/test/demo</parameter>"
|
||||
"</function></tool_call>"
|
||||
)
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
expected_reasoning = (
|
||||
"The user wants to see what files are in the current directory"
|
||||
" and their contents. Let me start by listing the directory."
|
||||
)
|
||||
assert reasoning == expected_reasoning
|
||||
assert "</think>" not in reasoning
|
||||
assert "<tool_call>" not in reasoning
|
||||
assert "<parameter=" not in reasoning
|
||||
|
||||
def test_no_terminal_text_in_reasoning(self, parser):
|
||||
"""Terminal text must never appear in reasoning output."""
|
||||
text = "Reasoning here.</think>Content here."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert "</think>" not in (reasoning or "")
|
||||
assert "<think>" not in (reasoning or "")
|
||||
|
||||
def test_no_terminal_text_in_content(self, parser):
|
||||
"""Terminal text must never appear in content output."""
|
||||
text = "Reasoning here.</think>Content here."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert "</think>" not in (content or "")
|
||||
assert "<think>" not in (content or "")
|
||||
|
||||
def test_duplicate_think_end_absorbed(self, parser):
|
||||
"""Duplicate </think> in CONTENT state must not leak."""
|
||||
text = "Reasoning here.</think>Content here.</think>More content."
|
||||
reasoning, content = parser.extract_reasoning(text, None)
|
||||
assert reasoning == "Reasoning here."
|
||||
assert content == "Content here.More content."
|
||||
|
||||
|
||||
class TestIsReasoningEnd:
|
||||
def test_think_end_token(self, parser):
|
||||
assert parser.is_reasoning_end([_THINK_START_ID, 1, _THINK_END_ID])
|
||||
|
||||
def test_no_end_token(self, parser):
|
||||
assert not parser.is_reasoning_end([_THINK_START_ID, 1, 2])
|
||||
|
||||
def test_start_after_end_means_not_ended(self, parser):
|
||||
assert not parser.is_reasoning_end([_THINK_END_ID, _THINK_START_ID, 1])
|
||||
|
||||
def test_tool_call_as_implicit_end(self, parser):
|
||||
"""Unpaired <tool_call> is implicit reasoning end."""
|
||||
assert parser.is_reasoning_end([_THINK_START_ID, 1, _TOOL_CALL_ID])
|
||||
|
||||
def test_paired_tool_call_not_end(self, parser):
|
||||
"""Paired <tool_call>...</tool_call> (from template) is NOT end."""
|
||||
assert not parser.is_reasoning_end(
|
||||
[_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
|
||||
)
|
||||
|
||||
def test_tool_call_after_think_end(self, parser):
|
||||
"""<tool_call> after </think> — already ended."""
|
||||
assert parser.is_reasoning_end(
|
||||
[_THINK_START_ID, 1, _THINK_END_ID, _TOOL_CALL_ID]
|
||||
)
|
||||
|
||||
def test_empty_ids(self, parser):
|
||||
assert not parser.is_reasoning_end([])
|
||||
|
||||
|
||||
class TestStreaming:
|
||||
def test_basic_streaming(self, parser):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["<think>", "thinking", " hard", "</think>", "done"],
|
||||
[
|
||||
(_THINK_START_ID,),
|
||||
(1,),
|
||||
(2,),
|
||||
(_THINK_END_ID,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "thinking hard"
|
||||
assert content == "done"
|
||||
|
||||
def test_streaming_no_start_token(self, parser):
|
||||
"""Qwen3.5 style: no <think> in output, just reasoning then </think>."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["reasoning ", "text", "</think>", "content"],
|
||||
[
|
||||
(1,),
|
||||
(2,),
|
||||
(_THINK_END_ID,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "reasoning text"
|
||||
assert content == "content"
|
||||
|
||||
def test_streaming_start_token_stripped(self, parser):
|
||||
"""<think> in output (old template) should be stripped."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["<think>reasoning", "</think>", "content"],
|
||||
[
|
||||
(_THINK_START_ID, 1),
|
||||
(_THINK_END_ID,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "reasoning"
|
||||
assert content == "content"
|
||||
|
||||
def test_streaming_tool_call_implicit_end(self, parser):
|
||||
"""<tool_call> ends reasoning implicitly during streaming."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["I need to check.", "<tool_call>", "\n<function=test>"],
|
||||
[
|
||||
(1,),
|
||||
(_TOOL_CALL_ID,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "I need to check."
|
||||
assert "<tool_call>" not in reasoning
|
||||
assert "</think>" not in reasoning
|
||||
assert content is not None
|
||||
|
||||
def test_streaming_content_after_think_end(self, parser):
|
||||
"""Content deltas after </think> are routed as content."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["reasoning", "</think>", "content1", " content2"],
|
||||
[
|
||||
(1,),
|
||||
(_THINK_END_ID,),
|
||||
(2,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "reasoning"
|
||||
assert content == "content1 content2"
|
||||
|
||||
def test_streaming_content_after_tool_call(self, parser):
|
||||
"""Content deltas after <tool_call> are routed as content."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["thinking", "<tool_call>", "<function=f>"],
|
||||
[
|
||||
(1,),
|
||||
(_TOOL_CALL_ID,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "thinking"
|
||||
assert "<tool_call>" not in reasoning
|
||||
assert content is not None
|
||||
|
||||
def test_streaming_end_grouped_with_content(self, parser):
|
||||
"""</think> grouped with following content in one delta."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["reasoning", "</think>the answer"],
|
||||
[
|
||||
(1,),
|
||||
(_THINK_END_ID, 2),
|
||||
],
|
||||
)
|
||||
assert reasoning == "reasoning"
|
||||
assert content == "the answer"
|
||||
|
||||
def test_streaming_think_and_end_in_one_delta(self, parser):
|
||||
"""<think> and </think> in the same delta."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["<think>reasoning</think>"],
|
||||
[
|
||||
(_THINK_START_ID, 1, _THINK_END_ID),
|
||||
],
|
||||
)
|
||||
assert reasoning == "reasoning"
|
||||
assert content == ""
|
||||
|
||||
def test_streaming_pure_content_no_think(self, parser):
|
||||
"""No think tokens at all — everything is reasoning (truncated)."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["hello ", "world"],
|
||||
[
|
||||
(1,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "hello world"
|
||||
assert content == ""
|
||||
|
||||
def test_streaming_think_end_and_tool_call_same_delta(self, parser):
|
||||
"""</think> and <tool_call> in the same delta — no leakage.
|
||||
|
||||
Regression test: the old override split at <tool_call> without
|
||||
stripping </think>, causing </think> to leak into reasoning.
|
||||
"""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
[
|
||||
"Let me list the directory.",
|
||||
"</think><tool_call>",
|
||||
"<function=read>",
|
||||
"<parameter=filePath>/tmp</parameter>",
|
||||
],
|
||||
[
|
||||
(1,),
|
||||
(_THINK_END_ID, _TOOL_CALL_ID),
|
||||
(2,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "Let me list the directory."
|
||||
assert "</think>" not in reasoning
|
||||
assert "<tool_call>" not in reasoning
|
||||
assert "<parameter=" not in reasoning
|
||||
assert content is not None
|
||||
|
||||
def test_streaming_no_terminal_text_leaks(self, parser):
|
||||
"""Terminal text must never appear in reasoning or content."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["reasoning", "</think>", "content"],
|
||||
[
|
||||
(1,),
|
||||
(_THINK_END_ID,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert "</think>" not in reasoning
|
||||
assert "</think>" not in content
|
||||
assert "<think>" not in reasoning
|
||||
|
||||
def test_streaming_duplicate_think_end_absorbed(self, parser):
|
||||
"""Duplicate </think> token in CONTENT state must not leak."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser,
|
||||
["reasoning", "</think>", "content", "</think>", "more"],
|
||||
[
|
||||
(1,),
|
||||
(_THINK_END_ID,),
|
||||
(2,),
|
||||
(_THINK_END_ID,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "reasoning"
|
||||
assert content == "contentmore"
|
||||
|
||||
|
||||
class TestTrailingWhitespaceStripping:
|
||||
"""When strip_trailing_reasoning_whitespace is True,
|
||||
trailing whitespace before </think> must be stripped.
|
||||
|
||||
Models often generate trailing newlines before </think>, and these
|
||||
accumulate across multi-turn conversations via a feedback loop.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def parser_with_strip(self):
|
||||
cfg = dataclasses.replace(
|
||||
qwen3_config(),
|
||||
strip_trailing_reasoning_whitespace=True,
|
||||
)
|
||||
return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
|
||||
|
||||
def test_non_streaming_trailing_newline(self, parser_with_strip):
|
||||
text = "Reasoning here.\n</think>Content."
|
||||
reasoning, content = parser_with_strip.extract_reasoning(text, None)
|
||||
assert reasoning == "Reasoning here."
|
||||
assert content == "Content."
|
||||
|
||||
def test_non_streaming_multiple_trailing_newlines(self, parser_with_strip):
|
||||
text = "Reasoning here.\n\n\n</think>Content."
|
||||
reasoning, content = parser_with_strip.extract_reasoning(text, None)
|
||||
assert reasoning == "Reasoning here."
|
||||
assert content == "Content."
|
||||
|
||||
def test_non_streaming_internal_newlines_preserved(self, parser_with_strip):
|
||||
text = "Step 1.\n\nStep 2.\n\nStep 3.</think>Answer."
|
||||
reasoning, content = parser_with_strip.extract_reasoning(text, None)
|
||||
assert reasoning == "Step 1.\n\nStep 2.\n\nStep 3."
|
||||
assert content == "Answer."
|
||||
|
||||
def test_non_streaming_only_newlines_becomes_none(self, parser_with_strip):
|
||||
text = "\n\n\n</think>Content."
|
||||
reasoning, content = parser_with_strip.extract_reasoning(text, None)
|
||||
assert reasoning is None
|
||||
assert content == "Content."
|
||||
|
||||
def test_streaming_trailing_newline_stripped(self, parser_with_strip):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser_with_strip,
|
||||
["thinking.\n", "</think>", "done"],
|
||||
[
|
||||
(1,),
|
||||
(_THINK_END_ID,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "thinking."
|
||||
assert content == "done"
|
||||
|
||||
def test_streaming_multiple_trailing_newlines_stripped(self, parser_with_strip):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser_with_strip,
|
||||
["thinking.\n", "\n", "\n", "</think>", "done"],
|
||||
[
|
||||
(1,),
|
||||
(2,),
|
||||
(3,),
|
||||
(_THINK_END_ID,),
|
||||
(4,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "thinking."
|
||||
assert content == "done"
|
||||
|
||||
def test_streaming_internal_newlines_preserved(self, parser_with_strip):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser_with_strip,
|
||||
["Step 1.\n", "\nStep 2.\n", "</think>", "Answer"],
|
||||
[
|
||||
(1,),
|
||||
(2,),
|
||||
(_THINK_END_ID,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "Step 1.\n\nStep 2."
|
||||
assert content == "Answer"
|
||||
|
||||
def test_streaming_trailing_newlines_before_tool_call(self, parser_with_strip):
|
||||
"""Trailing newlines before implicit <tool_call> end are stripped."""
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser_with_strip,
|
||||
["I'll check.\n\n", "<tool_call>", "<function=test>"],
|
||||
[
|
||||
(1,),
|
||||
(_TOOL_CALL_ID,),
|
||||
(2,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "I'll check."
|
||||
assert "<tool_call>" not in reasoning
|
||||
|
||||
|
||||
class TestWhitespaceStrippingDisabled:
|
||||
"""When strip_trailing_reasoning_whitespace is False,
|
||||
trailing whitespace in reasoning must be preserved."""
|
||||
|
||||
@pytest.fixture
|
||||
def parser_no_strip(self):
|
||||
cfg = dataclasses.replace(
|
||||
qwen3_config(),
|
||||
strip_trailing_reasoning_whitespace=False,
|
||||
)
|
||||
return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
|
||||
|
||||
def test_non_streaming_preserves_trailing_newline(self, parser_no_strip):
|
||||
text = "Reasoning here.\n</think>Content."
|
||||
reasoning, content = parser_no_strip.extract_reasoning(text, None)
|
||||
assert reasoning == "Reasoning here.\n"
|
||||
assert content == "Content."
|
||||
|
||||
def test_streaming_preserves_trailing_newlines(self, parser_no_strip):
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
parser_no_strip,
|
||||
["thinking.\n", "\n", "</think>", "done"],
|
||||
[
|
||||
(1,),
|
||||
(2,),
|
||||
(_THINK_END_ID,),
|
||||
(3,),
|
||||
],
|
||||
)
|
||||
assert reasoning == "thinking.\n\n"
|
||||
assert content == "done"
|
||||
|
||||
|
||||
class TestThinkingDisabled:
|
||||
"""When ``enable_thinking=False``, the chat template pre-fills a closed
|
||||
``<think>\\n\\n</think>\\n\\n`` block. The model output starts in content
|
||||
state, so the parser's initial state must be CONTENT — not REASONING.
|
||||
"""
|
||||
|
||||
def test_thinking_disabled_initial_state_is_content(self, mock_tokenizer):
|
||||
p = Qwen3Parser(
|
||||
mock_tokenizer,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
assert p.parser_engine_config.initial_state == ParserState.CONTENT
|
||||
|
||||
def test_thinking_enabled_initial_state_is_reasoning(self, mock_tokenizer):
|
||||
p = Qwen3Parser(
|
||||
mock_tokenizer,
|
||||
chat_template_kwargs={"enable_thinking": True},
|
||||
)
|
||||
assert p.parser_engine_config.initial_state == ParserState.REASONING
|
||||
|
||||
def test_default_initial_state_is_reasoning(self, mock_tokenizer):
|
||||
p = Qwen3Parser(mock_tokenizer)
|
||||
assert p.parser_engine_config.initial_state == ParserState.REASONING
|
||||
|
||||
def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
|
||||
"""Plain text with thinking disabled must stream as content, not
|
||||
reasoning. Before the fix, the REASONING initial state caused all
|
||||
output to be emitted as reasoning chunks."""
|
||||
p = Qwen3Parser(
|
||||
mock_tokenizer,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
reasoning, content = simulate_reasoning_streaming(
|
||||
p,
|
||||
["The answer", " is 42."],
|
||||
[
|
||||
(_TEXT_ID,),
|
||||
(_TEXT_ID,),
|
||||
],
|
||||
)
|
||||
assert content == "The answer is 42."
|
||||
assert reasoning == ""
|
||||
|
||||
def test_thinking_disabled_non_streaming(self, mock_tokenizer):
|
||||
p = Qwen3Parser(
|
||||
mock_tokenizer,
|
||||
chat_template_kwargs={"enable_thinking": False},
|
||||
)
|
||||
reasoning, content = p.extract_reasoning("The answer is 42.", None)
|
||||
assert reasoning is None
|
||||
assert content == "The answer is 42."
|
||||
@@ -0,0 +1,189 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
|
||||
|
||||
Replays dynamically built token sequences at different chunk sizes and
|
||||
holdback depths to verify chunk-size invariance and terminal-token hygiene.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
_test_request,
|
||||
assert_no_terminal_leakage,
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
replay_streaming,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import build_samples
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Qwen3Parser,
|
||||
)
|
||||
|
||||
_ENGINE_PARSERS: dict[str, type[Parser]] = {
|
||||
"qwen3_engine": Qwen3Parser,
|
||||
}
|
||||
|
||||
_qwen3_samples = build_samples("qwen3")
|
||||
|
||||
_QWEN3_TERMINALS = [
|
||||
"<think>",
|
||||
"</think>",
|
||||
"<tool_call>",
|
||||
"</tool_call>",
|
||||
"<function=",
|
||||
"</function>",
|
||||
]
|
||||
|
||||
HOLDBACK_CONFIGS = [6, 12, 24]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
|
||||
@pytest.mark.parametrize("chunk_size", [5, 10], ids=lambda c: f"chunk{c}")
|
||||
@pytest.mark.parametrize("sample", _qwen3_samples, ids=lambda s: s.id)
|
||||
class TestQwen3ReplayWithHoldback:
|
||||
"""Replay Qwen3 with simulated detokenizer holdback."""
|
||||
|
||||
def test_replay(self, sample, chunk_size, holdback):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = Qwen3Parser(tokenizer, sample.tools)
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
holdback_chars=holdback,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert_parse_output(output, sample)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
_QWEN3_TERMINALS,
|
||||
context=f"chunk_size={chunk_size}, holdback={holdback}",
|
||||
)
|
||||
|
||||
|
||||
_TOOL_CALL_SAMPLES = [
|
||||
(Qwen3Parser, s)
|
||||
for s in _qwen3_samples
|
||||
if s.expected_tool_calls and s.expected_reasoning
|
||||
]
|
||||
|
||||
|
||||
def _suppressed_expectations(sample) -> tuple[str, str]:
|
||||
"""Compute expected (reasoning, content) when tools are suppressed.
|
||||
|
||||
When an explicit reasoning-end delimiter (``</think>``, ``<channel|>``)
|
||||
is present, reasoning ends there and the tool call block becomes content.
|
||||
When reasoning ends implicitly (the tool-start token triggers both
|
||||
REASONING_END and TOOL_CALL_START), reasoning still ends at the tool
|
||||
start and the raw tool call block becomes content text — only the
|
||||
structured tool parsing is suppressed, not the reasoning boundary.
|
||||
"""
|
||||
full_text = "".join(text for _, text in sample.tokens)
|
||||
reasoning = sample.expected_reasoning
|
||||
idx = full_text.find(reasoning)
|
||||
if idx < 0:
|
||||
return (full_text, "")
|
||||
after_reasoning = full_text[idx + len(reasoning) :]
|
||||
for delim in ("</think>", "<channel|>"):
|
||||
pos = after_reasoning.find(delim)
|
||||
if pos >= 0:
|
||||
return (reasoning, after_reasoning[pos + len(delim) :])
|
||||
for delim in ("<tool_call>",):
|
||||
pos = after_reasoning.find(delim)
|
||||
if pos >= 0:
|
||||
return (reasoning, after_reasoning[pos:])
|
||||
return (full_text, "")
|
||||
|
||||
|
||||
_DUMMY_TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "stub", "parameters": {"type": "object"}},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
|
||||
@pytest.mark.parametrize(
|
||||
"parser_cls,sample",
|
||||
_TOOL_CALL_SAMPLES,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else v.__name__,
|
||||
)
|
||||
class TestSkipToolParsingReplay:
|
||||
"""Replay with skip_tool_parsing=True (tool_choice='none').
|
||||
|
||||
Verifies that reasoning is extracted normally and the raw tool call
|
||||
block appears as content text with no tool calls parsed.
|
||||
"""
|
||||
|
||||
def test_replay(self, parser_cls, sample, chunk_size):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
kwargs = {}
|
||||
if sample.chat_template_kwargs:
|
||||
kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
|
||||
parser = parser_cls(tokenizer, **kwargs)
|
||||
|
||||
request = _test_request()
|
||||
request.tool_choice = "none"
|
||||
request.tools = _DUMMY_TOOLS
|
||||
|
||||
all_ids = [tid for tid, _ in sample.tokens]
|
||||
all_texts = [text for _, text in sample.tokens]
|
||||
if chunk_size is None:
|
||||
chunk_size = len(all_ids)
|
||||
|
||||
results = []
|
||||
chunks = list(range(0, len(all_ids), chunk_size))
|
||||
for i, start in enumerate(chunks):
|
||||
end = min(start + chunk_size, len(all_ids))
|
||||
is_last = i == len(chunks) - 1
|
||||
result = parser.parse_delta(
|
||||
"".join(all_texts[start:end]),
|
||||
all_ids[start:end],
|
||||
request,
|
||||
prompt_token_ids=[] if start == 0 else None,
|
||||
finished=is_last,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
output = collect_output(results)
|
||||
|
||||
expected_reasoning, expected_content = _suppressed_expectations(sample)
|
||||
|
||||
assert output.reasoning == expected_reasoning, (
|
||||
f"Reasoning mismatch:\n"
|
||||
f" expected: {expected_reasoning!r}\n"
|
||||
f" actual: {output.reasoning!r}"
|
||||
)
|
||||
assert output.tool_calls == [], (
|
||||
f"Expected no tool calls but got {output.tool_calls}"
|
||||
)
|
||||
assert output.content == expected_content, (
|
||||
f"Content mismatch:\n"
|
||||
f" expected: {expected_content!r}\n"
|
||||
f" actual: {output.content!r}"
|
||||
)
|
||||
|
||||
|
||||
class TestAdapterReferences:
|
||||
"""Verify make_adapters sets reasoning/tool parser class refs on parser engine
|
||||
parser classes so the serving layer finds them and calls adjust_request."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"parser_name",
|
||||
list(_ENGINE_PARSERS.keys()),
|
||||
)
|
||||
def test_adapter_cls_refs_set(self, parser_name):
|
||||
parser_cls = _ENGINE_PARSERS[parser_name]
|
||||
assert parser_cls.reasoning_parser_cls is not None, (
|
||||
f"{parser_name}: reasoning_parser_cls is None"
|
||||
)
|
||||
assert parser_cls.tool_parser_cls is not None, (
|
||||
f"{parser_name}: tool_parser_cls is None"
|
||||
)
|
||||
@@ -0,0 +1,631 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for TokenIDScanner, focusing on hold-back text recovery.
|
||||
|
||||
Uses gemma4_config for all end-to-end engine tests, covering
|
||||
reasoning channels, tool calls, and combined flows."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.token_id_scanner import (
|
||||
PreLexedTerminal,
|
||||
TextChunk,
|
||||
TokenIDScanner,
|
||||
)
|
||||
|
||||
CHANNEL_START = "<|channel>"
|
||||
CHANNEL_END = "<channel|>"
|
||||
CHANNEL_START_ID = 100
|
||||
CHANNEL_END_ID = 101
|
||||
REGULAR_TOKEN_ID = 200
|
||||
TOOL_START = "<tool_call>"
|
||||
TOOL_END = "</tool_call>"
|
||||
TOOL_START_ID = 110
|
||||
TOOL_END_ID = 111
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tokenizer():
|
||||
tok = MagicMock()
|
||||
tok.get_vocab.return_value = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
CHANNEL_END: CHANNEL_END_ID,
|
||||
}
|
||||
tok.decode.side_effect = lambda ids: {
|
||||
CHANNEL_START_ID: CHANNEL_START,
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
REGULAR_TOKEN_ID: "regular",
|
||||
}.get(ids[0], f"<unk:{ids[0]}>")
|
||||
return tok
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def scanner(tokenizer):
|
||||
return TokenIDScanner(
|
||||
token_id_to_terminal={
|
||||
CHANNEL_START_ID: "THINK_START",
|
||||
CHANNEL_END_ID: "THINK_END",
|
||||
},
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
|
||||
|
||||
class TestJoinDecodedTextReturnsStr:
|
||||
"""_join_decoded_text now returns str unconditionally (was
|
||||
str | None when an isinstance guard made a branch unreachable)."""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
|
||||
|
||||
def test_mixed_items(self, bare_scanner):
|
||||
items = [
|
||||
TextChunk("hello "),
|
||||
PreLexedTerminal("TOOL_START", 42, "<tool_call>"),
|
||||
TextChunk(" world"),
|
||||
]
|
||||
result = bare_scanner._join_decoded_text(items)
|
||||
assert isinstance(result, str)
|
||||
assert result == "hello <tool_call> world"
|
||||
|
||||
def test_empty_list(self, bare_scanner):
|
||||
result = bare_scanner._join_decoded_text([])
|
||||
assert isinstance(result, str)
|
||||
assert result == ""
|
||||
|
||||
def test_only_text_chunks(self, bare_scanner):
|
||||
result = bare_scanner._join_decoded_text([TextChunk("abc"), TextChunk("def")])
|
||||
assert result == "abcdef"
|
||||
|
||||
|
||||
class TestHoldbackTextRecovery:
|
||||
def test_holdback_text_with_special_token_text_absent(self, scanner):
|
||||
"""delta_text has hold-back text but the special token's text is
|
||||
NOT in delta_text (held back by the detokenizer). Terminal is
|
||||
deferred until the text arrives in a subsequent delta."""
|
||||
result = scanner.scan(
|
||||
delta_text="processed is appropriate.",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Second scan: terminal text arrives (detokenizer flushes).
|
||||
# Deferred terminal resolves with holdback text before it.
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>Understood.",
|
||||
delta_token_ids=[20, 21],
|
||||
)
|
||||
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
|
||||
assert len(pre_lexed) == 1
|
||||
assert pre_lexed[0].terminal == "THINK_END"
|
||||
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
|
||||
combined = "".join(texts)
|
||||
assert "processed is appropriate." in combined
|
||||
assert "Understood." in combined
|
||||
|
||||
def test_holdback_text_with_special_token_text_present(self, scanner):
|
||||
"""delta_text includes hold-back text AND the special token text."""
|
||||
result = scanner.scan(
|
||||
delta_text="holdback text<channel|>",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
assert isinstance(result[0], TextChunk)
|
||||
assert result[0].text == "holdback text"
|
||||
assert isinstance(result[1], PreLexedTerminal)
|
||||
assert result[1].terminal == "THINK_END"
|
||||
|
||||
def test_no_holdback_text(self, scanner):
|
||||
"""delta_text is exactly the special token text — no hold-back."""
|
||||
result = scanner.scan(
|
||||
delta_text="<channel|>",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 1
|
||||
assert isinstance(result[0], PreLexedTerminal)
|
||||
assert result[0].terminal == "THINK_END"
|
||||
|
||||
def test_empty_delta_text(self, scanner):
|
||||
"""delta_text is empty — terminal deferred until text arrives."""
|
||||
result = scanner.scan(
|
||||
delta_text="",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
flushed = scanner.flush_pending()
|
||||
assert len(flushed) == 1
|
||||
assert isinstance(flushed[0], PreLexedTerminal)
|
||||
assert flushed[0].terminal == "THINK_END"
|
||||
|
||||
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
|
||||
"""delta_text="" with multiple tokens including special: all
|
||||
results deferred — individually-decoded TextChunks are unreliable
|
||||
and PreLexedTerminals wait for text confirmation."""
|
||||
tool_start_id = 400
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
tool_start_id: "<|tool_call>",
|
||||
tok_a: "call:",
|
||||
tok_b: "get_weather",
|
||||
}.get(ids[0], "?")
|
||||
|
||||
scanner = TokenIDScanner(
|
||||
token_id_to_terminal={tool_start_id: "TOOL_START"},
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
|
||||
result = scanner.scan(
|
||||
delta_text="",
|
||||
delta_token_ids=[tool_start_id, tok_a, tok_b],
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
flushed = scanner.flush_pending()
|
||||
assert len(flushed) == 1
|
||||
assert isinstance(flushed[0], PreLexedTerminal)
|
||||
assert flushed[0].terminal == "TOOL_START"
|
||||
|
||||
def test_holdback_before_start_tag(self, scanner):
|
||||
"""Hold-back text before a reasoning start tag."""
|
||||
result = scanner.scan(
|
||||
delta_text="prefix text<|channel>",
|
||||
delta_token_ids=[CHANNEL_START_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
assert isinstance(result[0], TextChunk)
|
||||
assert result[0].text == "prefix text"
|
||||
assert isinstance(result[1], PreLexedTerminal)
|
||||
assert result[1].terminal == "THINK_START"
|
||||
|
||||
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
|
||||
"""Stream-interval > 1: batch has regular tokens + special token.
|
||||
delta_text differs from individual decodes (context-dependent)."""
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
tok_a: "wordA",
|
||||
tok_b: "wordB",
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
}.get(ids[0], "?")
|
||||
|
||||
scanner_multi = TokenIDScanner(
|
||||
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
|
||||
result = scanner_multi.scan(
|
||||
delta_text="holdback wordA<channel|> wordB",
|
||||
delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
|
||||
)
|
||||
|
||||
texts = [r.text for r in result if isinstance(r, TextChunk)]
|
||||
terminals = [r.terminal for r in result if isinstance(r, PreLexedTerminal)]
|
||||
assert "THINK_END" in terminals
|
||||
assert "holdback wordA" in "".join(texts)
|
||||
|
||||
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
|
||||
"""Stream-interval > 1: batch has regular + special token, but
|
||||
delta_text doesn't contain the special token text at all
|
||||
(held back by detokenizer along with trailing regular tokens).
|
||||
Terminal is deferred until text arrives."""
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
tok_a: "alpha",
|
||||
tok_b: "beta",
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
}.get(ids[0], "?")
|
||||
|
||||
scanner_multi = TokenIDScanner(
|
||||
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
|
||||
result = scanner_multi.scan(
|
||||
delta_text="holdback alpha",
|
||||
delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Next delta: terminal text arrives (detokenizer flushes).
|
||||
# Deferred terminal resolves with holdback text before it.
|
||||
result2 = scanner_multi.scan(
|
||||
delta_text="<channel|> more text",
|
||||
delta_token_ids=[300],
|
||||
)
|
||||
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
|
||||
assert len(pre_lexed) == 1
|
||||
assert pre_lexed[0].terminal == "THINK_END"
|
||||
text_chunks = [r for r in result2 if isinstance(r, TextChunk)]
|
||||
combined = "".join(t.text for t in text_chunks)
|
||||
assert "holdback alpha" in combined
|
||||
assert "more text" in combined
|
||||
|
||||
def test_holdback_with_content_after_special_token(self, tokenizer):
|
||||
"""delta_text has hold-back + special token + content after,
|
||||
with corresponding token IDs for all parts."""
|
||||
tok_content = 210
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
tok_content: "content start",
|
||||
}.get(ids[0], "?")
|
||||
|
||||
scanner = TokenIDScanner(
|
||||
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
|
||||
result = scanner.scan(
|
||||
delta_text="reasoning end.<channel|>content start",
|
||||
delta_token_ids=[CHANNEL_END_ID, tok_content],
|
||||
)
|
||||
|
||||
pre_lexed = [r for r in result if isinstance(r, PreLexedTerminal)]
|
||||
assert len(pre_lexed) == 1
|
||||
assert pre_lexed[0].terminal == "THINK_END"
|
||||
|
||||
text_chunks = [r for r in result if isinstance(r, TextChunk)]
|
||||
combined = "".join(t.text for t in text_chunks)
|
||||
assert "reasoning end." in combined
|
||||
|
||||
|
||||
class TestDropTokens:
|
||||
def test_drop_token_with_holdback(self, tokenizer):
|
||||
"""Drop tokens stripped from delta_text, hold-back text preserved.
|
||||
Terminal is deferred when its text is absent from delta_text."""
|
||||
drop_id = 300
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
drop_id: "<eos>",
|
||||
}.get(ids[0], "?")
|
||||
|
||||
scanner = TokenIDScanner(
|
||||
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
|
||||
tokenizer=tokenizer,
|
||||
drop_token_ids={drop_id},
|
||||
)
|
||||
|
||||
result = scanner.scan(
|
||||
delta_text="holdback<eos>",
|
||||
delta_token_ids=[drop_id, CHANNEL_END_ID],
|
||||
)
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Terminal text arrives in next delta; deferred terminal resolves.
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>content",
|
||||
delta_token_ids=[20],
|
||||
)
|
||||
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
|
||||
assert len(pre_lexed) == 1
|
||||
assert pre_lexed[0].terminal == "THINK_END"
|
||||
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
|
||||
combined = "".join(texts)
|
||||
assert "holdback" in combined
|
||||
assert "<eos>" not in combined
|
||||
|
||||
assert len(scanner.flush_pending()) == 0
|
||||
|
||||
|
||||
class TestEndToEndReasoningHoldback:
|
||||
"""End-to-end tests through the full parser engine simulating
|
||||
stream-interval > 1 and detokenizer hold-back, using
|
||||
gemma4_config."""
|
||||
|
||||
def test_reasoning_content_not_truncated(self):
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
config = ParserEngineConfig(
|
||||
name="test_channel",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
tok = MagicMock()
|
||||
vocab = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
CHANNEL_END: CHANNEL_END_ID,
|
||||
}
|
||||
tok.get_vocab.return_value = vocab
|
||||
tok.decode.side_effect = lambda ids: {
|
||||
CHANNEL_START_ID: CHANNEL_START,
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
}.get(ids[0], f"tok{ids[0]}")
|
||||
|
||||
engine = StreamingParserEngine(config, tok)
|
||||
all_events = []
|
||||
|
||||
# Delta 1: channel start token (text includes start tag)
|
||||
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
|
||||
|
||||
# Delta 2: reasoning text (normal content, no special tokens)
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"thought\nThe request was received and ",
|
||||
[10, 11, 12, 13, 14],
|
||||
)
|
||||
)
|
||||
|
||||
# Delta 3: MORE reasoning text, the detokenizer held some back.
|
||||
# Then channel end token arrives in token_ids, but its text
|
||||
# is NOT in delta_text (held back by detokenizer).
|
||||
# delta_text = previously held-back reasoning text only.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"processed is appropriate.",
|
||||
[CHANNEL_END_ID],
|
||||
)
|
||||
)
|
||||
|
||||
# Delta 4: detokenizer flushes held-back channel end text
|
||||
# plus new content tokens.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"<channel|>Understood.",
|
||||
[20, 21],
|
||||
)
|
||||
)
|
||||
|
||||
all_events.extend(engine.finish())
|
||||
|
||||
reasoning_text = "".join(
|
||||
e.value for e in all_events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
content_text = "".join(
|
||||
e.value for e in all_events if e.type == EventType.TEXT_CHUNK
|
||||
)
|
||||
|
||||
assert "processed is appropriate." in reasoning_text
|
||||
assert "Understood." in content_text
|
||||
|
||||
def test_backtick_content_not_truncated(self):
|
||||
"""Reproduces the hostname backtick truncation case."""
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
config = ParserEngineConfig(
|
||||
name="test_channel",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
tok = MagicMock()
|
||||
vocab = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
CHANNEL_END: CHANNEL_END_ID,
|
||||
}
|
||||
tok.get_vocab.return_value = vocab
|
||||
tok.decode.side_effect = lambda ids: {
|
||||
CHANNEL_START_ID: CHANNEL_START,
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
}.get(ids[0], f"tok{ids[0]}")
|
||||
|
||||
engine = StreamingParserEngine(config, tok)
|
||||
all_events = []
|
||||
|
||||
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"thought\n1/10 completed. Next: ",
|
||||
[10, 11, 12, 13],
|
||||
)
|
||||
)
|
||||
|
||||
# Hold-back text includes backtick content; channel end text
|
||||
# absent from delta_text.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"`hostname`.\n",
|
||||
[CHANNEL_END_ID],
|
||||
)
|
||||
)
|
||||
|
||||
# Next delta flushes channel end + tool call start
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"<channel|>tool output",
|
||||
[20, 21],
|
||||
)
|
||||
)
|
||||
|
||||
all_events.extend(engine.finish())
|
||||
|
||||
reasoning_text = "".join(
|
||||
e.value for e in all_events if e.type == EventType.REASONING_CHUNK
|
||||
)
|
||||
|
||||
assert "`hostname`." in reasoning_text
|
||||
|
||||
|
||||
class TestRebuildFromAnchorsLiteralLookalike:
|
||||
"""When delta_text contains a literal mention of a special token's
|
||||
text before the real special token, _rebuild_from_anchors must
|
||||
anchor at the real occurrence, not the literal one."""
|
||||
|
||||
@pytest.fixture
|
||||
def tool_scanner(self):
|
||||
tok = MagicMock()
|
||||
tok.get_vocab.return_value = {
|
||||
TOOL_START: TOOL_START_ID,
|
||||
TOOL_END: TOOL_END_ID,
|
||||
}
|
||||
tok.decode.side_effect = lambda ids: {
|
||||
TOOL_START_ID: TOOL_START,
|
||||
TOOL_END_ID: TOOL_END,
|
||||
}.get(ids[0], f"t{ids[0]}")
|
||||
return TokenIDScanner(
|
||||
{TOOL_START_ID: "TOOL_START", TOOL_END_ID: "TOOL_END"},
|
||||
tok,
|
||||
)
|
||||
|
||||
def test_literal_before_real_anchor(self, tool_scanner):
|
||||
"""Literal <tool_call> in prose followed by a real <tool_call>
|
||||
special token — the scanner must split at the real one."""
|
||||
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
|
||||
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
|
||||
items = tool_scanner.scan(delta_text, delta_token_ids)
|
||||
|
||||
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
|
||||
terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
|
||||
|
||||
assert len(terminals) == 2
|
||||
assert terminals[0].terminal == "TOOL_START"
|
||||
assert terminals[1].terminal == "TOOL_END"
|
||||
|
||||
# The literal mention must appear in a text chunk, not be
|
||||
# consumed by the TOOL_START anchor.
|
||||
joined_text = "".join(text_parts)
|
||||
assert "<tool_call>" in joined_text
|
||||
assert '{"name":"f"}' in joined_text
|
||||
|
||||
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
|
||||
"""Two real tool calls with a literal mention between them."""
|
||||
delta_text = (
|
||||
'<tool_call>{"name":"a"}</tool_call>'
|
||||
" see <tool_call> syntax "
|
||||
'<tool_call>{"name":"b"}</tool_call>'
|
||||
)
|
||||
delta_token_ids = [
|
||||
TOOL_START_ID,
|
||||
1,
|
||||
TOOL_END_ID,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
TOOL_START_ID,
|
||||
5,
|
||||
TOOL_END_ID,
|
||||
]
|
||||
items = tool_scanner.scan(delta_text, delta_token_ids)
|
||||
|
||||
terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
|
||||
assert len(terminals) == 4
|
||||
|
||||
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
|
||||
joined_text = "".join(text_parts)
|
||||
# The literal mention between the two real calls must be in text
|
||||
assert "<tool_call> syntax" in joined_text
|
||||
|
||||
|
||||
class TestRebuildFromAnchorsCascadingDeferral:
|
||||
"""When a middle anchor's text is absent from delta_text,
|
||||
only that anchor should be deferred — not subsequent ones
|
||||
with valid positions."""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
tok = MagicMock()
|
||||
tok.decode.side_effect = lambda ids: f"t{ids[0]}"
|
||||
return TokenIDScanner({}, tok)
|
||||
|
||||
def test_middle_anchor_missing_does_not_cascade(self, bare_scanner):
|
||||
a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
|
||||
b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
|
||||
c = PreLexedTerminal("TOOL_END", TOOL_END_ID, TOOL_END)
|
||||
delta_text = f"prefix{TOOL_START}middle{TOOL_END}suffix"
|
||||
results = [a, b, c]
|
||||
|
||||
rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
|
||||
|
||||
terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
|
||||
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
|
||||
joined = "".join(t.text for t in texts)
|
||||
|
||||
assert len(terminals) == 2
|
||||
assert terminals[0].terminal == "TOOL_START"
|
||||
assert terminals[1].terminal == "TOOL_END"
|
||||
assert "prefix" in joined
|
||||
assert "middle" in joined
|
||||
assert "suffix" in joined
|
||||
assert len(bare_scanner._deferred_terminals) == 1
|
||||
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
|
||||
assert bare_scanner._deferred_post_text == ""
|
||||
|
||||
def test_first_anchor_missing_rest_still_emitted(self, bare_scanner):
|
||||
a = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
|
||||
b = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
|
||||
delta_text = f"text{TOOL_START}more"
|
||||
results = [a, b]
|
||||
|
||||
rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
|
||||
|
||||
terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
|
||||
assert len(terminals) == 1
|
||||
assert terminals[0].terminal == "TOOL_START"
|
||||
assert len(bare_scanner._deferred_terminals) == 1
|
||||
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
|
||||
|
||||
def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner):
|
||||
a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
|
||||
b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
|
||||
delta_text = f"text{TOOL_START}more"
|
||||
results = [a, b]
|
||||
|
||||
rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
|
||||
|
||||
terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
|
||||
assert len(terminals) == 1
|
||||
assert terminals[0].terminal == "TOOL_START"
|
||||
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
|
||||
joined = "".join(t.text for t in texts)
|
||||
assert "text" in joined
|
||||
# "more" is deferred along with the missing terminal —
|
||||
# it will be resolved in the next scan when the terminal
|
||||
# text arrives.
|
||||
assert bare_scanner._deferred_post_text == "more"
|
||||
assert len(bare_scanner._deferred_terminals) == 1
|
||||
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
|
||||
@@ -0,0 +1,410 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""On-demand trace builder for parser engine testing and benchmarks.
|
||||
|
||||
Generates token sequences programmatically from model-agnostic scenario
|
||||
definitions. Each model format handler knows how to render scenarios
|
||||
into the model's output format, tokenize them with correct special token
|
||||
IDs, and compute expected parse outputs.
|
||||
|
||||
Every generated sample is self-validated by replaying it through the
|
||||
real parser before being returned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from tests.parser.engine.replay_harness import (
|
||||
MockTokenizer,
|
||||
Sample,
|
||||
assert_parse_output,
|
||||
collect_output,
|
||||
replay_streaming,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Qwen3Parser,
|
||||
)
|
||||
|
||||
# ── Data structures ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolCallSpec:
|
||||
name: str
|
||||
arguments: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
class Scenario:
|
||||
id: str
|
||||
description: str
|
||||
reasoning: str | None = None
|
||||
content: str | None = None
|
||||
tool_calls: list[ToolCallSpec] | None = None
|
||||
|
||||
|
||||
# ── Scenarios ────────────────────────────────────────────────────────
|
||||
|
||||
_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"})
|
||||
_BASH_TOOL = ToolCallSpec(
|
||||
"bash", {"command": "hostname", "description": "Get hostname"}
|
||||
)
|
||||
_WEATHER_TOOL = ToolCallSpec(
|
||||
"get_weather",
|
||||
{"city": "Dallas", "state": "TX", "unit": "fahrenheit"},
|
||||
)
|
||||
_COMPLEX_TOOL = ToolCallSpec(
|
||||
"search",
|
||||
{
|
||||
"query": "vllm parser",
|
||||
"filters": {"language": "python", "min_stars": 100},
|
||||
"tags": ["ml", "inference"],
|
||||
"limit": 10,
|
||||
"verbose": True,
|
||||
},
|
||||
)
|
||||
|
||||
SCENARIOS: list[Scenario] = [
|
||||
Scenario(
|
||||
id="think-then-tool",
|
||||
description="Reasoning then single tool call",
|
||||
reasoning="Let me check the file.",
|
||||
tool_calls=[_READ_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="think-then-parallel-tools",
|
||||
description="Reasoning then two parallel tool calls",
|
||||
reasoning="I need to run both commands.",
|
||||
tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="think-then-content",
|
||||
description="Reasoning then content response",
|
||||
reasoning="Let me think about this carefully.",
|
||||
content="The answer is 42.",
|
||||
),
|
||||
Scenario(
|
||||
id="content-only",
|
||||
description="Plain content response without reasoning",
|
||||
content="Hello! How can I help you today?",
|
||||
),
|
||||
Scenario(
|
||||
id="tool-only",
|
||||
description="Tool call without reasoning",
|
||||
tool_calls=[_READ_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="complex-json-args",
|
||||
description="Tool call with nested objects, arrays, numbers, booleans",
|
||||
reasoning="This needs a complex query.",
|
||||
tool_calls=[_COMPLEX_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="whitespace-before-tool",
|
||||
description="Whitespace-only content before tool call",
|
||||
content="\n\n",
|
||||
tool_calls=[_WEATHER_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="think-content-tool",
|
||||
description="Reasoning, content, then tool call",
|
||||
reasoning="Let me analyze and then fetch data.",
|
||||
content="Checking the weather now.",
|
||||
tool_calls=[_WEATHER_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="think-whitespace-tool",
|
||||
description="Reasoning, whitespace-only gap, then tool call",
|
||||
reasoning="Let me check the file contents.",
|
||||
content="\n\n",
|
||||
tool_calls=[_READ_TOOL],
|
||||
),
|
||||
Scenario(
|
||||
id="empty-reasoning-content",
|
||||
description="Empty reasoning section followed by content",
|
||||
reasoning="",
|
||||
content="The epoch timestamp is 1779111346.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
# ── Tokenization ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _word_split(text: str) -> list[str]:
|
||||
"""Split text into word-like tokens, preserving all characters."""
|
||||
if not text:
|
||||
return []
|
||||
parts: list[str] = []
|
||||
current = ""
|
||||
for ch in text:
|
||||
if ch in " \t\n\r" and current and current[-1] not in " \t\n\r":
|
||||
parts.append(current)
|
||||
current = ch
|
||||
else:
|
||||
current += ch
|
||||
if current:
|
||||
parts.append(current)
|
||||
return parts
|
||||
|
||||
|
||||
def _tokenize(
|
||||
segments: list[tuple[str, bool]],
|
||||
vocab: dict[str, int],
|
||||
start_id: int = 100,
|
||||
) -> list[tuple[int, str]]:
|
||||
"""Build token list from segments.
|
||||
|
||||
Each segment is ``(text, is_special)``. Special segments use vocab
|
||||
IDs; content segments are word-split with sequential IDs.
|
||||
"""
|
||||
tokens: list[tuple[int, str]] = []
|
||||
next_id = start_id
|
||||
|
||||
for text, is_special in segments:
|
||||
if not text:
|
||||
continue
|
||||
if is_special:
|
||||
tid = vocab.get(text)
|
||||
if tid is None:
|
||||
raise ValueError(f"Special token {text!r} not in vocab")
|
||||
tokens.append((tid, text))
|
||||
else:
|
||||
for word in _word_split(text):
|
||||
tokens.append((next_id, word))
|
||||
next_id += 1
|
||||
|
||||
return tokens
|
||||
|
||||
|
||||
# ── Tool definitions ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _infer_schema(value: object) -> dict:
|
||||
"""Infer a JSON Schema from a Python value, recursing into dicts/lists."""
|
||||
if isinstance(value, bool):
|
||||
return {"type": "boolean"}
|
||||
if isinstance(value, int):
|
||||
return {"type": "integer"}
|
||||
if isinstance(value, float):
|
||||
return {"type": "number"}
|
||||
if isinstance(value, str):
|
||||
return {"type": "string"}
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
"type": "object",
|
||||
"properties": {k: _infer_schema(v) for k, v in value.items()},
|
||||
}
|
||||
if isinstance(value, list) and value:
|
||||
return {"type": "array", "items": _infer_schema(value[0])}
|
||||
if isinstance(value, list):
|
||||
return {"type": "array"}
|
||||
return {}
|
||||
|
||||
|
||||
def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]:
|
||||
"""Generate OpenAI-style tool definitions from tool call specs."""
|
||||
seen: set[str] = set()
|
||||
tools: list[dict] = []
|
||||
for tc in tool_calls:
|
||||
if tc.name in seen:
|
||||
continue
|
||||
seen.add(tc.name)
|
||||
properties = {k: _infer_schema(v) for k, v in tc.arguments.items()}
|
||||
tools.append(
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tc.name,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
},
|
||||
},
|
||||
}
|
||||
)
|
||||
return tools
|
||||
|
||||
|
||||
# ── Format handlers ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _expected_tc(scenario: Scenario) -> list[dict] | None:
|
||||
if not scenario.tool_calls:
|
||||
return None
|
||||
return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls]
|
||||
|
||||
|
||||
def _expected_tools(scenario: Scenario) -> list[dict] | None:
|
||||
return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None
|
||||
|
||||
|
||||
def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
|
||||
"""Replay sample through the real parser and assert correctness."""
|
||||
tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
|
||||
parser = parser_cls(tokenizer, sample.tools, **kwargs)
|
||||
deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
|
||||
output = collect_output(deltas)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
|
||||
def _validate_tools(
|
||||
tools: list[dict] | None,
|
||||
) -> list[ChatCompletionToolsParam] | None:
|
||||
if not tools:
|
||||
return None
|
||||
return [ChatCompletionToolsParam.model_validate(t) for t in tools]
|
||||
|
||||
|
||||
def _make_sample(
|
||||
sample_id: str,
|
||||
description: str,
|
||||
vocab: dict[str, int],
|
||||
segments: list[tuple[str, bool]],
|
||||
expected_reasoning: str | None,
|
||||
expected_content: str | None,
|
||||
expected_tool_calls: list[dict] | None,
|
||||
tools: list[dict] | None,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
) -> Sample:
|
||||
tokens = _tokenize(segments, vocab)
|
||||
return Sample(
|
||||
id=sample_id,
|
||||
description=description,
|
||||
source="trace-builder",
|
||||
vocab=dict(vocab),
|
||||
tokens=tokens,
|
||||
expected_reasoning=expected_reasoning,
|
||||
expected_content=expected_content,
|
||||
expected_tool_calls=expected_tool_calls,
|
||||
tools=_validate_tools(tools),
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
)
|
||||
|
||||
|
||||
# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
|
||||
|
||||
_QWEN3_VOCAB: dict[str, int] = {
|
||||
"<think>": 50,
|
||||
"</think>": 51,
|
||||
"<tool_call>": 60,
|
||||
"</tool_call>": 61,
|
||||
}
|
||||
|
||||
|
||||
def _qwen3_arg_value(value: Any) -> str:
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
|
||||
parts = [f"\n<function={tc.name}>"]
|
||||
for key, value in tc.arguments.items():
|
||||
parts.append(f"\n<parameter={key}>{_qwen3_arg_value(value)}</parameter>")
|
||||
parts.append("\n</function>\n")
|
||||
return [
|
||||
("<tool_call>", True),
|
||||
("".join(parts), False),
|
||||
("</tool_call>", True),
|
||||
]
|
||||
|
||||
|
||||
def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = []
|
||||
if scenario.reasoning is not None:
|
||||
segs.append((scenario.reasoning, False))
|
||||
if scenario.content is not None or scenario.tool_calls:
|
||||
segs.append(("</think>", True))
|
||||
if scenario.content is not None:
|
||||
segs.append((scenario.content, False))
|
||||
if scenario.tool_calls:
|
||||
for tc in scenario.tool_calls:
|
||||
segs.extend(_qwen3_tool_segments(tc))
|
||||
return segs
|
||||
|
||||
|
||||
def _qwen3_expected_content(scenario: Scenario) -> str | None:
|
||||
if (
|
||||
scenario.content is not None
|
||||
and scenario.tool_calls
|
||||
and not scenario.content.strip()
|
||||
):
|
||||
return ""
|
||||
return scenario.content
|
||||
|
||||
|
||||
def _build_qwen3(
|
||||
scenario: Scenario,
|
||||
name: str = "qwen3",
|
||||
parser_cls: type = Qwen3Parser,
|
||||
strip_trailing_ws: bool = False,
|
||||
validate: bool = True,
|
||||
) -> Sample:
|
||||
expected_reasoning: str | None
|
||||
if scenario.reasoning is not None:
|
||||
r = scenario.reasoning
|
||||
if strip_trailing_ws:
|
||||
r = r.rstrip()
|
||||
expected_reasoning = r
|
||||
else:
|
||||
expected_reasoning = ""
|
||||
|
||||
sample = _make_sample(
|
||||
sample_id=f"{name}-{scenario.id}",
|
||||
description=scenario.description,
|
||||
vocab=_QWEN3_VOCAB,
|
||||
segments=_qwen3_segments(scenario),
|
||||
expected_reasoning=expected_reasoning,
|
||||
expected_content=_qwen3_expected_content(scenario),
|
||||
expected_tool_calls=_expected_tc(scenario),
|
||||
tools=_expected_tools(scenario),
|
||||
)
|
||||
if validate:
|
||||
_validate_sample(sample, parser_cls)
|
||||
return sample
|
||||
|
||||
|
||||
# ── Registry and public API ──────────────────────────────────────────
|
||||
|
||||
_BUILDERS: dict[str, Any] = {
|
||||
"qwen3": _build_qwen3,
|
||||
}
|
||||
|
||||
|
||||
@functools.cache
|
||||
def build_samples(model: str) -> tuple[Sample, ...]:
|
||||
"""Build all scenario samples for a model, self-validated."""
|
||||
builder = _BUILDERS[model]
|
||||
return tuple(builder(s) for s in SCENARIOS)
|
||||
|
||||
|
||||
def build_sample(model: str, scenario: Scenario) -> Sample:
|
||||
"""Build a single sample for one model + scenario."""
|
||||
return _BUILDERS[model](scenario)
|
||||
|
||||
|
||||
def build_scaling_sample(
|
||||
model: str, token_count: int, validate: bool = False
|
||||
) -> Sample:
|
||||
"""Build a sample with approximately *token_count* tokens."""
|
||||
sentence = "The quick brown fox jumps over the lazy dog. "
|
||||
text = sentence * (token_count // 10 + 1)
|
||||
scenario = Scenario(
|
||||
id=f"scaling-{token_count}",
|
||||
description=f"Scaling test with ~{token_count} tokens",
|
||||
reasoning=text,
|
||||
tool_calls=[_READ_TOOL],
|
||||
)
|
||||
return _BUILDERS[model](scenario, validate=validate)
|
||||
Reference in New Issue
Block a user