[Frontend] Replace legacy Gemma4 parsers with engine-based implementation (#45588)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
25ee659db0
commit
76a373eff4
@@ -33,6 +33,7 @@ class Sample:
|
||||
expected_tool_calls: list[dict] | None
|
||||
tools: list[dict] | None = None
|
||||
chat_template_kwargs: dict | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -120,6 +121,7 @@ def replay_streaming(
|
||||
holdback_chars: int = 0,
|
||||
finished_on_last: bool = False,
|
||||
tools: list[dict] | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> list[DeltaMessage | None]:
|
||||
"""Feed tokens through ``parser.parse_delta()`` at a given chunk size.
|
||||
|
||||
@@ -146,6 +148,7 @@ def replay_streaming(
|
||||
all_texts = [text for _, text in tokens]
|
||||
|
||||
request = _test_request(tools=tools)
|
||||
first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
|
||||
|
||||
if holdback_chars <= 0:
|
||||
chunks = list(range(0, len(tokens), chunk_size))
|
||||
@@ -159,7 +162,7 @@ def replay_streaming(
|
||||
delta_text,
|
||||
batch_ids,
|
||||
request,
|
||||
prompt_token_ids=[] if start == 0 else None,
|
||||
prompt_token_ids=first_prompt_ids if start == 0 else None,
|
||||
finished=finished_on_last and is_last,
|
||||
)
|
||||
results.append(result)
|
||||
@@ -192,7 +195,7 @@ def replay_streaming(
|
||||
delta_text,
|
||||
batch_ids,
|
||||
request,
|
||||
prompt_token_ids=[] if is_first else None,
|
||||
prompt_token_ids=first_prompt_ids if is_first else None,
|
||||
finished=finished_on_last and is_last_chunk,
|
||||
)
|
||||
results.append(result)
|
||||
@@ -205,7 +208,7 @@ def replay_streaming(
|
||||
delta_text,
|
||||
batch_ids,
|
||||
request,
|
||||
prompt_token_ids=[] if is_first else None,
|
||||
prompt_token_ids=first_prompt_ids if is_first else None,
|
||||
finished=finished_on_last,
|
||||
)
|
||||
results.append(result)
|
||||
@@ -218,6 +221,7 @@ def replay_with_text_holdback(
|
||||
tokens: list[tuple[int, str]],
|
||||
text_delay: int = 1,
|
||||
tools: list[dict] | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> list[DeltaMessage | None]:
|
||||
"""Replay token-by-token with text arriving *text_delay* steps late.
|
||||
|
||||
@@ -235,6 +239,7 @@ def replay_with_text_holdback(
|
||||
"""
|
||||
results: list[DeltaMessage | None] = []
|
||||
request = _test_request(tools=tools)
|
||||
first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
|
||||
|
||||
n = len(tokens)
|
||||
held_texts: list[str] = []
|
||||
@@ -256,7 +261,7 @@ def replay_with_text_holdback(
|
||||
delta_text,
|
||||
[token_id],
|
||||
request,
|
||||
prompt_token_ids=[] if i == 0 else None,
|
||||
prompt_token_ids=first_prompt_ids if i == 0 else None,
|
||||
finished=is_last,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
@@ -29,8 +29,9 @@ from vllm.parser.parser_manager import ParserManager
|
||||
|
||||
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
|
||||
|
||||
_PAIRINGS: dict[str, tuple[str, str]] = {
|
||||
"engine": ("qwen3_coder", "qwen3"),
|
||||
_PAIRINGS: dict[str, tuple[str, str, str]] = {
|
||||
"engine": ("qwen3_coder", "qwen3", "qwen3"),
|
||||
"gemma4_engine": ("gemma4", "gemma4", "gemma4"),
|
||||
}
|
||||
|
||||
CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
|
||||
@@ -38,7 +39,7 @@ CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
|
||||
|
||||
@lru_cache
|
||||
def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
|
||||
tool_name, reasoning_name = _PAIRINGS[pairings]
|
||||
tool_name, reasoning_name, _ = _PAIRINGS[pairings]
|
||||
parser_cls = ParserManager.get_parser(
|
||||
tool_parser_name=tool_name,
|
||||
reasoning_parser_name=reasoning_name,
|
||||
@@ -48,16 +49,23 @@ def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
|
||||
return parser_cls
|
||||
|
||||
|
||||
_all_samples = build_samples("qwen3")
|
||||
def _pairing_samples() -> list[tuple[str, object]]:
|
||||
items: list[tuple[str, object]] = []
|
||||
for pairing_name, (_, _, model) in _PAIRINGS.items():
|
||||
for sample in build_samples(model):
|
||||
items.append((pairing_name, sample))
|
||||
return items
|
||||
|
||||
|
||||
_all_pairing_samples = _pairing_samples()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pairings",
|
||||
list(_PAIRINGS),
|
||||
ids=lambda p: f"mode={p}",
|
||||
)
|
||||
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
|
||||
@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
|
||||
@pytest.mark.parametrize(
|
||||
"pairings,sample",
|
||||
_all_pairing_samples,
|
||||
ids=lambda v: v.id if hasattr(v, "id") else v,
|
||||
)
|
||||
def test_delegating_replay(sample, chunk_size, pairings):
|
||||
parser_cls = _get_delegating_parser_cls(pairings=pairings)
|
||||
|
||||
@@ -77,6 +85,7 @@ def test_delegating_replay(sample, chunk_size, pairings):
|
||||
chunk_size=chunk_size,
|
||||
finished_on_last=True,
|
||||
tools=sample.tools,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -631,6 +631,103 @@ class TestFixArgTypes:
|
||||
original = '{"name": "Alice"}'
|
||||
assert engine._fix_arg_types(original, "f") == original
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"properties, input_json, expected_substr",
|
||||
[
|
||||
({"count": {"type": "integer"}}, '{"count": "42"}', '"count": 42'),
|
||||
({"score": {"type": "number"}}, '{"score": "3.14"}', '"score": 3.14'),
|
||||
({"flag": {"type": "boolean"}}, '{"flag": "true"}', '"flag": true'),
|
||||
({"flag": {"type": "boolean"}}, '{"flag": "false"}', '"flag": false'),
|
||||
({"val": {"type": "null"}}, '{"val": "null"}', '"val": null'),
|
||||
({"val": {"type": ["string", "null"]}}, '{"val": "null"}', '"val": null'),
|
||||
({"score": {"type": "number"}}, '{"score": "108."}', '"score": 108'),
|
||||
],
|
||||
ids=[
|
||||
"string_to_int",
|
||||
"string_to_float",
|
||||
"string_to_bool_true",
|
||||
"string_to_bool_false",
|
||||
"string_to_null",
|
||||
"string_to_null_union",
|
||||
"trailing_dot_float",
|
||||
],
|
||||
)
|
||||
def test_string_coerced_to_schema_type(
|
||||
self,
|
||||
properties,
|
||||
input_json,
|
||||
expected_substr,
|
||||
):
|
||||
tool = _make_tool("f", properties)
|
||||
engine = _make_engine(tools=[tool])
|
||||
result = engine._fix_arg_types(input_json, "f")
|
||||
assert expected_substr in result
|
||||
|
||||
def test_mixed_types_coerced(self):
|
||||
tool = _make_tool(
|
||||
"f",
|
||||
{
|
||||
"count": {"type": "integer"},
|
||||
"active": {"type": "boolean"},
|
||||
"score": {"type": "number"},
|
||||
},
|
||||
)
|
||||
engine = _make_engine(tools=[tool])
|
||||
result = engine._fix_arg_types(
|
||||
'{"count": "42", "active": "true", "score": "3.14"}', "f"
|
||||
)
|
||||
parsed = json.loads(result)
|
||||
assert parsed["count"] == 42
|
||||
assert parsed["active"] is True
|
||||
assert parsed["score"] == 3.14
|
||||
|
||||
def test_nested_object_coercion(self):
|
||||
tool = _make_tool(
|
||||
"f",
|
||||
{
|
||||
"inner": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"count": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
engine = _make_engine(tools=[tool])
|
||||
result = engine._fix_arg_types('{"inner": {"count": "42"}}', "f")
|
||||
parsed = json.loads(result)
|
||||
assert parsed["inner"]["count"] == 42
|
||||
|
||||
def test_array_item_coercion(self):
|
||||
tool = _make_tool(
|
||||
"f",
|
||||
{
|
||||
"nums": {
|
||||
"type": "array",
|
||||
"items": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
)
|
||||
engine = _make_engine(tools=[tool])
|
||||
result = engine._fix_arg_types('{"nums": ["42", "5"]}', "f")
|
||||
parsed = json.loads(result)
|
||||
assert parsed["nums"] == [42, 5]
|
||||
|
||||
def test_array_mixed_item_types(self):
|
||||
tool = _make_tool(
|
||||
"f",
|
||||
{
|
||||
"vals": {
|
||||
"type": "array",
|
||||
"items": {"type": "number"},
|
||||
},
|
||||
},
|
||||
)
|
||||
engine = _make_engine(tools=[tool])
|
||||
result = engine._fix_arg_types('{"vals": ["42", "3.14"]}', "f")
|
||||
parsed = json.loads(result)
|
||||
assert parsed["vals"] == [42, 3.14]
|
||||
|
||||
|
||||
# ── TestBuildExtractedResult ─────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -17,19 +17,25 @@ from tests.parser.engine.replay_harness import (
|
||||
collect_output,
|
||||
make_mock_tokenizer,
|
||||
replay_streaming,
|
||||
replay_with_text_holdback,
|
||||
)
|
||||
from tests.parser.engine.trace_builder import build_samples
|
||||
from vllm.parser.abstract_parser import Parser
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Gemma4Parser,
|
||||
Qwen3Parser,
|
||||
)
|
||||
|
||||
_ENGINE_PARSERS: dict[str, type[Parser]] = {
|
||||
"qwen3_engine": Qwen3Parser,
|
||||
"gemma4_engine": Gemma4Parser,
|
||||
}
|
||||
|
||||
_gemma4_samples = build_samples("gemma4")
|
||||
_qwen3_samples = build_samples("qwen3")
|
||||
|
||||
_GEMMA4_TERMINALS = ["<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>"]
|
||||
|
||||
_QWEN3_TERMINALS = [
|
||||
"<think>",
|
||||
"</think>",
|
||||
@@ -56,6 +62,7 @@ class TestQwen3ReplayWithHoldback:
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
holdback_chars=holdback,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
@@ -67,10 +74,85 @@ class TestQwen3ReplayWithHoldback:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
|
||||
@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
|
||||
@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
|
||||
class TestGemma4ReplayWithHoldback:
|
||||
"""Replay with simulated detokenizer holdback."""
|
||||
|
||||
def test_replay(self, sample, chunk_size, holdback):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = Gemma4Parser(tokenizer, sample.tools)
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=chunk_size,
|
||||
holdback_chars=holdback,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert_parse_output(output, sample)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
_GEMMA4_TERMINALS,
|
||||
context=f"chunk_size={chunk_size}, holdback={holdback}",
|
||||
)
|
||||
|
||||
|
||||
TEXT_HOLDBACK_DELAYS = [1, 2, 3]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
|
||||
@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
|
||||
class TestGemma4TextHoldback:
|
||||
"""Replay with production-like text/token-ID misalignment.
|
||||
|
||||
In production the detokenizer sends token IDs immediately but holds
|
||||
back text by N tokens. This exercises the TokenIDScanner deferred
|
||||
terminal path that aligned-holdback tests do not cover.
|
||||
"""
|
||||
|
||||
def test_replay(self, sample, delay):
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = Gemma4Parser(tokenizer, sample.tools)
|
||||
deltas = replay_with_text_holdback(
|
||||
parser,
|
||||
sample.tokens,
|
||||
text_delay=delay,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
|
||||
assert_parse_output(output, sample)
|
||||
assert_no_terminal_leakage(
|
||||
output,
|
||||
_GEMMA4_TERMINALS,
|
||||
context=f"text_delay={delay}",
|
||||
)
|
||||
|
||||
|
||||
class TestParserEngineAdjustRequest:
|
||||
"""Verify ParserEngine and its adapters set skip_special_tokens=False."""
|
||||
|
||||
def test_adjust_request_disables_skip_special_tokens(self):
|
||||
sample = _gemma4_samples[0]
|
||||
tokenizer = make_mock_tokenizer(sample)
|
||||
parser = Gemma4Parser(tokenizer, sample.tools)
|
||||
request = _test_request()
|
||||
assert request.skip_special_tokens is True
|
||||
adjusted = parser.adjust_request(request)
|
||||
assert adjusted.skip_special_tokens is False
|
||||
|
||||
|
||||
_TOOL_CALL_SAMPLES = [
|
||||
(Qwen3Parser, s)
|
||||
for s in _qwen3_samples
|
||||
if s.expected_tool_calls and s.expected_reasoning
|
||||
] + [
|
||||
(Gemma4Parser, s)
|
||||
for s in _gemma4_samples
|
||||
if s.expected_tool_calls and s.expected_reasoning
|
||||
]
|
||||
|
||||
|
||||
@@ -147,7 +229,9 @@ class TestSkipToolParsingReplay:
|
||||
"".join(all_texts[start:end]),
|
||||
all_ids[start:end],
|
||||
request,
|
||||
prompt_token_ids=[] if start == 0 else None,
|
||||
prompt_token_ids=(sample.prompt_token_ids or [])
|
||||
if start == 0
|
||||
else None,
|
||||
finished=is_last,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
@@ -1,20 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for TokenIDScanner, focusing on hold-back text recovery.
|
||||
|
||||
Uses gemma4_config for all end-to-end engine tests, covering
|
||||
reasoning channels, tool calls, and combined flows."""
|
||||
"""Tests for TokenIDScanner."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
from vllm.parser.engine.token_id_scanner import (
|
||||
PreLexedTerminal,
|
||||
TextChunk,
|
||||
TokenIDScanner,
|
||||
)
|
||||
from vllm.parser.gemma4 import gemma4_config
|
||||
|
||||
CHANNEL_START = "<|channel>"
|
||||
CHANNEL_END = "<channel|>"
|
||||
@@ -54,8 +53,7 @@ def scanner(tokenizer):
|
||||
|
||||
|
||||
class TestJoinDecodedTextReturnsStr:
|
||||
"""_join_decoded_text now returns str unconditionally (was
|
||||
str | None when an isinstance guard made a branch unreachable)."""
|
||||
"""_join_decoded_text always returns str."""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
@@ -83,9 +81,7 @@ class TestJoinDecodedTextReturnsStr:
|
||||
|
||||
class TestHoldbackTextRecovery:
|
||||
def test_holdback_text_with_special_token_text_absent(self, scanner):
|
||||
"""delta_text has hold-back text but the special token's text is
|
||||
NOT in delta_text (held back by the detokenizer). Terminal is
|
||||
deferred until the text arrives in a subsequent delta."""
|
||||
"""Terminal deferred when its text is absent from delta_text."""
|
||||
result = scanner.scan(
|
||||
delta_text="processed is appropriate.",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -93,8 +89,6 @@ class TestHoldbackTextRecovery:
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Second scan: terminal text arrives (detokenizer flushes).
|
||||
# Deferred terminal resolves with holdback text before it.
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>Understood.",
|
||||
delta_token_ids=[20, 21],
|
||||
@@ -108,7 +102,7 @@ class TestHoldbackTextRecovery:
|
||||
assert "Understood." in combined
|
||||
|
||||
def test_holdback_text_with_special_token_text_present(self, scanner):
|
||||
"""delta_text includes hold-back text AND the special token text."""
|
||||
"""Hold-back text + special token text both in delta_text."""
|
||||
result = scanner.scan(
|
||||
delta_text="holdback text<channel|>",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -121,7 +115,7 @@ class TestHoldbackTextRecovery:
|
||||
assert result[1].terminal == "THINK_END"
|
||||
|
||||
def test_no_holdback_text(self, scanner):
|
||||
"""delta_text is exactly the special token text — no hold-back."""
|
||||
"""delta_text is exactly the special token text."""
|
||||
result = scanner.scan(
|
||||
delta_text="<channel|>",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -132,7 +126,7 @@ class TestHoldbackTextRecovery:
|
||||
assert result[0].terminal == "THINK_END"
|
||||
|
||||
def test_empty_delta_text(self, scanner):
|
||||
"""delta_text is empty — terminal deferred until text arrives."""
|
||||
"""Empty delta_text defers the terminal until text arrives."""
|
||||
result = scanner.scan(
|
||||
delta_text="",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -146,9 +140,7 @@ class TestHoldbackTextRecovery:
|
||||
assert flushed[0].terminal == "THINK_END"
|
||||
|
||||
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
|
||||
"""delta_text="" with multiple tokens including special: all
|
||||
results deferred — individually-decoded TextChunks are unreliable
|
||||
and PreLexedTerminals wait for text confirmation."""
|
||||
"""Empty delta_text with multiple tokens: all results deferred."""
|
||||
tool_start_id = 400
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
@@ -176,7 +168,6 @@ class TestHoldbackTextRecovery:
|
||||
assert flushed[0].terminal == "TOOL_START"
|
||||
|
||||
def test_holdback_before_start_tag(self, scanner):
|
||||
"""Hold-back text before a reasoning start tag."""
|
||||
result = scanner.scan(
|
||||
delta_text="prefix text<|channel>",
|
||||
delta_token_ids=[CHANNEL_START_ID],
|
||||
@@ -189,8 +180,7 @@ class TestHoldbackTextRecovery:
|
||||
assert result[1].terminal == "THINK_START"
|
||||
|
||||
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
|
||||
"""Stream-interval > 1: batch has regular tokens + special token.
|
||||
delta_text differs from individual decodes (context-dependent)."""
|
||||
"""Multi-token batch with special token in the middle."""
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
@@ -215,10 +205,7 @@ class TestHoldbackTextRecovery:
|
||||
assert "holdback wordA" in "".join(texts)
|
||||
|
||||
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
|
||||
"""Stream-interval > 1: batch has regular + special token, but
|
||||
delta_text doesn't contain the special token text at all
|
||||
(held back by detokenizer along with trailing regular tokens).
|
||||
Terminal is deferred until text arrives."""
|
||||
"""Multi-token batch where special token text is absent."""
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
@@ -239,8 +226,6 @@ class TestHoldbackTextRecovery:
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Next delta: terminal text arrives (detokenizer flushes).
|
||||
# Deferred terminal resolves with holdback text before it.
|
||||
result2 = scanner_multi.scan(
|
||||
delta_text="<channel|> more text",
|
||||
delta_token_ids=[300],
|
||||
@@ -254,8 +239,7 @@ class TestHoldbackTextRecovery:
|
||||
assert "more text" in combined
|
||||
|
||||
def test_holdback_with_content_after_special_token(self, tokenizer):
|
||||
"""delta_text has hold-back + special token + content after,
|
||||
with corresponding token IDs for all parts."""
|
||||
"""Hold-back + special token + content after in one delta."""
|
||||
tok_content = 210
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
@@ -283,8 +267,7 @@ class TestHoldbackTextRecovery:
|
||||
|
||||
class TestDropTokens:
|
||||
def test_drop_token_with_holdback(self, tokenizer):
|
||||
"""Drop tokens stripped from delta_text, hold-back text preserved.
|
||||
Terminal is deferred when its text is absent from delta_text."""
|
||||
"""Drop tokens stripped; hold-back text preserved."""
|
||||
drop_id = 300
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
@@ -304,7 +287,6 @@ class TestDropTokens:
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Terminal text arrives in next delta; deferred terminal resolves.
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>content",
|
||||
delta_token_ids=[20],
|
||||
@@ -321,40 +303,10 @@ class TestDropTokens:
|
||||
|
||||
|
||||
class TestEndToEndReasoningHoldback:
|
||||
"""End-to-end tests through the full parser engine simulating
|
||||
stream-interval > 1 and detokenizer hold-back, using
|
||||
gemma4_config."""
|
||||
"""End-to-end engine tests with detokenizer hold-back."""
|
||||
|
||||
def test_reasoning_content_not_truncated(self):
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
config = ParserEngineConfig(
|
||||
name="test_channel",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
config = gemma4_config()
|
||||
tok = MagicMock()
|
||||
vocab = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
@@ -369,30 +321,21 @@ class TestEndToEndReasoningHoldback:
|
||||
engine = StreamingParserEngine(config, tok)
|
||||
all_events = []
|
||||
|
||||
# Delta 1: channel start token (text includes start tag)
|
||||
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
|
||||
|
||||
# Delta 2: reasoning text (normal content, no special tokens)
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"thought\nThe request was received and ",
|
||||
[10, 11, 12, 13, 14],
|
||||
)
|
||||
)
|
||||
|
||||
# Delta 3: MORE reasoning text, the detokenizer held some back.
|
||||
# Then channel end token arrives in token_ids, but its text
|
||||
# is NOT in delta_text (held back by detokenizer).
|
||||
# delta_text = previously held-back reasoning text only.
|
||||
# CHANNEL_END token arrives but its text is held back.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"processed is appropriate.",
|
||||
[CHANNEL_END_ID],
|
||||
)
|
||||
)
|
||||
|
||||
# Delta 4: detokenizer flushes held-back channel end text
|
||||
# plus new content tokens.
|
||||
# Detokenizer flushes the held-back text.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"<channel|>Understood.",
|
||||
@@ -413,36 +356,7 @@ class TestEndToEndReasoningHoldback:
|
||||
assert "Understood." in content_text
|
||||
|
||||
def test_backtick_content_not_truncated(self):
|
||||
"""Reproduces the hostname backtick truncation case."""
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
config = ParserEngineConfig(
|
||||
name="test_channel",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
config = gemma4_config()
|
||||
tok = MagicMock()
|
||||
vocab = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
@@ -464,17 +378,12 @@ class TestEndToEndReasoningHoldback:
|
||||
[10, 11, 12, 13],
|
||||
)
|
||||
)
|
||||
|
||||
# Hold-back text includes backtick content; channel end text
|
||||
# absent from delta_text.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"`hostname`.\n",
|
||||
[CHANNEL_END_ID],
|
||||
)
|
||||
)
|
||||
|
||||
# Next delta flushes channel end + tool call start
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"<channel|>tool output",
|
||||
@@ -491,10 +400,516 @@ class TestEndToEndReasoningHoldback:
|
||||
assert "`hostname`." in reasoning_text
|
||||
|
||||
|
||||
_CHANNEL_START_TAG = "<|channel>"
|
||||
_CHANNEL_END_TAG = "<channel|>"
|
||||
_TOOL_START_TAG = "<|tool_call>"
|
||||
_TOOL_END_TAG = "<tool_call|>"
|
||||
_QUOTE_TAG = '<|"|>'
|
||||
|
||||
_CHANNEL_START_TID = 100
|
||||
_CHANNEL_END_TID = 101
|
||||
_TOOL_START_TID = 102
|
||||
_TOOL_END_TID = 103
|
||||
_QUOTE_TID = 104
|
||||
_TOK = list(range(200, 215))
|
||||
|
||||
|
||||
def _gemma4_vocab() -> dict[str, int]:
|
||||
return {
|
||||
_CHANNEL_START_TAG: _CHANNEL_START_TID,
|
||||
_CHANNEL_END_TAG: _CHANNEL_END_TID,
|
||||
_TOOL_START_TAG: _TOOL_START_TID,
|
||||
_TOOL_END_TAG: _TOOL_END_TID,
|
||||
_QUOTE_TAG: _QUOTE_TID,
|
||||
}
|
||||
|
||||
|
||||
def _make_gemma4_tokenizer(
|
||||
extra_decode: dict[int, str] | None = None,
|
||||
) -> MagicMock:
|
||||
special = {
|
||||
_CHANNEL_START_TID: _CHANNEL_START_TAG,
|
||||
_CHANNEL_END_TID: _CHANNEL_END_TAG,
|
||||
_TOOL_START_TID: _TOOL_START_TAG,
|
||||
_TOOL_END_TID: _TOOL_END_TAG,
|
||||
_QUOTE_TID: _QUOTE_TAG,
|
||||
}
|
||||
decode_map = {**special, **(extra_decode or {})}
|
||||
|
||||
tok = MagicMock()
|
||||
tok.get_vocab.return_value = _gemma4_vocab()
|
||||
tok.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
|
||||
return tok
|
||||
|
||||
|
||||
def _collect_events(engine, deltas):
|
||||
from vllm.parser.engine.events import SemanticEvent
|
||||
|
||||
all_events: list[SemanticEvent] = []
|
||||
for delta_text, delta_token_ids in deltas:
|
||||
all_events.extend(engine.feed(delta_text, delta_token_ids))
|
||||
all_events.extend(engine.finish())
|
||||
return all_events
|
||||
|
||||
|
||||
def _reasoning_text(events) -> str:
|
||||
return "".join(e.value for e in events if e.type == EventType.REASONING_CHUNK)
|
||||
|
||||
|
||||
def _content_text(events) -> str:
|
||||
return "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
|
||||
|
||||
def _arg_text(events) -> str:
|
||||
return "".join(e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK)
|
||||
|
||||
|
||||
def _has_event(events, event_type) -> bool:
|
||||
return any(e.type == event_type for e in events)
|
||||
|
||||
|
||||
class TestMultiTokenBoundaryPreservation:
|
||||
"""No text lost at state boundaries with multi-token deltas."""
|
||||
|
||||
def test_empty_delta_text_at_channel_end_unified(self):
|
||||
"""Empty delta_text when CHANNEL_END arrives; text comes later."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
("", [_CHANNEL_START_TID]),
|
||||
("<|channel>thought\nSome reasoning.", [_TOK[0], _TOK[1]]),
|
||||
("", [_CHANNEL_END_TID]),
|
||||
("<channel|>Final answer.", [_TOK[2], _TOK[3]]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
assert "Some reasoning." in reasoning
|
||||
assert "Final answer." in content
|
||||
assert _has_event(events, EventType.REASONING_START)
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_deferred_channel_end_flushed_at_finish_unified(self):
|
||||
"""Deferred CHANNEL_END flushed at end-of-stream."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
|
||||
("thought\nReasoning text.", [_TOK[0]]),
|
||||
(" Final thought.", [_CHANNEL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
assert "Reasoning text. Final thought." in reasoning
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_reasoning_to_tool_call_handoff_unified(self):
|
||||
"""Full reasoning -> content -> tool call flow."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
|
||||
("thought\nI need to check the weather.", [_TOK[0], _TOK[1], _TOK[2]]),
|
||||
(_CHANNEL_END_TAG, [_CHANNEL_END_TID]),
|
||||
("Let me call a tool.", [_TOK[3], _TOK[4]]),
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_weather{city:", [_TOK[5], _TOK[6]]),
|
||||
('<|"|>SF<|"|>}', [_QUOTE_TID, _TOK[7], _QUOTE_TID, _TOK[8]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
assert "I need to check the weather." in reasoning
|
||||
assert "Let me call a tool." in content
|
||||
assert _has_event(events, EventType.REASONING_START)
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
assert _has_event(events, EventType.TOOL_CALL_START)
|
||||
assert _has_event(events, EventType.TOOL_CALL_END)
|
||||
assert "SF" in _arg_text(events)
|
||||
|
||||
def test_multiple_tool_calls_rapid_transitions_unified(self):
|
||||
"""Two back-to-back tool calls with correct tool_index tracking."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_weather{city:", [_TOK[0], _TOK[1]]),
|
||||
('<|"|>NYC<|"|>}', [_QUOTE_TID, _TOK[2], _QUOTE_TID, _TOK[3]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_time{tz:", [_TOK[4], _TOK[5]]),
|
||||
('<|"|>EST<|"|>}', [_QUOTE_TID, _TOK[6], _QUOTE_TID, _TOK[7]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
|
||||
ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
|
||||
assert len(starts) == 2
|
||||
assert len(ends) == 2
|
||||
assert starts[0].tool_index == 0
|
||||
assert starts[1].tool_index == 1
|
||||
|
||||
names = "".join(e.value for e in events if e.type == EventType.TOOL_NAME)
|
||||
assert "get_weather" in names
|
||||
assert "get_time" in names
|
||||
|
||||
def test_deferred_channel_end_before_tool_call_unified(self):
|
||||
"""Deferred CHANNEL_END followed by a tool call."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
|
||||
("thought\nNeed to call a tool.", [_TOK[0], _TOK[1]]),
|
||||
(" Let me proceed.", [_CHANNEL_END_TID]),
|
||||
(_CHANNEL_END_TAG, [_TOK[2]]),
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_weather{city:", [_TOK[3], _TOK[4]]),
|
||||
('<|"|>Tokyo<|"|>}', [_QUOTE_TID, _TOK[5], _QUOTE_TID, _TOK[6]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
assert "Need to call a tool. Let me proceed." in reasoning
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
assert _has_event(events, EventType.TOOL_CALL_START)
|
||||
assert _has_event(events, EventType.TOOL_CALL_END)
|
||||
assert "Tokyo" in _arg_text(events)
|
||||
|
||||
|
||||
class TestStreamInterval10:
|
||||
"""Tests with stream_interval=10 (large multi-token batches)."""
|
||||
|
||||
def test_channel_end_mid_batch_text_present(self):
|
||||
"""<channel|> mid-batch with its text present in delta_text."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nword0 word1 word2 word3 word4 "
|
||||
"word5 word6 word7 word8 ",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"word9 word10 word11 <channel|>word12 word13 word14 word0 word1 word2 ",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
for w in ("word9", "word10", "word11"):
|
||||
assert w in reasoning, f"{w!r} missing from reasoning"
|
||||
|
||||
for w in ("word12", "word13", "word14"):
|
||||
assert w in content, f"{w!r} missing from content"
|
||||
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_channel_end_and_tool_start_same_batch_unified(self):
|
||||
"""Both <channel|> and <|tool_call> in a single batch."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nw0 w1 w2 w3 w4 w5 w6 w7 w8 ",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"w9 w10 <channel|>w11 <|tool_call>",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[11],
|
||||
_TOOL_START_TID,
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
],
|
||||
)
|
||||
)
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
|
||||
assert "w9" in reasoning
|
||||
assert "w10" in reasoning
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
assert _has_event(events, EventType.TOOL_CALL_START)
|
||||
|
||||
def test_channel_end_mid_batch_text_absent(self):
|
||||
"""<channel|> mid-batch with its text absent from delta_text."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nword0 word1 word2 word3 word4 "
|
||||
"word5 word6 word7 word8 ",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"word9 word10 word11 ",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<channel|>word12 word13 word14 word0 word1 word2 ",
|
||||
[_TOK[3], _TOK[4], _TOK[5]],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
for w in ("word9", "word10", "word11"):
|
||||
assert w in reasoning, f"{w!r} missing from reasoning"
|
||||
|
||||
for w in ("word12", "word13", "word14"):
|
||||
assert w in content, f"{w!r} missing from content"
|
||||
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_tool_end_mid_batch_text_absent_unified(self):
|
||||
"""<tool_call|> mid-batch with text absent."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i}" for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
events.extend(
|
||||
engine.feed(
|
||||
_CHANNEL_START_TAG,
|
||||
[_CHANNEL_START_TID],
|
||||
)
|
||||
)
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"thought\nNeed a tool.",
|
||||
[_TOK[0], _TOK[1]],
|
||||
)
|
||||
)
|
||||
events.extend(
|
||||
engine.feed(
|
||||
_TOOL_START_TAG,
|
||||
[_TOOL_START_TID],
|
||||
)
|
||||
)
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"call:get_weather{city:",
|
||||
[_TOK[2], _TOK[3], _TOK[4]],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
'<|"|>San Francisco<|"|>}',
|
||||
[
|
||||
_QUOTE_TID,
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_QUOTE_TID,
|
||||
_TOK[7],
|
||||
_TOOL_END_TID,
|
||||
_TOK[8],
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<tool_call|>w8w9w10w11w12",
|
||||
[_TOK[12], _TOK[13]],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
assert _has_event(events, EventType.TOOL_CALL_END)
|
||||
assert "San Francisco" in _arg_text(events)
|
||||
|
||||
def test_large_batch_holdback_spans_two_batches(self):
|
||||
"""Holdback text spanning two batches with <channel|> in the second."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nThe user asked about machine learning "
|
||||
"and I need to think about the best approach to",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
" explain this complex topic. Let me organize my thoughts.",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<channel|>w0 w1 w2 Here is what I recommend: start with "
|
||||
"the fundamentals and build up from there.",
|
||||
[
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_TOK[12],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
assert "organize my thoughts." in reasoning
|
||||
assert "explain" in reasoning
|
||||
assert "recommend" in content
|
||||
|
||||
assert _has_event(events, EventType.REASONING_START)
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
|
||||
class TestRebuildFromAnchorsLiteralLookalike:
|
||||
"""When delta_text contains a literal mention of a special token's
|
||||
text before the real special token, _rebuild_from_anchors must
|
||||
anchor at the real occurrence, not the literal one."""
|
||||
"""Literal token text in prose must not be consumed as an anchor."""
|
||||
|
||||
@pytest.fixture
|
||||
def tool_scanner(self):
|
||||
@@ -513,8 +928,6 @@ class TestRebuildFromAnchorsLiteralLookalike:
|
||||
)
|
||||
|
||||
def test_literal_before_real_anchor(self, tool_scanner):
|
||||
"""Literal <tool_call> in prose followed by a real <tool_call>
|
||||
special token — the scanner must split at the real one."""
|
||||
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
|
||||
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
|
||||
items = tool_scanner.scan(delta_text, delta_token_ids)
|
||||
@@ -526,14 +939,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
|
||||
assert terminals[0].terminal == "TOOL_START"
|
||||
assert terminals[1].terminal == "TOOL_END"
|
||||
|
||||
# The literal mention must appear in a text chunk, not be
|
||||
# consumed by the TOOL_START anchor.
|
||||
joined_text = "".join(text_parts)
|
||||
assert "<tool_call>" in joined_text
|
||||
assert '{"name":"f"}' in joined_text
|
||||
|
||||
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
|
||||
"""Two real tool calls with a literal mention between them."""
|
||||
delta_text = (
|
||||
'<tool_call>{"name":"a"}</tool_call>'
|
||||
" see <tool_call> syntax "
|
||||
@@ -557,14 +967,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
|
||||
|
||||
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
|
||||
joined_text = "".join(text_parts)
|
||||
# The literal mention between the two real calls must be in text
|
||||
assert "<tool_call> syntax" in joined_text
|
||||
|
||||
|
||||
class TestRebuildFromAnchorsCascadingDeferral:
|
||||
"""When a middle anchor's text is absent from delta_text,
|
||||
only that anchor should be deferred — not subsequent ones
|
||||
with valid positions."""
|
||||
"""Missing middle anchor defers only itself, not subsequent ones."""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
@@ -623,9 +1030,6 @@ class TestRebuildFromAnchorsCascadingDeferral:
|
||||
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
|
||||
joined = "".join(t.text for t in texts)
|
||||
assert "text" in joined
|
||||
# "more" is deferred along with the missing terminal —
|
||||
# it will be resolved in the next scan when the terminal
|
||||
# text arrives.
|
||||
assert bare_scanner._deferred_post_text == "more"
|
||||
assert len(bare_scanner._deferred_terminals) == 1
|
||||
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionToolsParam,
|
||||
)
|
||||
from vllm.parser.engine.registered_adapters import (
|
||||
Gemma4Parser,
|
||||
Qwen3Parser,
|
||||
)
|
||||
|
||||
@@ -48,6 +49,7 @@ class Scenario:
|
||||
reasoning: str | None = None
|
||||
content: str | None = None
|
||||
tool_calls: list[ToolCallSpec] | None = None
|
||||
after_tool_response: bool = False
|
||||
|
||||
|
||||
# ── Scenarios ────────────────────────────────────────────────────────
|
||||
@@ -132,6 +134,12 @@ SCENARIOS: list[Scenario] = [
|
||||
reasoning="",
|
||||
content="The epoch timestamp is 1779111346.",
|
||||
),
|
||||
Scenario(
|
||||
id="tool-after-tool-response",
|
||||
description="Tool call immediately after tool response (agentic flow)",
|
||||
tool_calls=[_READ_TOOL],
|
||||
after_tool_response=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -250,7 +258,13 @@ def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
|
||||
"""Replay sample through the real parser and assert correctness."""
|
||||
tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
|
||||
parser = parser_cls(tokenizer, sample.tools, **kwargs)
|
||||
deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
|
||||
deltas = replay_streaming(
|
||||
parser,
|
||||
sample.tokens,
|
||||
chunk_size=1,
|
||||
tools=sample.tools,
|
||||
prompt_token_ids=sample.prompt_token_ids,
|
||||
)
|
||||
output = collect_output(deltas)
|
||||
assert_parse_output(output, sample)
|
||||
|
||||
@@ -273,6 +287,7 @@ def _make_sample(
|
||||
expected_tool_calls: list[dict] | None,
|
||||
tools: list[dict] | None,
|
||||
chat_template_kwargs: dict | None = None,
|
||||
prompt_token_ids: list[int] | None = None,
|
||||
) -> Sample:
|
||||
tokens = _tokenize(segments, vocab)
|
||||
return Sample(
|
||||
@@ -286,10 +301,11 @@ def _make_sample(
|
||||
expected_tool_calls=expected_tool_calls,
|
||||
tools=_validate_tools(tools),
|
||||
chat_template_kwargs=chat_template_kwargs,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
)
|
||||
|
||||
|
||||
# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
|
||||
# ── Qwen3 (XML tool format, starts in REASONING) ────────────────────
|
||||
|
||||
_QWEN3_VOCAB: dict[str, int] = {
|
||||
"<think>": 50,
|
||||
@@ -376,10 +392,105 @@ def _build_qwen3(
|
||||
return sample
|
||||
|
||||
|
||||
# ── Gemma4 (channel reasoning, custom arg format) ────────────────────
|
||||
|
||||
_GEMMA4_VOCAB: dict[str, int] = {
|
||||
"<|channel>": 50,
|
||||
"<channel|>": 51,
|
||||
"<|tool_call>": 48,
|
||||
"<tool_call|>": 49,
|
||||
'<|"|>': 52,
|
||||
"<|turn>": 53,
|
||||
"<|tool_response>": 54,
|
||||
}
|
||||
_GEMMA4_THOUGHT_PREFIX = "thought\n"
|
||||
_GEMMA4_QUOTE = '<|"|>'
|
||||
|
||||
|
||||
def _gemma4_value_segments(value: Any) -> list[tuple[str, bool]]:
|
||||
"""Render a value in Gemma4 arg format as segments."""
|
||||
if isinstance(value, str):
|
||||
return [(_GEMMA4_QUOTE, True), (value, False), (_GEMMA4_QUOTE, True)]
|
||||
if isinstance(value, bool):
|
||||
return [("true" if value else "false", False)]
|
||||
if isinstance(value, (int, float)):
|
||||
return [(str(value), False)]
|
||||
if isinstance(value, dict):
|
||||
segs: list[tuple[str, bool]] = [("{", False)]
|
||||
for i, (k, v) in enumerate(value.items()):
|
||||
if i > 0:
|
||||
segs.append((",", False))
|
||||
segs.append((f"{k}:", False))
|
||||
segs.extend(_gemma4_value_segments(v))
|
||||
segs.append(("}", False))
|
||||
return segs
|
||||
if isinstance(value, list):
|
||||
segs = [("[", False)]
|
||||
for i, item in enumerate(value):
|
||||
if i > 0:
|
||||
segs.append((",", False))
|
||||
segs.extend(_gemma4_value_segments(item))
|
||||
segs.append(("]", False))
|
||||
return segs
|
||||
return [(json.dumps(value, ensure_ascii=False), False)]
|
||||
|
||||
|
||||
def _gemma4_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = [
|
||||
("<|tool_call>", True),
|
||||
(f"call:{tc.name}", False),
|
||||
("{", False),
|
||||
]
|
||||
for i, (key, value) in enumerate(tc.arguments.items()):
|
||||
if i > 0:
|
||||
segs.append((",", False))
|
||||
segs.append((f"{key}:", False))
|
||||
segs.extend(_gemma4_value_segments(value))
|
||||
segs.append(("}", False))
|
||||
segs.append(("<tool_call|>", True))
|
||||
return segs
|
||||
|
||||
|
||||
def _gemma4_segments(scenario: Scenario) -> list[tuple[str, bool]]:
|
||||
segs: list[tuple[str, bool]] = []
|
||||
if scenario.reasoning is not None:
|
||||
segs.append(("<|channel>", True))
|
||||
segs.append((_GEMMA4_THOUGHT_PREFIX, False))
|
||||
segs.append((scenario.reasoning, False))
|
||||
segs.append(("<channel|>", True))
|
||||
if scenario.content is not None:
|
||||
segs.append((scenario.content, False))
|
||||
if scenario.tool_calls:
|
||||
for tc in scenario.tool_calls:
|
||||
segs.extend(_gemma4_tool_segments(tc))
|
||||
return segs
|
||||
|
||||
|
||||
def _build_gemma4(scenario: Scenario, validate: bool = True) -> Sample:
|
||||
prompt_token_ids = None
|
||||
if scenario.after_tool_response:
|
||||
prompt_token_ids = [_GEMMA4_VOCAB["<|tool_response>"]]
|
||||
sample = _make_sample(
|
||||
sample_id=f"gemma4-{scenario.id}",
|
||||
description=scenario.description,
|
||||
vocab=_GEMMA4_VOCAB,
|
||||
segments=_gemma4_segments(scenario),
|
||||
expected_reasoning=scenario.reasoning,
|
||||
expected_content=_qwen3_expected_content(scenario),
|
||||
expected_tool_calls=_expected_tc(scenario),
|
||||
tools=_expected_tools(scenario),
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
)
|
||||
if validate:
|
||||
_validate_sample(sample, Gemma4Parser)
|
||||
return sample
|
||||
|
||||
|
||||
# ── Registry and public API ──────────────────────────────────────────
|
||||
|
||||
_BUILDERS: dict[str, Any] = {
|
||||
"qwen3": _build_qwen3,
|
||||
"gemma4": _build_gemma4,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user