[Frontend] Replace legacy Gemma4 parsers with engine-based implementation (#45588)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-06-15 21:34:07 +00:00
committed by GitHub
co-authored by Flora Feng
parent 25ee659db0
commit 76a373eff4
20 changed files with 2809 additions and 1333 deletions
+9 -4
View File
@@ -33,6 +33,7 @@ class Sample:
expected_tool_calls: list[dict] | None
tools: list[dict] | None = None
chat_template_kwargs: dict | None = None
prompt_token_ids: list[int] | None = None
@dataclass
@@ -120,6 +121,7 @@ def replay_streaming(
holdback_chars: int = 0,
finished_on_last: bool = False,
tools: list[dict] | None = None,
prompt_token_ids: list[int] | None = None,
) -> list[DeltaMessage | None]:
"""Feed tokens through ``parser.parse_delta()`` at a given chunk size.
@@ -146,6 +148,7 @@ def replay_streaming(
all_texts = [text for _, text in tokens]
request = _test_request(tools=tools)
first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
if holdback_chars <= 0:
chunks = list(range(0, len(tokens), chunk_size))
@@ -159,7 +162,7 @@ def replay_streaming(
delta_text,
batch_ids,
request,
prompt_token_ids=[] if start == 0 else None,
prompt_token_ids=first_prompt_ids if start == 0 else None,
finished=finished_on_last and is_last,
)
results.append(result)
@@ -192,7 +195,7 @@ def replay_streaming(
delta_text,
batch_ids,
request,
prompt_token_ids=[] if is_first else None,
prompt_token_ids=first_prompt_ids if is_first else None,
finished=finished_on_last and is_last_chunk,
)
results.append(result)
@@ -205,7 +208,7 @@ def replay_streaming(
delta_text,
batch_ids,
request,
prompt_token_ids=[] if is_first else None,
prompt_token_ids=first_prompt_ids if is_first else None,
finished=finished_on_last,
)
results.append(result)
@@ -218,6 +221,7 @@ def replay_with_text_holdback(
tokens: list[tuple[int, str]],
text_delay: int = 1,
tools: list[dict] | None = None,
prompt_token_ids: list[int] | None = None,
) -> list[DeltaMessage | None]:
"""Replay token-by-token with text arriving *text_delay* steps late.
@@ -235,6 +239,7 @@ def replay_with_text_holdback(
"""
results: list[DeltaMessage | None] = []
request = _test_request(tools=tools)
first_prompt_ids = prompt_token_ids if prompt_token_ids is not None else []
n = len(tokens)
held_texts: list[str] = []
@@ -256,7 +261,7 @@ def replay_with_text_holdback(
delta_text,
[token_id],
request,
prompt_token_ids=[] if i == 0 else None,
prompt_token_ids=first_prompt_ids if i == 0 else None,
finished=is_last,
)
results.append(result)
+19 -10
View File
@@ -29,8 +29,9 @@ from vllm.parser.parser_manager import ParserManager
_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
_PAIRINGS: dict[str, tuple[str, str]] = {
"engine": ("qwen3_coder", "qwen3"),
_PAIRINGS: dict[str, tuple[str, str, str]] = {
"engine": ("qwen3_coder", "qwen3", "qwen3"),
"gemma4_engine": ("gemma4", "gemma4", "gemma4"),
}
CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
@@ -38,7 +39,7 @@ CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
@lru_cache
def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
tool_name, reasoning_name = _PAIRINGS[pairings]
tool_name, reasoning_name, _ = _PAIRINGS[pairings]
parser_cls = ParserManager.get_parser(
tool_parser_name=tool_name,
reasoning_parser_name=reasoning_name,
@@ -48,16 +49,23 @@ def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
return parser_cls
_all_samples = build_samples("qwen3")
def _pairing_samples() -> list[tuple[str, object]]:
items: list[tuple[str, object]] = []
for pairing_name, (_, _, model) in _PAIRINGS.items():
for sample in build_samples(model):
items.append((pairing_name, sample))
return items
_all_pairing_samples = _pairing_samples()
@pytest.mark.parametrize(
"pairings",
list(_PAIRINGS),
ids=lambda p: f"mode={p}",
)
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
@pytest.mark.parametrize(
"pairings,sample",
_all_pairing_samples,
ids=lambda v: v.id if hasattr(v, "id") else v,
)
def test_delegating_replay(sample, chunk_size, pairings):
parser_cls = _get_delegating_parser_cls(pairings=pairings)
@@ -77,6 +85,7 @@ def test_delegating_replay(sample, chunk_size, pairings):
chunk_size=chunk_size,
finished_on_last=True,
tools=sample.tools,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
File diff suppressed because it is too large Load Diff
+97
View File
@@ -631,6 +631,103 @@ class TestFixArgTypes:
original = '{"name": "Alice"}'
assert engine._fix_arg_types(original, "f") == original
@pytest.mark.parametrize(
"properties, input_json, expected_substr",
[
({"count": {"type": "integer"}}, '{"count": "42"}', '"count": 42'),
({"score": {"type": "number"}}, '{"score": "3.14"}', '"score": 3.14'),
({"flag": {"type": "boolean"}}, '{"flag": "true"}', '"flag": true'),
({"flag": {"type": "boolean"}}, '{"flag": "false"}', '"flag": false'),
({"val": {"type": "null"}}, '{"val": "null"}', '"val": null'),
({"val": {"type": ["string", "null"]}}, '{"val": "null"}', '"val": null'),
({"score": {"type": "number"}}, '{"score": "108."}', '"score": 108'),
],
ids=[
"string_to_int",
"string_to_float",
"string_to_bool_true",
"string_to_bool_false",
"string_to_null",
"string_to_null_union",
"trailing_dot_float",
],
)
def test_string_coerced_to_schema_type(
self,
properties,
input_json,
expected_substr,
):
tool = _make_tool("f", properties)
engine = _make_engine(tools=[tool])
result = engine._fix_arg_types(input_json, "f")
assert expected_substr in result
def test_mixed_types_coerced(self):
tool = _make_tool(
"f",
{
"count": {"type": "integer"},
"active": {"type": "boolean"},
"score": {"type": "number"},
},
)
engine = _make_engine(tools=[tool])
result = engine._fix_arg_types(
'{"count": "42", "active": "true", "score": "3.14"}', "f"
)
parsed = json.loads(result)
assert parsed["count"] == 42
assert parsed["active"] is True
assert parsed["score"] == 3.14
def test_nested_object_coercion(self):
tool = _make_tool(
"f",
{
"inner": {
"type": "object",
"properties": {
"count": {"type": "integer"},
},
},
},
)
engine = _make_engine(tools=[tool])
result = engine._fix_arg_types('{"inner": {"count": "42"}}', "f")
parsed = json.loads(result)
assert parsed["inner"]["count"] == 42
def test_array_item_coercion(self):
tool = _make_tool(
"f",
{
"nums": {
"type": "array",
"items": {"type": "integer"},
},
},
)
engine = _make_engine(tools=[tool])
result = engine._fix_arg_types('{"nums": ["42", "5"]}', "f")
parsed = json.loads(result)
assert parsed["nums"] == [42, 5]
def test_array_mixed_item_types(self):
tool = _make_tool(
"f",
{
"vals": {
"type": "array",
"items": {"type": "number"},
},
},
)
engine = _make_engine(tools=[tool])
result = engine._fix_arg_types('{"vals": ["42", "3.14"]}', "f")
parsed = json.loads(result)
assert parsed["vals"] == [42, 3.14]
# ── TestBuildExtractedResult ─────────────────────────────────────────
+85 -1
View File
@@ -17,19 +17,25 @@ from tests.parser.engine.replay_harness import (
collect_output,
make_mock_tokenizer,
replay_streaming,
replay_with_text_holdback,
)
from tests.parser.engine.trace_builder import build_samples
from vllm.parser.abstract_parser import Parser
from vllm.parser.engine.registered_adapters import (
Gemma4Parser,
Qwen3Parser,
)
_ENGINE_PARSERS: dict[str, type[Parser]] = {
"qwen3_engine": Qwen3Parser,
"gemma4_engine": Gemma4Parser,
}
_gemma4_samples = build_samples("gemma4")
_qwen3_samples = build_samples("qwen3")
_GEMMA4_TERMINALS = ["<|channel>", "<channel|>", "<|tool_call>", "<tool_call|>"]
_QWEN3_TERMINALS = [
"<think>",
"</think>",
@@ -56,6 +62,7 @@ class TestQwen3ReplayWithHoldback:
sample.tokens,
chunk_size=chunk_size,
holdback_chars=holdback,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
@@ -67,10 +74,85 @@ class TestQwen3ReplayWithHoldback:
)
@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
@pytest.mark.parametrize("chunk_size", [3, 5, 10], ids=lambda c: f"chunk{c}")
@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
class TestGemma4ReplayWithHoldback:
"""Replay with simulated detokenizer holdback."""
def test_replay(self, sample, chunk_size, holdback):
tokenizer = make_mock_tokenizer(sample)
parser = Gemma4Parser(tokenizer, sample.tools)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=chunk_size,
holdback_chars=holdback,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
assert_no_terminal_leakage(
output,
_GEMMA4_TERMINALS,
context=f"chunk_size={chunk_size}, holdback={holdback}",
)
TEXT_HOLDBACK_DELAYS = [1, 2, 3]
@pytest.mark.parametrize("delay", TEXT_HOLDBACK_DELAYS, ids=lambda d: f"delay{d}")
@pytest.mark.parametrize("sample", _gemma4_samples, ids=lambda s: s.id)
class TestGemma4TextHoldback:
"""Replay with production-like text/token-ID misalignment.
In production the detokenizer sends token IDs immediately but holds
back text by N tokens. This exercises the TokenIDScanner deferred
terminal path that aligned-holdback tests do not cover.
"""
def test_replay(self, sample, delay):
tokenizer = make_mock_tokenizer(sample)
parser = Gemma4Parser(tokenizer, sample.tools)
deltas = replay_with_text_holdback(
parser,
sample.tokens,
text_delay=delay,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
assert_no_terminal_leakage(
output,
_GEMMA4_TERMINALS,
context=f"text_delay={delay}",
)
class TestParserEngineAdjustRequest:
"""Verify ParserEngine and its adapters set skip_special_tokens=False."""
def test_adjust_request_disables_skip_special_tokens(self):
sample = _gemma4_samples[0]
tokenizer = make_mock_tokenizer(sample)
parser = Gemma4Parser(tokenizer, sample.tools)
request = _test_request()
assert request.skip_special_tokens is True
adjusted = parser.adjust_request(request)
assert adjusted.skip_special_tokens is False
_TOOL_CALL_SAMPLES = [
(Qwen3Parser, s)
for s in _qwen3_samples
if s.expected_tool_calls and s.expected_reasoning
] + [
(Gemma4Parser, s)
for s in _gemma4_samples
if s.expected_tool_calls and s.expected_reasoning
]
@@ -147,7 +229,9 @@ class TestSkipToolParsingReplay:
"".join(all_texts[start:end]),
all_ids[start:end],
request,
prompt_token_ids=[] if start == 0 else None,
prompt_token_ids=(sample.prompt_token_ids or [])
if start == 0
else None,
finished=is_last,
)
results.append(result)
+528 -124
View File
@@ -1,20 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for TokenIDScanner, focusing on hold-back text recovery.
Uses gemma4_config for all end-to-end engine tests, covering
reasoning channels, tool calls, and combined flows."""
"""Tests for TokenIDScanner."""
from unittest.mock import MagicMock
import pytest
from vllm.parser.engine.events import EventType
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
from vllm.parser.engine.token_id_scanner import (
PreLexedTerminal,
TextChunk,
TokenIDScanner,
)
from vllm.parser.gemma4 import gemma4_config
CHANNEL_START = "<|channel>"
CHANNEL_END = "<channel|>"
@@ -54,8 +53,7 @@ def scanner(tokenizer):
class TestJoinDecodedTextReturnsStr:
"""_join_decoded_text now returns str unconditionally (was
str | None when an isinstance guard made a branch unreachable)."""
"""_join_decoded_text always returns str."""
@pytest.fixture
def bare_scanner(self):
@@ -83,9 +81,7 @@ class TestJoinDecodedTextReturnsStr:
class TestHoldbackTextRecovery:
def test_holdback_text_with_special_token_text_absent(self, scanner):
"""delta_text has hold-back text but the special token's text is
NOT in delta_text (held back by the detokenizer). Terminal is
deferred until the text arrives in a subsequent delta."""
"""Terminal deferred when its text is absent from delta_text."""
result = scanner.scan(
delta_text="processed is appropriate.",
delta_token_ids=[CHANNEL_END_ID],
@@ -93,8 +89,6 @@ class TestHoldbackTextRecovery:
assert len(result) == 0
# Second scan: terminal text arrives (detokenizer flushes).
# Deferred terminal resolves with holdback text before it.
result2 = scanner.scan(
delta_text="<channel|>Understood.",
delta_token_ids=[20, 21],
@@ -108,7 +102,7 @@ class TestHoldbackTextRecovery:
assert "Understood." in combined
def test_holdback_text_with_special_token_text_present(self, scanner):
"""delta_text includes hold-back text AND the special token text."""
"""Hold-back text + special token text both in delta_text."""
result = scanner.scan(
delta_text="holdback text<channel|>",
delta_token_ids=[CHANNEL_END_ID],
@@ -121,7 +115,7 @@ class TestHoldbackTextRecovery:
assert result[1].terminal == "THINK_END"
def test_no_holdback_text(self, scanner):
"""delta_text is exactly the special token text — no hold-back."""
"""delta_text is exactly the special token text."""
result = scanner.scan(
delta_text="<channel|>",
delta_token_ids=[CHANNEL_END_ID],
@@ -132,7 +126,7 @@ class TestHoldbackTextRecovery:
assert result[0].terminal == "THINK_END"
def test_empty_delta_text(self, scanner):
"""delta_text is empty — terminal deferred until text arrives."""
"""Empty delta_text defers the terminal until text arrives."""
result = scanner.scan(
delta_text="",
delta_token_ids=[CHANNEL_END_ID],
@@ -146,9 +140,7 @@ class TestHoldbackTextRecovery:
assert flushed[0].terminal == "THINK_END"
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
"""delta_text="" with multiple tokens including special: all
results deferred — individually-decoded TextChunks are unreliable
and PreLexedTerminals wait for text confirmation."""
"""Empty delta_text with multiple tokens: all results deferred."""
tool_start_id = 400
tok_a = 201
tok_b = 202
@@ -176,7 +168,6 @@ class TestHoldbackTextRecovery:
assert flushed[0].terminal == "TOOL_START"
def test_holdback_before_start_tag(self, scanner):
"""Hold-back text before a reasoning start tag."""
result = scanner.scan(
delta_text="prefix text<|channel>",
delta_token_ids=[CHANNEL_START_ID],
@@ -189,8 +180,7 @@ class TestHoldbackTextRecovery:
assert result[1].terminal == "THINK_START"
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
"""Stream-interval > 1: batch has regular tokens + special token.
delta_text differs from individual decodes (context-dependent)."""
"""Multi-token batch with special token in the middle."""
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
@@ -215,10 +205,7 @@ class TestHoldbackTextRecovery:
assert "holdback wordA" in "".join(texts)
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
"""Stream-interval > 1: batch has regular + special token, but
delta_text doesn't contain the special token text at all
(held back by detokenizer along with trailing regular tokens).
Terminal is deferred until text arrives."""
"""Multi-token batch where special token text is absent."""
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
@@ -239,8 +226,6 @@ class TestHoldbackTextRecovery:
assert len(result) == 0
# Next delta: terminal text arrives (detokenizer flushes).
# Deferred terminal resolves with holdback text before it.
result2 = scanner_multi.scan(
delta_text="<channel|> more text",
delta_token_ids=[300],
@@ -254,8 +239,7 @@ class TestHoldbackTextRecovery:
assert "more text" in combined
def test_holdback_with_content_after_special_token(self, tokenizer):
"""delta_text has hold-back + special token + content after,
with corresponding token IDs for all parts."""
"""Hold-back + special token + content after in one delta."""
tok_content = 210
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
@@ -283,8 +267,7 @@ class TestHoldbackTextRecovery:
class TestDropTokens:
def test_drop_token_with_holdback(self, tokenizer):
"""Drop tokens stripped from delta_text, hold-back text preserved.
Terminal is deferred when its text is absent from delta_text."""
"""Drop tokens stripped; hold-back text preserved."""
drop_id = 300
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
@@ -304,7 +287,6 @@ class TestDropTokens:
assert len(result) == 0
# Terminal text arrives in next delta; deferred terminal resolves.
result2 = scanner.scan(
delta_text="<channel|>content",
delta_token_ids=[20],
@@ -321,40 +303,10 @@ class TestDropTokens:
class TestEndToEndReasoningHoldback:
"""End-to-end tests through the full parser engine simulating
stream-interval > 1 and detokenizer hold-back, using
gemma4_config."""
"""End-to-end engine tests with detokenizer hold-back."""
def test_reasoning_content_not_truncated(self):
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
config = ParserEngineConfig(
name="test_channel",
initial_state=ParserState.CONTENT,
terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
token_id_terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
config = gemma4_config()
tok = MagicMock()
vocab = {
CHANNEL_START: CHANNEL_START_ID,
@@ -369,30 +321,21 @@ class TestEndToEndReasoningHoldback:
engine = StreamingParserEngine(config, tok)
all_events = []
# Delta 1: channel start token (text includes start tag)
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
# Delta 2: reasoning text (normal content, no special tokens)
all_events.extend(
engine.feed(
"thought\nThe request was received and ",
[10, 11, 12, 13, 14],
)
)
# Delta 3: MORE reasoning text, the detokenizer held some back.
# Then channel end token arrives in token_ids, but its text
# is NOT in delta_text (held back by detokenizer).
# delta_text = previously held-back reasoning text only.
# CHANNEL_END token arrives but its text is held back.
all_events.extend(
engine.feed(
"processed is appropriate.",
[CHANNEL_END_ID],
)
)
# Delta 4: detokenizer flushes held-back channel end text
# plus new content tokens.
# Detokenizer flushes the held-back text.
all_events.extend(
engine.feed(
"<channel|>Understood.",
@@ -413,36 +356,7 @@ class TestEndToEndReasoningHoldback:
assert "Understood." in content_text
def test_backtick_content_not_truncated(self):
"""Reproduces the hostname backtick truncation case."""
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
config = ParserEngineConfig(
name="test_channel",
initial_state=ParserState.CONTENT,
terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
token_id_terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
config = gemma4_config()
tok = MagicMock()
vocab = {
CHANNEL_START: CHANNEL_START_ID,
@@ -464,17 +378,12 @@ class TestEndToEndReasoningHoldback:
[10, 11, 12, 13],
)
)
# Hold-back text includes backtick content; channel end text
# absent from delta_text.
all_events.extend(
engine.feed(
"`hostname`.\n",
[CHANNEL_END_ID],
)
)
# Next delta flushes channel end + tool call start
all_events.extend(
engine.feed(
"<channel|>tool output",
@@ -491,10 +400,516 @@ class TestEndToEndReasoningHoldback:
assert "`hostname`." in reasoning_text
_CHANNEL_START_TAG = "<|channel>"
_CHANNEL_END_TAG = "<channel|>"
_TOOL_START_TAG = "<|tool_call>"
_TOOL_END_TAG = "<tool_call|>"
_QUOTE_TAG = '<|"|>'
_CHANNEL_START_TID = 100
_CHANNEL_END_TID = 101
_TOOL_START_TID = 102
_TOOL_END_TID = 103
_QUOTE_TID = 104
_TOK = list(range(200, 215))
def _gemma4_vocab() -> dict[str, int]:
return {
_CHANNEL_START_TAG: _CHANNEL_START_TID,
_CHANNEL_END_TAG: _CHANNEL_END_TID,
_TOOL_START_TAG: _TOOL_START_TID,
_TOOL_END_TAG: _TOOL_END_TID,
_QUOTE_TAG: _QUOTE_TID,
}
def _make_gemma4_tokenizer(
extra_decode: dict[int, str] | None = None,
) -> MagicMock:
special = {
_CHANNEL_START_TID: _CHANNEL_START_TAG,
_CHANNEL_END_TID: _CHANNEL_END_TAG,
_TOOL_START_TID: _TOOL_START_TAG,
_TOOL_END_TID: _TOOL_END_TAG,
_QUOTE_TID: _QUOTE_TAG,
}
decode_map = {**special, **(extra_decode or {})}
tok = MagicMock()
tok.get_vocab.return_value = _gemma4_vocab()
tok.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
return tok
def _collect_events(engine, deltas):
from vllm.parser.engine.events import SemanticEvent
all_events: list[SemanticEvent] = []
for delta_text, delta_token_ids in deltas:
all_events.extend(engine.feed(delta_text, delta_token_ids))
all_events.extend(engine.finish())
return all_events
def _reasoning_text(events) -> str:
return "".join(e.value for e in events if e.type == EventType.REASONING_CHUNK)
def _content_text(events) -> str:
return "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
def _arg_text(events) -> str:
return "".join(e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK)
def _has_event(events, event_type) -> bool:
return any(e.type == event_type for e in events)
class TestMultiTokenBoundaryPreservation:
"""No text lost at state boundaries with multi-token deltas."""
def test_empty_delta_text_at_channel_end_unified(self):
"""Empty delta_text when CHANNEL_END arrives; text comes later."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
("", [_CHANNEL_START_TID]),
("<|channel>thought\nSome reasoning.", [_TOK[0], _TOK[1]]),
("", [_CHANNEL_END_TID]),
("<channel|>Final answer.", [_TOK[2], _TOK[3]]),
],
)
reasoning = _reasoning_text(events)
content = _content_text(events)
assert "Some reasoning." in reasoning
assert "Final answer." in content
assert _has_event(events, EventType.REASONING_START)
assert _has_event(events, EventType.REASONING_END)
def test_deferred_channel_end_flushed_at_finish_unified(self):
"""Deferred CHANNEL_END flushed at end-of-stream."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
("thought\nReasoning text.", [_TOK[0]]),
(" Final thought.", [_CHANNEL_END_TID]),
],
)
reasoning = _reasoning_text(events)
assert "Reasoning text. Final thought." in reasoning
assert _has_event(events, EventType.REASONING_END)
def test_reasoning_to_tool_call_handoff_unified(self):
"""Full reasoning -> content -> tool call flow."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
("thought\nI need to check the weather.", [_TOK[0], _TOK[1], _TOK[2]]),
(_CHANNEL_END_TAG, [_CHANNEL_END_TID]),
("Let me call a tool.", [_TOK[3], _TOK[4]]),
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_weather{city:", [_TOK[5], _TOK[6]]),
('<|"|>SF<|"|>}', [_QUOTE_TID, _TOK[7], _QUOTE_TID, _TOK[8]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
],
)
reasoning = _reasoning_text(events)
content = _content_text(events)
assert "I need to check the weather." in reasoning
assert "Let me call a tool." in content
assert _has_event(events, EventType.REASONING_START)
assert _has_event(events, EventType.REASONING_END)
assert _has_event(events, EventType.TOOL_CALL_START)
assert _has_event(events, EventType.TOOL_CALL_END)
assert "SF" in _arg_text(events)
def test_multiple_tool_calls_rapid_transitions_unified(self):
"""Two back-to-back tool calls with correct tool_index tracking."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_weather{city:", [_TOK[0], _TOK[1]]),
('<|"|>NYC<|"|>}', [_QUOTE_TID, _TOK[2], _QUOTE_TID, _TOK[3]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_time{tz:", [_TOK[4], _TOK[5]]),
('<|"|>EST<|"|>}', [_QUOTE_TID, _TOK[6], _QUOTE_TID, _TOK[7]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
],
)
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
assert len(starts) == 2
assert len(ends) == 2
assert starts[0].tool_index == 0
assert starts[1].tool_index == 1
names = "".join(e.value for e in events if e.type == EventType.TOOL_NAME)
assert "get_weather" in names
assert "get_time" in names
def test_deferred_channel_end_before_tool_call_unified(self):
"""Deferred CHANNEL_END followed by a tool call."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
("thought\nNeed to call a tool.", [_TOK[0], _TOK[1]]),
(" Let me proceed.", [_CHANNEL_END_TID]),
(_CHANNEL_END_TAG, [_TOK[2]]),
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_weather{city:", [_TOK[3], _TOK[4]]),
('<|"|>Tokyo<|"|>}', [_QUOTE_TID, _TOK[5], _QUOTE_TID, _TOK[6]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
],
)
reasoning = _reasoning_text(events)
assert "Need to call a tool. Let me proceed." in reasoning
assert _has_event(events, EventType.REASONING_END)
assert _has_event(events, EventType.TOOL_CALL_START)
assert _has_event(events, EventType.TOOL_CALL_END)
assert "Tokyo" in _arg_text(events)
class TestStreamInterval10:
"""Tests with stream_interval=10 (large multi-token batches)."""
def test_channel_end_mid_batch_text_present(self):
"""<channel|> mid-batch with its text present in delta_text."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nword0 word1 word2 word3 word4 "
"word5 word6 word7 word8 ",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
"word9 word10 word11 <channel|>word12 word13 word14 word0 word1 word2 ",
[
_TOK[9],
_TOK[10],
_TOK[11],
_CHANNEL_END_TID,
_TOK[12],
_TOK[13],
_TOK[14],
_TOK[0],
_TOK[1],
_TOK[2],
],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
content = _content_text(events)
for w in ("word9", "word10", "word11"):
assert w in reasoning, f"{w!r} missing from reasoning"
for w in ("word12", "word13", "word14"):
assert w in content, f"{w!r} missing from content"
assert _has_event(events, EventType.REASONING_END)
def test_channel_end_and_tool_start_same_batch_unified(self):
"""Both <channel|> and <|tool_call> in a single batch."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nw0 w1 w2 w3 w4 w5 w6 w7 w8 ",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
"w9 w10 <channel|>w11 <|tool_call>",
[
_TOK[9],
_TOK[10],
_CHANNEL_END_TID,
_TOK[11],
_TOOL_START_TID,
_TOK[12],
_TOK[13],
_TOK[14],
_TOK[0],
_TOK[1],
],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
assert "w9" in reasoning
assert "w10" in reasoning
assert _has_event(events, EventType.REASONING_END)
assert _has_event(events, EventType.TOOL_CALL_START)
def test_channel_end_mid_batch_text_absent(self):
"""<channel|> mid-batch with its text absent from delta_text."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nword0 word1 word2 word3 word4 "
"word5 word6 word7 word8 ",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
"word9 word10 word11 ",
[
_TOK[9],
_TOK[10],
_TOK[11],
_CHANNEL_END_TID,
_TOK[12],
_TOK[13],
_TOK[14],
_TOK[0],
_TOK[1],
_TOK[2],
],
)
)
events.extend(
engine.feed(
"<channel|>word12 word13 word14 word0 word1 word2 ",
[_TOK[3], _TOK[4], _TOK[5]],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
content = _content_text(events)
for w in ("word9", "word10", "word11"):
assert w in reasoning, f"{w!r} missing from reasoning"
for w in ("word12", "word13", "word14"):
assert w in content, f"{w!r} missing from content"
assert _has_event(events, EventType.REASONING_END)
def test_tool_end_mid_batch_text_absent_unified(self):
"""<tool_call|> mid-batch with text absent."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i}" for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
_CHANNEL_START_TAG,
[_CHANNEL_START_TID],
)
)
events.extend(
engine.feed(
"thought\nNeed a tool.",
[_TOK[0], _TOK[1]],
)
)
events.extend(
engine.feed(
_TOOL_START_TAG,
[_TOOL_START_TID],
)
)
events.extend(
engine.feed(
"call:get_weather{city:",
[_TOK[2], _TOK[3], _TOK[4]],
)
)
events.extend(
engine.feed(
'<|"|>San Francisco<|"|>}',
[
_QUOTE_TID,
_TOK[5],
_TOK[6],
_QUOTE_TID,
_TOK[7],
_TOOL_END_TID,
_TOK[8],
_TOK[9],
_TOK[10],
_TOK[11],
],
)
)
events.extend(
engine.feed(
"<tool_call|>w8w9w10w11w12",
[_TOK[12], _TOK[13]],
)
)
events.extend(engine.finish())
assert _has_event(events, EventType.TOOL_CALL_END)
assert "San Francisco" in _arg_text(events)
def test_large_batch_holdback_spans_two_batches(self):
"""Holdback text spanning two batches with <channel|> in the second."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nThe user asked about machine learning "
"and I need to think about the best approach to",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
" explain this complex topic. Let me organize my thoughts.",
[
_TOK[9],
_TOK[10],
_TOK[11],
_TOK[12],
_TOK[13],
_TOK[14],
_CHANNEL_END_TID,
_TOK[0],
_TOK[1],
_TOK[2],
],
)
)
events.extend(
engine.feed(
"<channel|>w0 w1 w2 Here is what I recommend: start with "
"the fundamentals and build up from there.",
[
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
_TOK[9],
_TOK[10],
_TOK[11],
_TOK[12],
],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
content = _content_text(events)
assert "organize my thoughts." in reasoning
assert "explain" in reasoning
assert "recommend" in content
assert _has_event(events, EventType.REASONING_START)
assert _has_event(events, EventType.REASONING_END)
class TestRebuildFromAnchorsLiteralLookalike:
"""When delta_text contains a literal mention of a special token's
text before the real special token, _rebuild_from_anchors must
anchor at the real occurrence, not the literal one."""
"""Literal token text in prose must not be consumed as an anchor."""
@pytest.fixture
def tool_scanner(self):
@@ -513,8 +928,6 @@ class TestRebuildFromAnchorsLiteralLookalike:
)
def test_literal_before_real_anchor(self, tool_scanner):
"""Literal <tool_call> in prose followed by a real <tool_call>
special token — the scanner must split at the real one."""
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
items = tool_scanner.scan(delta_text, delta_token_ids)
@@ -526,14 +939,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
assert terminals[0].terminal == "TOOL_START"
assert terminals[1].terminal == "TOOL_END"
# The literal mention must appear in a text chunk, not be
# consumed by the TOOL_START anchor.
joined_text = "".join(text_parts)
assert "<tool_call>" in joined_text
assert '{"name":"f"}' in joined_text
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
"""Two real tool calls with a literal mention between them."""
delta_text = (
'<tool_call>{"name":"a"}</tool_call>'
" see <tool_call> syntax "
@@ -557,14 +967,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
joined_text = "".join(text_parts)
# The literal mention between the two real calls must be in text
assert "<tool_call> syntax" in joined_text
class TestRebuildFromAnchorsCascadingDeferral:
"""When a middle anchor's text is absent from delta_text,
only that anchor should be deferred — not subsequent ones
with valid positions."""
"""Missing middle anchor defers only itself, not subsequent ones."""
@pytest.fixture
def bare_scanner(self):
@@ -623,9 +1030,6 @@ class TestRebuildFromAnchorsCascadingDeferral:
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
joined = "".join(t.text for t in texts)
assert "text" in joined
# "more" is deferred along with the missing terminal —
# it will be resolved in the next scan when the terminal
# text arrives.
assert bare_scanner._deferred_post_text == "more"
assert len(bare_scanner._deferred_terminals) == 1
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
+113 -2
View File
@@ -29,6 +29,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.parser.engine.registered_adapters import (
Gemma4Parser,
Qwen3Parser,
)
@@ -48,6 +49,7 @@ class Scenario:
reasoning: str | None = None
content: str | None = None
tool_calls: list[ToolCallSpec] | None = None
after_tool_response: bool = False
# ── Scenarios ────────────────────────────────────────────────────────
@@ -132,6 +134,12 @@ SCENARIOS: list[Scenario] = [
reasoning="",
content="The epoch timestamp is 1779111346.",
),
Scenario(
id="tool-after-tool-response",
description="Tool call immediately after tool response (agentic flow)",
tool_calls=[_READ_TOOL],
after_tool_response=True,
),
]
@@ -250,7 +258,13 @@ def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
"""Replay sample through the real parser and assert correctness."""
tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
parser = parser_cls(tokenizer, sample.tools, **kwargs)
deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
deltas = replay_streaming(
parser,
sample.tokens,
chunk_size=1,
tools=sample.tools,
prompt_token_ids=sample.prompt_token_ids,
)
output = collect_output(deltas)
assert_parse_output(output, sample)
@@ -273,6 +287,7 @@ def _make_sample(
expected_tool_calls: list[dict] | None,
tools: list[dict] | None,
chat_template_kwargs: dict | None = None,
prompt_token_ids: list[int] | None = None,
) -> Sample:
tokens = _tokenize(segments, vocab)
return Sample(
@@ -286,10 +301,11 @@ def _make_sample(
expected_tool_calls=expected_tool_calls,
tools=_validate_tools(tools),
chat_template_kwargs=chat_template_kwargs,
prompt_token_ids=prompt_token_ids,
)
# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
# ── Qwen3 (XML tool format, starts in REASONING) ────────────────────
_QWEN3_VOCAB: dict[str, int] = {
"<think>": 50,
@@ -376,10 +392,105 @@ def _build_qwen3(
return sample
# ── Gemma4 (channel reasoning, custom arg format) ────────────────────
_GEMMA4_VOCAB: dict[str, int] = {
"<|channel>": 50,
"<channel|>": 51,
"<|tool_call>": 48,
"<tool_call|>": 49,
'<|"|>': 52,
"<|turn>": 53,
"<|tool_response>": 54,
}
_GEMMA4_THOUGHT_PREFIX = "thought\n"
_GEMMA4_QUOTE = '<|"|>'
def _gemma4_value_segments(value: Any) -> list[tuple[str, bool]]:
"""Render a value in Gemma4 arg format as segments."""
if isinstance(value, str):
return [(_GEMMA4_QUOTE, True), (value, False), (_GEMMA4_QUOTE, True)]
if isinstance(value, bool):
return [("true" if value else "false", False)]
if isinstance(value, (int, float)):
return [(str(value), False)]
if isinstance(value, dict):
segs: list[tuple[str, bool]] = [("{", False)]
for i, (k, v) in enumerate(value.items()):
if i > 0:
segs.append((",", False))
segs.append((f"{k}:", False))
segs.extend(_gemma4_value_segments(v))
segs.append(("}", False))
return segs
if isinstance(value, list):
segs = [("[", False)]
for i, item in enumerate(value):
if i > 0:
segs.append((",", False))
segs.extend(_gemma4_value_segments(item))
segs.append(("]", False))
return segs
return [(json.dumps(value, ensure_ascii=False), False)]
def _gemma4_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = [
("<|tool_call>", True),
(f"call:{tc.name}", False),
("{", False),
]
for i, (key, value) in enumerate(tc.arguments.items()):
if i > 0:
segs.append((",", False))
segs.append((f"{key}:", False))
segs.extend(_gemma4_value_segments(value))
segs.append(("}", False))
segs.append(("<tool_call|>", True))
return segs
def _gemma4_segments(scenario: Scenario) -> list[tuple[str, bool]]:
segs: list[tuple[str, bool]] = []
if scenario.reasoning is not None:
segs.append(("<|channel>", True))
segs.append((_GEMMA4_THOUGHT_PREFIX, False))
segs.append((scenario.reasoning, False))
segs.append(("<channel|>", True))
if scenario.content is not None:
segs.append((scenario.content, False))
if scenario.tool_calls:
for tc in scenario.tool_calls:
segs.extend(_gemma4_tool_segments(tc))
return segs
def _build_gemma4(scenario: Scenario, validate: bool = True) -> Sample:
prompt_token_ids = None
if scenario.after_tool_response:
prompt_token_ids = [_GEMMA4_VOCAB["<|tool_response>"]]
sample = _make_sample(
sample_id=f"gemma4-{scenario.id}",
description=scenario.description,
vocab=_GEMMA4_VOCAB,
segments=_gemma4_segments(scenario),
expected_reasoning=scenario.reasoning,
expected_content=_qwen3_expected_content(scenario),
expected_tool_calls=_expected_tc(scenario),
tools=_expected_tools(scenario),
prompt_token_ids=prompt_token_ids,
)
if validate:
_validate_sample(sample, Gemma4Parser)
return sample
# ── Registry and public API ──────────────────────────────────────────
_BUILDERS: dict[str, Any] = {
"qwen3": _build_qwen3,
"gemma4": _build_gemma4,
}