[Frontend] Replace legacy Gemma4 parsers with engine-based implementation (#45588)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ben Browning
2026-06-15 21:34:07 +00:00
committed by GitHub
co-authored by Flora Feng
parent 25ee659db0
commit 76a373eff4
20 changed files with 2809 additions and 1333 deletions
+528 -124
View File
@@ -1,20 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for TokenIDScanner, focusing on hold-back text recovery.
Uses gemma4_config for all end-to-end engine tests, covering
reasoning channels, tool calls, and combined flows."""
"""Tests for TokenIDScanner."""
from unittest.mock import MagicMock
import pytest
from vllm.parser.engine.events import EventType
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
from vllm.parser.engine.token_id_scanner import (
PreLexedTerminal,
TextChunk,
TokenIDScanner,
)
from vllm.parser.gemma4 import gemma4_config
CHANNEL_START = "<|channel>"
CHANNEL_END = "<channel|>"
@@ -54,8 +53,7 @@ def scanner(tokenizer):
class TestJoinDecodedTextReturnsStr:
"""_join_decoded_text now returns str unconditionally (was
str | None when an isinstance guard made a branch unreachable)."""
"""_join_decoded_text always returns str."""
@pytest.fixture
def bare_scanner(self):
@@ -83,9 +81,7 @@ class TestJoinDecodedTextReturnsStr:
class TestHoldbackTextRecovery:
def test_holdback_text_with_special_token_text_absent(self, scanner):
"""delta_text has hold-back text but the special token's text is
NOT in delta_text (held back by the detokenizer). Terminal is
deferred until the text arrives in a subsequent delta."""
"""Terminal deferred when its text is absent from delta_text."""
result = scanner.scan(
delta_text="processed is appropriate.",
delta_token_ids=[CHANNEL_END_ID],
@@ -93,8 +89,6 @@ class TestHoldbackTextRecovery:
assert len(result) == 0
# Second scan: terminal text arrives (detokenizer flushes).
# Deferred terminal resolves with holdback text before it.
result2 = scanner.scan(
delta_text="<channel|>Understood.",
delta_token_ids=[20, 21],
@@ -108,7 +102,7 @@ class TestHoldbackTextRecovery:
assert "Understood." in combined
def test_holdback_text_with_special_token_text_present(self, scanner):
"""delta_text includes hold-back text AND the special token text."""
"""Hold-back text + special token text both in delta_text."""
result = scanner.scan(
delta_text="holdback text<channel|>",
delta_token_ids=[CHANNEL_END_ID],
@@ -121,7 +115,7 @@ class TestHoldbackTextRecovery:
assert result[1].terminal == "THINK_END"
def test_no_holdback_text(self, scanner):
"""delta_text is exactly the special token text — no hold-back."""
"""delta_text is exactly the special token text."""
result = scanner.scan(
delta_text="<channel|>",
delta_token_ids=[CHANNEL_END_ID],
@@ -132,7 +126,7 @@ class TestHoldbackTextRecovery:
assert result[0].terminal == "THINK_END"
def test_empty_delta_text(self, scanner):
"""delta_text is empty — terminal deferred until text arrives."""
"""Empty delta_text defers the terminal until text arrives."""
result = scanner.scan(
delta_text="",
delta_token_ids=[CHANNEL_END_ID],
@@ -146,9 +140,7 @@ class TestHoldbackTextRecovery:
assert flushed[0].terminal == "THINK_END"
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
"""delta_text="" with multiple tokens including special: all
results deferred — individually-decoded TextChunks are unreliable
and PreLexedTerminals wait for text confirmation."""
"""Empty delta_text with multiple tokens: all results deferred."""
tool_start_id = 400
tok_a = 201
tok_b = 202
@@ -176,7 +168,6 @@ class TestHoldbackTextRecovery:
assert flushed[0].terminal == "TOOL_START"
def test_holdback_before_start_tag(self, scanner):
"""Hold-back text before a reasoning start tag."""
result = scanner.scan(
delta_text="prefix text<|channel>",
delta_token_ids=[CHANNEL_START_ID],
@@ -189,8 +180,7 @@ class TestHoldbackTextRecovery:
assert result[1].terminal == "THINK_START"
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
"""Stream-interval > 1: batch has regular tokens + special token.
delta_text differs from individual decodes (context-dependent)."""
"""Multi-token batch with special token in the middle."""
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
@@ -215,10 +205,7 @@ class TestHoldbackTextRecovery:
assert "holdback wordA" in "".join(texts)
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
"""Stream-interval > 1: batch has regular + special token, but
delta_text doesn't contain the special token text at all
(held back by detokenizer along with trailing regular tokens).
Terminal is deferred until text arrives."""
"""Multi-token batch where special token text is absent."""
tok_a = 201
tok_b = 202
tokenizer.decode.side_effect = lambda ids: {
@@ -239,8 +226,6 @@ class TestHoldbackTextRecovery:
assert len(result) == 0
# Next delta: terminal text arrives (detokenizer flushes).
# Deferred terminal resolves with holdback text before it.
result2 = scanner_multi.scan(
delta_text="<channel|> more text",
delta_token_ids=[300],
@@ -254,8 +239,7 @@ class TestHoldbackTextRecovery:
assert "more text" in combined
def test_holdback_with_content_after_special_token(self, tokenizer):
"""delta_text has hold-back + special token + content after,
with corresponding token IDs for all parts."""
"""Hold-back + special token + content after in one delta."""
tok_content = 210
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
@@ -283,8 +267,7 @@ class TestHoldbackTextRecovery:
class TestDropTokens:
def test_drop_token_with_holdback(self, tokenizer):
"""Drop tokens stripped from delta_text, hold-back text preserved.
Terminal is deferred when its text is absent from delta_text."""
"""Drop tokens stripped; hold-back text preserved."""
drop_id = 300
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
@@ -304,7 +287,6 @@ class TestDropTokens:
assert len(result) == 0
# Terminal text arrives in next delta; deferred terminal resolves.
result2 = scanner.scan(
delta_text="<channel|>content",
delta_token_ids=[20],
@@ -321,40 +303,10 @@ class TestDropTokens:
class TestEndToEndReasoningHoldback:
"""End-to-end tests through the full parser engine simulating
stream-interval > 1 and detokenizer hold-back, using
gemma4_config."""
"""End-to-end engine tests with detokenizer hold-back."""
def test_reasoning_content_not_truncated(self):
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
config = ParserEngineConfig(
name="test_channel",
initial_state=ParserState.CONTENT,
terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
token_id_terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
config = gemma4_config()
tok = MagicMock()
vocab = {
CHANNEL_START: CHANNEL_START_ID,
@@ -369,30 +321,21 @@ class TestEndToEndReasoningHoldback:
engine = StreamingParserEngine(config, tok)
all_events = []
# Delta 1: channel start token (text includes start tag)
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
# Delta 2: reasoning text (normal content, no special tokens)
all_events.extend(
engine.feed(
"thought\nThe request was received and ",
[10, 11, 12, 13, 14],
)
)
# Delta 3: MORE reasoning text, the detokenizer held some back.
# Then channel end token arrives in token_ids, but its text
# is NOT in delta_text (held back by detokenizer).
# delta_text = previously held-back reasoning text only.
# CHANNEL_END token arrives but its text is held back.
all_events.extend(
engine.feed(
"processed is appropriate.",
[CHANNEL_END_ID],
)
)
# Delta 4: detokenizer flushes held-back channel end text
# plus new content tokens.
# Detokenizer flushes the held-back text.
all_events.extend(
engine.feed(
"<channel|>Understood.",
@@ -413,36 +356,7 @@ class TestEndToEndReasoningHoldback:
assert "Understood." in content_text
def test_backtick_content_not_truncated(self):
"""Reproduces the hostname backtick truncation case."""
from vllm.parser.engine.parser_engine_config import (
ParserEngineConfig,
ParserState,
Transition,
)
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
config = ParserEngineConfig(
name="test_channel",
initial_state=ParserState.CONTENT,
terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
token_id_terminals={
"THINK_START": CHANNEL_START,
"THINK_END": CHANNEL_END,
},
transitions={
(ParserState.CONTENT, "THINK_START"): Transition(
ParserState.REASONING,
(EventType.REASONING_START,),
),
(ParserState.REASONING, "THINK_END"): Transition(
ParserState.CONTENT,
(EventType.REASONING_END,),
),
},
)
config = gemma4_config()
tok = MagicMock()
vocab = {
CHANNEL_START: CHANNEL_START_ID,
@@ -464,17 +378,12 @@ class TestEndToEndReasoningHoldback:
[10, 11, 12, 13],
)
)
# Hold-back text includes backtick content; channel end text
# absent from delta_text.
all_events.extend(
engine.feed(
"`hostname`.\n",
[CHANNEL_END_ID],
)
)
# Next delta flushes channel end + tool call start
all_events.extend(
engine.feed(
"<channel|>tool output",
@@ -491,10 +400,516 @@ class TestEndToEndReasoningHoldback:
assert "`hostname`." in reasoning_text
_CHANNEL_START_TAG = "<|channel>"
_CHANNEL_END_TAG = "<channel|>"
_TOOL_START_TAG = "<|tool_call>"
_TOOL_END_TAG = "<tool_call|>"
_QUOTE_TAG = '<|"|>'
_CHANNEL_START_TID = 100
_CHANNEL_END_TID = 101
_TOOL_START_TID = 102
_TOOL_END_TID = 103
_QUOTE_TID = 104
_TOK = list(range(200, 215))
def _gemma4_vocab() -> dict[str, int]:
return {
_CHANNEL_START_TAG: _CHANNEL_START_TID,
_CHANNEL_END_TAG: _CHANNEL_END_TID,
_TOOL_START_TAG: _TOOL_START_TID,
_TOOL_END_TAG: _TOOL_END_TID,
_QUOTE_TAG: _QUOTE_TID,
}
def _make_gemma4_tokenizer(
extra_decode: dict[int, str] | None = None,
) -> MagicMock:
special = {
_CHANNEL_START_TID: _CHANNEL_START_TAG,
_CHANNEL_END_TID: _CHANNEL_END_TAG,
_TOOL_START_TID: _TOOL_START_TAG,
_TOOL_END_TID: _TOOL_END_TAG,
_QUOTE_TID: _QUOTE_TAG,
}
decode_map = {**special, **(extra_decode or {})}
tok = MagicMock()
tok.get_vocab.return_value = _gemma4_vocab()
tok.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
return tok
def _collect_events(engine, deltas):
from vllm.parser.engine.events import SemanticEvent
all_events: list[SemanticEvent] = []
for delta_text, delta_token_ids in deltas:
all_events.extend(engine.feed(delta_text, delta_token_ids))
all_events.extend(engine.finish())
return all_events
def _reasoning_text(events) -> str:
return "".join(e.value for e in events if e.type == EventType.REASONING_CHUNK)
def _content_text(events) -> str:
return "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
def _arg_text(events) -> str:
return "".join(e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK)
def _has_event(events, event_type) -> bool:
return any(e.type == event_type for e in events)
class TestMultiTokenBoundaryPreservation:
"""No text lost at state boundaries with multi-token deltas."""
def test_empty_delta_text_at_channel_end_unified(self):
"""Empty delta_text when CHANNEL_END arrives; text comes later."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
("", [_CHANNEL_START_TID]),
("<|channel>thought\nSome reasoning.", [_TOK[0], _TOK[1]]),
("", [_CHANNEL_END_TID]),
("<channel|>Final answer.", [_TOK[2], _TOK[3]]),
],
)
reasoning = _reasoning_text(events)
content = _content_text(events)
assert "Some reasoning." in reasoning
assert "Final answer." in content
assert _has_event(events, EventType.REASONING_START)
assert _has_event(events, EventType.REASONING_END)
def test_deferred_channel_end_flushed_at_finish_unified(self):
"""Deferred CHANNEL_END flushed at end-of-stream."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
("thought\nReasoning text.", [_TOK[0]]),
(" Final thought.", [_CHANNEL_END_TID]),
],
)
reasoning = _reasoning_text(events)
assert "Reasoning text. Final thought." in reasoning
assert _has_event(events, EventType.REASONING_END)
def test_reasoning_to_tool_call_handoff_unified(self):
"""Full reasoning -> content -> tool call flow."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
("thought\nI need to check the weather.", [_TOK[0], _TOK[1], _TOK[2]]),
(_CHANNEL_END_TAG, [_CHANNEL_END_TID]),
("Let me call a tool.", [_TOK[3], _TOK[4]]),
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_weather{city:", [_TOK[5], _TOK[6]]),
('<|"|>SF<|"|>}', [_QUOTE_TID, _TOK[7], _QUOTE_TID, _TOK[8]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
],
)
reasoning = _reasoning_text(events)
content = _content_text(events)
assert "I need to check the weather." in reasoning
assert "Let me call a tool." in content
assert _has_event(events, EventType.REASONING_START)
assert _has_event(events, EventType.REASONING_END)
assert _has_event(events, EventType.TOOL_CALL_START)
assert _has_event(events, EventType.TOOL_CALL_END)
assert "SF" in _arg_text(events)
def test_multiple_tool_calls_rapid_transitions_unified(self):
"""Two back-to-back tool calls with correct tool_index tracking."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_weather{city:", [_TOK[0], _TOK[1]]),
('<|"|>NYC<|"|>}', [_QUOTE_TID, _TOK[2], _QUOTE_TID, _TOK[3]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_time{tz:", [_TOK[4], _TOK[5]]),
('<|"|>EST<|"|>}', [_QUOTE_TID, _TOK[6], _QUOTE_TID, _TOK[7]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
],
)
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
assert len(starts) == 2
assert len(ends) == 2
assert starts[0].tool_index == 0
assert starts[1].tool_index == 1
names = "".join(e.value for e in events if e.type == EventType.TOOL_NAME)
assert "get_weather" in names
assert "get_time" in names
def test_deferred_channel_end_before_tool_call_unified(self):
"""Deferred CHANNEL_END followed by a tool call."""
tok = _make_gemma4_tokenizer()
engine = StreamingParserEngine(gemma4_config(), tok)
events = _collect_events(
engine,
[
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
("thought\nNeed to call a tool.", [_TOK[0], _TOK[1]]),
(" Let me proceed.", [_CHANNEL_END_TID]),
(_CHANNEL_END_TAG, [_TOK[2]]),
(_TOOL_START_TAG, [_TOOL_START_TID]),
("call:get_weather{city:", [_TOK[3], _TOK[4]]),
('<|"|>Tokyo<|"|>}', [_QUOTE_TID, _TOK[5], _QUOTE_TID, _TOK[6]]),
(_TOOL_END_TAG, [_TOOL_END_TID]),
],
)
reasoning = _reasoning_text(events)
assert "Need to call a tool. Let me proceed." in reasoning
assert _has_event(events, EventType.REASONING_END)
assert _has_event(events, EventType.TOOL_CALL_START)
assert _has_event(events, EventType.TOOL_CALL_END)
assert "Tokyo" in _arg_text(events)
class TestStreamInterval10:
"""Tests with stream_interval=10 (large multi-token batches)."""
def test_channel_end_mid_batch_text_present(self):
"""<channel|> mid-batch with its text present in delta_text."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nword0 word1 word2 word3 word4 "
"word5 word6 word7 word8 ",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
"word9 word10 word11 <channel|>word12 word13 word14 word0 word1 word2 ",
[
_TOK[9],
_TOK[10],
_TOK[11],
_CHANNEL_END_TID,
_TOK[12],
_TOK[13],
_TOK[14],
_TOK[0],
_TOK[1],
_TOK[2],
],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
content = _content_text(events)
for w in ("word9", "word10", "word11"):
assert w in reasoning, f"{w!r} missing from reasoning"
for w in ("word12", "word13", "word14"):
assert w in content, f"{w!r} missing from content"
assert _has_event(events, EventType.REASONING_END)
def test_channel_end_and_tool_start_same_batch_unified(self):
"""Both <channel|> and <|tool_call> in a single batch."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nw0 w1 w2 w3 w4 w5 w6 w7 w8 ",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
"w9 w10 <channel|>w11 <|tool_call>",
[
_TOK[9],
_TOK[10],
_CHANNEL_END_TID,
_TOK[11],
_TOOL_START_TID,
_TOK[12],
_TOK[13],
_TOK[14],
_TOK[0],
_TOK[1],
],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
assert "w9" in reasoning
assert "w10" in reasoning
assert _has_event(events, EventType.REASONING_END)
assert _has_event(events, EventType.TOOL_CALL_START)
def test_channel_end_mid_batch_text_absent(self):
"""<channel|> mid-batch with its text absent from delta_text."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nword0 word1 word2 word3 word4 "
"word5 word6 word7 word8 ",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
"word9 word10 word11 ",
[
_TOK[9],
_TOK[10],
_TOK[11],
_CHANNEL_END_TID,
_TOK[12],
_TOK[13],
_TOK[14],
_TOK[0],
_TOK[1],
_TOK[2],
],
)
)
events.extend(
engine.feed(
"<channel|>word12 word13 word14 word0 word1 word2 ",
[_TOK[3], _TOK[4], _TOK[5]],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
content = _content_text(events)
for w in ("word9", "word10", "word11"):
assert w in reasoning, f"{w!r} missing from reasoning"
for w in ("word12", "word13", "word14"):
assert w in content, f"{w!r} missing from content"
assert _has_event(events, EventType.REASONING_END)
def test_tool_end_mid_batch_text_absent_unified(self):
"""<tool_call|> mid-batch with text absent."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i}" for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
_CHANNEL_START_TAG,
[_CHANNEL_START_TID],
)
)
events.extend(
engine.feed(
"thought\nNeed a tool.",
[_TOK[0], _TOK[1]],
)
)
events.extend(
engine.feed(
_TOOL_START_TAG,
[_TOOL_START_TID],
)
)
events.extend(
engine.feed(
"call:get_weather{city:",
[_TOK[2], _TOK[3], _TOK[4]],
)
)
events.extend(
engine.feed(
'<|"|>San Francisco<|"|>}',
[
_QUOTE_TID,
_TOK[5],
_TOK[6],
_QUOTE_TID,
_TOK[7],
_TOOL_END_TID,
_TOK[8],
_TOK[9],
_TOK[10],
_TOK[11],
],
)
)
events.extend(
engine.feed(
"<tool_call|>w8w9w10w11w12",
[_TOK[12], _TOK[13]],
)
)
events.extend(engine.finish())
assert _has_event(events, EventType.TOOL_CALL_END)
assert "San Francisco" in _arg_text(events)
def test_large_batch_holdback_spans_two_batches(self):
"""Holdback text spanning two batches with <channel|> in the second."""
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
engine = StreamingParserEngine(gemma4_config(), tok)
events: list = []
events.extend(
engine.feed(
"<|channel>thought\nThe user asked about machine learning "
"and I need to think about the best approach to",
[
_CHANNEL_START_TID,
_TOK[0],
_TOK[1],
_TOK[2],
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
],
)
)
events.extend(
engine.feed(
" explain this complex topic. Let me organize my thoughts.",
[
_TOK[9],
_TOK[10],
_TOK[11],
_TOK[12],
_TOK[13],
_TOK[14],
_CHANNEL_END_TID,
_TOK[0],
_TOK[1],
_TOK[2],
],
)
)
events.extend(
engine.feed(
"<channel|>w0 w1 w2 Here is what I recommend: start with "
"the fundamentals and build up from there.",
[
_TOK[3],
_TOK[4],
_TOK[5],
_TOK[6],
_TOK[7],
_TOK[8],
_TOK[9],
_TOK[10],
_TOK[11],
_TOK[12],
],
)
)
events.extend(engine.finish())
reasoning = _reasoning_text(events)
content = _content_text(events)
assert "organize my thoughts." in reasoning
assert "explain" in reasoning
assert "recommend" in content
assert _has_event(events, EventType.REASONING_START)
assert _has_event(events, EventType.REASONING_END)
class TestRebuildFromAnchorsLiteralLookalike:
"""When delta_text contains a literal mention of a special token's
text before the real special token, _rebuild_from_anchors must
anchor at the real occurrence, not the literal one."""
"""Literal token text in prose must not be consumed as an anchor."""
@pytest.fixture
def tool_scanner(self):
@@ -513,8 +928,6 @@ class TestRebuildFromAnchorsLiteralLookalike:
)
def test_literal_before_real_anchor(self, tool_scanner):
"""Literal <tool_call> in prose followed by a real <tool_call>
special token — the scanner must split at the real one."""
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
items = tool_scanner.scan(delta_text, delta_token_ids)
@@ -526,14 +939,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
assert terminals[0].terminal == "TOOL_START"
assert terminals[1].terminal == "TOOL_END"
# The literal mention must appear in a text chunk, not be
# consumed by the TOOL_START anchor.
joined_text = "".join(text_parts)
assert "<tool_call>" in joined_text
assert '{"name":"f"}' in joined_text
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
"""Two real tool calls with a literal mention between them."""
delta_text = (
'<tool_call>{"name":"a"}</tool_call>'
" see <tool_call> syntax "
@@ -557,14 +967,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
joined_text = "".join(text_parts)
# The literal mention between the two real calls must be in text
assert "<tool_call> syntax" in joined_text
class TestRebuildFromAnchorsCascadingDeferral:
"""When a middle anchor's text is absent from delta_text,
only that anchor should be deferred — not subsequent ones
with valid positions."""
"""Missing middle anchor defers only itself, not subsequent ones."""
@pytest.fixture
def bare_scanner(self):
@@ -623,9 +1030,6 @@ class TestRebuildFromAnchorsCascadingDeferral:
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
joined = "".join(t.text for t in texts)
assert "text" in joined
# "more" is deferred along with the missing terminal —
# it will be resolved in the next scan when the terminal
# text arrives.
assert bare_scanner._deferred_post_text == "more"
assert len(bare_scanner._deferred_terminals) == 1
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"