[Frontend] Replace legacy Gemma4 parsers with engine-based implementation (#45588)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Flora Feng
parent
25ee659db0
commit
76a373eff4
@@ -1,20 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for TokenIDScanner, focusing on hold-back text recovery.
|
||||
|
||||
Uses gemma4_config for all end-to-end engine tests, covering
|
||||
reasoning channels, tool calls, and combined flows."""
|
||||
"""Tests for TokenIDScanner."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.parser.engine.events import EventType
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
from vllm.parser.engine.token_id_scanner import (
|
||||
PreLexedTerminal,
|
||||
TextChunk,
|
||||
TokenIDScanner,
|
||||
)
|
||||
from vllm.parser.gemma4 import gemma4_config
|
||||
|
||||
CHANNEL_START = "<|channel>"
|
||||
CHANNEL_END = "<channel|>"
|
||||
@@ -54,8 +53,7 @@ def scanner(tokenizer):
|
||||
|
||||
|
||||
class TestJoinDecodedTextReturnsStr:
|
||||
"""_join_decoded_text now returns str unconditionally (was
|
||||
str | None when an isinstance guard made a branch unreachable)."""
|
||||
"""_join_decoded_text always returns str."""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
@@ -83,9 +81,7 @@ class TestJoinDecodedTextReturnsStr:
|
||||
|
||||
class TestHoldbackTextRecovery:
|
||||
def test_holdback_text_with_special_token_text_absent(self, scanner):
|
||||
"""delta_text has hold-back text but the special token's text is
|
||||
NOT in delta_text (held back by the detokenizer). Terminal is
|
||||
deferred until the text arrives in a subsequent delta."""
|
||||
"""Terminal deferred when its text is absent from delta_text."""
|
||||
result = scanner.scan(
|
||||
delta_text="processed is appropriate.",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -93,8 +89,6 @@ class TestHoldbackTextRecovery:
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Second scan: terminal text arrives (detokenizer flushes).
|
||||
# Deferred terminal resolves with holdback text before it.
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>Understood.",
|
||||
delta_token_ids=[20, 21],
|
||||
@@ -108,7 +102,7 @@ class TestHoldbackTextRecovery:
|
||||
assert "Understood." in combined
|
||||
|
||||
def test_holdback_text_with_special_token_text_present(self, scanner):
|
||||
"""delta_text includes hold-back text AND the special token text."""
|
||||
"""Hold-back text + special token text both in delta_text."""
|
||||
result = scanner.scan(
|
||||
delta_text="holdback text<channel|>",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -121,7 +115,7 @@ class TestHoldbackTextRecovery:
|
||||
assert result[1].terminal == "THINK_END"
|
||||
|
||||
def test_no_holdback_text(self, scanner):
|
||||
"""delta_text is exactly the special token text — no hold-back."""
|
||||
"""delta_text is exactly the special token text."""
|
||||
result = scanner.scan(
|
||||
delta_text="<channel|>",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -132,7 +126,7 @@ class TestHoldbackTextRecovery:
|
||||
assert result[0].terminal == "THINK_END"
|
||||
|
||||
def test_empty_delta_text(self, scanner):
|
||||
"""delta_text is empty — terminal deferred until text arrives."""
|
||||
"""Empty delta_text defers the terminal until text arrives."""
|
||||
result = scanner.scan(
|
||||
delta_text="",
|
||||
delta_token_ids=[CHANNEL_END_ID],
|
||||
@@ -146,9 +140,7 @@ class TestHoldbackTextRecovery:
|
||||
assert flushed[0].terminal == "THINK_END"
|
||||
|
||||
def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
|
||||
"""delta_text="" with multiple tokens including special: all
|
||||
results deferred — individually-decoded TextChunks are unreliable
|
||||
and PreLexedTerminals wait for text confirmation."""
|
||||
"""Empty delta_text with multiple tokens: all results deferred."""
|
||||
tool_start_id = 400
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
@@ -176,7 +168,6 @@ class TestHoldbackTextRecovery:
|
||||
assert flushed[0].terminal == "TOOL_START"
|
||||
|
||||
def test_holdback_before_start_tag(self, scanner):
|
||||
"""Hold-back text before a reasoning start tag."""
|
||||
result = scanner.scan(
|
||||
delta_text="prefix text<|channel>",
|
||||
delta_token_ids=[CHANNEL_START_ID],
|
||||
@@ -189,8 +180,7 @@ class TestHoldbackTextRecovery:
|
||||
assert result[1].terminal == "THINK_START"
|
||||
|
||||
def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
|
||||
"""Stream-interval > 1: batch has regular tokens + special token.
|
||||
delta_text differs from individual decodes (context-dependent)."""
|
||||
"""Multi-token batch with special token in the middle."""
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
@@ -215,10 +205,7 @@ class TestHoldbackTextRecovery:
|
||||
assert "holdback wordA" in "".join(texts)
|
||||
|
||||
def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
|
||||
"""Stream-interval > 1: batch has regular + special token, but
|
||||
delta_text doesn't contain the special token text at all
|
||||
(held back by detokenizer along with trailing regular tokens).
|
||||
Terminal is deferred until text arrives."""
|
||||
"""Multi-token batch where special token text is absent."""
|
||||
tok_a = 201
|
||||
tok_b = 202
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
@@ -239,8 +226,6 @@ class TestHoldbackTextRecovery:
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Next delta: terminal text arrives (detokenizer flushes).
|
||||
# Deferred terminal resolves with holdback text before it.
|
||||
result2 = scanner_multi.scan(
|
||||
delta_text="<channel|> more text",
|
||||
delta_token_ids=[300],
|
||||
@@ -254,8 +239,7 @@ class TestHoldbackTextRecovery:
|
||||
assert "more text" in combined
|
||||
|
||||
def test_holdback_with_content_after_special_token(self, tokenizer):
|
||||
"""delta_text has hold-back + special token + content after,
|
||||
with corresponding token IDs for all parts."""
|
||||
"""Hold-back + special token + content after in one delta."""
|
||||
tok_content = 210
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
@@ -283,8 +267,7 @@ class TestHoldbackTextRecovery:
|
||||
|
||||
class TestDropTokens:
|
||||
def test_drop_token_with_holdback(self, tokenizer):
|
||||
"""Drop tokens stripped from delta_text, hold-back text preserved.
|
||||
Terminal is deferred when its text is absent from delta_text."""
|
||||
"""Drop tokens stripped; hold-back text preserved."""
|
||||
drop_id = 300
|
||||
tokenizer.decode.side_effect = lambda ids: {
|
||||
CHANNEL_END_ID: CHANNEL_END,
|
||||
@@ -304,7 +287,6 @@ class TestDropTokens:
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
# Terminal text arrives in next delta; deferred terminal resolves.
|
||||
result2 = scanner.scan(
|
||||
delta_text="<channel|>content",
|
||||
delta_token_ids=[20],
|
||||
@@ -321,40 +303,10 @@ class TestDropTokens:
|
||||
|
||||
|
||||
class TestEndToEndReasoningHoldback:
|
||||
"""End-to-end tests through the full parser engine simulating
|
||||
stream-interval > 1 and detokenizer hold-back, using
|
||||
gemma4_config."""
|
||||
"""End-to-end engine tests with detokenizer hold-back."""
|
||||
|
||||
def test_reasoning_content_not_truncated(self):
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
config = ParserEngineConfig(
|
||||
name="test_channel",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
config = gemma4_config()
|
||||
tok = MagicMock()
|
||||
vocab = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
@@ -369,30 +321,21 @@ class TestEndToEndReasoningHoldback:
|
||||
engine = StreamingParserEngine(config, tok)
|
||||
all_events = []
|
||||
|
||||
# Delta 1: channel start token (text includes start tag)
|
||||
all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
|
||||
|
||||
# Delta 2: reasoning text (normal content, no special tokens)
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"thought\nThe request was received and ",
|
||||
[10, 11, 12, 13, 14],
|
||||
)
|
||||
)
|
||||
|
||||
# Delta 3: MORE reasoning text, the detokenizer held some back.
|
||||
# Then channel end token arrives in token_ids, but its text
|
||||
# is NOT in delta_text (held back by detokenizer).
|
||||
# delta_text = previously held-back reasoning text only.
|
||||
# CHANNEL_END token arrives but its text is held back.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"processed is appropriate.",
|
||||
[CHANNEL_END_ID],
|
||||
)
|
||||
)
|
||||
|
||||
# Delta 4: detokenizer flushes held-back channel end text
|
||||
# plus new content tokens.
|
||||
# Detokenizer flushes the held-back text.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"<channel|>Understood.",
|
||||
@@ -413,36 +356,7 @@ class TestEndToEndReasoningHoldback:
|
||||
assert "Understood." in content_text
|
||||
|
||||
def test_backtick_content_not_truncated(self):
|
||||
"""Reproduces the hostname backtick truncation case."""
|
||||
from vllm.parser.engine.parser_engine_config import (
|
||||
ParserEngineConfig,
|
||||
ParserState,
|
||||
Transition,
|
||||
)
|
||||
from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
|
||||
|
||||
config = ParserEngineConfig(
|
||||
name="test_channel",
|
||||
initial_state=ParserState.CONTENT,
|
||||
terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
token_id_terminals={
|
||||
"THINK_START": CHANNEL_START,
|
||||
"THINK_END": CHANNEL_END,
|
||||
},
|
||||
transitions={
|
||||
(ParserState.CONTENT, "THINK_START"): Transition(
|
||||
ParserState.REASONING,
|
||||
(EventType.REASONING_START,),
|
||||
),
|
||||
(ParserState.REASONING, "THINK_END"): Transition(
|
||||
ParserState.CONTENT,
|
||||
(EventType.REASONING_END,),
|
||||
),
|
||||
},
|
||||
)
|
||||
config = gemma4_config()
|
||||
tok = MagicMock()
|
||||
vocab = {
|
||||
CHANNEL_START: CHANNEL_START_ID,
|
||||
@@ -464,17 +378,12 @@ class TestEndToEndReasoningHoldback:
|
||||
[10, 11, 12, 13],
|
||||
)
|
||||
)
|
||||
|
||||
# Hold-back text includes backtick content; channel end text
|
||||
# absent from delta_text.
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"`hostname`.\n",
|
||||
[CHANNEL_END_ID],
|
||||
)
|
||||
)
|
||||
|
||||
# Next delta flushes channel end + tool call start
|
||||
all_events.extend(
|
||||
engine.feed(
|
||||
"<channel|>tool output",
|
||||
@@ -491,10 +400,516 @@ class TestEndToEndReasoningHoldback:
|
||||
assert "`hostname`." in reasoning_text
|
||||
|
||||
|
||||
_CHANNEL_START_TAG = "<|channel>"
|
||||
_CHANNEL_END_TAG = "<channel|>"
|
||||
_TOOL_START_TAG = "<|tool_call>"
|
||||
_TOOL_END_TAG = "<tool_call|>"
|
||||
_QUOTE_TAG = '<|"|>'
|
||||
|
||||
_CHANNEL_START_TID = 100
|
||||
_CHANNEL_END_TID = 101
|
||||
_TOOL_START_TID = 102
|
||||
_TOOL_END_TID = 103
|
||||
_QUOTE_TID = 104
|
||||
_TOK = list(range(200, 215))
|
||||
|
||||
|
||||
def _gemma4_vocab() -> dict[str, int]:
|
||||
return {
|
||||
_CHANNEL_START_TAG: _CHANNEL_START_TID,
|
||||
_CHANNEL_END_TAG: _CHANNEL_END_TID,
|
||||
_TOOL_START_TAG: _TOOL_START_TID,
|
||||
_TOOL_END_TAG: _TOOL_END_TID,
|
||||
_QUOTE_TAG: _QUOTE_TID,
|
||||
}
|
||||
|
||||
|
||||
def _make_gemma4_tokenizer(
|
||||
extra_decode: dict[int, str] | None = None,
|
||||
) -> MagicMock:
|
||||
special = {
|
||||
_CHANNEL_START_TID: _CHANNEL_START_TAG,
|
||||
_CHANNEL_END_TID: _CHANNEL_END_TAG,
|
||||
_TOOL_START_TID: _TOOL_START_TAG,
|
||||
_TOOL_END_TID: _TOOL_END_TAG,
|
||||
_QUOTE_TID: _QUOTE_TAG,
|
||||
}
|
||||
decode_map = {**special, **(extra_decode or {})}
|
||||
|
||||
tok = MagicMock()
|
||||
tok.get_vocab.return_value = _gemma4_vocab()
|
||||
tok.decode.side_effect = lambda ids: decode_map.get(ids[0], f"tok{ids[0]}")
|
||||
return tok
|
||||
|
||||
|
||||
def _collect_events(engine, deltas):
|
||||
from vllm.parser.engine.events import SemanticEvent
|
||||
|
||||
all_events: list[SemanticEvent] = []
|
||||
for delta_text, delta_token_ids in deltas:
|
||||
all_events.extend(engine.feed(delta_text, delta_token_ids))
|
||||
all_events.extend(engine.finish())
|
||||
return all_events
|
||||
|
||||
|
||||
def _reasoning_text(events) -> str:
|
||||
return "".join(e.value for e in events if e.type == EventType.REASONING_CHUNK)
|
||||
|
||||
|
||||
def _content_text(events) -> str:
|
||||
return "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
|
||||
|
||||
|
||||
def _arg_text(events) -> str:
|
||||
return "".join(e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK)
|
||||
|
||||
|
||||
def _has_event(events, event_type) -> bool:
|
||||
return any(e.type == event_type for e in events)
|
||||
|
||||
|
||||
class TestMultiTokenBoundaryPreservation:
|
||||
"""No text lost at state boundaries with multi-token deltas."""
|
||||
|
||||
def test_empty_delta_text_at_channel_end_unified(self):
|
||||
"""Empty delta_text when CHANNEL_END arrives; text comes later."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
("", [_CHANNEL_START_TID]),
|
||||
("<|channel>thought\nSome reasoning.", [_TOK[0], _TOK[1]]),
|
||||
("", [_CHANNEL_END_TID]),
|
||||
("<channel|>Final answer.", [_TOK[2], _TOK[3]]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
assert "Some reasoning." in reasoning
|
||||
assert "Final answer." in content
|
||||
assert _has_event(events, EventType.REASONING_START)
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_deferred_channel_end_flushed_at_finish_unified(self):
|
||||
"""Deferred CHANNEL_END flushed at end-of-stream."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
|
||||
("thought\nReasoning text.", [_TOK[0]]),
|
||||
(" Final thought.", [_CHANNEL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
assert "Reasoning text. Final thought." in reasoning
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_reasoning_to_tool_call_handoff_unified(self):
|
||||
"""Full reasoning -> content -> tool call flow."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
|
||||
("thought\nI need to check the weather.", [_TOK[0], _TOK[1], _TOK[2]]),
|
||||
(_CHANNEL_END_TAG, [_CHANNEL_END_TID]),
|
||||
("Let me call a tool.", [_TOK[3], _TOK[4]]),
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_weather{city:", [_TOK[5], _TOK[6]]),
|
||||
('<|"|>SF<|"|>}', [_QUOTE_TID, _TOK[7], _QUOTE_TID, _TOK[8]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
assert "I need to check the weather." in reasoning
|
||||
assert "Let me call a tool." in content
|
||||
assert _has_event(events, EventType.REASONING_START)
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
assert _has_event(events, EventType.TOOL_CALL_START)
|
||||
assert _has_event(events, EventType.TOOL_CALL_END)
|
||||
assert "SF" in _arg_text(events)
|
||||
|
||||
def test_multiple_tool_calls_rapid_transitions_unified(self):
|
||||
"""Two back-to-back tool calls with correct tool_index tracking."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_weather{city:", [_TOK[0], _TOK[1]]),
|
||||
('<|"|>NYC<|"|>}', [_QUOTE_TID, _TOK[2], _QUOTE_TID, _TOK[3]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_time{tz:", [_TOK[4], _TOK[5]]),
|
||||
('<|"|>EST<|"|>}', [_QUOTE_TID, _TOK[6], _QUOTE_TID, _TOK[7]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
|
||||
ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
|
||||
assert len(starts) == 2
|
||||
assert len(ends) == 2
|
||||
assert starts[0].tool_index == 0
|
||||
assert starts[1].tool_index == 1
|
||||
|
||||
names = "".join(e.value for e in events if e.type == EventType.TOOL_NAME)
|
||||
assert "get_weather" in names
|
||||
assert "get_time" in names
|
||||
|
||||
def test_deferred_channel_end_before_tool_call_unified(self):
|
||||
"""Deferred CHANNEL_END followed by a tool call."""
|
||||
tok = _make_gemma4_tokenizer()
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events = _collect_events(
|
||||
engine,
|
||||
[
|
||||
(_CHANNEL_START_TAG, [_CHANNEL_START_TID]),
|
||||
("thought\nNeed to call a tool.", [_TOK[0], _TOK[1]]),
|
||||
(" Let me proceed.", [_CHANNEL_END_TID]),
|
||||
(_CHANNEL_END_TAG, [_TOK[2]]),
|
||||
(_TOOL_START_TAG, [_TOOL_START_TID]),
|
||||
("call:get_weather{city:", [_TOK[3], _TOK[4]]),
|
||||
('<|"|>Tokyo<|"|>}', [_QUOTE_TID, _TOK[5], _QUOTE_TID, _TOK[6]]),
|
||||
(_TOOL_END_TAG, [_TOOL_END_TID]),
|
||||
],
|
||||
)
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
assert "Need to call a tool. Let me proceed." in reasoning
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
assert _has_event(events, EventType.TOOL_CALL_START)
|
||||
assert _has_event(events, EventType.TOOL_CALL_END)
|
||||
assert "Tokyo" in _arg_text(events)
|
||||
|
||||
|
||||
class TestStreamInterval10:
|
||||
"""Tests with stream_interval=10 (large multi-token batches)."""
|
||||
|
||||
def test_channel_end_mid_batch_text_present(self):
|
||||
"""<channel|> mid-batch with its text present in delta_text."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nword0 word1 word2 word3 word4 "
|
||||
"word5 word6 word7 word8 ",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"word9 word10 word11 <channel|>word12 word13 word14 word0 word1 word2 ",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
for w in ("word9", "word10", "word11"):
|
||||
assert w in reasoning, f"{w!r} missing from reasoning"
|
||||
|
||||
for w in ("word12", "word13", "word14"):
|
||||
assert w in content, f"{w!r} missing from content"
|
||||
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_channel_end_and_tool_start_same_batch_unified(self):
|
||||
"""Both <channel|> and <|tool_call> in a single batch."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nw0 w1 w2 w3 w4 w5 w6 w7 w8 ",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"w9 w10 <channel|>w11 <|tool_call>",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[11],
|
||||
_TOOL_START_TID,
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
],
|
||||
)
|
||||
)
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
|
||||
assert "w9" in reasoning
|
||||
assert "w10" in reasoning
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
assert _has_event(events, EventType.TOOL_CALL_START)
|
||||
|
||||
def test_channel_end_mid_batch_text_absent(self):
|
||||
"""<channel|> mid-batch with its text absent from delta_text."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"word{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nword0 word1 word2 word3 word4 "
|
||||
"word5 word6 word7 word8 ",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"word9 word10 word11 ",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<channel|>word12 word13 word14 word0 word1 word2 ",
|
||||
[_TOK[3], _TOK[4], _TOK[5]],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
for w in ("word9", "word10", "word11"):
|
||||
assert w in reasoning, f"{w!r} missing from reasoning"
|
||||
|
||||
for w in ("word12", "word13", "word14"):
|
||||
assert w in content, f"{w!r} missing from content"
|
||||
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
def test_tool_end_mid_batch_text_absent_unified(self):
|
||||
"""<tool_call|> mid-batch with text absent."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i}" for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
events.extend(
|
||||
engine.feed(
|
||||
_CHANNEL_START_TAG,
|
||||
[_CHANNEL_START_TID],
|
||||
)
|
||||
)
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"thought\nNeed a tool.",
|
||||
[_TOK[0], _TOK[1]],
|
||||
)
|
||||
)
|
||||
events.extend(
|
||||
engine.feed(
|
||||
_TOOL_START_TAG,
|
||||
[_TOOL_START_TID],
|
||||
)
|
||||
)
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"call:get_weather{city:",
|
||||
[_TOK[2], _TOK[3], _TOK[4]],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
'<|"|>San Francisco<|"|>}',
|
||||
[
|
||||
_QUOTE_TID,
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_QUOTE_TID,
|
||||
_TOK[7],
|
||||
_TOOL_END_TID,
|
||||
_TOK[8],
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<tool_call|>w8w9w10w11w12",
|
||||
[_TOK[12], _TOK[13]],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
assert _has_event(events, EventType.TOOL_CALL_END)
|
||||
assert "San Francisco" in _arg_text(events)
|
||||
|
||||
def test_large_batch_holdback_spans_two_batches(self):
|
||||
"""Holdback text spanning two batches with <channel|> in the second."""
|
||||
tok = _make_gemma4_tokenizer({_TOK[i]: f"w{i} " for i in range(15)})
|
||||
engine = StreamingParserEngine(gemma4_config(), tok)
|
||||
|
||||
events: list = []
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<|channel>thought\nThe user asked about machine learning "
|
||||
"and I need to think about the best approach to",
|
||||
[
|
||||
_CHANNEL_START_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
" explain this complex topic. Let me organize my thoughts.",
|
||||
[
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_TOK[12],
|
||||
_TOK[13],
|
||||
_TOK[14],
|
||||
_CHANNEL_END_TID,
|
||||
_TOK[0],
|
||||
_TOK[1],
|
||||
_TOK[2],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(
|
||||
engine.feed(
|
||||
"<channel|>w0 w1 w2 Here is what I recommend: start with "
|
||||
"the fundamentals and build up from there.",
|
||||
[
|
||||
_TOK[3],
|
||||
_TOK[4],
|
||||
_TOK[5],
|
||||
_TOK[6],
|
||||
_TOK[7],
|
||||
_TOK[8],
|
||||
_TOK[9],
|
||||
_TOK[10],
|
||||
_TOK[11],
|
||||
_TOK[12],
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
events.extend(engine.finish())
|
||||
|
||||
reasoning = _reasoning_text(events)
|
||||
content = _content_text(events)
|
||||
|
||||
assert "organize my thoughts." in reasoning
|
||||
assert "explain" in reasoning
|
||||
assert "recommend" in content
|
||||
|
||||
assert _has_event(events, EventType.REASONING_START)
|
||||
assert _has_event(events, EventType.REASONING_END)
|
||||
|
||||
|
||||
class TestRebuildFromAnchorsLiteralLookalike:
|
||||
"""When delta_text contains a literal mention of a special token's
|
||||
text before the real special token, _rebuild_from_anchors must
|
||||
anchor at the real occurrence, not the literal one."""
|
||||
"""Literal token text in prose must not be consumed as an anchor."""
|
||||
|
||||
@pytest.fixture
|
||||
def tool_scanner(self):
|
||||
@@ -513,8 +928,6 @@ class TestRebuildFromAnchorsLiteralLookalike:
|
||||
)
|
||||
|
||||
def test_literal_before_real_anchor(self, tool_scanner):
|
||||
"""Literal <tool_call> in prose followed by a real <tool_call>
|
||||
special token — the scanner must split at the real one."""
|
||||
delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
|
||||
delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
|
||||
items = tool_scanner.scan(delta_text, delta_token_ids)
|
||||
@@ -526,14 +939,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
|
||||
assert terminals[0].terminal == "TOOL_START"
|
||||
assert terminals[1].terminal == "TOOL_END"
|
||||
|
||||
# The literal mention must appear in a text chunk, not be
|
||||
# consumed by the TOOL_START anchor.
|
||||
joined_text = "".join(text_parts)
|
||||
assert "<tool_call>" in joined_text
|
||||
assert '{"name":"f"}' in joined_text
|
||||
|
||||
def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
|
||||
"""Two real tool calls with a literal mention between them."""
|
||||
delta_text = (
|
||||
'<tool_call>{"name":"a"}</tool_call>'
|
||||
" see <tool_call> syntax "
|
||||
@@ -557,14 +967,11 @@ class TestRebuildFromAnchorsLiteralLookalike:
|
||||
|
||||
text_parts = [it.text for it in items if isinstance(it, TextChunk)]
|
||||
joined_text = "".join(text_parts)
|
||||
# The literal mention between the two real calls must be in text
|
||||
assert "<tool_call> syntax" in joined_text
|
||||
|
||||
|
||||
class TestRebuildFromAnchorsCascadingDeferral:
|
||||
"""When a middle anchor's text is absent from delta_text,
|
||||
only that anchor should be deferred — not subsequent ones
|
||||
with valid positions."""
|
||||
"""Missing middle anchor defers only itself, not subsequent ones."""
|
||||
|
||||
@pytest.fixture
|
||||
def bare_scanner(self):
|
||||
@@ -623,9 +1030,6 @@ class TestRebuildFromAnchorsCascadingDeferral:
|
||||
texts = [r for r in rebuilt if isinstance(r, TextChunk)]
|
||||
joined = "".join(t.text for t in texts)
|
||||
assert "text" in joined
|
||||
# "more" is deferred along with the missing terminal —
|
||||
# it will be resolved in the next scan when the terminal
|
||||
# text arrives.
|
||||
assert bare_scanner._deferred_post_text == "more"
|
||||
assert len(bare_scanner._deferred_terminals) == 1
|
||||
assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
|
||||
|
||||
Reference in New Issue
Block a user