[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+1 -38
View File
@@ -57,7 +57,7 @@ class TestJoinDecodedTextReturnsStr:
@pytest.fixture
def bare_scanner(self):
return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
return TokenIDScanner({}, tokenizer=None)
def test_mixed_items(self, bare_scanner):
items = [
@@ -265,43 +265,6 @@ class TestHoldbackTextRecovery:
assert "reasoning end." in combined
class TestDropTokens:
def test_drop_token_with_holdback(self, tokenizer):
"""Drop tokens stripped; hold-back text preserved."""
drop_id = 300
tokenizer.decode.side_effect = lambda ids: {
CHANNEL_END_ID: CHANNEL_END,
drop_id: "<eos>",
}.get(ids[0], "?")
scanner = TokenIDScanner(
token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
tokenizer=tokenizer,
drop_token_ids={drop_id},
)
result = scanner.scan(
delta_text="holdback<eos>",
delta_token_ids=[drop_id, CHANNEL_END_ID],
)
assert len(result) == 0
result2 = scanner.scan(
delta_text="<channel|>content",
delta_token_ids=[20],
)
pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
assert len(pre_lexed) == 1
assert pre_lexed[0].terminal == "THINK_END"
texts = [r.text for r in result2 if isinstance(r, TextChunk)]
combined = "".join(texts)
assert "holdback" in combined
assert "<eos>" not in combined
assert len(scanner.flush_pending()) == 0
class TestEndToEndReasoningHoldback:
"""End-to-end engine tests with detokenizer hold-back."""