[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
@@ -100,14 +100,24 @@ def _build_chat_request(
class _StubTokenizer:
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
_VOCAB: dict[str, int] = {
"<|tool_call>": 256_000,
"<tool_call|>": 256_001,
'<|"|>': 52,
"<|channel>": 256_002,
"<channel|>": 256_003,
}
def get_vocab(self) -> dict[str, int]:
return {
"<|tool_call>": 256_000,
"<tool_call|>": 256_001,
'<|"|>': 52,
"<|channel>": 256_002,
"<channel|>": 256_003,
}
return dict(self._VOCAB)
@property
def all_special_tokens(self) -> list[str]:
return list(self._VOCAB.keys())
@property
def all_special_ids(self) -> list[int]:
return list(self._VOCAB.values())
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
@@ -235,10 +245,10 @@ def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
assert request.skip_special_tokens is False
def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
"""No active tools + thinking disabled: keep the default
(``skip_special_tokens=True``) so stray delimiters do not leak into
content.
def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:
"""No active tools + thinking disabled: ``skip_special_tokens`` stays
``False`` because the parser engine's ``__DROP__`` terminal mechanism
strips unconfigured special tokens automatically.
"""
parser = Gemma4ToolParser(_StubTokenizer())
request = _build_chat_request(
@@ -247,4 +257,4 @@ def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
parser.adjust_request(request)
assert request.skip_special_tokens is True
assert request.skip_special_tokens is False