[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
27d5f78b63
commit
25671cb520
@@ -100,14 +100,24 @@ def _build_chat_request(
|
||||
class _StubTokenizer:
|
||||
"""Minimal tokenizer stub to satisfy ``Gemma4EngineToolParser.__init__``."""
|
||||
|
||||
_VOCAB: dict[str, int] = {
|
||||
"<|tool_call>": 256_000,
|
||||
"<tool_call|>": 256_001,
|
||||
'<|"|>': 52,
|
||||
"<|channel>": 256_002,
|
||||
"<channel|>": 256_003,
|
||||
}
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return {
|
||||
"<|tool_call>": 256_000,
|
||||
"<tool_call|>": 256_001,
|
||||
'<|"|>': 52,
|
||||
"<|channel>": 256_002,
|
||||
"<channel|>": 256_003,
|
||||
}
|
||||
return dict(self._VOCAB)
|
||||
|
||||
@property
|
||||
def all_special_tokens(self) -> list[str]:
|
||||
return list(self._VOCAB.keys())
|
||||
|
||||
@property
|
||||
def all_special_ids(self) -> list[int]:
|
||||
return list(self._VOCAB.values())
|
||||
|
||||
|
||||
def test_gemma4_adjust_request_sets_skip_special_tokens_on_responses() -> None:
|
||||
@@ -235,10 +245,10 @@ def test_gemma4_keeps_special_tokens_with_tools_thinking_disabled() -> None:
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
|
||||
def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
|
||||
"""No active tools + thinking disabled: keep the default
|
||||
(``skip_special_tokens=True``) so stray delimiters do not leak into
|
||||
content.
|
||||
def test_gemma4_keeps_skip_special_tokens_false_when_nothing_to_preserve() -> None:
|
||||
"""No active tools + thinking disabled: ``skip_special_tokens`` stays
|
||||
``False`` because the parser engine's ``__DROP__`` terminal mechanism
|
||||
strips unconfigured special tokens automatically.
|
||||
"""
|
||||
parser = Gemma4ToolParser(_StubTokenizer())
|
||||
request = _build_chat_request(
|
||||
@@ -247,4 +257,4 @@ def test_gemma4_strips_special_tokens_when_nothing_to_preserve() -> None:
|
||||
|
||||
parser.adjust_request(request)
|
||||
|
||||
assert request.skip_special_tokens is True
|
||||
assert request.skip_special_tokens is False
|
||||
|
||||
Reference in New Issue
Block a user