[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)

Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ben Browning
2026-06-30 13:46:53 -04:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 27d5f78b63
commit 25671cb520
14 changed files with 783 additions and 219 deletions
+14 -2
View File
@@ -15,8 +15,17 @@ def should_do_global_cleanup_after_test() -> bool:
return False
def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
"""Create a mock tokenizer with the given special-token vocabulary.
def make_mock_tokenizer(
vocab: dict[str, int],
special_tokens: list[str] | None = None,
) -> MagicMock:
"""Create a mock tokenizer with the given vocabulary.
Args:
vocab: Mapping of token text to token ID.
special_tokens: Which tokens to mark as special. When ``None``
(the default), every key in *vocab* is treated as special —
convenient when the vocab only contains delimiter tokens.
The returned mock supports get_vocab(), encode(), and decode().
decode() maps known token IDs back to their text and falls back to
@@ -29,6 +38,9 @@ def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
tokenizer.decode.side_effect = lambda ids: "".join(
id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
)
st = special_tokens if special_tokens is not None else list(vocab.keys())
tokenizer.all_special_tokens = st
tokenizer.all_special_ids = [vocab[t] for t in st if t in vocab]
return tokenizer