[Parser][Bugfix] Ensure tool call or other special tokens don't leak in non-streaming tool parsing (#46875)
Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
27d5f78b63
commit
25671cb520
@@ -15,8 +15,17 @@ def should_do_global_cleanup_after_test() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
|
||||
"""Create a mock tokenizer with the given special-token vocabulary.
|
||||
def make_mock_tokenizer(
|
||||
vocab: dict[str, int],
|
||||
special_tokens: list[str] | None = None,
|
||||
) -> MagicMock:
|
||||
"""Create a mock tokenizer with the given vocabulary.
|
||||
|
||||
Args:
|
||||
vocab: Mapping of token text to token ID.
|
||||
special_tokens: Which tokens to mark as special. When ``None``
|
||||
(the default), every key in *vocab* is treated as special —
|
||||
convenient when the vocab only contains delimiter tokens.
|
||||
|
||||
The returned mock supports get_vocab(), encode(), and decode().
|
||||
decode() maps known token IDs back to their text and falls back to
|
||||
@@ -29,6 +38,9 @@ def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
|
||||
tokenizer.decode.side_effect = lambda ids: "".join(
|
||||
id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
|
||||
)
|
||||
st = special_tokens if special_tokens is not None else list(vocab.keys())
|
||||
tokenizer.all_special_tokens = st
|
||||
tokenizer.all_special_ids = [vocab[t] for t in st if t in vocab]
|
||||
return tokenizer
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user