[Bugfix] Fix byte fallback handling when using outlines (#31391)

Signed-off-by: Shinichi Hemmi <50256998+Alnusjaponica@users.noreply.github.com>
Co-authored-by: Kenichi Maehashi <maehashi@preferred.jp>
This commit is contained in:
Shinichi Hemmi
2026-01-20 19:48:08 +00:00
committed by GitHub
co-authored by Kenichi Maehashi
parent 7c5dedc247
commit 86c69dc54c
2 changed files with 9 additions and 2 deletions
+3 -1
View File
@@ -226,7 +226,9 @@ def _reduced_vocabulary(
# by this point.
token_bytes = bytes(token_str) # type: ignore[arg-type]
elif "\ufffd" in token_str and not re_replacement_seq.match(token_str):
elif (token_str == "\ufffd" and token != "\ufffd") or (
"\ufffd" in token_str and not re_replacement_seq.match(token_str)
):
# Handle tokens with invalid UTF-8 sequences.
if re_llama_byte_token.match(token):
# Llama-like tokenizers use <0xXX> for incomplete sequences.