[Bugfix] Fix byte fallback handling when using outlines (#31391)
Signed-off-by: Shinichi Hemmi <50256998+Alnusjaponica@users.noreply.github.com> Co-authored-by: Kenichi Maehashi <maehashi@preferred.jp>
This commit is contained in:
co-authored by
Kenichi Maehashi
parent
7c5dedc247
commit
86c69dc54c
@@ -226,7 +226,9 @@ def _reduced_vocabulary(
|
||||
# by this point.
|
||||
token_bytes = bytes(token_str) # type: ignore[arg-type]
|
||||
|
||||
elif "\ufffd" in token_str and not re_replacement_seq.match(token_str):
|
||||
elif (token_str == "\ufffd" and token != "\ufffd") or (
|
||||
"\ufffd" in token_str and not re_replacement_seq.match(token_str)
|
||||
):
|
||||
# Handle tokens with invalid UTF-8 sequences.
|
||||
if re_llama_byte_token.match(token):
|
||||
# Llama-like tokenizers use <0xXX> for incomplete sequences.
|
||||
|
||||
Reference in New Issue
Block a user