Fix Baichuan tokenizer error (#1874)

This commit is contained in:
Woosuk Kwon
2023-11-30 18:35:50 -08:00
committed by GitHub
parent 66785cc05c
commit d06980dfa7
3 changed files with 281 additions and 0 deletions
+13
View File
@@ -4,6 +4,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
PreTrainedTokenizerFast)
from vllm.logger import init_logger
from vllm.transformers_utils.tokenizers import *
logger = init_logger(__name__)
@@ -61,6 +62,18 @@ def get_tokenizer(
raise RuntimeError(err_msg) from e
else:
raise e
except AttributeError as e:
if "BaichuanTokenizer" in str(e):
# This is for the error "'BaichuanTokenizer' object has no
# attribute 'sp_model'".
tokenizer = BaichuanTokenizer.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
tokenizer_revision=tokenizer_revision,
**kwargs)
else:
raise e
if not isinstance(tokenizer, PreTrainedTokenizerFast):
logger.warning(