Fix Baichuan tokenizer error (#1874)
This commit is contained in:
@@ -4,6 +4,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
|
||||
PreTrainedTokenizerFast)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.tokenizers import *
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -61,6 +62,18 @@ def get_tokenizer(
|
||||
raise RuntimeError(err_msg) from e
|
||||
else:
|
||||
raise e
|
||||
except AttributeError as e:
|
||||
if "BaichuanTokenizer" in str(e):
|
||||
# This is for the error "'BaichuanTokenizer' object has no
|
||||
# attribute 'sp_model'".
|
||||
tokenizer = BaichuanTokenizer.from_pretrained(
|
||||
tokenizer_name,
|
||||
*args,
|
||||
trust_remote_code=trust_remote_code,
|
||||
tokenizer_revision=tokenizer_revision,
|
||||
**kwargs)
|
||||
else:
|
||||
raise e
|
||||
|
||||
if not isinstance(tokenizer, PreTrainedTokenizerFast):
|
||||
logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user