[Tokenizer] Add tokenizer mode (#298)

This commit is contained in:
Woosuk Kwon
2023-06-28 14:19:22 -07:00
committed by GitHub
parent 425040d4c1
commit 998d9d1509
6 changed files with 38 additions and 5 deletions
+7
View File
@@ -13,10 +13,17 @@ _FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
def get_tokenizer(
tokenizer_name: str,
tokenizer_mode: str = "auto",
*args,
**kwargs,
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
"""Gets a tokenizer for the given model name via Huggingface."""
if tokenizer_mode == "slow":
if kwargs.get("use_fast", False):
raise ValueError(
"Cannot use the fast tokenizer in slow tokenizer mode.")
kwargs["use_fast"] = False
if "llama" in tokenizer_name.lower() and kwargs.get("use_fast", True):
logger.info(
"For some LLaMA-based models, initializing the fast tokenizer may "