add support for tokenizer revision (#1163)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
Federico Cassano
2023-10-02 19:19:46 -07:00
committed by GitHub
co-authored by Zhuohan Li
parent ba0bfd40e2
commit 66d18a7fb0
5 changed files with 23 additions and 1 deletions
+2
View File
@@ -16,6 +16,7 @@ def get_tokenizer(
*args,
tokenizer_mode: str = "auto",
trust_remote_code: bool = False,
tokenizer_revision: Optional[str] = None,
**kwargs,
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
"""Gets a tokenizer for the given model name via Huggingface."""
@@ -37,6 +38,7 @@ def get_tokenizer(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
tokenizer_revision=tokenizer_revision,
**kwargs)
except TypeError as e:
# The LLaMA tokenizer causes a protobuf error in some environments.