[Model] Add Mistral Tokenization to improve robustness and chat encoding (#7739)

This commit is contained in:
Patrick von Platen
2024-08-27 12:40:02 +00:00
committed by GitHub
parent 9606c7197d
commit 6fc4e6e07a
12 changed files with 275 additions and 60 deletions
+3 -2
View File
@@ -198,10 +198,11 @@ class EngineArgs:
'--tokenizer-mode',
type=str,
default=EngineArgs.tokenizer_mode,
choices=['auto', 'slow'],
choices=['auto', 'slow', 'mistral'],
help='The tokenizer mode.\n\n* "auto" will use the '
'fast tokenizer if available.\n* "slow" will '
'always use the slow tokenizer.')
'always use the slow tokenizer. \n* '
'"mistral" will always use the `mistral_common` tokenizer.')
parser.add_argument('--trust-remote-code',
action='store_true',
help='Trust remote code from huggingface.')