[Misc] Unify tokenizer registration (#29767)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -23,7 +23,7 @@ class _HfExamplesInfo:
|
||||
tokenizer: str | None = None
|
||||
"""Set the tokenizer to load for this architecture."""
|
||||
|
||||
tokenizer_mode: TokenizerMode = "auto"
|
||||
tokenizer_mode: TokenizerMode | str = "auto"
|
||||
"""Set the tokenizer type for this architecture."""
|
||||
|
||||
speculative_model: str | None = None
|
||||
|
||||
Reference in New Issue
Block a user