[Misc] Update TokenizerLike interface and move get_cached_tokenizer (#29730)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .hf import HfTokenizer
|
||||
from .mistral import MistralTokenizer
|
||||
from .protocol import TokenizerLike
|
||||
from .registry import TokenizerRegistry
|
||||
|
||||
__all__ = ["TokenizerLike", "MistralTokenizer", "TokenizerRegistry"]
|
||||
__all__ = ["TokenizerLike", "HfTokenizer", "MistralTokenizer", "TokenizerRegistry"]
|
||||
|
||||
Reference in New Issue
Block a user