[Refactor] TokenizerRegistry only uses lazy imports (#30609)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,9 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .deepseekv32 import DeepseekV32Tokenizer
|
||||
from .hf import HfTokenizer
|
||||
from .mistral import MistralTokenizer
|
||||
from .protocol import TokenizerLike
|
||||
from .registry import (
|
||||
TokenizerRegistry,
|
||||
@@ -15,12 +12,9 @@ from .registry import (
|
||||
|
||||
__all__ = [
|
||||
"TokenizerLike",
|
||||
"HfTokenizer",
|
||||
"MistralTokenizer",
|
||||
"TokenizerRegistry",
|
||||
"cached_get_tokenizer",
|
||||
"get_tokenizer",
|
||||
"cached_tokenizer_from_config",
|
||||
"init_tokenizer_from_config",
|
||||
"DeepseekV32Tokenizer",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user