[Feature] Lazy import for the "mistral" tokenizer module. (#34651)

Signed-off-by: Neil Schemenauer <nas@arctrix.com>
This commit is contained in:
Neil Schemenauer
2026-02-23 00:43:01 -08:00
committed by GitHub
parent e631f8e78e
commit 54e2f83d0a
14 changed files with 68 additions and 48 deletions
+2
View File
@@ -210,6 +210,8 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int:
class MistralTokenizer(TokenizerLike):
IS_MISTRAL_TOKENIZER = True # used by vllm.utils.mistral
@classmethod
def from_pretrained(
cls,