diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 82df4339e8..84f3ccce55 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -205,17 +205,27 @@ def get_tokenizer( **kwargs, ) + if tokenizer_cls == TokenizerLike: + tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode) + else: + tokenizer_cls_ = tokenizer_cls + # Ensure that, if the config were to come from vllm.transformers_utils.config, it is # registered with AutoConfig before the tokenizer is loaded. This is necessary since # tokenizer_cls_.from_pretrained will call AutoConfig.from_pretrained internally. # This may fail for paths that don't have a model config (e.g. LoRA adapters), # which is fine — those don't need custom config registration. + # HF-backed tokenizers must receive the HF config. In a dual-format Mistral + # repository, auto detection intentionally prefers params.json, but passing + # that generic config to AutoTokenizer can select the wrong tokenizer class. + config_format = "hf" if tokenizer_cls_ is CachedHfTokenizer else "auto" config = None with contextlib.suppress(ValueError, OSError): config = get_config( tokenizer_name, trust_remote_code=trust_remote_code, revision=revision, + config_format=config_format, ) # Some models have an incorrect tokenizer_class on the hub. @@ -229,10 +239,6 @@ def get_tokenizer( model_type, ) tokenizer_cls_ = TokenizersBackend - elif tokenizer_cls == TokenizerLike: - tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode) - else: - tokenizer_cls_ = tokenizer_cls if config is not None and tokenizer_cls_ is CachedHfTokenizer: # AutoTokenizer otherwise reloads config.json internally. Reuse the