diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index 9635e9963b..0a47426dc9 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -109,6 +109,8 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path): try: def fake_from_pretrained(path_or_repo_id: str, *args, **kwargs): + passed_config = kwargs.pop("config") + assert isinstance(passed_config, Qwen3_5MoeConfig) loaded_config = AutoConfig.from_pretrained( path_or_repo_id, trust_remote_code=False, diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index e6c12ccc3b..82df4339e8 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -18,6 +18,7 @@ from vllm.transformers_utils.repo_utils import ( ) from vllm.utils.import_utils import resolve_obj_by_qualname +from .hf import CachedHfTokenizer from .protocol import TokenizerLike if TYPE_CHECKING: @@ -233,6 +234,12 @@ def get_tokenizer( else: tokenizer_cls_ = tokenizer_cls + if config is not None and tokenizer_cls_ is CachedHfTokenizer: + # AutoTokenizer otherwise reloads config.json internally. Reuse the + # config that get_config just loaded successfully so a concurrent Hub + # cache refresh cannot invalidate the file between the two reads. + kwargs.setdefault("config", config) + tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs) if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS: from vllm.tokenizers.hf import get_cached_tokenizer