From 49cb2bbf71cfd36ee5aa33d1e184b1cddff55294 Mon Sep 17 00:00:00 2001 From: dskwe Date: Fri, 16 Oct 2026 13:21:12 +0800 Subject: [PATCH] [Bugfix] Honor code_revision in config-based tokenizer loading (#61611) Signed-off-by: Yuan Li --- tests/tokenizers_/test_basic.py | 148 ++++++++++++++++++++++++++++- tests/tokenizers_/test_registry.py | 119 ++++++++++++++++++++++- vllm/tokenizers/registry.py | 13 +++ 3 files changed, 277 insertions(+), 3 deletions(-) diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index a911245494..922820b89c 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from pathlib import Path +from types import SimpleNamespace from typing import _get_protocol_attrs # type: ignore from unittest.mock import patch @@ -15,7 +16,11 @@ from transformers import ( from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers.hf import HfTokenizer from vllm.tokenizers.mistral import MistralTokenizer -from vllm.tokenizers.registry import resolve_tokenizer_args +from vllm.tokenizers.registry import ( + cached_tokenizer_from_config, + resolve_tokenizer_args, + tokenizer_args_from_config, +) def _get_missing_attrs(obj: object, target: type): @@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str): @pytest.mark.parametrize( "code_revision, error_match", [ - pytest.param("never", "not a valid git identifier", id="unknown-revision"), + pytest.param( + "never", + "not a valid git identifier|couldn't find them in the cached files", + id="unknown-revision", + ), pytest.param( "73293ac504e43f4a895688b57c65cebe24f01a38", r"tokenization_fast\.py", @@ -143,6 +152,141 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None): _assert_tokenizer_like(tokenizer) +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize( + "code_revision", + [ + None, + "c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2", + "2efddce40dddaccd37bae208c3c7ca66dbedf68a", + ], +) +def test_tokenizer_config_code_revision(code_revision): + """Load real remote config code at its pin, not the data revision.""" + revision = "c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2" + + with patch( + "vllm.tokenizers.registry.CachedHfTokenizer.from_pretrained", + return_value=SimpleNamespace(is_fast=True), + ) as load_tokenizer: + get_tokenizer( + "hf-internal-testing/test_dynamic_model", + tokenizer_mode="hf", + trust_remote_code=True, + revision=revision, + code_revision=code_revision, + ) + + config = load_tokenizer.call_args.kwargs["config"] + assert (code_revision or revision) in type(config).__module__.split(".") + assert config.new_hidden_size == 12 + + +@pytest.mark.skip_global_cleanup +@pytest.mark.parametrize( + "loader", (cached_tokenizer_from_config, tokenizer_args_from_config) +) +@pytest.mark.parametrize( + "model, code_revision, kwargs, error_match", + [ + pytest.param( + None, + "never", + {}, + "not a valid git identifier|couldn't find them in the cached files", + id="same-repo", + ), + pytest.param( + None, + "73293ac504e43f4a895688b57c65cebe24f01a38", + {}, + r"tokenization_fast\.py", + id="same-repo-missing-code", + ), + pytest.param( + "facebook/opt-125m", + "27dcfa74d334bc871f3234de431e71c6eeba5dd6", + {}, + None, + id="separate-repo", + ), + pytest.param( + None, + "never", + {"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"}, + None, + id="explicit-revision", + ), + pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"), + ], +) +def test_config_tokenizer_code_revision( + loader, model, code_revision, kwargs, error_match +): + """Only same-repo tokenizers inherit model code revisions; overrides win.""" + revision = "08a35780b7a78794436d8654aafa26ee0165c149" + repo = "hf-internal-testing/test_dynamic_tokenizer" + config = SimpleNamespace( + model=model or repo, + tokenizer=repo, + runner_type="generate", + tokenizer_mode="hf", + tokenizer_revision=revision, + code_revision=code_revision, + trust_remote_code=True, + skip_tokenizer_init=False, + ) + + def load(): + result = loader(config, **kwargs) + if loader is tokenizer_args_from_config: + mode, name, args, resolved_kwargs = result + return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs) + return result + + if error_match is not None: + with pytest.raises(OSError, match=error_match): + load() + else: + tokenizer = load() + reference = AutoTokenizer.from_pretrained( + repo, + revision=revision, + code_revision=kwargs.get("code_revision"), + trust_remote_code=True, + ) + assert tokenizer.encode("hello world") == reference.encode("hello world") + _assert_tokenizer_like(tokenizer) + + +@pytest.mark.parametrize( + "loader", (cached_tokenizer_from_config, tokenizer_args_from_config) +) +@pytest.mark.parametrize("tokenizer_mode", ["mistral", "auto"]) +@pytest.mark.parametrize("code_revision", [None, "never"]) +def test_config_mistral_code_revision(loader, tokenizer_mode, code_revision): + """Config loading must not pass HF-only revisions to the Mistral backend.""" + repo = "mistralai/Mistral-7B-Instruct-v0.3" + config = SimpleNamespace( + model=repo, + tokenizer=repo, + runner_type="generate", + tokenizer_mode=tokenizer_mode, + tokenizer_revision=None, + code_revision=code_revision, + trust_remote_code=False, + skip_tokenizer_init=False, + ) + tokenizer = loader(config) + if loader is tokenizer_args_from_config: + mode, name, args, kwargs = tokenizer + tokenizer = get_tokenizer(name, *args, tokenizer_mode=mode, **kwargs) + + reference = get_tokenizer(repo, tokenizer_mode="mistral") + assert isinstance(tokenizer, MistralTokenizer) + assert tokenizer.encode("hello world") == reference.encode("hello world") + + def test_hub_tokenizer_loads_from_local_snapshot(): """Transformers makes a Hub request per file when given a repo ID.""" with patch.object( diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index b9d9c732f3..73ff4bc5df 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -9,7 +9,7 @@ import pytest from transformers import AutoConfig, Qwen3_5MoeConfig from transformers.models.auto.configuration_auto import CONFIG_MAPPING -from vllm.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike, registry from vllm.tokenizers.registry import ( TokenizerRegistry, cached_get_tokenizer, @@ -17,6 +17,7 @@ from vllm.tokenizers.registry import ( cached_tokenizer_from_config, get_tokenizer, resolve_tokenizer_args, + tokenizer_args_from_config, ) @@ -116,11 +117,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path): ) model_config = SimpleNamespace( + model=str(tmp_path), skip_tokenizer_init=False, tokenizer=str(tmp_path), runner_type="generate", tokenizer_mode="hf", tokenizer_revision=None, + code_revision=None, trust_remote_code=True, hf_config=Qwen3_5MoeConfig(), ) @@ -164,3 +167,117 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path): CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None) if registered_config is not None: CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config + + +@pytest.fixture +def config_tokenizer_backend(monkeypatch): + """Keep config helpers and dispatch real; replace Hub/backend loading.""" + calls = [] + + class Backend: + @classmethod + def from_pretrained(cls, name, *args, **kwargs): + calls.append(kwargs) + return SimpleNamespace(is_fast=True) + + monkeypatch.setattr(registry, "get_config", lambda *a, **kw: None) + monkeypatch.setattr(registry, "_maybe_register_hf_config", lambda config: None) + monkeypatch.setattr(registry, "is_mistral_model_repo", lambda **kw: True) + monkeypatch.setattr(registry, "any_pattern_in_repo_files", lambda **kw: True) + monkeypatch.setattr( + registry, "get_non_weight_snapshot_path", lambda name, **kw: name + ) + monkeypatch.setattr(TokenizerRegistry, "load_tokenizer_cls", lambda mode: Backend) + cached_get_tokenizer.cache_clear() + cached_resolve_tokenizer_args.cache_clear() + yield calls + cached_get_tokenizer.cache_clear() + cached_resolve_tokenizer_args.cache_clear() + + +def _load_config_tokenizer(loader, config, **kwargs): + result = loader(config, **kwargs) + if loader is tokenizer_args_from_config: + mode, name, args, resolved_kwargs = result + return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs) + return result + + +@pytest.mark.parametrize( + "loader", (tokenizer_args_from_config, cached_tokenizer_from_config) +) +@pytest.mark.parametrize("mode", ["mistral", "auto"]) +@pytest.mark.parametrize("code_revision", [None, "model-code"]) +@pytest.mark.parametrize("overrides", [{}, {"code_revision": None}]) +def test_config_mistral_does_not_forward_code_revision( + loader, mode, code_revision, overrides, config_tokenizer_backend +): + """HF code revisions must not reach MistralCommonBackend's strict kwargs.""" + config = SimpleNamespace( + model="test-model", + tokenizer="test-model", + runner_type="generate", + tokenizer_mode=mode, + tokenizer_revision=None, + code_revision=code_revision, + trust_remote_code=False, + skip_tokenizer_init=False, + ) + _load_config_tokenizer(loader, config, **overrides) + assert len(config_tokenizer_backend) == 1 + assert "code_revision" not in config_tokenizer_backend[0] + + +@pytest.mark.parametrize( + "loader", (tokenizer_args_from_config, cached_tokenizer_from_config) +) +@pytest.mark.parametrize("missing", [False, True]) +def test_config_tokenizer_without_code_revision( + loader, missing, config_tokenizer_backend +): + """Default or minimal configs must not introduce an HF-only plugin kwarg.""" + config = SimpleNamespace( + model="test-model", + tokenizer="test-model", + runner_type="generate", + tokenizer_mode="test_tokenizer", + tokenizer_revision=None, + trust_remote_code=False, + skip_tokenizer_init=False, + ) + if not missing: + config.code_revision = None + _load_config_tokenizer(loader, config) + assert len(config_tokenizer_backend) == 1 + assert "code_revision" not in config_tokenizer_backend[0] + + +@pytest.mark.parametrize( + "loader", (tokenizer_args_from_config, cached_tokenizer_from_config) +) +@pytest.mark.parametrize("mode", ["hf", "cohere", "kimi_k3", "inkling", "deepseek_v4"]) +@pytest.mark.parametrize( + "overrides, expected", + [ + ({}, "model-code"), + ({"code_revision": None}, None), + ({"code_revision": "override"}, "override"), + ], +) +def test_config_hf_backed_tokenizer_preserves_code_revision( + loader, mode, overrides, expected, config_tokenizer_backend +): + """Filtering Mistral kwargs must preserve HF wrappers and caller overrides.""" + config = SimpleNamespace( + model="test-model", + tokenizer="test-model", + runner_type="generate", + tokenizer_mode=mode, + tokenizer_revision=None, + code_revision="model-code", + trust_remote_code=True, + skip_tokenizer_init=False, + ) + _load_config_tokenizer(loader, config, **overrides) + assert len(config_tokenizer_backend) == 1 + assert config_tokenizer_backend[0]["code_revision"] == expected diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 972300cad1..a52bf18401 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -174,6 +174,10 @@ def resolve_tokenizer_args( token=kwargs.get("token"), ) + if tokenizer_mode == "mistral": + # MistralCommonBackend rejects HF-only keyword arguments, even None. + kwargs.pop("code_revision", None) + return tokenizer_mode, tokenizer_name, args, kwargs @@ -181,6 +185,10 @@ cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args) def tokenizer_args_from_config(config: "ModelConfig", **kwargs): + if config.tokenizer == config.model: + code_revision = getattr(config, "code_revision", None) + if code_revision is not None: + kwargs.setdefault("code_revision", code_revision) return cached_resolve_tokenizer_args( config.tokenizer, runner_type=config.runner_type, @@ -241,6 +249,7 @@ def get_tokenizer( tokenizer_name, trust_remote_code=trust_remote_code, revision=revision, + code_revision=kwargs.get("code_revision"), config_format=config_format, token=kwargs.get("token"), ) @@ -288,6 +297,10 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs): return None _maybe_register_hf_config(getattr(model_config, "hf_config", None)) + if model_config.tokenizer == model_config.model: + code_revision = getattr(model_config, "code_revision", None) + if code_revision is not None: + kwargs.setdefault("code_revision", code_revision) return cached_get_tokenizer( model_config.tokenizer,