[Bugfix] Honor code_revision in config-based tokenizer loading (#61611)
Signed-off-by: Yuan Li <dskwelmcy@163.com>
This commit is contained in:
@@ -9,7 +9,7 @@ import pytest
|
||||
from transformers import AutoConfig, Qwen3_5MoeConfig
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers import TokenizerLike, registry
|
||||
from vllm.tokenizers.registry import (
|
||||
TokenizerRegistry,
|
||||
cached_get_tokenizer,
|
||||
@@ -17,6 +17,7 @@ from vllm.tokenizers.registry import (
|
||||
cached_tokenizer_from_config,
|
||||
get_tokenizer,
|
||||
resolve_tokenizer_args,
|
||||
tokenizer_args_from_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -116,11 +117,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
||||
)
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
model=str(tmp_path),
|
||||
skip_tokenizer_init=False,
|
||||
tokenizer=str(tmp_path),
|
||||
runner_type="generate",
|
||||
tokenizer_mode="hf",
|
||||
tokenizer_revision=None,
|
||||
code_revision=None,
|
||||
trust_remote_code=True,
|
||||
hf_config=Qwen3_5MoeConfig(),
|
||||
)
|
||||
@@ -164,3 +167,117 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
||||
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
|
||||
if registered_config is not None:
|
||||
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def config_tokenizer_backend(monkeypatch):
|
||||
"""Keep config helpers and dispatch real; replace Hub/backend loading."""
|
||||
calls = []
|
||||
|
||||
class Backend:
|
||||
@classmethod
|
||||
def from_pretrained(cls, name, *args, **kwargs):
|
||||
calls.append(kwargs)
|
||||
return SimpleNamespace(is_fast=True)
|
||||
|
||||
monkeypatch.setattr(registry, "get_config", lambda *a, **kw: None)
|
||||
monkeypatch.setattr(registry, "_maybe_register_hf_config", lambda config: None)
|
||||
monkeypatch.setattr(registry, "is_mistral_model_repo", lambda **kw: True)
|
||||
monkeypatch.setattr(registry, "any_pattern_in_repo_files", lambda **kw: True)
|
||||
monkeypatch.setattr(
|
||||
registry, "get_non_weight_snapshot_path", lambda name, **kw: name
|
||||
)
|
||||
monkeypatch.setattr(TokenizerRegistry, "load_tokenizer_cls", lambda mode: Backend)
|
||||
cached_get_tokenizer.cache_clear()
|
||||
cached_resolve_tokenizer_args.cache_clear()
|
||||
yield calls
|
||||
cached_get_tokenizer.cache_clear()
|
||||
cached_resolve_tokenizer_args.cache_clear()
|
||||
|
||||
|
||||
def _load_config_tokenizer(loader, config, **kwargs):
|
||||
result = loader(config, **kwargs)
|
||||
if loader is tokenizer_args_from_config:
|
||||
mode, name, args, resolved_kwargs = result
|
||||
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("mode", ["mistral", "auto"])
|
||||
@pytest.mark.parametrize("code_revision", [None, "model-code"])
|
||||
@pytest.mark.parametrize("overrides", [{}, {"code_revision": None}])
|
||||
def test_config_mistral_does_not_forward_code_revision(
|
||||
loader, mode, code_revision, overrides, config_tokenizer_backend
|
||||
):
|
||||
"""HF code revisions must not reach MistralCommonBackend's strict kwargs."""
|
||||
config = SimpleNamespace(
|
||||
model="test-model",
|
||||
tokenizer="test-model",
|
||||
runner_type="generate",
|
||||
tokenizer_mode=mode,
|
||||
tokenizer_revision=None,
|
||||
code_revision=code_revision,
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
_load_config_tokenizer(loader, config, **overrides)
|
||||
assert len(config_tokenizer_backend) == 1
|
||||
assert "code_revision" not in config_tokenizer_backend[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("missing", [False, True])
|
||||
def test_config_tokenizer_without_code_revision(
|
||||
loader, missing, config_tokenizer_backend
|
||||
):
|
||||
"""Default or minimal configs must not introduce an HF-only plugin kwarg."""
|
||||
config = SimpleNamespace(
|
||||
model="test-model",
|
||||
tokenizer="test-model",
|
||||
runner_type="generate",
|
||||
tokenizer_mode="test_tokenizer",
|
||||
tokenizer_revision=None,
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
if not missing:
|
||||
config.code_revision = None
|
||||
_load_config_tokenizer(loader, config)
|
||||
assert len(config_tokenizer_backend) == 1
|
||||
assert "code_revision" not in config_tokenizer_backend[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("mode", ["hf", "cohere", "kimi_k3", "inkling", "deepseek_v4"])
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, expected",
|
||||
[
|
||||
({}, "model-code"),
|
||||
({"code_revision": None}, None),
|
||||
({"code_revision": "override"}, "override"),
|
||||
],
|
||||
)
|
||||
def test_config_hf_backed_tokenizer_preserves_code_revision(
|
||||
loader, mode, overrides, expected, config_tokenizer_backend
|
||||
):
|
||||
"""Filtering Mistral kwargs must preserve HF wrappers and caller overrides."""
|
||||
config = SimpleNamespace(
|
||||
model="test-model",
|
||||
tokenizer="test-model",
|
||||
runner_type="generate",
|
||||
tokenizer_mode=mode,
|
||||
tokenizer_revision=None,
|
||||
code_revision="model-code",
|
||||
trust_remote_code=True,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
_load_config_tokenizer(loader, config, **overrides)
|
||||
assert len(config_tokenizer_backend) == 1
|
||||
assert config_tokenizer_backend[0]["code_revision"] == expected
|
||||
|
||||
Reference in New Issue
Block a user