[Bugfix] Honor code_revision in config-based tokenizer loading (#61611)
Signed-off-by: Yuan Li <dskwelmcy@163.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
from typing import _get_protocol_attrs # type: ignore
|
from typing import _get_protocol_attrs # type: ignore
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
@@ -15,7 +16,11 @@ from transformers import (
|
|||||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||||
from vllm.tokenizers.hf import HfTokenizer
|
from vllm.tokenizers.hf import HfTokenizer
|
||||||
from vllm.tokenizers.mistral import MistralTokenizer
|
from vllm.tokenizers.mistral import MistralTokenizer
|
||||||
from vllm.tokenizers.registry import resolve_tokenizer_args
|
from vllm.tokenizers.registry import (
|
||||||
|
cached_tokenizer_from_config,
|
||||||
|
resolve_tokenizer_args,
|
||||||
|
tokenizer_args_from_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _get_missing_attrs(obj: object, target: type):
|
def _get_missing_attrs(obj: object, target: type):
|
||||||
@@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str):
|
|||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"code_revision, error_match",
|
"code_revision, error_match",
|
||||||
[
|
[
|
||||||
pytest.param("never", "not a valid git identifier", id="unknown-revision"),
|
pytest.param(
|
||||||
|
"never",
|
||||||
|
"not a valid git identifier|couldn't find them in the cached files",
|
||||||
|
id="unknown-revision",
|
||||||
|
),
|
||||||
pytest.param(
|
pytest.param(
|
||||||
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
||||||
r"tokenization_fast\.py",
|
r"tokenization_fast\.py",
|
||||||
@@ -143,6 +152,141 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
|
|||||||
_assert_tokenizer_like(tokenizer)
|
_assert_tokenizer_like(tokenizer)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"code_revision",
|
||||||
|
[
|
||||||
|
None,
|
||||||
|
"c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2",
|
||||||
|
"2efddce40dddaccd37bae208c3c7ca66dbedf68a",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_tokenizer_config_code_revision(code_revision):
|
||||||
|
"""Load real remote config code at its pin, not the data revision."""
|
||||||
|
revision = "c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2"
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"vllm.tokenizers.registry.CachedHfTokenizer.from_pretrained",
|
||||||
|
return_value=SimpleNamespace(is_fast=True),
|
||||||
|
) as load_tokenizer:
|
||||||
|
get_tokenizer(
|
||||||
|
"hf-internal-testing/test_dynamic_model",
|
||||||
|
tokenizer_mode="hf",
|
||||||
|
trust_remote_code=True,
|
||||||
|
revision=revision,
|
||||||
|
code_revision=code_revision,
|
||||||
|
)
|
||||||
|
|
||||||
|
config = load_tokenizer.call_args.kwargs["config"]
|
||||||
|
assert (code_revision or revision) in type(config).__module__.split(".")
|
||||||
|
assert config.new_hidden_size == 12
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skip_global_cleanup
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"model, code_revision, kwargs, error_match",
|
||||||
|
[
|
||||||
|
pytest.param(
|
||||||
|
None,
|
||||||
|
"never",
|
||||||
|
{},
|
||||||
|
"not a valid git identifier|couldn't find them in the cached files",
|
||||||
|
id="same-repo",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
None,
|
||||||
|
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
||||||
|
{},
|
||||||
|
r"tokenization_fast\.py",
|
||||||
|
id="same-repo-missing-code",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
"facebook/opt-125m",
|
||||||
|
"27dcfa74d334bc871f3234de431e71c6eeba5dd6",
|
||||||
|
{},
|
||||||
|
None,
|
||||||
|
id="separate-repo",
|
||||||
|
),
|
||||||
|
pytest.param(
|
||||||
|
None,
|
||||||
|
"never",
|
||||||
|
{"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"},
|
||||||
|
None,
|
||||||
|
id="explicit-revision",
|
||||||
|
),
|
||||||
|
pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_config_tokenizer_code_revision(
|
||||||
|
loader, model, code_revision, kwargs, error_match
|
||||||
|
):
|
||||||
|
"""Only same-repo tokenizers inherit model code revisions; overrides win."""
|
||||||
|
revision = "08a35780b7a78794436d8654aafa26ee0165c149"
|
||||||
|
repo = "hf-internal-testing/test_dynamic_tokenizer"
|
||||||
|
config = SimpleNamespace(
|
||||||
|
model=model or repo,
|
||||||
|
tokenizer=repo,
|
||||||
|
runner_type="generate",
|
||||||
|
tokenizer_mode="hf",
|
||||||
|
tokenizer_revision=revision,
|
||||||
|
code_revision=code_revision,
|
||||||
|
trust_remote_code=True,
|
||||||
|
skip_tokenizer_init=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def load():
|
||||||
|
result = loader(config, **kwargs)
|
||||||
|
if loader is tokenizer_args_from_config:
|
||||||
|
mode, name, args, resolved_kwargs = result
|
||||||
|
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
|
||||||
|
return result
|
||||||
|
|
||||||
|
if error_match is not None:
|
||||||
|
with pytest.raises(OSError, match=error_match):
|
||||||
|
load()
|
||||||
|
else:
|
||||||
|
tokenizer = load()
|
||||||
|
reference = AutoTokenizer.from_pretrained(
|
||||||
|
repo,
|
||||||
|
revision=revision,
|
||||||
|
code_revision=kwargs.get("code_revision"),
|
||||||
|
trust_remote_code=True,
|
||||||
|
)
|
||||||
|
assert tokenizer.encode("hello world") == reference.encode("hello world")
|
||||||
|
_assert_tokenizer_like(tokenizer)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("tokenizer_mode", ["mistral", "auto"])
|
||||||
|
@pytest.mark.parametrize("code_revision", [None, "never"])
|
||||||
|
def test_config_mistral_code_revision(loader, tokenizer_mode, code_revision):
|
||||||
|
"""Config loading must not pass HF-only revisions to the Mistral backend."""
|
||||||
|
repo = "mistralai/Mistral-7B-Instruct-v0.3"
|
||||||
|
config = SimpleNamespace(
|
||||||
|
model=repo,
|
||||||
|
tokenizer=repo,
|
||||||
|
runner_type="generate",
|
||||||
|
tokenizer_mode=tokenizer_mode,
|
||||||
|
tokenizer_revision=None,
|
||||||
|
code_revision=code_revision,
|
||||||
|
trust_remote_code=False,
|
||||||
|
skip_tokenizer_init=False,
|
||||||
|
)
|
||||||
|
tokenizer = loader(config)
|
||||||
|
if loader is tokenizer_args_from_config:
|
||||||
|
mode, name, args, kwargs = tokenizer
|
||||||
|
tokenizer = get_tokenizer(name, *args, tokenizer_mode=mode, **kwargs)
|
||||||
|
|
||||||
|
reference = get_tokenizer(repo, tokenizer_mode="mistral")
|
||||||
|
assert isinstance(tokenizer, MistralTokenizer)
|
||||||
|
assert tokenizer.encode("hello world") == reference.encode("hello world")
|
||||||
|
|
||||||
|
|
||||||
def test_hub_tokenizer_loads_from_local_snapshot():
|
def test_hub_tokenizer_loads_from_local_snapshot():
|
||||||
"""Transformers makes a Hub request per file when given a repo ID."""
|
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||||
with patch.object(
|
with patch.object(
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import pytest
|
|||||||
from transformers import AutoConfig, Qwen3_5MoeConfig
|
from transformers import AutoConfig, Qwen3_5MoeConfig
|
||||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||||
|
|
||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike, registry
|
||||||
from vllm.tokenizers.registry import (
|
from vllm.tokenizers.registry import (
|
||||||
TokenizerRegistry,
|
TokenizerRegistry,
|
||||||
cached_get_tokenizer,
|
cached_get_tokenizer,
|
||||||
@@ -17,6 +17,7 @@ from vllm.tokenizers.registry import (
|
|||||||
cached_tokenizer_from_config,
|
cached_tokenizer_from_config,
|
||||||
get_tokenizer,
|
get_tokenizer,
|
||||||
resolve_tokenizer_args,
|
resolve_tokenizer_args,
|
||||||
|
tokenizer_args_from_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -116,11 +117,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
|||||||
)
|
)
|
||||||
|
|
||||||
model_config = SimpleNamespace(
|
model_config = SimpleNamespace(
|
||||||
|
model=str(tmp_path),
|
||||||
skip_tokenizer_init=False,
|
skip_tokenizer_init=False,
|
||||||
tokenizer=str(tmp_path),
|
tokenizer=str(tmp_path),
|
||||||
runner_type="generate",
|
runner_type="generate",
|
||||||
tokenizer_mode="hf",
|
tokenizer_mode="hf",
|
||||||
tokenizer_revision=None,
|
tokenizer_revision=None,
|
||||||
|
code_revision=None,
|
||||||
trust_remote_code=True,
|
trust_remote_code=True,
|
||||||
hf_config=Qwen3_5MoeConfig(),
|
hf_config=Qwen3_5MoeConfig(),
|
||||||
)
|
)
|
||||||
@@ -164,3 +167,117 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
|
|||||||
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
|
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
|
||||||
if registered_config is not None:
|
if registered_config is not None:
|
||||||
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config
|
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def config_tokenizer_backend(monkeypatch):
|
||||||
|
"""Keep config helpers and dispatch real; replace Hub/backend loading."""
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class Backend:
|
||||||
|
@classmethod
|
||||||
|
def from_pretrained(cls, name, *args, **kwargs):
|
||||||
|
calls.append(kwargs)
|
||||||
|
return SimpleNamespace(is_fast=True)
|
||||||
|
|
||||||
|
monkeypatch.setattr(registry, "get_config", lambda *a, **kw: None)
|
||||||
|
monkeypatch.setattr(registry, "_maybe_register_hf_config", lambda config: None)
|
||||||
|
monkeypatch.setattr(registry, "is_mistral_model_repo", lambda **kw: True)
|
||||||
|
monkeypatch.setattr(registry, "any_pattern_in_repo_files", lambda **kw: True)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
registry, "get_non_weight_snapshot_path", lambda name, **kw: name
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(TokenizerRegistry, "load_tokenizer_cls", lambda mode: Backend)
|
||||||
|
cached_get_tokenizer.cache_clear()
|
||||||
|
cached_resolve_tokenizer_args.cache_clear()
|
||||||
|
yield calls
|
||||||
|
cached_get_tokenizer.cache_clear()
|
||||||
|
cached_resolve_tokenizer_args.cache_clear()
|
||||||
|
|
||||||
|
|
||||||
|
def _load_config_tokenizer(loader, config, **kwargs):
|
||||||
|
result = loader(config, **kwargs)
|
||||||
|
if loader is tokenizer_args_from_config:
|
||||||
|
mode, name, args, resolved_kwargs = result
|
||||||
|
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("mode", ["mistral", "auto"])
|
||||||
|
@pytest.mark.parametrize("code_revision", [None, "model-code"])
|
||||||
|
@pytest.mark.parametrize("overrides", [{}, {"code_revision": None}])
|
||||||
|
def test_config_mistral_does_not_forward_code_revision(
|
||||||
|
loader, mode, code_revision, overrides, config_tokenizer_backend
|
||||||
|
):
|
||||||
|
"""HF code revisions must not reach MistralCommonBackend's strict kwargs."""
|
||||||
|
config = SimpleNamespace(
|
||||||
|
model="test-model",
|
||||||
|
tokenizer="test-model",
|
||||||
|
runner_type="generate",
|
||||||
|
tokenizer_mode=mode,
|
||||||
|
tokenizer_revision=None,
|
||||||
|
code_revision=code_revision,
|
||||||
|
trust_remote_code=False,
|
||||||
|
skip_tokenizer_init=False,
|
||||||
|
)
|
||||||
|
_load_config_tokenizer(loader, config, **overrides)
|
||||||
|
assert len(config_tokenizer_backend) == 1
|
||||||
|
assert "code_revision" not in config_tokenizer_backend[0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("missing", [False, True])
|
||||||
|
def test_config_tokenizer_without_code_revision(
|
||||||
|
loader, missing, config_tokenizer_backend
|
||||||
|
):
|
||||||
|
"""Default or minimal configs must not introduce an HF-only plugin kwarg."""
|
||||||
|
config = SimpleNamespace(
|
||||||
|
model="test-model",
|
||||||
|
tokenizer="test-model",
|
||||||
|
runner_type="generate",
|
||||||
|
tokenizer_mode="test_tokenizer",
|
||||||
|
tokenizer_revision=None,
|
||||||
|
trust_remote_code=False,
|
||||||
|
skip_tokenizer_init=False,
|
||||||
|
)
|
||||||
|
if not missing:
|
||||||
|
config.code_revision = None
|
||||||
|
_load_config_tokenizer(loader, config)
|
||||||
|
assert len(config_tokenizer_backend) == 1
|
||||||
|
assert "code_revision" not in config_tokenizer_backend[0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
|
||||||
|
)
|
||||||
|
@pytest.mark.parametrize("mode", ["hf", "cohere", "kimi_k3", "inkling", "deepseek_v4"])
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"overrides, expected",
|
||||||
|
[
|
||||||
|
({}, "model-code"),
|
||||||
|
({"code_revision": None}, None),
|
||||||
|
({"code_revision": "override"}, "override"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_config_hf_backed_tokenizer_preserves_code_revision(
|
||||||
|
loader, mode, overrides, expected, config_tokenizer_backend
|
||||||
|
):
|
||||||
|
"""Filtering Mistral kwargs must preserve HF wrappers and caller overrides."""
|
||||||
|
config = SimpleNamespace(
|
||||||
|
model="test-model",
|
||||||
|
tokenizer="test-model",
|
||||||
|
runner_type="generate",
|
||||||
|
tokenizer_mode=mode,
|
||||||
|
tokenizer_revision=None,
|
||||||
|
code_revision="model-code",
|
||||||
|
trust_remote_code=True,
|
||||||
|
skip_tokenizer_init=False,
|
||||||
|
)
|
||||||
|
_load_config_tokenizer(loader, config, **overrides)
|
||||||
|
assert len(config_tokenizer_backend) == 1
|
||||||
|
assert config_tokenizer_backend[0]["code_revision"] == expected
|
||||||
|
|||||||
@@ -174,6 +174,10 @@ def resolve_tokenizer_args(
|
|||||||
token=kwargs.get("token"),
|
token=kwargs.get("token"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if tokenizer_mode == "mistral":
|
||||||
|
# MistralCommonBackend rejects HF-only keyword arguments, even None.
|
||||||
|
kwargs.pop("code_revision", None)
|
||||||
|
|
||||||
return tokenizer_mode, tokenizer_name, args, kwargs
|
return tokenizer_mode, tokenizer_name, args, kwargs
|
||||||
|
|
||||||
|
|
||||||
@@ -181,6 +185,10 @@ cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args)
|
|||||||
|
|
||||||
|
|
||||||
def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
|
def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
|
||||||
|
if config.tokenizer == config.model:
|
||||||
|
code_revision = getattr(config, "code_revision", None)
|
||||||
|
if code_revision is not None:
|
||||||
|
kwargs.setdefault("code_revision", code_revision)
|
||||||
return cached_resolve_tokenizer_args(
|
return cached_resolve_tokenizer_args(
|
||||||
config.tokenizer,
|
config.tokenizer,
|
||||||
runner_type=config.runner_type,
|
runner_type=config.runner_type,
|
||||||
@@ -241,6 +249,7 @@ def get_tokenizer(
|
|||||||
tokenizer_name,
|
tokenizer_name,
|
||||||
trust_remote_code=trust_remote_code,
|
trust_remote_code=trust_remote_code,
|
||||||
revision=revision,
|
revision=revision,
|
||||||
|
code_revision=kwargs.get("code_revision"),
|
||||||
config_format=config_format,
|
config_format=config_format,
|
||||||
token=kwargs.get("token"),
|
token=kwargs.get("token"),
|
||||||
)
|
)
|
||||||
@@ -288,6 +297,10 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
_maybe_register_hf_config(getattr(model_config, "hf_config", None))
|
_maybe_register_hf_config(getattr(model_config, "hf_config", None))
|
||||||
|
if model_config.tokenizer == model_config.model:
|
||||||
|
code_revision = getattr(model_config, "code_revision", None)
|
||||||
|
if code_revision is not None:
|
||||||
|
kwargs.setdefault("code_revision", code_revision)
|
||||||
|
|
||||||
return cached_get_tokenizer(
|
return cached_get_tokenizer(
|
||||||
model_config.tokenizer,
|
model_config.tokenizer,
|
||||||
|
|||||||
Reference in New Issue
Block a user