[Bugfix] Honor code_revision in config-based tokenizer loading (#61611)
Signed-off-by: Yuan Li <dskwelmcy@163.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import _get_protocol_attrs # type: ignore
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -15,7 +16,11 @@ from transformers import (
|
||||
from vllm.tokenizers import TokenizerLike, get_tokenizer
|
||||
from vllm.tokenizers.hf import HfTokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tokenizers.registry import resolve_tokenizer_args
|
||||
from vllm.tokenizers.registry import (
|
||||
cached_tokenizer_from_config,
|
||||
resolve_tokenizer_args,
|
||||
tokenizer_args_from_config,
|
||||
)
|
||||
|
||||
|
||||
def _get_missing_attrs(obj: object, target: type):
|
||||
@@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str):
|
||||
@pytest.mark.parametrize(
|
||||
"code_revision, error_match",
|
||||
[
|
||||
pytest.param("never", "not a valid git identifier", id="unknown-revision"),
|
||||
pytest.param(
|
||||
"never",
|
||||
"not a valid git identifier|couldn't find them in the cached files",
|
||||
id="unknown-revision",
|
||||
),
|
||||
pytest.param(
|
||||
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
||||
r"tokenization_fast\.py",
|
||||
@@ -143,6 +152,141 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"code_revision",
|
||||
[
|
||||
None,
|
||||
"c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2",
|
||||
"2efddce40dddaccd37bae208c3c7ca66dbedf68a",
|
||||
],
|
||||
)
|
||||
def test_tokenizer_config_code_revision(code_revision):
|
||||
"""Load real remote config code at its pin, not the data revision."""
|
||||
revision = "c8ae8a2e5845fc3ad4e5237e4346e5a9ba1e87d2"
|
||||
|
||||
with patch(
|
||||
"vllm.tokenizers.registry.CachedHfTokenizer.from_pretrained",
|
||||
return_value=SimpleNamespace(is_fast=True),
|
||||
) as load_tokenizer:
|
||||
get_tokenizer(
|
||||
"hf-internal-testing/test_dynamic_model",
|
||||
tokenizer_mode="hf",
|
||||
trust_remote_code=True,
|
||||
revision=revision,
|
||||
code_revision=code_revision,
|
||||
)
|
||||
|
||||
config = load_tokenizer.call_args.kwargs["config"]
|
||||
assert (code_revision or revision) in type(config).__module__.split(".")
|
||||
assert config.new_hidden_size == 12
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model, code_revision, kwargs, error_match",
|
||||
[
|
||||
pytest.param(
|
||||
None,
|
||||
"never",
|
||||
{},
|
||||
"not a valid git identifier|couldn't find them in the cached files",
|
||||
id="same-repo",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
"73293ac504e43f4a895688b57c65cebe24f01a38",
|
||||
{},
|
||||
r"tokenization_fast\.py",
|
||||
id="same-repo-missing-code",
|
||||
),
|
||||
pytest.param(
|
||||
"facebook/opt-125m",
|
||||
"27dcfa74d334bc871f3234de431e71c6eeba5dd6",
|
||||
{},
|
||||
None,
|
||||
id="separate-repo",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
"never",
|
||||
{"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"},
|
||||
None,
|
||||
id="explicit-revision",
|
||||
),
|
||||
pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"),
|
||||
],
|
||||
)
|
||||
def test_config_tokenizer_code_revision(
|
||||
loader, model, code_revision, kwargs, error_match
|
||||
):
|
||||
"""Only same-repo tokenizers inherit model code revisions; overrides win."""
|
||||
revision = "08a35780b7a78794436d8654aafa26ee0165c149"
|
||||
repo = "hf-internal-testing/test_dynamic_tokenizer"
|
||||
config = SimpleNamespace(
|
||||
model=model or repo,
|
||||
tokenizer=repo,
|
||||
runner_type="generate",
|
||||
tokenizer_mode="hf",
|
||||
tokenizer_revision=revision,
|
||||
code_revision=code_revision,
|
||||
trust_remote_code=True,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
|
||||
def load():
|
||||
result = loader(config, **kwargs)
|
||||
if loader is tokenizer_args_from_config:
|
||||
mode, name, args, resolved_kwargs = result
|
||||
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
|
||||
return result
|
||||
|
||||
if error_match is not None:
|
||||
with pytest.raises(OSError, match=error_match):
|
||||
load()
|
||||
else:
|
||||
tokenizer = load()
|
||||
reference = AutoTokenizer.from_pretrained(
|
||||
repo,
|
||||
revision=revision,
|
||||
code_revision=kwargs.get("code_revision"),
|
||||
trust_remote_code=True,
|
||||
)
|
||||
assert tokenizer.encode("hello world") == reference.encode("hello world")
|
||||
_assert_tokenizer_like(tokenizer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
|
||||
)
|
||||
@pytest.mark.parametrize("tokenizer_mode", ["mistral", "auto"])
|
||||
@pytest.mark.parametrize("code_revision", [None, "never"])
|
||||
def test_config_mistral_code_revision(loader, tokenizer_mode, code_revision):
|
||||
"""Config loading must not pass HF-only revisions to the Mistral backend."""
|
||||
repo = "mistralai/Mistral-7B-Instruct-v0.3"
|
||||
config = SimpleNamespace(
|
||||
model=repo,
|
||||
tokenizer=repo,
|
||||
runner_type="generate",
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
tokenizer_revision=None,
|
||||
code_revision=code_revision,
|
||||
trust_remote_code=False,
|
||||
skip_tokenizer_init=False,
|
||||
)
|
||||
tokenizer = loader(config)
|
||||
if loader is tokenizer_args_from_config:
|
||||
mode, name, args, kwargs = tokenizer
|
||||
tokenizer = get_tokenizer(name, *args, tokenizer_mode=mode, **kwargs)
|
||||
|
||||
reference = get_tokenizer(repo, tokenizer_mode="mistral")
|
||||
assert isinstance(tokenizer, MistralTokenizer)
|
||||
assert tokenizer.encode("hello world") == reference.encode("hello world")
|
||||
|
||||
|
||||
def test_hub_tokenizer_loads_from_local_snapshot():
|
||||
"""Transformers makes a Hub request per file when given a repo ID."""
|
||||
with patch.object(
|
||||
|
||||
Reference in New Issue
Block a user