[Bugfix] Preserve independent tokenizer code_revision in snapshot resolution (#61272)

Signed-off-by: Yuan Li <dskwelmcy@163.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

tests/tokenizers_/test_basic.py

@@ -1,10 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from pathlib import Path
 from typing import _get_protocol_attrs  # type: ignore
 from unittest.mock import patch
 
 import pytest
 from transformers import (
+    AutoTokenizer,
     PreTrainedTokenizerBase,
     TokenizersBackend,
     tokenization_utils_base,
@@ -13,6 +15,7 @@ from transformers import (
 from vllm.tokenizers import TokenizerLike, get_tokenizer
 from vllm.tokenizers.hf import HfTokenizer
 from vllm.tokenizers.mistral import MistralTokenizer
+from vllm.tokenizers.registry import resolve_tokenizer_args
 
 
 def _get_missing_attrs(obj: object, target: type):
@@ -58,6 +61,88 @@ def test_tokenizer_revision(tokenizer_name: str):
         get_tokenizer(tokenizer_name, revision="never")
 
 
+@pytest.mark.skip_global_cleanup
+@pytest.mark.parametrize(
+    "code_revision, error_match",
+    [
+        pytest.param("never", "not a valid git identifier", id="unknown-revision"),
+        pytest.param(
+            "73293ac504e43f4a895688b57c65cebe24f01a38",
+            r"tokenization_fast\.py",
+            id="revision-without-code",
+        ),
+    ],
+)
+def test_tokenizer_code_revision_rejects_missing_code(
+    code_revision: str, error_match: str
+):
+    """A data snapshot must not shadow the requested remote-code revision."""
+    with pytest.raises(OSError, match=error_match):
+        get_tokenizer(
+            "hf-internal-testing/test_dynamic_tokenizer",
+            tokenizer_mode="hf",
+            revision="08a35780b7a78794436d8654aafa26ee0165c149",
+            code_revision=code_revision,
+            trust_remote_code=True,
+        )
+
+
+@pytest.mark.skip_global_cleanup
+def test_tokenizer_code_revision_distinct_from_revision():
+    """Distinct code/data revisions: the resolver must keep the Hub ID.
+
+    Dynamic code is read from a local directory as-is, so resolving to a
+    data-revision snapshot would silently shadow an independent code
+    revision (the same contract already enforced for processors).
+    """
+    revision = "08a35780b7a78794436d8654aafa26ee0165c149"
+    repo = "hf-internal-testing/test_dynamic_tokenizer"
+    _, name, _, kwargs = resolve_tokenizer_args(
+        repo,
+        tokenizer_mode="hf",
+        revision=revision,
+        code_revision=revision,
+        trust_remote_code=True,
+    )
+    assert Path(name).is_dir()
+    assert kwargs["code_revision"] == revision
+
+    _, name, _, kwargs = resolve_tokenizer_args(
+        repo,
+        tokenizer_mode="hf",
+        revision=revision,
+        code_revision="73293ac504e43f4a895688b57c65cebe24f01a38",
+        trust_remote_code=True,
+    )
+    assert name == repo
+    assert kwargs["code_revision"] == "73293ac504e43f4a895688b57c65cebe24f01a38"
+
+
+@pytest.mark.skip_global_cleanup
+@pytest.mark.parametrize(
+    "code_revision", [None, "08a35780b7a78794436d8654aafa26ee0165c149"]
+)
+def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
+    revision = "08a35780b7a78794436d8654aafa26ee0165c149"
+    tokenizer = get_tokenizer(
+        "hf-internal-testing/test_dynamic_tokenizer",
+        tokenizer_mode="hf",
+        revision=revision,
+        code_revision=code_revision,
+        trust_remote_code=True,
+    )
+    reference = AutoTokenizer.from_pretrained(
+        "hf-internal-testing/test_dynamic_tokenizer",
+        revision=revision,
+        code_revision=code_revision,
+        trust_remote_code=True,
+    )
+    assert isinstance(tokenizer, PreTrainedTokenizerBase)
+    assert tokenizer.encode("hello world") == reference.encode("hello world")
+    assert tokenizer.name_or_path == "hf-internal-testing/test_dynamic_tokenizer"
+    _assert_tokenizer_like(tokenizer)
+
+
 def test_hub_tokenizer_loads_from_local_snapshot():
     """Transformers makes a Hub request per file when given a repo ID."""
     with patch.object(

vllm/tokenizers/registry.py

@@ -165,12 +165,13 @@ def resolve_tokenizer_args(
                 "mistral_format=True is not supported with tokenizer_mode='hf'"
             )
         kwargs["mistral_format"] = False
-        tokenizer_name = get_non_weight_snapshot_path(
-            str(tokenizer_name),
-            revision=revision,
-            cache_dir=download_dir,
-            token=kwargs.get("token"),
-        )
+        if kwargs.get("code_revision") in (None, revision):
+            tokenizer_name = get_non_weight_snapshot_path(
+                str(tokenizer_name),
+                revision=revision,
+                cache_dir=download_dir,
+                token=kwargs.get("token"),
+            )
 
     return tokenizer_mode, tokenizer_name, args, kwargs