[Bugfix] Avoid MistralCommonBackend for HF tokenizers (#54192)
Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
This commit is contained in:
@@ -10,6 +10,7 @@ from transformers import AutoConfig
|
||||
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
|
||||
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tokenizers.hf import CachedHfTokenizer
|
||||
from vllm.tokenizers.registry import (
|
||||
TokenizerRegistry,
|
||||
cached_get_tokenizer,
|
||||
@@ -68,6 +69,89 @@ def test_resolve_tokenizer_args_idempotent(runner_type):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tokenizer_mode", "input_kwargs"),
|
||||
[
|
||||
("hf", {}),
|
||||
("hf", {"mistral_format": False}),
|
||||
("slow", {}),
|
||||
("slow", {"mistral_format": False}),
|
||||
],
|
||||
)
|
||||
def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
|
||||
tokenizer_mode, input_kwargs
|
||||
):
|
||||
resolved_mode, _, _, kwargs = resolve_tokenizer_args(
|
||||
"mistralai/Mistral-Nemo-Instruct-2407",
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
**input_kwargs,
|
||||
)
|
||||
|
||||
assert resolved_mode == "hf"
|
||||
assert kwargs["mistral_format"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"])
|
||||
def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode):
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="mistral_format=True is not supported with tokenizer_mode='hf'",
|
||||
):
|
||||
resolve_tokenizer_args(
|
||||
"mistralai/Mistral-Nemo-Instruct-2407",
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
mistral_format=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("transformers_version", "model_type", "native_filename", "expect_error"),
|
||||
[
|
||||
pytest.param("5.14.0", "mistral", "tekken.json", True),
|
||||
pytest.param("5.15.0", "mistral", "tekken.json", False),
|
||||
pytest.param("5.14.0", "mistral3", "tekken.json", True),
|
||||
pytest.param("5.15.0", "mistral3", "tekken.json", False),
|
||||
pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
|
||||
pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
|
||||
pytest.param("5.14.0", "mistral", None, False),
|
||||
pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
|
||||
],
|
||||
)
|
||||
def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
|
||||
tmp_path: Path,
|
||||
transformers_version: str,
|
||||
model_type: str,
|
||||
native_filename: str | None,
|
||||
expect_error: bool,
|
||||
):
|
||||
if native_filename is not None:
|
||||
(tmp_path / native_filename).touch()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"vllm.tokenizers.registry.get_config",
|
||||
return_value=SimpleNamespace(model_type=model_type),
|
||||
),
|
||||
patch(
|
||||
"vllm.tokenizers.registry.transformers.__version__",
|
||||
transformers_version,
|
||||
),
|
||||
patch.object(
|
||||
CachedHfTokenizer,
|
||||
"from_pretrained",
|
||||
return_value=SimpleNamespace(is_fast=True),
|
||||
) as from_pretrained,
|
||||
):
|
||||
if expect_error:
|
||||
with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
|
||||
get_tokenizer(str(tmp_path), tokenizer_mode="hf")
|
||||
else:
|
||||
tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
|
||||
assert tokenizer.is_fast is True
|
||||
|
||||
assert from_pretrained.called == (not expect_error)
|
||||
|
||||
|
||||
def test_customized_tokenizer():
|
||||
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
|
||||
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Structured output with a `MistralCommonBackend` tokenizer."""
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer, MistralCommonBackend
|
||||
|
||||
from vllm.config import StructuredOutputsConfig, VllmConfig
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.v1.structured_output.backend_outlines import OutlinesBackend
|
||||
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
|
||||
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
from vllm.v1.structured_output.utils import (
|
||||
_reduced_vocabulary,
|
||||
maybe_wrap_mistral_common_tokenizer,
|
||||
)
|
||||
|
||||
TOKENIZER = "mistralai/Mistral-Nemo-Instruct-2407"
|
||||
JSON_SCHEMA = (
|
||||
'{"type": "object", "properties": {"x": {"type": "integer"}}, '
|
||||
'"required": ["x"], "additionalProperties": false}'
|
||||
)
|
||||
VALID_DOCUMENT = '{"x": 1}'
|
||||
|
||||
BACKENDS = [("xgrammar", XgrammarBackend), ("outlines", OutlinesBackend)]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def loaded_tokenizer():
|
||||
tokenizer = get_tokenizer(tokenizer_name=TOKENIZER, tokenizer_mode="hf")
|
||||
assert isinstance(tokenizer, MistralCommonBackend)
|
||||
return tokenizer
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def wrapped_tokenizer(loaded_tokenizer):
|
||||
return maybe_wrap_mistral_common_tokenizer(loaded_tokenizer)
|
||||
|
||||
|
||||
def _encode(tokenizer, text):
|
||||
special = set(tokenizer.all_special_ids)
|
||||
return [tid for tid in tokenizer.encode(text) if tid not in special]
|
||||
|
||||
|
||||
def _backend(cls, name, tokenizer):
|
||||
vllm_config = VllmConfig(
|
||||
structured_outputs_config=StructuredOutputsConfig(backend=name)
|
||||
)
|
||||
return cls(vllm_config, tokenizer=tokenizer, vocab_size=tokenizer.vocab_size)
|
||||
|
||||
|
||||
def test_wrapping_produces_a_mistral_tokenizer(loaded_tokenizer, wrapped_tokenizer):
|
||||
assert isinstance(wrapped_tokenizer, MistralTokenizer)
|
||||
assert len(wrapped_tokenizer.vocab) == loaded_tokenizer.vocab_size
|
||||
|
||||
|
||||
def test_other_tokenizers_are_left_alone():
|
||||
tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
|
||||
assert maybe_wrap_mistral_common_tokenizer(tokenizer) is tokenizer
|
||||
|
||||
|
||||
def test_wrapping_fixes_the_outlines_vocabulary(loaded_tokenizer, wrapped_tokenizer):
|
||||
unwrapped_vocab = _reduced_vocabulary(loaded_tokenizer)
|
||||
wrapped_vocab = _reduced_vocabulary(wrapped_tokenizer)
|
||||
|
||||
assert unwrapped_vocab.keys() == wrapped_vocab.keys()
|
||||
|
||||
mismapped = {
|
||||
token for token, ids in unwrapped_vocab.items() if ids != wrapped_vocab[token]
|
||||
}
|
||||
assert mismapped
|
||||
|
||||
assert len({tuple(wrapped_vocab[t]) for t in mismapped}) == len(mismapped)
|
||||
assert len({tuple(unwrapped_vocab[t]) for t in mismapped}) < len(mismapped)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,cls", BACKENDS)
|
||||
def test_backend_accepts_a_valid_document(wrapped_tokenizer, name, cls):
|
||||
backend = _backend(cls, name, wrapped_tokenizer)
|
||||
grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
|
||||
|
||||
for token in _encode(wrapped_tokenizer, VALID_DOCUMENT):
|
||||
assert grammar.accept_tokens("req", [token])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name,cls", BACKENDS)
|
||||
def test_backend_rejects_an_invalid_token(wrapped_tokenizer, name, cls):
|
||||
backend = _backend(cls, name, wrapped_tokenizer)
|
||||
grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
|
||||
|
||||
assert not grammar.accept_tokens("req", [_encode(wrapped_tokenizer, "]")[-1]])
|
||||
|
||||
|
||||
def test_xgrammar_vocab_size_matches_the_tokenizer(wrapped_tokenizer):
|
||||
backend = _backend(XgrammarBackend, "xgrammar", wrapped_tokenizer)
|
||||
assert backend.vocab_size == len(wrapped_tokenizer.vocab)
|
||||
|
||||
|
||||
def test_xgrammar_rejects_the_unwrapped_tokenizer(loaded_tokenizer):
|
||||
with pytest.raises(ValueError, match="Unsupported tokenizer type"):
|
||||
_backend(XgrammarBackend, "xgrammar", loaded_tokenizer)
|
||||
+2
-30
@@ -116,17 +116,10 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
tokenizer_all_special_tokens = tokenizer.all_special_tokens
|
||||
tokenizer_vocab = tokenizer.get_vocab()
|
||||
tokenizer_len = len(tokenizer)
|
||||
# The underlying tokenizer class could be MistralCommonBackend,
|
||||
# which does not implement is_fast in Transformers
|
||||
# The underlying tokenizer class could be a specific backend,
|
||||
# which does not always implement is_fast in Transformers
|
||||
tokenizer_is_fast = getattr(tokenizer, "is_fast", True)
|
||||
|
||||
# MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization.
|
||||
mistral_tekkenizer = None
|
||||
if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None):
|
||||
from vllm.tokenizers.mistral import mistral_common_tekkenizer
|
||||
|
||||
mistral_tekkenizer = mistral_common_tekkenizer(tokenizer)
|
||||
|
||||
max_token_id = max(tokenizer_vocab.values())
|
||||
max_chars_per_token = max(len(tok) for tok in tokenizer_vocab)
|
||||
|
||||
@@ -159,27 +152,6 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
|
||||
def is_fast(self) -> bool:
|
||||
return tokenizer_is_fast
|
||||
|
||||
def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False):
|
||||
if mistral_tekkenizer is not None:
|
||||
from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens
|
||||
|
||||
return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids)
|
||||
return super().convert_ids_to_tokens(
|
||||
ids, skip_special_tokens=skip_special_tokens
|
||||
)
|
||||
|
||||
def convert_tokens_to_string(self, tokens: list[str]) -> str:
|
||||
if mistral_tekkenizer is not None:
|
||||
from vllm.tokenizers.mistral import tekken_convert_tokens_to_string
|
||||
|
||||
return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens)
|
||||
try:
|
||||
return super().convert_tokens_to_string(tokens)
|
||||
except NotImplementedError:
|
||||
# The underlying tokenizer class could be MistralCommonBackend,
|
||||
# which does not implement convert_tokens_to_string in Transformers
|
||||
return "".join(tokens)
|
||||
|
||||
def get_vocab(self) -> dict[str, int]:
|
||||
return tokenizer_vocab
|
||||
|
||||
|
||||
@@ -7,9 +7,15 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import huggingface_hub
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers.models.auto.tokenization_auto import (
|
||||
TOKENIZER_MAPPING_NAMES as _HF_TOKENIZER_MAPPING_NAMES,
|
||||
)
|
||||
from transformers.utils.import_utils import is_mistral_common_available
|
||||
from typing_extensions import TypeVar, assert_never
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.config import _maybe_register_hf_config, get_config
|
||||
from vllm.transformers_utils.repo_utils import (
|
||||
@@ -26,6 +32,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL = Version("5.15.0")
|
||||
|
||||
|
||||
# Model types whose hub tokenizer_class is incorrect and should be overridden with
|
||||
# TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a
|
||||
@@ -74,8 +82,6 @@ class _TokenizerRegistry:
|
||||
|
||||
self.tokenizers[tokenizer_mode] = (module, class_name)
|
||||
|
||||
return None
|
||||
|
||||
def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]:
|
||||
if tokenizer_mode not in self.tokenizers:
|
||||
raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.")
|
||||
@@ -150,12 +156,15 @@ def resolve_tokenizer_args(
|
||||
if (
|
||||
tokenizer_mode == "auto"
|
||||
and is_mistral_model_repo(
|
||||
model_name_or_path=str(tokenizer_name), revision=revision
|
||||
model_name_or_path=str(tokenizer_name),
|
||||
revision=revision,
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
and any_pattern_in_repo_files(
|
||||
model_name_or_path=str(tokenizer_name),
|
||||
allow_patterns=["tekken.json", "tokenizer.model.v*"],
|
||||
revision=revision,
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
):
|
||||
tokenizer_mode = "mistral"
|
||||
@@ -164,6 +173,13 @@ def resolve_tokenizer_args(
|
||||
if tokenizer_mode == "auto":
|
||||
tokenizer_mode = "hf"
|
||||
|
||||
if tokenizer_mode == "hf":
|
||||
if kwargs.pop("mistral_format", False):
|
||||
raise ValueError(
|
||||
"mistral_format=True is not supported with tokenizer_mode='hf'"
|
||||
)
|
||||
kwargs["mistral_format"] = False
|
||||
|
||||
return tokenizer_mode, tokenizer_name, args, kwargs
|
||||
|
||||
|
||||
@@ -231,6 +247,27 @@ def get_tokenizer(
|
||||
trust_remote_code=trust_remote_code,
|
||||
revision=revision,
|
||||
config_format=config_format,
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
|
||||
# TODO: delete when Transformers version dependency is bumped >= 5.15.0
|
||||
if (
|
||||
tokenizer_mode == "hf"
|
||||
and Version(transformers.__version__) < _MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL
|
||||
and is_mistral_common_available()
|
||||
and _HF_TOKENIZER_MAPPING_NAMES.get(getattr(config, "model_type", None))
|
||||
== "MistralCommonBackend"
|
||||
and any_pattern_in_repo_files(
|
||||
model_name_or_path=str(tokenizer_name),
|
||||
allow_patterns=["tekken.json", "tokenizer.model.v*"],
|
||||
revision=revision,
|
||||
token=kwargs.get("token"),
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Loading Mistral models with tokenizer_mode='hf' requires "
|
||||
f"transformers>={_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL}. "
|
||||
"Please upgrade transformers or delete Mistral tokenizer files."
|
||||
)
|
||||
|
||||
# Some models have an incorrect tokenizer_class on the hub.
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputGrammar,
|
||||
)
|
||||
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
|
||||
from vllm.v1.structured_output.utils import maybe_wrap_mistral_common_tokenizer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import numpy as np
|
||||
@@ -77,8 +76,8 @@ class StructuredOutputManager:
|
||||
# of CPUs.
|
||||
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
|
||||
self.executor = ThreadPoolExecutor(max_workers=max_workers)
|
||||
self.tokenizer = maybe_wrap_mistral_common_tokenizer(
|
||||
cached_tokenizer_from_config(model_config=self.vllm_config.model_config)
|
||||
self.tokenizer = cached_tokenizer_from_config(
|
||||
model_config=self.vllm_config.model_config
|
||||
)
|
||||
reasoning_parser_plugin = (
|
||||
self.vllm_config.structured_outputs_config.reasoning_parser_plugin
|
||||
|
||||
@@ -8,7 +8,6 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from transformers import MistralCommonBackend
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
@@ -98,10 +97,6 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
|
||||
if is_mistral_tokenizer(self.tokenizer):
|
||||
self.ll_tokenizer = self.tokenizer.llg_tokenizer
|
||||
elif isinstance(self.tokenizer, MistralCommonBackend):
|
||||
from mistral_common.guidance.tokenizer import from_mistral_tokenizer
|
||||
|
||||
self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer)
|
||||
else:
|
||||
self.ll_tokenizer = llguidance_hf.from_tokenizer(
|
||||
self.tokenizer, max(self.vocab_size, len(self.tokenizer))
|
||||
|
||||
@@ -14,7 +14,6 @@ from typing import TYPE_CHECKING, TypeVar
|
||||
import regex as re
|
||||
import torch
|
||||
from cachetools import LRUCache
|
||||
from transformers import MistralCommonBackend
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
@@ -320,18 +319,6 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
|
||||
re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")
|
||||
|
||||
|
||||
def maybe_wrap_mistral_common_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
|
||||
"""The grammar backends cannot consume a `MistralCommonBackend` directly."""
|
||||
if not isinstance(tokenizer, MistralCommonBackend):
|
||||
return tokenizer
|
||||
|
||||
# Deferred: `vllm.tokenizers.mistral` pulls in a large dependency tree, and
|
||||
# this module is imported by `vllm.v1.worker.gpu_model_runner`.
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
return MistralTokenizer(tokenizer)
|
||||
|
||||
|
||||
def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]:
|
||||
"""Create a map from vocabulary tokens to lists of equivalent token ids.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user