From 983b7e28c9f87c3a150081640e0fdeafec5d553a Mon Sep 17 00:00:00 2001 From: Julien Denize <40604584+juliendenize@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:57:44 +0200 Subject: [PATCH] [Bugfix] Avoid MistralCommonBackend for HF tokenizers (#54192) Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> --- tests/tokenizers_/test_registry.py | 84 +++++++++++++++ .../test_mistral_common_tokenizer.py | 102 ------------------ vllm/tokenizers/hf.py | 32 +----- vllm/tokenizers/registry.py | 45 +++++++- vllm/v1/structured_output/__init__.py | 5 +- vllm/v1/structured_output/backend_guidance.py | 5 - vllm/v1/structured_output/utils.py | 13 --- 7 files changed, 129 insertions(+), 157 deletions(-) delete mode 100644 tests/v1/structured_output/test_mistral_common_tokenizer.py diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index 0a47426dc9..b7201c1cfe 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -10,6 +10,7 @@ from transformers import AutoConfig from transformers.models.auto.configuration_auto import CONFIG_MAPPING from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.hf import CachedHfTokenizer from vllm.tokenizers.registry import ( TokenizerRegistry, cached_get_tokenizer, @@ -68,6 +69,89 @@ def test_resolve_tokenizer_args_idempotent(runner_type): ) +@pytest.mark.parametrize( + ("tokenizer_mode", "input_kwargs"), + [ + ("hf", {}), + ("hf", {"mistral_format": False}), + ("slow", {}), + ("slow", {"mistral_format": False}), + ], +) +def test_resolve_tokenizer_args_forces_hf_mistral_format_false( + tokenizer_mode, input_kwargs +): + resolved_mode, _, _, kwargs = resolve_tokenizer_args( + "mistralai/Mistral-Nemo-Instruct-2407", + tokenizer_mode=tokenizer_mode, + **input_kwargs, + ) + + assert resolved_mode == "hf" + assert kwargs["mistral_format"] is False + + +@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"]) +def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode): + with pytest.raises( + ValueError, + match="mistral_format=True is not supported with tokenizer_mode='hf'", + ): + resolve_tokenizer_args( + "mistralai/Mistral-Nemo-Instruct-2407", + tokenizer_mode=tokenizer_mode, + mistral_format=True, + ) + + +@pytest.mark.parametrize( + ("transformers_version", "model_type", "native_filename", "expect_error"), + [ + pytest.param("5.14.0", "mistral", "tekken.json", True), + pytest.param("5.15.0", "mistral", "tekken.json", False), + pytest.param("5.14.0", "mistral3", "tekken.json", True), + pytest.param("5.15.0", "mistral3", "tekken.json", False), + pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True), + pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False), + pytest.param("5.14.0", "mistral", None, False), + pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False), + ], +) +def test_get_tokenizer_handles_hf_mistral_transformers_compatibility( + tmp_path: Path, + transformers_version: str, + model_type: str, + native_filename: str | None, + expect_error: bool, +): + if native_filename is not None: + (tmp_path / native_filename).touch() + + with ( + patch( + "vllm.tokenizers.registry.get_config", + return_value=SimpleNamespace(model_type=model_type), + ), + patch( + "vllm.tokenizers.registry.transformers.__version__", + transformers_version, + ), + patch.object( + CachedHfTokenizer, + "from_pretrained", + return_value=SimpleNamespace(is_fast=True), + ) as from_pretrained, + ): + if expect_error: + with pytest.raises(ValueError, match="requires transformers>=5.15.0"): + get_tokenizer(str(tmp_path), tokenizer_mode="hf") + else: + tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf") + assert tokenizer.is_fast is True + + assert from_pretrained.called == (not expect_error) + + def test_customized_tokenizer(): TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__) diff --git a/tests/v1/structured_output/test_mistral_common_tokenizer.py b/tests/v1/structured_output/test_mistral_common_tokenizer.py deleted file mode 100644 index 74d7cb085c..0000000000 --- a/tests/v1/structured_output/test_mistral_common_tokenizer.py +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Structured output with a `MistralCommonBackend` tokenizer.""" - -import pytest -from transformers import AutoTokenizer, MistralCommonBackend - -from vllm.config import StructuredOutputsConfig, VllmConfig -from vllm.tokenizers import get_tokenizer -from vllm.tokenizers.mistral import MistralTokenizer -from vllm.v1.structured_output.backend_outlines import OutlinesBackend -from vllm.v1.structured_output.backend_types import StructuredOutputOptions -from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend -from vllm.v1.structured_output.utils import ( - _reduced_vocabulary, - maybe_wrap_mistral_common_tokenizer, -) - -TOKENIZER = "mistralai/Mistral-Nemo-Instruct-2407" -JSON_SCHEMA = ( - '{"type": "object", "properties": {"x": {"type": "integer"}}, ' - '"required": ["x"], "additionalProperties": false}' -) -VALID_DOCUMENT = '{"x": 1}' - -BACKENDS = [("xgrammar", XgrammarBackend), ("outlines", OutlinesBackend)] - - -@pytest.fixture(scope="module") -def loaded_tokenizer(): - tokenizer = get_tokenizer(tokenizer_name=TOKENIZER, tokenizer_mode="hf") - assert isinstance(tokenizer, MistralCommonBackend) - return tokenizer - - -@pytest.fixture(scope="module") -def wrapped_tokenizer(loaded_tokenizer): - return maybe_wrap_mistral_common_tokenizer(loaded_tokenizer) - - -def _encode(tokenizer, text): - special = set(tokenizer.all_special_ids) - return [tid for tid in tokenizer.encode(text) if tid not in special] - - -def _backend(cls, name, tokenizer): - vllm_config = VllmConfig( - structured_outputs_config=StructuredOutputsConfig(backend=name) - ) - return cls(vllm_config, tokenizer=tokenizer, vocab_size=tokenizer.vocab_size) - - -def test_wrapping_produces_a_mistral_tokenizer(loaded_tokenizer, wrapped_tokenizer): - assert isinstance(wrapped_tokenizer, MistralTokenizer) - assert len(wrapped_tokenizer.vocab) == loaded_tokenizer.vocab_size - - -def test_other_tokenizers_are_left_alone(): - tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") - assert maybe_wrap_mistral_common_tokenizer(tokenizer) is tokenizer - - -def test_wrapping_fixes_the_outlines_vocabulary(loaded_tokenizer, wrapped_tokenizer): - unwrapped_vocab = _reduced_vocabulary(loaded_tokenizer) - wrapped_vocab = _reduced_vocabulary(wrapped_tokenizer) - - assert unwrapped_vocab.keys() == wrapped_vocab.keys() - - mismapped = { - token for token, ids in unwrapped_vocab.items() if ids != wrapped_vocab[token] - } - assert mismapped - - assert len({tuple(wrapped_vocab[t]) for t in mismapped}) == len(mismapped) - assert len({tuple(unwrapped_vocab[t]) for t in mismapped}) < len(mismapped) - - -@pytest.mark.parametrize("name,cls", BACKENDS) -def test_backend_accepts_a_valid_document(wrapped_tokenizer, name, cls): - backend = _backend(cls, name, wrapped_tokenizer) - grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA) - - for token in _encode(wrapped_tokenizer, VALID_DOCUMENT): - assert grammar.accept_tokens("req", [token]) - - -@pytest.mark.parametrize("name,cls", BACKENDS) -def test_backend_rejects_an_invalid_token(wrapped_tokenizer, name, cls): - backend = _backend(cls, name, wrapped_tokenizer) - grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA) - - assert not grammar.accept_tokens("req", [_encode(wrapped_tokenizer, "]")[-1]]) - - -def test_xgrammar_vocab_size_matches_the_tokenizer(wrapped_tokenizer): - backend = _backend(XgrammarBackend, "xgrammar", wrapped_tokenizer) - assert backend.vocab_size == len(wrapped_tokenizer.vocab) - - -def test_xgrammar_rejects_the_unwrapped_tokenizer(loaded_tokenizer): - with pytest.raises(ValueError, match="Unsupported tokenizer type"): - _backend(XgrammarBackend, "xgrammar", loaded_tokenizer) diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index bdc767acd6..3d35952c5f 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -116,17 +116,10 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: tokenizer_all_special_tokens = tokenizer.all_special_tokens tokenizer_vocab = tokenizer.get_vocab() tokenizer_len = len(tokenizer) - # The underlying tokenizer class could be MistralCommonBackend, - # which does not implement is_fast in Transformers + # The underlying tokenizer class could be a specific backend, + # which does not always implement is_fast in Transformers tokenizer_is_fast = getattr(tokenizer, "is_fast", True) - # MistralCommonBackend is tekken-backed and needs byte-fallback-aware tokenization. - mistral_tekkenizer = None - if getattr(getattr(tokenizer, "tokenizer", None), "instruct_tokenizer", None): - from vllm.tokenizers.mistral import mistral_common_tekkenizer - - mistral_tekkenizer = mistral_common_tekkenizer(tokenizer) - max_token_id = max(tokenizer_vocab.values()) max_chars_per_token = max(len(tok) for tok in tokenizer_vocab) @@ -159,27 +152,6 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: def is_fast(self) -> bool: return tokenizer_is_fast - def convert_ids_to_tokens(self, ids, skip_special_tokens: bool = False): - if mistral_tekkenizer is not None: - from vllm.tokenizers.mistral import tekken_convert_ids_to_tokens - - return tekken_convert_ids_to_tokens(mistral_tekkenizer, ids) - return super().convert_ids_to_tokens( - ids, skip_special_tokens=skip_special_tokens - ) - - def convert_tokens_to_string(self, tokens: list[str]) -> str: - if mistral_tekkenizer is not None: - from vllm.tokenizers.mistral import tekken_convert_tokens_to_string - - return tekken_convert_tokens_to_string(mistral_tekkenizer, tokens) - try: - return super().convert_tokens_to_string(tokens) - except NotImplementedError: - # The underlying tokenizer class could be MistralCommonBackend, - # which does not implement convert_tokens_to_string in Transformers - return "".join(tokens) - def get_vocab(self) -> dict[str, int]: return tokenizer_vocab diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index b9c9226675..46446ecd6d 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -7,9 +7,15 @@ from pathlib import Path from typing import TYPE_CHECKING import huggingface_hub +import transformers +from packaging.version import Version +from transformers.models.auto.tokenization_auto import ( + TOKENIZER_MAPPING_NAMES as _HF_TOKENIZER_MAPPING_NAMES, +) +from transformers.utils.import_utils import is_mistral_common_available from typing_extensions import TypeVar, assert_never -import vllm.envs as envs +from vllm import envs from vllm.logger import init_logger from vllm.transformers_utils.config import _maybe_register_hf_config, get_config from vllm.transformers_utils.repo_utils import ( @@ -26,6 +32,8 @@ if TYPE_CHECKING: logger = init_logger(__name__) +_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL = Version("5.15.0") + # Model types whose hub tokenizer_class is incorrect and should be overridden with # TokenizersBackend (the generic fast tokenizer). Adding a model type here is always a @@ -74,8 +82,6 @@ class _TokenizerRegistry: self.tokenizers[tokenizer_mode] = (module, class_name) - return None - def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]: if tokenizer_mode not in self.tokenizers: raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.") @@ -150,12 +156,15 @@ def resolve_tokenizer_args( if ( tokenizer_mode == "auto" and is_mistral_model_repo( - model_name_or_path=str(tokenizer_name), revision=revision + model_name_or_path=str(tokenizer_name), + revision=revision, + token=kwargs.get("token"), ) and any_pattern_in_repo_files( model_name_or_path=str(tokenizer_name), allow_patterns=["tekken.json", "tokenizer.model.v*"], revision=revision, + token=kwargs.get("token"), ) ): tokenizer_mode = "mistral" @@ -164,6 +173,13 @@ def resolve_tokenizer_args( if tokenizer_mode == "auto": tokenizer_mode = "hf" + if tokenizer_mode == "hf": + if kwargs.pop("mistral_format", False): + raise ValueError( + "mistral_format=True is not supported with tokenizer_mode='hf'" + ) + kwargs["mistral_format"] = False + return tokenizer_mode, tokenizer_name, args, kwargs @@ -231,6 +247,27 @@ def get_tokenizer( trust_remote_code=trust_remote_code, revision=revision, config_format=config_format, + token=kwargs.get("token"), + ) + + # TODO: delete when Transformers version dependency is bumped >= 5.15.0 + if ( + tokenizer_mode == "hf" + and Version(transformers.__version__) < _MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL + and is_mistral_common_available() + and _HF_TOKENIZER_MAPPING_NAMES.get(getattr(config, "model_type", None)) + == "MistralCommonBackend" + and any_pattern_in_repo_files( + model_name_or_path=str(tokenizer_name), + allow_patterns=["tekken.json", "tokenizer.model.v*"], + revision=revision, + token=kwargs.get("token"), + ) + ): + raise ValueError( + "Loading Mistral models with tokenizer_mode='hf' requires " + f"transformers>={_MIN_TRANSFORMERS_VERSION_FOR_HF_MISTRAL}. " + "Please upgrade transformers or delete Mistral tokenizer files." ) # Some models have an incorrect tokenizer_class on the hub. diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index fd48c99d1c..ca6adc736d 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -18,7 +18,6 @@ from vllm.v1.structured_output.backend_types import ( StructuredOutputGrammar, ) from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend -from vllm.v1.structured_output.utils import maybe_wrap_mistral_common_tokenizer if TYPE_CHECKING: import numpy as np @@ -77,8 +76,8 @@ class StructuredOutputManager: # of CPUs. max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2) self.executor = ThreadPoolExecutor(max_workers=max_workers) - self.tokenizer = maybe_wrap_mistral_common_tokenizer( - cached_tokenizer_from_config(model_config=self.vllm_config.model_config) + self.tokenizer = cached_tokenizer_from_config( + model_config=self.vllm_config.model_config ) reasoning_parser_plugin = ( self.vllm_config.structured_outputs_config.reasoning_parser_plugin diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 16e1f54be6..0d8b7377c5 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -8,7 +8,6 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import torch -from transformers import MistralCommonBackend from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger @@ -98,10 +97,6 @@ class GuidanceBackend(StructuredOutputBackend): if is_mistral_tokenizer(self.tokenizer): self.ll_tokenizer = self.tokenizer.llg_tokenizer - elif isinstance(self.tokenizer, MistralCommonBackend): - from mistral_common.guidance.tokenizer import from_mistral_tokenizer - - self.ll_tokenizer = from_mistral_tokenizer(self.tokenizer.tokenizer) else: self.ll_tokenizer = llguidance_hf.from_tokenizer( self.tokenizer, max(self.vocab_size, len(self.tokenizer)) diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index aa5d59c9c5..92122f85d0 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -14,7 +14,6 @@ from typing import TYPE_CHECKING, TypeVar import regex as re import torch from cachetools import LRUCache -from transformers import MistralCommonBackend import vllm.envs as envs from vllm.logger import init_logger @@ -320,18 +319,6 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$") re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$") -def maybe_wrap_mistral_common_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike: - """The grammar backends cannot consume a `MistralCommonBackend` directly.""" - if not isinstance(tokenizer, MistralCommonBackend): - return tokenizer - - # Deferred: `vllm.tokenizers.mistral` pulls in a large dependency tree, and - # this module is imported by `vllm.v1.worker.gpu_model_runner`. - from vllm.tokenizers.mistral import MistralTokenizer - - return MistralTokenizer(tokenizer) - - def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]: """Create a map from vocabulary tokens to lists of equivalent token ids.