[Bugfix] Avoid MistralCommonBackend for HF tokenizers (#54192)

Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>
This commit is contained in:
Julien Denize
2026-09-10 17:57:44 +00:00
committed by GitHub
parent 8359e15aae
commit 983b7e28c9
7 changed files with 129 additions and 157 deletions
+84
View File
@@ -10,6 +10,7 @@ from transformers import AutoConfig
from transformers.models.auto.configuration_auto import CONFIG_MAPPING
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.hf import CachedHfTokenizer
from vllm.tokenizers.registry import (
TokenizerRegistry,
cached_get_tokenizer,
@@ -68,6 +69,89 @@ def test_resolve_tokenizer_args_idempotent(runner_type):
)
@pytest.mark.parametrize(
("tokenizer_mode", "input_kwargs"),
[
("hf", {}),
("hf", {"mistral_format": False}),
("slow", {}),
("slow", {"mistral_format": False}),
],
)
def test_resolve_tokenizer_args_forces_hf_mistral_format_false(
tokenizer_mode, input_kwargs
):
resolved_mode, _, _, kwargs = resolve_tokenizer_args(
"mistralai/Mistral-Nemo-Instruct-2407",
tokenizer_mode=tokenizer_mode,
**input_kwargs,
)
assert resolved_mode == "hf"
assert kwargs["mistral_format"] is False
@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"])
def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode):
with pytest.raises(
ValueError,
match="mistral_format=True is not supported with tokenizer_mode='hf'",
):
resolve_tokenizer_args(
"mistralai/Mistral-Nemo-Instruct-2407",
tokenizer_mode=tokenizer_mode,
mistral_format=True,
)
@pytest.mark.parametrize(
("transformers_version", "model_type", "native_filename", "expect_error"),
[
pytest.param("5.14.0", "mistral", "tekken.json", True),
pytest.param("5.15.0", "mistral", "tekken.json", False),
pytest.param("5.14.0", "mistral3", "tekken.json", True),
pytest.param("5.15.0", "mistral3", "tekken.json", False),
pytest.param("5.14.0", "mixtral", "tokenizer.model.v1", True),
pytest.param("5.15.0", "mixtral", "tokenizer.model.v1", False),
pytest.param("5.14.0", "mistral", None, False),
pytest.param("5.14.0", "qwen3_5_moe", "tekken.json", False),
],
)
def test_get_tokenizer_handles_hf_mistral_transformers_compatibility(
tmp_path: Path,
transformers_version: str,
model_type: str,
native_filename: str | None,
expect_error: bool,
):
if native_filename is not None:
(tmp_path / native_filename).touch()
with (
patch(
"vllm.tokenizers.registry.get_config",
return_value=SimpleNamespace(model_type=model_type),
),
patch(
"vllm.tokenizers.registry.transformers.__version__",
transformers_version,
),
patch.object(
CachedHfTokenizer,
"from_pretrained",
return_value=SimpleNamespace(is_fast=True),
) as from_pretrained,
):
if expect_error:
with pytest.raises(ValueError, match="requires transformers>=5.15.0"):
get_tokenizer(str(tmp_path), tokenizer_mode="hf")
else:
tokenizer = get_tokenizer(str(tmp_path), tokenizer_mode="hf")
assert tokenizer.is_fast is True
assert from_pretrained.called == (not expect_error)
def test_customized_tokenizer():
TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Structured output with a `MistralCommonBackend` tokenizer."""
import pytest
from transformers import AutoTokenizer, MistralCommonBackend
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.v1.structured_output.backend_outlines import OutlinesBackend
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
from vllm.v1.structured_output.utils import (
_reduced_vocabulary,
maybe_wrap_mistral_common_tokenizer,
)
TOKENIZER = "mistralai/Mistral-Nemo-Instruct-2407"
JSON_SCHEMA = (
'{"type": "object", "properties": {"x": {"type": "integer"}}, '
'"required": ["x"], "additionalProperties": false}'
)
VALID_DOCUMENT = '{"x": 1}'
BACKENDS = [("xgrammar", XgrammarBackend), ("outlines", OutlinesBackend)]
@pytest.fixture(scope="module")
def loaded_tokenizer():
tokenizer = get_tokenizer(tokenizer_name=TOKENIZER, tokenizer_mode="hf")
assert isinstance(tokenizer, MistralCommonBackend)
return tokenizer
@pytest.fixture(scope="module")
def wrapped_tokenizer(loaded_tokenizer):
return maybe_wrap_mistral_common_tokenizer(loaded_tokenizer)
def _encode(tokenizer, text):
special = set(tokenizer.all_special_ids)
return [tid for tid in tokenizer.encode(text) if tid not in special]
def _backend(cls, name, tokenizer):
vllm_config = VllmConfig(
structured_outputs_config=StructuredOutputsConfig(backend=name)
)
return cls(vllm_config, tokenizer=tokenizer, vocab_size=tokenizer.vocab_size)
def test_wrapping_produces_a_mistral_tokenizer(loaded_tokenizer, wrapped_tokenizer):
assert isinstance(wrapped_tokenizer, MistralTokenizer)
assert len(wrapped_tokenizer.vocab) == loaded_tokenizer.vocab_size
def test_other_tokenizers_are_left_alone():
tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
assert maybe_wrap_mistral_common_tokenizer(tokenizer) is tokenizer
def test_wrapping_fixes_the_outlines_vocabulary(loaded_tokenizer, wrapped_tokenizer):
unwrapped_vocab = _reduced_vocabulary(loaded_tokenizer)
wrapped_vocab = _reduced_vocabulary(wrapped_tokenizer)
assert unwrapped_vocab.keys() == wrapped_vocab.keys()
mismapped = {
token for token, ids in unwrapped_vocab.items() if ids != wrapped_vocab[token]
}
assert mismapped
assert len({tuple(wrapped_vocab[t]) for t in mismapped}) == len(mismapped)
assert len({tuple(unwrapped_vocab[t]) for t in mismapped}) < len(mismapped)
@pytest.mark.parametrize("name,cls", BACKENDS)
def test_backend_accepts_a_valid_document(wrapped_tokenizer, name, cls):
backend = _backend(cls, name, wrapped_tokenizer)
grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
for token in _encode(wrapped_tokenizer, VALID_DOCUMENT):
assert grammar.accept_tokens("req", [token])
@pytest.mark.parametrize("name,cls", BACKENDS)
def test_backend_rejects_an_invalid_token(wrapped_tokenizer, name, cls):
backend = _backend(cls, name, wrapped_tokenizer)
grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
assert not grammar.accept_tokens("req", [_encode(wrapped_tokenizer, "]")[-1]])
def test_xgrammar_vocab_size_matches_the_tokenizer(wrapped_tokenizer):
backend = _backend(XgrammarBackend, "xgrammar", wrapped_tokenizer)
assert backend.vocab_size == len(wrapped_tokenizer.vocab)
def test_xgrammar_rejects_the_unwrapped_tokenizer(loaded_tokenizer):
with pytest.raises(ValueError, match="Unsupported tokenizer type"):
_backend(XgrammarBackend, "xgrammar", loaded_tokenizer)