[Bugfix] Support MistralCommonBackend tokenizers in structured output (#52720)

Signed-off-by: Thanh Phan <thanhphantuan1110@gmail.com>
This commit is contained in:
Thanh Phan
2026-08-20 19:08:50 +01:00
committed by GitHub
parent 7c8b68b9ce
commit d56bbf3995
3 changed files with 118 additions and 2 deletions
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Structured output with a `MistralCommonBackend` tokenizer."""
import pytest
from transformers import AutoTokenizer, MistralCommonBackend
from vllm.config import StructuredOutputsConfig, VllmConfig
from vllm.tokenizers import get_tokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.v1.structured_output.backend_outlines import OutlinesBackend
from vllm.v1.structured_output.backend_types import StructuredOutputOptions
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
from vllm.v1.structured_output.utils import (
_reduced_vocabulary,
maybe_wrap_mistral_common_tokenizer,
)
TOKENIZER = "mistralai/Mistral-Nemo-Instruct-2407"
JSON_SCHEMA = (
'{"type": "object", "properties": {"x": {"type": "integer"}}, '
'"required": ["x"], "additionalProperties": false}'
)
VALID_DOCUMENT = '{"x": 1}'
BACKENDS = [("xgrammar", XgrammarBackend), ("outlines", OutlinesBackend)]
@pytest.fixture(scope="module")
def loaded_tokenizer():
tokenizer = get_tokenizer(tokenizer_name=TOKENIZER, tokenizer_mode="hf")
assert isinstance(tokenizer, MistralCommonBackend)
return tokenizer
@pytest.fixture(scope="module")
def wrapped_tokenizer(loaded_tokenizer):
return maybe_wrap_mistral_common_tokenizer(loaded_tokenizer)
def _encode(tokenizer, text):
special = set(tokenizer.all_special_ids)
return [tid for tid in tokenizer.encode(text) if tid not in special]
def _backend(cls, name, tokenizer):
vllm_config = VllmConfig(
structured_outputs_config=StructuredOutputsConfig(backend=name)
)
return cls(vllm_config, tokenizer=tokenizer, vocab_size=tokenizer.vocab_size)
def test_wrapping_produces_a_mistral_tokenizer(loaded_tokenizer, wrapped_tokenizer):
assert isinstance(wrapped_tokenizer, MistralTokenizer)
assert len(wrapped_tokenizer.vocab) == loaded_tokenizer.vocab_size
def test_other_tokenizers_are_left_alone():
tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
assert maybe_wrap_mistral_common_tokenizer(tokenizer) is tokenizer
def test_wrapping_fixes_the_outlines_vocabulary(loaded_tokenizer, wrapped_tokenizer):
unwrapped_vocab = _reduced_vocabulary(loaded_tokenizer)
wrapped_vocab = _reduced_vocabulary(wrapped_tokenizer)
assert unwrapped_vocab.keys() == wrapped_vocab.keys()
mismapped = {
token for token, ids in unwrapped_vocab.items() if ids != wrapped_vocab[token]
}
assert mismapped
assert len({tuple(wrapped_vocab[t]) for t in mismapped}) == len(mismapped)
assert len({tuple(unwrapped_vocab[t]) for t in mismapped}) < len(mismapped)
@pytest.mark.parametrize("name,cls", BACKENDS)
def test_backend_accepts_a_valid_document(wrapped_tokenizer, name, cls):
backend = _backend(cls, name, wrapped_tokenizer)
grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
for token in _encode(wrapped_tokenizer, VALID_DOCUMENT):
assert grammar.accept_tokens("req", [token])
@pytest.mark.parametrize("name,cls", BACKENDS)
def test_backend_rejects_an_invalid_token(wrapped_tokenizer, name, cls):
backend = _backend(cls, name, wrapped_tokenizer)
grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA)
assert not grammar.accept_tokens("req", [_encode(wrapped_tokenizer, "]")[-1]])
def test_xgrammar_vocab_size_matches_the_tokenizer(wrapped_tokenizer):
backend = _backend(XgrammarBackend, "xgrammar", wrapped_tokenizer)
assert backend.vocab_size == len(wrapped_tokenizer.vocab)
def test_xgrammar_rejects_the_unwrapped_tokenizer(loaded_tokenizer):
with pytest.raises(ValueError, match="Unsupported tokenizer type"):
_backend(XgrammarBackend, "xgrammar", loaded_tokenizer)
+3 -2
View File
@@ -17,6 +17,7 @@ from vllm.v1.structured_output.backend_types import (
StructuredOutputGrammar,
)
from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
from vllm.v1.structured_output.utils import maybe_wrap_mistral_common_tokenizer
if TYPE_CHECKING:
import numpy as np
@@ -75,8 +76,8 @@ class StructuredOutputManager:
# of CPUs.
max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2)
self.executor = ThreadPoolExecutor(max_workers=max_workers)
self.tokenizer = cached_tokenizer_from_config(
model_config=self.vllm_config.model_config
self.tokenizer = maybe_wrap_mistral_common_tokenizer(
cached_tokenizer_from_config(model_config=self.vllm_config.model_config)
)
reasoning_parser_plugin = (
self.vllm_config.structured_outputs_config.reasoning_parser_plugin
+13
View File
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, TypeVar
import regex as re
import torch
from cachetools import LRUCache
from transformers import MistralCommonBackend
import vllm.envs as envs
from vllm.logger import init_logger
@@ -305,6 +306,18 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$")
re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$")
def maybe_wrap_mistral_common_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike:
"""The grammar backends cannot consume a `MistralCommonBackend` directly."""
if not isinstance(tokenizer, MistralCommonBackend):
return tokenizer
# Deferred: `vllm.tokenizers.mistral` pulls in a large dependency tree, and
# this module is imported by `vllm.v1.worker.gpu_model_runner`.
from vllm.tokenizers.mistral import MistralTokenizer
return MistralTokenizer(tokenizer)
def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]:
"""Create a map from vocabulary tokens to lists of equivalent token ids.