diff --git a/tests/v1/structured_output/test_mistral_common_tokenizer.py b/tests/v1/structured_output/test_mistral_common_tokenizer.py new file mode 100644 index 0000000000..74d7cb085c --- /dev/null +++ b/tests/v1/structured_output/test_mistral_common_tokenizer.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Structured output with a `MistralCommonBackend` tokenizer.""" + +import pytest +from transformers import AutoTokenizer, MistralCommonBackend + +from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.mistral import MistralTokenizer +from vllm.v1.structured_output.backend_outlines import OutlinesBackend +from vllm.v1.structured_output.backend_types import StructuredOutputOptions +from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend +from vllm.v1.structured_output.utils import ( + _reduced_vocabulary, + maybe_wrap_mistral_common_tokenizer, +) + +TOKENIZER = "mistralai/Mistral-Nemo-Instruct-2407" +JSON_SCHEMA = ( + '{"type": "object", "properties": {"x": {"type": "integer"}}, ' + '"required": ["x"], "additionalProperties": false}' +) +VALID_DOCUMENT = '{"x": 1}' + +BACKENDS = [("xgrammar", XgrammarBackend), ("outlines", OutlinesBackend)] + + +@pytest.fixture(scope="module") +def loaded_tokenizer(): + tokenizer = get_tokenizer(tokenizer_name=TOKENIZER, tokenizer_mode="hf") + assert isinstance(tokenizer, MistralCommonBackend) + return tokenizer + + +@pytest.fixture(scope="module") +def wrapped_tokenizer(loaded_tokenizer): + return maybe_wrap_mistral_common_tokenizer(loaded_tokenizer) + + +def _encode(tokenizer, text): + special = set(tokenizer.all_special_ids) + return [tid for tid in tokenizer.encode(text) if tid not in special] + + +def _backend(cls, name, tokenizer): + vllm_config = VllmConfig( + structured_outputs_config=StructuredOutputsConfig(backend=name) + ) + return cls(vllm_config, tokenizer=tokenizer, vocab_size=tokenizer.vocab_size) + + +def test_wrapping_produces_a_mistral_tokenizer(loaded_tokenizer, wrapped_tokenizer): + assert isinstance(wrapped_tokenizer, MistralTokenizer) + assert len(wrapped_tokenizer.vocab) == loaded_tokenizer.vocab_size + + +def test_other_tokenizers_are_left_alone(): + tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2") + assert maybe_wrap_mistral_common_tokenizer(tokenizer) is tokenizer + + +def test_wrapping_fixes_the_outlines_vocabulary(loaded_tokenizer, wrapped_tokenizer): + unwrapped_vocab = _reduced_vocabulary(loaded_tokenizer) + wrapped_vocab = _reduced_vocabulary(wrapped_tokenizer) + + assert unwrapped_vocab.keys() == wrapped_vocab.keys() + + mismapped = { + token for token, ids in unwrapped_vocab.items() if ids != wrapped_vocab[token] + } + assert mismapped + + assert len({tuple(wrapped_vocab[t]) for t in mismapped}) == len(mismapped) + assert len({tuple(unwrapped_vocab[t]) for t in mismapped}) < len(mismapped) + + +@pytest.mark.parametrize("name,cls", BACKENDS) +def test_backend_accepts_a_valid_document(wrapped_tokenizer, name, cls): + backend = _backend(cls, name, wrapped_tokenizer) + grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA) + + for token in _encode(wrapped_tokenizer, VALID_DOCUMENT): + assert grammar.accept_tokens("req", [token]) + + +@pytest.mark.parametrize("name,cls", BACKENDS) +def test_backend_rejects_an_invalid_token(wrapped_tokenizer, name, cls): + backend = _backend(cls, name, wrapped_tokenizer) + grammar = backend.compile_grammar(StructuredOutputOptions.JSON, JSON_SCHEMA) + + assert not grammar.accept_tokens("req", [_encode(wrapped_tokenizer, "]")[-1]]) + + +def test_xgrammar_vocab_size_matches_the_tokenizer(wrapped_tokenizer): + backend = _backend(XgrammarBackend, "xgrammar", wrapped_tokenizer) + assert backend.vocab_size == len(wrapped_tokenizer.vocab) + + +def test_xgrammar_rejects_the_unwrapped_tokenizer(loaded_tokenizer): + with pytest.raises(ValueError, match="Unsupported tokenizer type"): + _backend(XgrammarBackend, "xgrammar", loaded_tokenizer) diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 6a2409c8a5..2fe1399d4d 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -17,6 +17,7 @@ from vllm.v1.structured_output.backend_types import ( StructuredOutputGrammar, ) from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend +from vllm.v1.structured_output.utils import maybe_wrap_mistral_common_tokenizer if TYPE_CHECKING: import numpy as np @@ -75,8 +76,8 @@ class StructuredOutputManager: # of CPUs. max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2) self.executor = ThreadPoolExecutor(max_workers=max_workers) - self.tokenizer = cached_tokenizer_from_config( - model_config=self.vllm_config.model_config + self.tokenizer = maybe_wrap_mistral_common_tokenizer( + cached_tokenizer_from_config(model_config=self.vllm_config.model_config) ) reasoning_parser_plugin = ( self.vllm_config.structured_outputs_config.reasoning_parser_plugin diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 0629a6d2e0..0633c55fe6 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, TypeVar import regex as re import torch from cachetools import LRUCache +from transformers import MistralCommonBackend import vllm.envs as envs from vllm.logger import init_logger @@ -305,6 +306,18 @@ re_llama_byte_token = re.compile(r"^<0x[0-9A-F]{2}>$") re_replacement_seq = re.compile(r"^.{0,6}�+.{0,6}$") +def maybe_wrap_mistral_common_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike: + """The grammar backends cannot consume a `MistralCommonBackend` directly.""" + if not isinstance(tokenizer, MistralCommonBackend): + return tokenizer + + # Deferred: `vllm.tokenizers.mistral` pulls in a large dependency tree, and + # this module is imported by `vllm.v1.worker.gpu_model_runner`. + from vllm.tokenizers.mistral import MistralTokenizer + + return MistralTokenizer(tokenizer) + + def _reduced_vocabulary(tokenizer: TokenizerLike) -> dict[bytes, list[int]]: """Create a map from vocabulary tokens to lists of equivalent token ids.