[Bugfix] Validate guidance tokenizer compatibility before EngineCore (#52298)

Signed-off-by: Oxygen56 <jiangth99@163.com>
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>

tests/v1/structured_output/test_validation.py

@@ -5,6 +5,7 @@
 import json
 
 import pytest
+from transformers import MistralCommonBackend, TokenizersBackend
 
 from vllm.config import StructuredOutputsConfig
 from vllm.exceptions import VLLMClientError, VLLMValidationError
@@ -14,6 +15,8 @@ from vllm.sampling_params import (
     SamplingParams,
     StructuredOutputsParams,
 )
+from vllm.tokenizers import mistral as mistral_tokenizers
+from vllm.v1.structured_output import backend_guidance
 
 pytestmark = pytest.mark.cpu_test
 
@@ -278,10 +281,102 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):
 def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):
     """`auto` falls back on rejection, so it must catch what the validators raise."""
     params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
+    # trick to create tokenizer that guidance backend accepts
+    tokenizer = object.__new__(TokenizersBackend)
     params._validate_structured_outputs(
         _StubModelConfig(is_diffusion=False),
         StructuredOutputsConfig(backend="auto"),
-        tokenizer=object(),
+        tokenizer=tokenizer,
     )
     assert params.structured_outputs is not None
     assert params.structured_outputs._backend == expected_backend
+
+
+# ================================================
+# Test validate_structured_outputs functionality
+# ================================================
+
+
+class TestValidateStructuredOutputs:
+    class TestPR52298Regression:
+        class _StubSlowTokenizer:
+            is_fast = False
+
+        class _StubVllmMistralTokenizer:
+            IS_MISTRAL_TOKENIZER = True
+
+            def __init__(self, is_tekken: bool):
+                self.is_tekken = is_tekken
+                self.llg_tokenizer = object()
+
+        @staticmethod
+        def _validate_guidance(tokenizer):
+            params = SamplingParams(
+                structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
+            )
+            params._validate_structured_outputs(
+                _StubModelConfig(is_diffusion=False),
+                StructuredOutputsConfig(backend="guidance"),
+                tokenizer=tokenizer,
+            )
+            return params
+
+        def _make_tokenizer(self, tokenizer_id: str):
+            if tokenizer_id == "slow-hf":
+                return self._StubSlowTokenizer()
+            if tokenizer_id == "plain-object":
+                return object()
+            if tokenizer_id == "raw-mistral-common":
+                return object.__new__(MistralCommonBackend)
+            if tokenizer_id == "vllm-mistral-non-tekken":
+                return self._StubVllmMistralTokenizer(is_tekken=False)
+            if tokenizer_id == "fast-hf":
+                return object.__new__(TokenizersBackend)
+            if tokenizer_id == "vllm-mistral-tekken":
+                return self._StubVllmMistralTokenizer(is_tekken=True)
+            raise ValueError(tokenizer_id)
+
+        @pytest.mark.parametrize(
+            "tokenizer_id, patch_mistral",
+            [
+                pytest.param("slow-hf", False, id="slow-hf"),
+                pytest.param("plain-object", False, id="plain-object"),
+                pytest.param("raw-mistral-common", False, id="raw-mistral-common"),
+                pytest.param(
+                    "vllm-mistral-non-tekken",
+                    True,
+                    id="vllm-mistral-non-tekken",
+                ),
+            ],
+        )
+        def test_unsupported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral):
+            if patch_mistral:
+                monkeypatch.setattr(
+                    mistral_tokenizers,
+                    "MistralTokenizer",
+                    self._StubVllmMistralTokenizer,
+                )
+            with pytest.raises(VLLMValidationError):
+                self._validate_guidance(self._make_tokenizer(tokenizer_id))
+
+        @pytest.mark.parametrize(
+            "tokenizer_id, patch_mistral",
+            [
+                pytest.param("fast-hf", False, id="fast-hf"),
+                pytest.param("vllm-mistral-tekken", True, id="vllm-mistral-tekken"),
+            ],
+        )
+        def test_supported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral):
+            if patch_mistral:
+                monkeypatch.setattr(
+                    mistral_tokenizers,
+                    "MistralTokenizer",
+                    self._StubVllmMistralTokenizer,
+                )
+                monkeypatch.setattr(
+                    backend_guidance,
+                    "validate_guidance_grammar",
+                    lambda *_args, **_kwargs: None,
+                )
+            params = self._validate_guidance(self._make_tokenizer(tokenizer_id))
+            assert params.structured_outputs._backend == "guidance"

vllm/sampling_params.py

@@ -1317,6 +1317,7 @@ class SamplingParams(
             )
 
         from vllm.v1.structured_output.backend_guidance import (
+            GuidanceBackend,
             has_guidance_unsupported_json_features,
             validate_guidance_grammar,
         )
@@ -1340,6 +1341,13 @@ class SamplingParams(
                     "Mistral model, the ['xgrammar', 'outlines'] "
                     "backends or tokenizer_mode='hf' instead."
                 )
+            if not GuidanceBackend.is_tokenizer_supported(tokenizer):
+                raise VLLMValidationError(
+                    "The 'guidance' structured output backend only supports fast "
+                    "Hugging Face tokenizers and Tekken-based Mistral tokenizers. "
+                    "Please use the 'xgrammar' backend or configure a fast "
+                    "tokenizer instead."
+                )
             # TODO: ideally we would have the LLTokenizer here as Lark syntax
             # allows <|special_token|> and similar, see
             # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
@@ -1408,12 +1416,19 @@ class SamplingParams(
                     skip_guidance = has_guidance_unsupported_json_features(schema)
 
                 if skip_guidance:
-                    # Fall back to outlines if the tokenizer is non-tekken Mistral or
-                    # the schema contains features unsupported by guidance
+                    # Fall back to outlines for non-Tekken Mistral tokenizers or
+                    # schemas containing features unsupported by guidance.
                     validate_structured_output_request_outlines(self)
                     self.structured_outputs._backend = "outlines"
                 else:
                     # Fall back to guidance by default.
+                    if not GuidanceBackend.is_tokenizer_supported(tokenizer):
+                        raise VLLMValidationError(
+                            "No compatible structured output backend was found for "
+                            "this request. XGrammar rejected the constraint, and "
+                            "Guidance does not support this tokenizer. Configure a "
+                            "fast tokenizer or select a compatible backend explicitly."
+                        ) from None
                     validate_guidance_grammar(
                         self,
                         tokenizer=_get_llg_tokenizer(tokenizer),

vllm/v1/structured_output/backend_guidance.py

@@ -8,10 +8,12 @@ from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 
 import torch
+from transformers import TokenizersBackend
 
 from vllm.exceptions import VLLMValidationError
 from vllm.logger import init_logger
 from vllm.sampling_params import SamplingParams
+from vllm.tokenizers import TokenizerLike
 from vllm.utils.import_utils import LazyLoader
 from vllm.utils.mistral import is_mistral_tokenizer
 from vllm.v1.structured_output.backend_types import (
@@ -128,6 +130,37 @@ def process_for_additional_properties(
 
 @dataclass
 class GuidanceBackend(StructuredOutputBackend):
+    @staticmethod
+    def is_tokenizer_supported(tokenizer: TokenizerLike) -> bool:
+        """Check if tokenizer is supported by guidance backend.
+
+        TODO(arpera):
+        guidance backend does NOT support some types of tokenizers
+        so, we MUST verify tokenizer before using guidance
+        The problem is that guidance API does not provide
+        a way to validate tokenizer before building LLTokenizer
+        So, we have to do it manually here in vLLM.
+        This is ofcourse very bad, we duplicate complex logic
+        that can change arbitrary in future.
+        Our check here is based on guidance source code
+        on function `from_tokenizer` implementation:
+        https://github.com/guidance-ai/llguidance/blob/main/python/llguidance/hf.py#L29-L47
+        If you want to contribute to vLLM, please, add to llguidance
+        API function to implement this check there.
+        In that case we will remove this check from vLLM.
+        That kind of work is really appreciated!
+        """
+        if is_mistral_tokenizer(tokenizer):
+            return tokenizer.is_tekken
+        # PreTrainedTokenizerFast is the same as TokenizersBackend
+        return isinstance(tokenizer, TokenizersBackend)
+
+    @staticmethod
+    def _build_ll_tokenizer(tokenizer: TokenizerLike, vocab_size: int) -> Any:
+        if is_mistral_tokenizer(tokenizer):
+            return tokenizer.llg_tokenizer
+        return llguidance_hf.from_tokenizer(tokenizer, max(vocab_size, len(tokenizer)))
+
     def __post_init__(self):
         self.disable_any_whitespace = (
             self.vllm_config.structured_outputs_config.disable_any_whitespace
@@ -136,12 +169,13 @@ class GuidanceBackend(StructuredOutputBackend):
             self.vllm_config.structured_outputs_config.disable_additional_properties
         )
 
-        if is_mistral_tokenizer(self.tokenizer):
-            self.ll_tokenizer = self.tokenizer.llg_tokenizer
-        else:
-            self.ll_tokenizer = llguidance_hf.from_tokenizer(
-                self.tokenizer, max(self.vocab_size, len(self.tokenizer))
+        if not self.is_tokenizer_supported(self.tokenizer):
+            raise ValueError(
+                "Only fast tokenizers are supported."
+                "This error should NOT happen,"
+                "if you see this, please report an issue."
             )
+        self.ll_tokenizer = self._build_ll_tokenizer(self.tokenizer, self.vocab_size)
 
     def compile_grammar(
         self,