From ad8a6065f4bcbdfafd38828a4b3f1c6fac4dbf89 Mon Sep 17 00:00:00 2001 From: Willow Lopez <100782273+Oxygen56@users.noreply.github.com> Date: Sat, 10 Oct 2026 02:59:02 +0800 Subject: [PATCH] [Bugfix] Validate guidance tokenizer compatibility before EngineCore (#52298) Signed-off-by: Oxygen56 Signed-off-by: Artem Perevedentsev Co-authored-by: Artem Perevedentsev --- tests/v1/structured_output/test_validation.py | 97 ++++++++++++++++++- vllm/sampling_params.py | 19 +++- vllm/v1/structured_output/backend_guidance.py | 44 ++++++++- 3 files changed, 152 insertions(+), 8 deletions(-) diff --git a/tests/v1/structured_output/test_validation.py b/tests/v1/structured_output/test_validation.py index cd5526c7b7..276056b501 100644 --- a/tests/v1/structured_output/test_validation.py +++ b/tests/v1/structured_output/test_validation.py @@ -5,6 +5,7 @@ import json import pytest +from transformers import MistralCommonBackend, TokenizersBackend from vllm.config import StructuredOutputsConfig from vllm.exceptions import VLLMClientError, VLLMValidationError @@ -14,6 +15,8 @@ from vllm.sampling_params import ( SamplingParams, StructuredOutputsParams, ) +from vllm.tokenizers import mistral as mistral_tokenizers +from vllm.v1.structured_output import backend_guidance pytestmark = pytest.mark.cpu_test @@ -278,10 +281,102 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs): def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend): """`auto` falls back on rejection, so it must catch what the validators raise.""" params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema)) + # trick to create tokenizer that guidance backend accepts + tokenizer = object.__new__(TokenizersBackend) params._validate_structured_outputs( _StubModelConfig(is_diffusion=False), StructuredOutputsConfig(backend="auto"), - tokenizer=object(), + tokenizer=tokenizer, ) assert params.structured_outputs is not None assert params.structured_outputs._backend == expected_backend + + +# ================================================ +# Test validate_structured_outputs functionality +# ================================================ + + +class TestValidateStructuredOutputs: + class TestPR52298Regression: + class _StubSlowTokenizer: + is_fast = False + + class _StubVllmMistralTokenizer: + IS_MISTRAL_TOKENIZER = True + + def __init__(self, is_tekken: bool): + self.is_tekken = is_tekken + self.llg_tokenizer = object() + + @staticmethod + def _validate_guidance(tokenizer): + params = SamplingParams( + structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA) + ) + params._validate_structured_outputs( + _StubModelConfig(is_diffusion=False), + StructuredOutputsConfig(backend="guidance"), + tokenizer=tokenizer, + ) + return params + + def _make_tokenizer(self, tokenizer_id: str): + if tokenizer_id == "slow-hf": + return self._StubSlowTokenizer() + if tokenizer_id == "plain-object": + return object() + if tokenizer_id == "raw-mistral-common": + return object.__new__(MistralCommonBackend) + if tokenizer_id == "vllm-mistral-non-tekken": + return self._StubVllmMistralTokenizer(is_tekken=False) + if tokenizer_id == "fast-hf": + return object.__new__(TokenizersBackend) + if tokenizer_id == "vllm-mistral-tekken": + return self._StubVllmMistralTokenizer(is_tekken=True) + raise ValueError(tokenizer_id) + + @pytest.mark.parametrize( + "tokenizer_id, patch_mistral", + [ + pytest.param("slow-hf", False, id="slow-hf"), + pytest.param("plain-object", False, id="plain-object"), + pytest.param("raw-mistral-common", False, id="raw-mistral-common"), + pytest.param( + "vllm-mistral-non-tekken", + True, + id="vllm-mistral-non-tekken", + ), + ], + ) + def test_unsupported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral): + if patch_mistral: + monkeypatch.setattr( + mistral_tokenizers, + "MistralTokenizer", + self._StubVllmMistralTokenizer, + ) + with pytest.raises(VLLMValidationError): + self._validate_guidance(self._make_tokenizer(tokenizer_id)) + + @pytest.mark.parametrize( + "tokenizer_id, patch_mistral", + [ + pytest.param("fast-hf", False, id="fast-hf"), + pytest.param("vllm-mistral-tekken", True, id="vllm-mistral-tekken"), + ], + ) + def test_supported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral): + if patch_mistral: + monkeypatch.setattr( + mistral_tokenizers, + "MistralTokenizer", + self._StubVllmMistralTokenizer, + ) + monkeypatch.setattr( + backend_guidance, + "validate_guidance_grammar", + lambda *_args, **_kwargs: None, + ) + params = self._validate_guidance(self._make_tokenizer(tokenizer_id)) + assert params.structured_outputs._backend == "guidance" diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 8a37c5a3fd..8331a1707b 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -1317,6 +1317,7 @@ class SamplingParams( ) from vllm.v1.structured_output.backend_guidance import ( + GuidanceBackend, has_guidance_unsupported_json_features, validate_guidance_grammar, ) @@ -1340,6 +1341,13 @@ class SamplingParams( "Mistral model, the ['xgrammar', 'outlines'] " "backends or tokenizer_mode='hf' instead." ) + if not GuidanceBackend.is_tokenizer_supported(tokenizer): + raise VLLMValidationError( + "The 'guidance' structured output backend only supports fast " + "Hugging Face tokenizers and Tekken-based Mistral tokenizers. " + "Please use the 'xgrammar' backend or configure a fast " + "tokenizer instead." + ) # TODO: ideally we would have the LLTokenizer here as Lark syntax # allows <|special_token|> and similar, see # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens @@ -1408,12 +1416,19 @@ class SamplingParams( skip_guidance = has_guidance_unsupported_json_features(schema) if skip_guidance: - # Fall back to outlines if the tokenizer is non-tekken Mistral or - # the schema contains features unsupported by guidance + # Fall back to outlines for non-Tekken Mistral tokenizers or + # schemas containing features unsupported by guidance. validate_structured_output_request_outlines(self) self.structured_outputs._backend = "outlines" else: # Fall back to guidance by default. + if not GuidanceBackend.is_tokenizer_supported(tokenizer): + raise VLLMValidationError( + "No compatible structured output backend was found for " + "this request. XGrammar rejected the constraint, and " + "Guidance does not support this tokenizer. Configure a " + "fast tokenizer or select a compatible backend explicitly." + ) from None validate_guidance_grammar( self, tokenizer=_get_llg_tokenizer(tokenizer), diff --git a/vllm/v1/structured_output/backend_guidance.py b/vllm/v1/structured_output/backend_guidance.py index 6dc2dc91ff..4af5ca2da7 100644 --- a/vllm/v1/structured_output/backend_guidance.py +++ b/vllm/v1/structured_output/backend_guidance.py @@ -8,10 +8,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import torch +from transformers import TokenizersBackend from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.sampling_params import SamplingParams +from vllm.tokenizers import TokenizerLike from vllm.utils.import_utils import LazyLoader from vllm.utils.mistral import is_mistral_tokenizer from vllm.v1.structured_output.backend_types import ( @@ -128,6 +130,37 @@ def process_for_additional_properties( @dataclass class GuidanceBackend(StructuredOutputBackend): + @staticmethod + def is_tokenizer_supported(tokenizer: TokenizerLike) -> bool: + """Check if tokenizer is supported by guidance backend. + + TODO(arpera): + guidance backend does NOT support some types of tokenizers + so, we MUST verify tokenizer before using guidance + The problem is that guidance API does not provide + a way to validate tokenizer before building LLTokenizer + So, we have to do it manually here in vLLM. + This is ofcourse very bad, we duplicate complex logic + that can change arbitrary in future. + Our check here is based on guidance source code + on function `from_tokenizer` implementation: + https://github.com/guidance-ai/llguidance/blob/main/python/llguidance/hf.py#L29-L47 + If you want to contribute to vLLM, please, add to llguidance + API function to implement this check there. + In that case we will remove this check from vLLM. + That kind of work is really appreciated! + """ + if is_mistral_tokenizer(tokenizer): + return tokenizer.is_tekken + # PreTrainedTokenizerFast is the same as TokenizersBackend + return isinstance(tokenizer, TokenizersBackend) + + @staticmethod + def _build_ll_tokenizer(tokenizer: TokenizerLike, vocab_size: int) -> Any: + if is_mistral_tokenizer(tokenizer): + return tokenizer.llg_tokenizer + return llguidance_hf.from_tokenizer(tokenizer, max(vocab_size, len(tokenizer))) + def __post_init__(self): self.disable_any_whitespace = ( self.vllm_config.structured_outputs_config.disable_any_whitespace @@ -136,12 +169,13 @@ class GuidanceBackend(StructuredOutputBackend): self.vllm_config.structured_outputs_config.disable_additional_properties ) - if is_mistral_tokenizer(self.tokenizer): - self.ll_tokenizer = self.tokenizer.llg_tokenizer - else: - self.ll_tokenizer = llguidance_hf.from_tokenizer( - self.tokenizer, max(self.vocab_size, len(self.tokenizer)) + if not self.is_tokenizer_supported(self.tokenizer): + raise ValueError( + "Only fast tokenizers are supported." + "This error should NOT happen," + "if you see this, please report an issue." ) + self.ll_tokenizer = self._build_ll_tokenizer(self.tokenizer, self.vocab_size) def compile_grammar( self,