[Bugfix] Validate guidance tokenizer compatibility before EngineCore (#52298)
Signed-off-by: Oxygen56 <jiangth99@163.com> Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
co-authored by
Artem Perevedentsev
parent
1dc60dd367
commit
ad8a6065f4
@@ -5,6 +5,7 @@
|
|||||||
import json
|
import json
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from transformers import MistralCommonBackend, TokenizersBackend
|
||||||
|
|
||||||
from vllm.config import StructuredOutputsConfig
|
from vllm.config import StructuredOutputsConfig
|
||||||
from vllm.exceptions import VLLMClientError, VLLMValidationError
|
from vllm.exceptions import VLLMClientError, VLLMValidationError
|
||||||
@@ -14,6 +15,8 @@ from vllm.sampling_params import (
|
|||||||
SamplingParams,
|
SamplingParams,
|
||||||
StructuredOutputsParams,
|
StructuredOutputsParams,
|
||||||
)
|
)
|
||||||
|
from vllm.tokenizers import mistral as mistral_tokenizers
|
||||||
|
from vllm.v1.structured_output import backend_guidance
|
||||||
|
|
||||||
pytestmark = pytest.mark.cpu_test
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
@@ -278,10 +281,102 @@ def test_unsupported_grammar_is_a_client_error(backend, structured_outputs):
|
|||||||
def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):
|
def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):
|
||||||
"""`auto` falls back on rejection, so it must catch what the validators raise."""
|
"""`auto` falls back on rejection, so it must catch what the validators raise."""
|
||||||
params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
|
params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
|
||||||
|
# trick to create tokenizer that guidance backend accepts
|
||||||
|
tokenizer = object.__new__(TokenizersBackend)
|
||||||
params._validate_structured_outputs(
|
params._validate_structured_outputs(
|
||||||
_StubModelConfig(is_diffusion=False),
|
_StubModelConfig(is_diffusion=False),
|
||||||
StructuredOutputsConfig(backend="auto"),
|
StructuredOutputsConfig(backend="auto"),
|
||||||
tokenizer=object(),
|
tokenizer=tokenizer,
|
||||||
)
|
)
|
||||||
assert params.structured_outputs is not None
|
assert params.structured_outputs is not None
|
||||||
assert params.structured_outputs._backend == expected_backend
|
assert params.structured_outputs._backend == expected_backend
|
||||||
|
|
||||||
|
|
||||||
|
# ================================================
|
||||||
|
# Test validate_structured_outputs functionality
|
||||||
|
# ================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidateStructuredOutputs:
|
||||||
|
class TestPR52298Regression:
|
||||||
|
class _StubSlowTokenizer:
|
||||||
|
is_fast = False
|
||||||
|
|
||||||
|
class _StubVllmMistralTokenizer:
|
||||||
|
IS_MISTRAL_TOKENIZER = True
|
||||||
|
|
||||||
|
def __init__(self, is_tekken: bool):
|
||||||
|
self.is_tekken = is_tekken
|
||||||
|
self.llg_tokenizer = object()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_guidance(tokenizer):
|
||||||
|
params = SamplingParams(
|
||||||
|
structured_outputs=StructuredOutputsParams(json=JSON_SCHEMA)
|
||||||
|
)
|
||||||
|
params._validate_structured_outputs(
|
||||||
|
_StubModelConfig(is_diffusion=False),
|
||||||
|
StructuredOutputsConfig(backend="guidance"),
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
)
|
||||||
|
return params
|
||||||
|
|
||||||
|
def _make_tokenizer(self, tokenizer_id: str):
|
||||||
|
if tokenizer_id == "slow-hf":
|
||||||
|
return self._StubSlowTokenizer()
|
||||||
|
if tokenizer_id == "plain-object":
|
||||||
|
return object()
|
||||||
|
if tokenizer_id == "raw-mistral-common":
|
||||||
|
return object.__new__(MistralCommonBackend)
|
||||||
|
if tokenizer_id == "vllm-mistral-non-tekken":
|
||||||
|
return self._StubVllmMistralTokenizer(is_tekken=False)
|
||||||
|
if tokenizer_id == "fast-hf":
|
||||||
|
return object.__new__(TokenizersBackend)
|
||||||
|
if tokenizer_id == "vllm-mistral-tekken":
|
||||||
|
return self._StubVllmMistralTokenizer(is_tekken=True)
|
||||||
|
raise ValueError(tokenizer_id)
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"tokenizer_id, patch_mistral",
|
||||||
|
[
|
||||||
|
pytest.param("slow-hf", False, id="slow-hf"),
|
||||||
|
pytest.param("plain-object", False, id="plain-object"),
|
||||||
|
pytest.param("raw-mistral-common", False, id="raw-mistral-common"),
|
||||||
|
pytest.param(
|
||||||
|
"vllm-mistral-non-tekken",
|
||||||
|
True,
|
||||||
|
id="vllm-mistral-non-tekken",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_unsupported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral):
|
||||||
|
if patch_mistral:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
mistral_tokenizers,
|
||||||
|
"MistralTokenizer",
|
||||||
|
self._StubVllmMistralTokenizer,
|
||||||
|
)
|
||||||
|
with pytest.raises(VLLMValidationError):
|
||||||
|
self._validate_guidance(self._make_tokenizer(tokenizer_id))
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"tokenizer_id, patch_mistral",
|
||||||
|
[
|
||||||
|
pytest.param("fast-hf", False, id="fast-hf"),
|
||||||
|
pytest.param("vllm-mistral-tekken", True, id="vllm-mistral-tekken"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_supported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral):
|
||||||
|
if patch_mistral:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
mistral_tokenizers,
|
||||||
|
"MistralTokenizer",
|
||||||
|
self._StubVllmMistralTokenizer,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
backend_guidance,
|
||||||
|
"validate_guidance_grammar",
|
||||||
|
lambda *_args, **_kwargs: None,
|
||||||
|
)
|
||||||
|
params = self._validate_guidance(self._make_tokenizer(tokenizer_id))
|
||||||
|
assert params.structured_outputs._backend == "guidance"
|
||||||
|
|||||||
+17
-2
@@ -1317,6 +1317,7 @@ class SamplingParams(
|
|||||||
)
|
)
|
||||||
|
|
||||||
from vllm.v1.structured_output.backend_guidance import (
|
from vllm.v1.structured_output.backend_guidance import (
|
||||||
|
GuidanceBackend,
|
||||||
has_guidance_unsupported_json_features,
|
has_guidance_unsupported_json_features,
|
||||||
validate_guidance_grammar,
|
validate_guidance_grammar,
|
||||||
)
|
)
|
||||||
@@ -1340,6 +1341,13 @@ class SamplingParams(
|
|||||||
"Mistral model, the ['xgrammar', 'outlines'] "
|
"Mistral model, the ['xgrammar', 'outlines'] "
|
||||||
"backends or tokenizer_mode='hf' instead."
|
"backends or tokenizer_mode='hf' instead."
|
||||||
)
|
)
|
||||||
|
if not GuidanceBackend.is_tokenizer_supported(tokenizer):
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"The 'guidance' structured output backend only supports fast "
|
||||||
|
"Hugging Face tokenizers and Tekken-based Mistral tokenizers. "
|
||||||
|
"Please use the 'xgrammar' backend or configure a fast "
|
||||||
|
"tokenizer instead."
|
||||||
|
)
|
||||||
# TODO: ideally we would have the LLTokenizer here as Lark syntax
|
# TODO: ideally we would have the LLTokenizer here as Lark syntax
|
||||||
# allows <|special_token|> and similar, see
|
# allows <|special_token|> and similar, see
|
||||||
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
|
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
|
||||||
@@ -1408,12 +1416,19 @@ class SamplingParams(
|
|||||||
skip_guidance = has_guidance_unsupported_json_features(schema)
|
skip_guidance = has_guidance_unsupported_json_features(schema)
|
||||||
|
|
||||||
if skip_guidance:
|
if skip_guidance:
|
||||||
# Fall back to outlines if the tokenizer is non-tekken Mistral or
|
# Fall back to outlines for non-Tekken Mistral tokenizers or
|
||||||
# the schema contains features unsupported by guidance
|
# schemas containing features unsupported by guidance.
|
||||||
validate_structured_output_request_outlines(self)
|
validate_structured_output_request_outlines(self)
|
||||||
self.structured_outputs._backend = "outlines"
|
self.structured_outputs._backend = "outlines"
|
||||||
else:
|
else:
|
||||||
# Fall back to guidance by default.
|
# Fall back to guidance by default.
|
||||||
|
if not GuidanceBackend.is_tokenizer_supported(tokenizer):
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"No compatible structured output backend was found for "
|
||||||
|
"this request. XGrammar rejected the constraint, and "
|
||||||
|
"Guidance does not support this tokenizer. Configure a "
|
||||||
|
"fast tokenizer or select a compatible backend explicitly."
|
||||||
|
) from None
|
||||||
validate_guidance_grammar(
|
validate_guidance_grammar(
|
||||||
self,
|
self,
|
||||||
tokenizer=_get_llg_tokenizer(tokenizer),
|
tokenizer=_get_llg_tokenizer(tokenizer),
|
||||||
|
|||||||
@@ -8,10 +8,12 @@ from dataclasses import dataclass
|
|||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
from transformers import TokenizersBackend
|
||||||
|
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.sampling_params import SamplingParams
|
from vllm.sampling_params import SamplingParams
|
||||||
|
from vllm.tokenizers import TokenizerLike
|
||||||
from vllm.utils.import_utils import LazyLoader
|
from vllm.utils.import_utils import LazyLoader
|
||||||
from vllm.utils.mistral import is_mistral_tokenizer
|
from vllm.utils.mistral import is_mistral_tokenizer
|
||||||
from vllm.v1.structured_output.backend_types import (
|
from vllm.v1.structured_output.backend_types import (
|
||||||
@@ -128,6 +130,37 @@ def process_for_additional_properties(
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class GuidanceBackend(StructuredOutputBackend):
|
class GuidanceBackend(StructuredOutputBackend):
|
||||||
|
@staticmethod
|
||||||
|
def is_tokenizer_supported(tokenizer: TokenizerLike) -> bool:
|
||||||
|
"""Check if tokenizer is supported by guidance backend.
|
||||||
|
|
||||||
|
TODO(arpera):
|
||||||
|
guidance backend does NOT support some types of tokenizers
|
||||||
|
so, we MUST verify tokenizer before using guidance
|
||||||
|
The problem is that guidance API does not provide
|
||||||
|
a way to validate tokenizer before building LLTokenizer
|
||||||
|
So, we have to do it manually here in vLLM.
|
||||||
|
This is ofcourse very bad, we duplicate complex logic
|
||||||
|
that can change arbitrary in future.
|
||||||
|
Our check here is based on guidance source code
|
||||||
|
on function `from_tokenizer` implementation:
|
||||||
|
https://github.com/guidance-ai/llguidance/blob/main/python/llguidance/hf.py#L29-L47
|
||||||
|
If you want to contribute to vLLM, please, add to llguidance
|
||||||
|
API function to implement this check there.
|
||||||
|
In that case we will remove this check from vLLM.
|
||||||
|
That kind of work is really appreciated!
|
||||||
|
"""
|
||||||
|
if is_mistral_tokenizer(tokenizer):
|
||||||
|
return tokenizer.is_tekken
|
||||||
|
# PreTrainedTokenizerFast is the same as TokenizersBackend
|
||||||
|
return isinstance(tokenizer, TokenizersBackend)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _build_ll_tokenizer(tokenizer: TokenizerLike, vocab_size: int) -> Any:
|
||||||
|
if is_mistral_tokenizer(tokenizer):
|
||||||
|
return tokenizer.llg_tokenizer
|
||||||
|
return llguidance_hf.from_tokenizer(tokenizer, max(vocab_size, len(tokenizer)))
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
self.disable_any_whitespace = (
|
self.disable_any_whitespace = (
|
||||||
self.vllm_config.structured_outputs_config.disable_any_whitespace
|
self.vllm_config.structured_outputs_config.disable_any_whitespace
|
||||||
@@ -136,12 +169,13 @@ class GuidanceBackend(StructuredOutputBackend):
|
|||||||
self.vllm_config.structured_outputs_config.disable_additional_properties
|
self.vllm_config.structured_outputs_config.disable_additional_properties
|
||||||
)
|
)
|
||||||
|
|
||||||
if is_mistral_tokenizer(self.tokenizer):
|
if not self.is_tokenizer_supported(self.tokenizer):
|
||||||
self.ll_tokenizer = self.tokenizer.llg_tokenizer
|
raise ValueError(
|
||||||
else:
|
"Only fast tokenizers are supported."
|
||||||
self.ll_tokenizer = llguidance_hf.from_tokenizer(
|
"This error should NOT happen,"
|
||||||
self.tokenizer, max(self.vocab_size, len(self.tokenizer))
|
"if you see this, please report an issue."
|
||||||
)
|
)
|
||||||
|
self.ll_tokenizer = self._build_ll_tokenizer(self.tokenizer, self.vocab_size)
|
||||||
|
|
||||||
def compile_grammar(
|
def compile_grammar(
|
||||||
self,
|
self,
|
||||||
|
|||||||
Reference in New Issue
Block a user