[Bugfix] Validate guidance tokenizer compatibility before EngineCore (#52298)
Signed-off-by: Oxygen56 <jiangth99@163.com> Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
co-authored by
Artem Perevedentsev
parent
1dc60dd367
commit
ad8a6065f4
+17
-2
@@ -1317,6 +1317,7 @@ class SamplingParams(
|
||||
)
|
||||
|
||||
from vllm.v1.structured_output.backend_guidance import (
|
||||
GuidanceBackend,
|
||||
has_guidance_unsupported_json_features,
|
||||
validate_guidance_grammar,
|
||||
)
|
||||
@@ -1340,6 +1341,13 @@ class SamplingParams(
|
||||
"Mistral model, the ['xgrammar', 'outlines'] "
|
||||
"backends or tokenizer_mode='hf' instead."
|
||||
)
|
||||
if not GuidanceBackend.is_tokenizer_supported(tokenizer):
|
||||
raise VLLMValidationError(
|
||||
"The 'guidance' structured output backend only supports fast "
|
||||
"Hugging Face tokenizers and Tekken-based Mistral tokenizers. "
|
||||
"Please use the 'xgrammar' backend or configure a fast "
|
||||
"tokenizer instead."
|
||||
)
|
||||
# TODO: ideally we would have the LLTokenizer here as Lark syntax
|
||||
# allows <|special_token|> and similar, see
|
||||
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
|
||||
@@ -1408,12 +1416,19 @@ class SamplingParams(
|
||||
skip_guidance = has_guidance_unsupported_json_features(schema)
|
||||
|
||||
if skip_guidance:
|
||||
# Fall back to outlines if the tokenizer is non-tekken Mistral or
|
||||
# the schema contains features unsupported by guidance
|
||||
# Fall back to outlines for non-Tekken Mistral tokenizers or
|
||||
# schemas containing features unsupported by guidance.
|
||||
validate_structured_output_request_outlines(self)
|
||||
self.structured_outputs._backend = "outlines"
|
||||
else:
|
||||
# Fall back to guidance by default.
|
||||
if not GuidanceBackend.is_tokenizer_supported(tokenizer):
|
||||
raise VLLMValidationError(
|
||||
"No compatible structured output backend was found for "
|
||||
"this request. XGrammar rejected the constraint, and "
|
||||
"Guidance does not support this tokenizer. Configure a "
|
||||
"fast tokenizer or select a compatible backend explicitly."
|
||||
) from None
|
||||
validate_guidance_grammar(
|
||||
self,
|
||||
tokenizer=_get_llg_tokenizer(tokenizer),
|
||||
|
||||
@@ -8,10 +8,12 @@ from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from transformers import TokenizersBackend
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from vllm.v1.structured_output.backend_types import (
|
||||
@@ -128,6 +130,37 @@ def process_for_additional_properties(
|
||||
|
||||
@dataclass
|
||||
class GuidanceBackend(StructuredOutputBackend):
|
||||
@staticmethod
|
||||
def is_tokenizer_supported(tokenizer: TokenizerLike) -> bool:
|
||||
"""Check if tokenizer is supported by guidance backend.
|
||||
|
||||
TODO(arpera):
|
||||
guidance backend does NOT support some types of tokenizers
|
||||
so, we MUST verify tokenizer before using guidance
|
||||
The problem is that guidance API does not provide
|
||||
a way to validate tokenizer before building LLTokenizer
|
||||
So, we have to do it manually here in vLLM.
|
||||
This is ofcourse very bad, we duplicate complex logic
|
||||
that can change arbitrary in future.
|
||||
Our check here is based on guidance source code
|
||||
on function `from_tokenizer` implementation:
|
||||
https://github.com/guidance-ai/llguidance/blob/main/python/llguidance/hf.py#L29-L47
|
||||
If you want to contribute to vLLM, please, add to llguidance
|
||||
API function to implement this check there.
|
||||
In that case we will remove this check from vLLM.
|
||||
That kind of work is really appreciated!
|
||||
"""
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
return tokenizer.is_tekken
|
||||
# PreTrainedTokenizerFast is the same as TokenizersBackend
|
||||
return isinstance(tokenizer, TokenizersBackend)
|
||||
|
||||
@staticmethod
|
||||
def _build_ll_tokenizer(tokenizer: TokenizerLike, vocab_size: int) -> Any:
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
return tokenizer.llg_tokenizer
|
||||
return llguidance_hf.from_tokenizer(tokenizer, max(vocab_size, len(tokenizer)))
|
||||
|
||||
def __post_init__(self):
|
||||
self.disable_any_whitespace = (
|
||||
self.vllm_config.structured_outputs_config.disable_any_whitespace
|
||||
@@ -136,12 +169,13 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
self.vllm_config.structured_outputs_config.disable_additional_properties
|
||||
)
|
||||
|
||||
if is_mistral_tokenizer(self.tokenizer):
|
||||
self.ll_tokenizer = self.tokenizer.llg_tokenizer
|
||||
else:
|
||||
self.ll_tokenizer = llguidance_hf.from_tokenizer(
|
||||
self.tokenizer, max(self.vocab_size, len(self.tokenizer))
|
||||
if not self.is_tokenizer_supported(self.tokenizer):
|
||||
raise ValueError(
|
||||
"Only fast tokenizers are supported."
|
||||
"This error should NOT happen,"
|
||||
"if you see this, please report an issue."
|
||||
)
|
||||
self.ll_tokenizer = self._build_ll_tokenizer(self.tokenizer, self.vocab_size)
|
||||
|
||||
def compile_grammar(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user