[Bugfix] Validate guidance tokenizer compatibility before EngineCore (#52298)
Signed-off-by: Oxygen56 <jiangth99@163.com> Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com> Co-authored-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
co-authored by
Artem Perevedentsev
parent
1dc60dd367
commit
ad8a6065f4
+17
-2
@@ -1317,6 +1317,7 @@ class SamplingParams(
|
||||
)
|
||||
|
||||
from vllm.v1.structured_output.backend_guidance import (
|
||||
GuidanceBackend,
|
||||
has_guidance_unsupported_json_features,
|
||||
validate_guidance_grammar,
|
||||
)
|
||||
@@ -1340,6 +1341,13 @@ class SamplingParams(
|
||||
"Mistral model, the ['xgrammar', 'outlines'] "
|
||||
"backends or tokenizer_mode='hf' instead."
|
||||
)
|
||||
if not GuidanceBackend.is_tokenizer_supported(tokenizer):
|
||||
raise VLLMValidationError(
|
||||
"The 'guidance' structured output backend only supports fast "
|
||||
"Hugging Face tokenizers and Tekken-based Mistral tokenizers. "
|
||||
"Please use the 'xgrammar' backend or configure a fast "
|
||||
"tokenizer instead."
|
||||
)
|
||||
# TODO: ideally we would have the LLTokenizer here as Lark syntax
|
||||
# allows <|special_token|> and similar, see
|
||||
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
|
||||
@@ -1408,12 +1416,19 @@ class SamplingParams(
|
||||
skip_guidance = has_guidance_unsupported_json_features(schema)
|
||||
|
||||
if skip_guidance:
|
||||
# Fall back to outlines if the tokenizer is non-tekken Mistral or
|
||||
# the schema contains features unsupported by guidance
|
||||
# Fall back to outlines for non-Tekken Mistral tokenizers or
|
||||
# schemas containing features unsupported by guidance.
|
||||
validate_structured_output_request_outlines(self)
|
||||
self.structured_outputs._backend = "outlines"
|
||||
else:
|
||||
# Fall back to guidance by default.
|
||||
if not GuidanceBackend.is_tokenizer_supported(tokenizer):
|
||||
raise VLLMValidationError(
|
||||
"No compatible structured output backend was found for "
|
||||
"this request. XGrammar rejected the constraint, and "
|
||||
"Guidance does not support this tokenizer. Configure a "
|
||||
"fast tokenizer or select a compatible backend explicitly."
|
||||
) from None
|
||||
validate_guidance_grammar(
|
||||
self,
|
||||
tokenizer=_get_llg_tokenizer(tokenizer),
|
||||
|
||||
Reference in New Issue
Block a user