Default model load/config/tokenizer to mistral format if relevant files exist (#28659)
Signed-off-by: Julien Denize <julien.denize@mistral.ai> Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
mgoin
parent
c68c7b403d
commit
57430fc95c
@@ -20,6 +20,7 @@ from vllm.multimodal.utils import argsort_mm_positions
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.metrics.stats import MultiModalCacheStats
|
||||
@@ -300,12 +301,24 @@ class Processor:
|
||||
# allows <|special_token|> and similar, see
|
||||
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
|
||||
# Without tokenizer these are disallowed in grammars.
|
||||
if isinstance(self.tokenizer, MistralTokenizer):
|
||||
raise ValueError(
|
||||
"Mistral tokenizer is not supported for the 'guidance' "
|
||||
"structured output backend. Please use ['xgrammar', 'outlines'] "
|
||||
"backends or tokenizer_mode='hf' instead."
|
||||
)
|
||||
validate_guidance_grammar(params, tokenizer=None)
|
||||
elif backend == "outlines":
|
||||
# outlines backend
|
||||
validate_structured_output_request_outlines(params)
|
||||
elif backend == "lm-format-enforcer":
|
||||
# lm format enforcer backend
|
||||
if isinstance(self.tokenizer, MistralTokenizer):
|
||||
raise ValueError(
|
||||
"Mistral tokenizer is not supported for the 'lm-format-enforcer' "
|
||||
"structured output backend. Please use ['xgrammar', 'outlines'] "
|
||||
"backends or tokenizer_mode='hf' instead."
|
||||
)
|
||||
validate_structured_output_request_lm_format_enforcer(params)
|
||||
else:
|
||||
# NOTE: backend must be "auto" here, because we have
|
||||
@@ -320,9 +333,15 @@ class Processor:
|
||||
except ValueError:
|
||||
# The request either failed validation
|
||||
# or includes some jsonschema feature(s) that
|
||||
# are not supported in xgrammar. Fall back to guidance.
|
||||
validate_guidance_grammar(params, tokenizer=None)
|
||||
params.structured_outputs._backend = "guidance"
|
||||
# are not supported in xgrammar.
|
||||
if isinstance(self.tokenizer, MistralTokenizer):
|
||||
# Fall back to outlines if the tokenizer is Mistral
|
||||
validate_structured_output_request_outlines(params)
|
||||
params.structured_outputs._backend = "outlines"
|
||||
else:
|
||||
# Fall back to guidance by default.
|
||||
validate_guidance_grammar(params, tokenizer=None)
|
||||
params.structured_outputs._backend = "guidance"
|
||||
# Remember that this backend was set automatically
|
||||
params.structured_outputs._backend_was_auto = True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user