[V1] Support disable_any_whtespace for guidance backend (#15584)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
@@ -1561,7 +1561,8 @@ class EngineArgs:
|
||||
|
||||
# Xgrammar and Guidance are supported.
|
||||
SUPPORTED_GUIDED_DECODING = [
|
||||
"xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto"
|
||||
"xgrammar", "xgrammar:disable-any-whitespace", "guidance",
|
||||
"guidance:disable-any-whitespace", "auto"
|
||||
]
|
||||
if self.guided_decoding_backend not in SUPPORTED_GUIDED_DECODING:
|
||||
_raise_or_fallback(feature_name="--guided-decoding-backend",
|
||||
|
||||
@@ -18,14 +18,22 @@ def get_local_guidance_guided_decoding_logits_processor(
|
||||
"""
|
||||
|
||||
grm = ""
|
||||
any_whitespace = 'disable-any-whitespace' not in \
|
||||
guided_params.backend_options()
|
||||
if guided_params.json:
|
||||
grm = llguidance.LLMatcher.grammar_from_json_schema(
|
||||
guided_params.json,
|
||||
overrides={"whitespace_pattern": guided_params.whitespace_pattern})
|
||||
overrides={"whitespace_pattern": guided_params.whitespace_pattern},
|
||||
defaults={
|
||||
"whitespace_flexible": any_whitespace,
|
||||
})
|
||||
elif guided_params.json_object:
|
||||
grm = llguidance.LLMatcher.grammar_from_json_schema(
|
||||
'{"type": "object"}',
|
||||
overrides={"whitespace_pattern": guided_params.whitespace_pattern})
|
||||
overrides={"whitespace_pattern": guided_params.whitespace_pattern},
|
||||
defaults={
|
||||
"whitespace_flexible": any_whitespace,
|
||||
})
|
||||
elif guided_params.regex:
|
||||
grm = llguidance.grammar_from("regex", guided_params.regex)
|
||||
elif guided_params.choice:
|
||||
|
||||
@@ -121,7 +121,8 @@ class Processor:
|
||||
return
|
||||
|
||||
supported_backends = [
|
||||
"xgrammar", "xgrammar:disable-any-whitespace", "guidance", "auto"
|
||||
"xgrammar", "xgrammar:disable-any-whitespace", "guidance",
|
||||
"guidance:disable-any-whitespace", "auto"
|
||||
]
|
||||
engine_level_backend = self.decoding_config.guided_decoding_backend
|
||||
if engine_level_backend not in supported_backends:
|
||||
@@ -140,11 +141,10 @@ class Processor:
|
||||
raise ValueError("Structured output is not supported on TPU.")
|
||||
|
||||
# Request content validation
|
||||
|
||||
if engine_level_backend == "xgrammar":
|
||||
if engine_level_backend.startswith("xgrammar"):
|
||||
# xgrammar with no fallback
|
||||
validate_structured_output_request_xgrammar(params)
|
||||
params.guided_decoding.backend = "xgrammar"
|
||||
params.guided_decoding.backend = engine_level_backend
|
||||
elif engine_level_backend == "auto":
|
||||
# "auto" is an opt-in to opinionated behavior where we try to
|
||||
# choose a backend based on request contents. This is not the
|
||||
@@ -158,12 +158,13 @@ class Processor:
|
||||
# are not supported in xgrammar. Fall back to guidance.
|
||||
params.guided_decoding.backend = "guidance"
|
||||
|
||||
if params.guided_decoding.backend == "guidance":
|
||||
if engine_level_backend.startswith("guidance"):
|
||||
# TODO ideally we would have the LLTokenizer here as Lark syntax
|
||||
# allows <|special_token|> and similar, see
|
||||
# https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens
|
||||
# Without tokenizer these are disallowed in grammars.
|
||||
validate_guidance_grammar(params, tokenizer=None)
|
||||
params.guided_decoding.backend = engine_level_backend
|
||||
|
||||
def process_inputs(
|
||||
self,
|
||||
|
||||
@@ -41,6 +41,9 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
tokenizer_group.ping()
|
||||
self.vllm_config = vllm_config
|
||||
self.vocab_size = vllm_config.model_config.get_vocab_size()
|
||||
self.disable_any_whitespace = (
|
||||
"disable-any-whitespace"
|
||||
in vllm_config.decoding_config.guided_decoding_backend)
|
||||
|
||||
tokenizer = tokenizer_group.get_lora_tokenizer(None)
|
||||
self.ll_tokenizer = llguidance_hf.from_tokenizer(tokenizer, None)
|
||||
@@ -48,7 +51,7 @@ class GuidanceBackend(StructuredOutputBackend):
|
||||
def compile_grammar(self, request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> StructuredOutputGrammar:
|
||||
self.serialized_grammar = serialize_guidance_grammar(
|
||||
request_type, grammar_spec)
|
||||
request_type, grammar_spec, self.disable_any_whitespace)
|
||||
|
||||
ll_matcher = llguidance.LLMatcher(
|
||||
self.ll_tokenizer,
|
||||
@@ -126,17 +129,19 @@ class GuidanceGrammar(StructuredOutputGrammar):
|
||||
|
||||
|
||||
def serialize_guidance_grammar(request_type: StructuredOutputOptions,
|
||||
grammar_spec: str) -> str:
|
||||
grammar_spec: str,
|
||||
disable_any_whitespace: bool = False) -> str:
|
||||
if request_type == StructuredOutputOptions.JSON:
|
||||
# TODO: make whitespace_flexible configurable
|
||||
return llguidance.LLMatcher.grammar_from_json_schema(
|
||||
grammar_spec, defaults={
|
||||
"whitespace_flexible": True,
|
||||
grammar_spec,
|
||||
defaults={
|
||||
"whitespace_flexible": not disable_any_whitespace,
|
||||
})
|
||||
elif request_type == StructuredOutputOptions.JSON_OBJECT:
|
||||
return llguidance.LLMatcher.grammar_from_json_schema(
|
||||
'{"type": "object"}', defaults={
|
||||
"whitespace_flexible": True,
|
||||
'{"type": "object"}',
|
||||
defaults={
|
||||
"whitespace_flexible": not disable_any_whitespace,
|
||||
})
|
||||
else:
|
||||
if request_type == StructuredOutputOptions.REGEX:
|
||||
|
||||
Reference in New Issue
Block a user