Simplify (and fix) passing of guided decoding backend options (#17008)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -26,8 +26,8 @@ def maybe_backend_fallback(
|
||||
def fallback_or_error(guided_params: GuidedDecodingParams, message: str,
|
||||
fallback: str) -> None:
|
||||
"""Change the backend to the specified fallback with a warning log,
|
||||
or raise a ValueError if the `no-fallback` option is specified."""
|
||||
if guided_params.no_fallback():
|
||||
or raise a ValueError if the `disable_fallback` option is specified."""
|
||||
if guided_params.disable_fallback:
|
||||
raise ValueError(message)
|
||||
|
||||
logger.warning("%s Falling back to use %s instead.", message, fallback)
|
||||
@@ -40,7 +40,7 @@ def maybe_backend_fallback(
|
||||
guided_params.backend = "xgrammar"
|
||||
|
||||
# lm-format-enforce doesn't support grammar, fallback to xgrammar
|
||||
if guided_params.backend_name == "lm-format-enforcer":
|
||||
if guided_params.backend == "lm-format-enforcer":
|
||||
if guided_params.grammar is not None:
|
||||
fallback_or_error(
|
||||
guided_params,
|
||||
@@ -55,7 +55,7 @@ def maybe_backend_fallback(
|
||||
"lm-format-enforcer does not support advanced JSON schema "
|
||||
"features like patterns or numeric ranges.", "outlines")
|
||||
|
||||
if guided_params.backend_name == "xgrammar":
|
||||
if guided_params.backend == "xgrammar":
|
||||
from vllm.model_executor.guided_decoding.xgrammar_decoding import (
|
||||
xgr_installed)
|
||||
|
||||
@@ -87,7 +87,7 @@ def maybe_backend_fallback(
|
||||
guided_params,
|
||||
"xgrammar module cannot be imported successfully.", "outlines")
|
||||
|
||||
if (guided_params.backend_name == "outlines"
|
||||
if (guided_params.backend == "outlines"
|
||||
and guided_params.json_object is not None):
|
||||
# outlines doesn't support json_object, fallback to guidance
|
||||
fallback_or_error(guided_params,
|
||||
@@ -111,7 +111,7 @@ async def get_guided_decoding_logits_processor(
|
||||
guided_params = maybe_backend_fallback(guided_params)
|
||||
|
||||
# CFG grammar not supported by LMFE, so we use outlines instead
|
||||
if guided_params.backend_name == 'outlines':
|
||||
if guided_params.backend == 'outlines':
|
||||
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
|
||||
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
|
||||
get_outlines_guided_decoding_logits_processor)
|
||||
@@ -122,12 +122,12 @@ async def get_guided_decoding_logits_processor(
|
||||
get_local_lm_format_enforcer_guided_decoding_logits_processor)
|
||||
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
|
||||
guided_params, tokenizer)
|
||||
if guided_params.backend_name == 'xgrammar':
|
||||
if guided_params.backend == 'xgrammar':
|
||||
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
|
||||
get_local_xgrammar_guided_decoding_logits_processor)
|
||||
return get_local_xgrammar_guided_decoding_logits_processor(
|
||||
guided_params, tokenizer, model_config, reasoner)
|
||||
if guided_params.backend_name == 'guidance':
|
||||
if guided_params.backend == 'guidance':
|
||||
from vllm.model_executor.guided_decoding.guidance_decoding import (
|
||||
get_local_guidance_guided_decoding_logits_processor)
|
||||
return get_local_guidance_guided_decoding_logits_processor(
|
||||
@@ -152,23 +152,23 @@ def get_local_guided_decoding_logits_processor(
|
||||
reasoner = reasoner_class(tokenizer)
|
||||
|
||||
# CFG grammar not supported by LMFE, so we use outlines instead
|
||||
if guided_params.backend_name == 'outlines':
|
||||
if guided_params.backend == 'outlines':
|
||||
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
|
||||
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
|
||||
get_local_outlines_guided_decoding_logits_processor)
|
||||
return get_local_outlines_guided_decoding_logits_processor(
|
||||
guided_params, tokenizer, reasoner)
|
||||
if guided_params.backend_name == 'lm-format-enforcer':
|
||||
if guided_params.backend == 'lm-format-enforcer':
|
||||
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
|
||||
get_local_lm_format_enforcer_guided_decoding_logits_processor)
|
||||
return get_local_lm_format_enforcer_guided_decoding_logits_processor(
|
||||
guided_params, tokenizer)
|
||||
if guided_params.backend_name == 'xgrammar':
|
||||
if guided_params.backend == 'xgrammar':
|
||||
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
|
||||
get_local_xgrammar_guided_decoding_logits_processor)
|
||||
return get_local_xgrammar_guided_decoding_logits_processor(
|
||||
guided_params, tokenizer, model_config, reasoner)
|
||||
if guided_params.backend_name == 'guidance':
|
||||
if guided_params.backend == 'guidance':
|
||||
from vllm.model_executor.guided_decoding.guidance_decoding import (
|
||||
get_local_guidance_guided_decoding_logits_processor)
|
||||
return get_local_guidance_guided_decoding_logits_processor(
|
||||
|
||||
@@ -21,13 +21,12 @@ def get_local_guidance_guided_decoding_logits_processor(
|
||||
"""
|
||||
|
||||
grm = ""
|
||||
any_whitespace = 'disable-any-whitespace' not in \
|
||||
guided_params.backend_options()
|
||||
any_whitespace = not guided_params.disable_any_whitespace
|
||||
if (guide_json := guided_params.json) is not None:
|
||||
# Optionally set additionalProperties to False at the top-level
|
||||
# By default, other backends do not allow additional top-level
|
||||
# properties, so this makes guidance more similar to other backends
|
||||
if 'no-additional-properties' in guided_params.backend_options():
|
||||
if guided_params.disable_additional_properties:
|
||||
if not isinstance(guide_json, str):
|
||||
guide_json = json.dumps(guide_json)
|
||||
guide_json = process_for_additional_properties(guide_json)
|
||||
|
||||
@@ -175,8 +175,7 @@ class GrammarConfig:
|
||||
else:
|
||||
json_str = guided_params.json
|
||||
|
||||
any_whitespace = 'disable-any-whitespace' not in \
|
||||
guided_params.backend_options()
|
||||
any_whitespace = not guided_params.disable_any_whitespace
|
||||
|
||||
# Check and log if model with xgrammar and whitespace have history
|
||||
# of runaway generation of whitespaces.
|
||||
@@ -191,11 +190,10 @@ class GrammarConfig:
|
||||
model_with_warn = 'Qwen'
|
||||
|
||||
if model_with_warn is not None and any_whitespace:
|
||||
msg = (f"{model_with_warn} "
|
||||
f"model detected, consider set "
|
||||
f"`guided_backend=xgrammar:disable-any-whitespace` "
|
||||
f"to prevent runaway generation of whitespaces.")
|
||||
logger.info_once(msg)
|
||||
logger.info_once(
|
||||
"%s model detected, consider setting "
|
||||
"`disable_any_whitespace` to prevent runaway generation "
|
||||
"of whitespaces.", model_with_warn)
|
||||
# Validate the schema and raise ValueError here if it is invalid.
|
||||
# This is to avoid exceptions in model execution, which will crash
|
||||
# the engine worker process.
|
||||
|
||||
Reference in New Issue
Block a user