[Core][V0] Add guidance backend for structured output (#14589)
Signed-off-by: Russell Bryant <rbryant@redhat.com> Co-authored-by: Loc Huynh <lohuynh@microsoft.com> Co-authored-by: Michal Moskal <michal@moskal.me> Co-authored-by: Aaron Pham <contact@aarnphm.xyz>
This commit is contained in:
co-authored by
Loc Huynh
Michal Moskal
Aaron Pham
parent
b88be22165
commit
1f16b7fe74
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from re import escape as regex_escape
|
||||
|
||||
import llguidance
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.model_executor.guided_decoding.guidance_logits_processors import (
|
||||
GuidanceLogitsProcessor)
|
||||
from vllm.sampling_params import GuidedDecodingParams
|
||||
|
||||
|
||||
def get_local_guidance_guided_decoding_logits_processor(
|
||||
guided_params: GuidedDecodingParams,
|
||||
tokenizer: PreTrainedTokenizerBase) -> GuidanceLogitsProcessor:
|
||||
"""
|
||||
Given an OpenAI-compatible request, check for guided decoding parameters
|
||||
and get the necessary logits processor for the given guide.
|
||||
"""
|
||||
|
||||
grm = ""
|
||||
if guided_params.json:
|
||||
grm = llguidance.LLMatcher.grammar_from_json_schema(
|
||||
guided_params.json,
|
||||
overrides={"whitespace_pattern": guided_params.whitespace_pattern})
|
||||
elif guided_params.json_object:
|
||||
grm = llguidance.LLMatcher.grammar_from_json_schema(
|
||||
'{"type": "object"}',
|
||||
overrides={"whitespace_pattern": guided_params.whitespace_pattern})
|
||||
elif guided_params.regex:
|
||||
grm = llguidance.grammar_from("regex", guided_params.regex)
|
||||
elif guided_params.choice:
|
||||
# choice just uses regex
|
||||
choices = (regex_escape(str(choice))
|
||||
for choice in guided_params.choice)
|
||||
choices_regex = "(" + "|".join(choices) + ")"
|
||||
grm = llguidance.grammar_from("regex", choices_regex)
|
||||
elif guided_params.grammar:
|
||||
# this supports Lark and GBNF
|
||||
grm = llguidance.grammar_from("grammar", guided_params.grammar)
|
||||
|
||||
if grm:
|
||||
return GuidanceLogitsProcessor(grm, tokenizer)
|
||||
|
||||
raise ValueError("Unknown guided decoding mode")
|
||||
Reference in New Issue
Block a user