[Frontend][Core] Move guided decoding params into sampling params (#8252)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
This commit is contained in:
@@ -5,16 +5,11 @@ from json import dumps as json_dumps
|
||||
from re import escape as regex_escape
|
||||
from typing import Tuple, Union
|
||||
|
||||
from pydantic import BaseModel
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionNamedToolChoiceParam, ChatCompletionRequest,
|
||||
CompletionRequest)
|
||||
from vllm.model_executor.guided_decoding.guided_fields import (
|
||||
GuidedDecodingRequest)
|
||||
from vllm.model_executor.guided_decoding.outlines_logits_processors import (
|
||||
CFGLogitsProcessor, JSONLogitsProcessor, RegexLogitsProcessor)
|
||||
from vllm.sampling_params import GuidedDecodingParams
|
||||
|
||||
|
||||
class GuidedDecodingMode(Enum):
|
||||
@@ -55,8 +50,7 @@ global_thread_pool = None # used for generating logits processor fsm
|
||||
|
||||
|
||||
async def get_outlines_guided_decoding_logits_processor(
|
||||
request: Union[CompletionRequest,
|
||||
ChatCompletionRequest], tokenizer: PreTrainedTokenizerBase
|
||||
guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
|
||||
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
|
||||
None]:
|
||||
"""
|
||||
@@ -66,7 +60,7 @@ async def get_outlines_guided_decoding_logits_processor(
|
||||
we make a shallow copy to reuse the same underlying FSM.
|
||||
"""
|
||||
global global_thread_pool
|
||||
guide, mode = _get_guide_and_mode(request)
|
||||
guide, mode = _get_guide_and_mode(guided_params)
|
||||
if not guide or not mode:
|
||||
return None
|
||||
|
||||
@@ -77,11 +71,11 @@ async def get_outlines_guided_decoding_logits_processor(
|
||||
|
||||
return await loop.run_in_executor(global_thread_pool,
|
||||
_get_logits_processor, guide, tokenizer,
|
||||
mode, request.guided_whitespace_pattern)
|
||||
mode, guided_params.whitespace_pattern)
|
||||
|
||||
|
||||
def get_local_outlines_guided_decoding_logits_processor(
|
||||
guided_options: GuidedDecodingRequest, tokenizer: PreTrainedTokenizerBase
|
||||
guided_params: GuidedDecodingParams, tokenizer: PreTrainedTokenizerBase
|
||||
) -> Union[JSONLogitsProcessor, RegexLogitsProcessor, CFGLogitsProcessor,
|
||||
None]:
|
||||
"""
|
||||
@@ -90,65 +84,37 @@ def get_local_outlines_guided_decoding_logits_processor(
|
||||
We cache logit processors by (guide, tokenizer), and on cache hit
|
||||
we make a shallow copy to reuse the same underlying FSM.
|
||||
"""
|
||||
guide, mode = _get_guide_and_mode(guided_options)
|
||||
guide, mode = _get_guide_and_mode(guided_params)
|
||||
if not guide or not mode:
|
||||
return None
|
||||
|
||||
return _get_logits_processor(guide, tokenizer, mode,
|
||||
guided_options.guided_whitespace_pattern)
|
||||
guided_params.whitespace_pattern)
|
||||
|
||||
|
||||
def _get_guide_and_mode(
|
||||
request: Union[CompletionRequest, ChatCompletionRequest,
|
||||
GuidedDecodingRequest]
|
||||
guided_params: GuidedDecodingParams
|
||||
) -> Union[Tuple[str, GuidedDecodingMode], Tuple[None, None]]:
|
||||
# if the request is a chat completion request, AND the tool choice is a
|
||||
# named tool choice, do guided decoding
|
||||
# using that tool as the JSON schema
|
||||
if isinstance(request, ChatCompletionRequest) and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||
# Guided generation for tools/functions parameters
|
||||
if request.tool_choice.type == "function":
|
||||
for tool in request.tools:
|
||||
if (tool.type == "function" and tool.function.name
|
||||
== request.tool_choice.function.name):
|
||||
json = json_dumps(tool.function.parameters, sort_keys=True)
|
||||
return json, GuidedDecodingMode.JSON
|
||||
return None, None
|
||||
|
||||
elif request.guided_json:
|
||||
if isinstance(request.guided_json, dict):
|
||||
if guided_params.json:
|
||||
if isinstance(guided_params.json, dict):
|
||||
# turn dict into hashable string
|
||||
json = json_dumps(request.guided_json)
|
||||
elif isinstance(request.guided_json, BaseModel):
|
||||
# use pydantic signature so that different model classes
|
||||
# with the same fields will get hashed the same
|
||||
json = str(request.guided_json.__signature__)
|
||||
json = json_dumps(guided_params.json)
|
||||
else:
|
||||
json = request.guided_json
|
||||
json = guided_params.json
|
||||
return json, GuidedDecodingMode.JSON
|
||||
elif request.guided_regex:
|
||||
return request.guided_regex, GuidedDecodingMode.REGEX
|
||||
elif request.guided_choice:
|
||||
elif guided_params.regex:
|
||||
return guided_params.regex, GuidedDecodingMode.REGEX
|
||||
elif guided_params.choice:
|
||||
# choice just uses regex
|
||||
choices = [
|
||||
regex_escape(str(choice)) for choice in request.guided_choice
|
||||
regex_escape(str(choice)) for choice in guided_params.choice
|
||||
]
|
||||
choices_regex = "(" + "|".join(choices) + ")"
|
||||
return choices_regex, GuidedDecodingMode.CHOICE
|
||||
elif request.guided_grammar:
|
||||
return request.guided_grammar, GuidedDecodingMode.GRAMMAR
|
||||
elif (not isinstance(request, GuidedDecodingRequest)
|
||||
and request.response_format is not None
|
||||
and request.response_format.type == "json_object"):
|
||||
elif guided_params.grammar:
|
||||
return guided_params.grammar, GuidedDecodingMode.GRAMMAR
|
||||
elif guided_params.json_object:
|
||||
return JSON_GRAMMAR, GuidedDecodingMode.GRAMMAR
|
||||
elif (not isinstance(request, GuidedDecodingRequest)
|
||||
and request.response_format is not None
|
||||
and request.response_format.type == "json_schema"
|
||||
and request.response_format.json_schema is not None
|
||||
and request.response_format.json_schema.json_schema is not None):
|
||||
json = json_dumps(request.response_format.json_schema.json_schema)
|
||||
return json, GuidedDecodingMode.JSON
|
||||
else:
|
||||
return None, None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user