[Frontend]Responses API supports Tool/Function calling with streaming with required (#40700)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -9,10 +9,7 @@ from collections.abc import Sequence as GenericSequence
|
||||
from http import HTTPStatus
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
import partial_json_parser
|
||||
import regex as re
|
||||
from fastapi import Request
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
@@ -73,7 +70,7 @@ from vllm.reasoning import ReasoningParser
|
||||
from vllm.renderers import ChatParams
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.utils import partial_json_loads
|
||||
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
|
||||
@@ -383,45 +380,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
return self.response_role
|
||||
return request.messages[-1]["role"]
|
||||
|
||||
@staticmethod
|
||||
def _bracket_level(s: str, opening="{", closing="}") -> int:
|
||||
"""
|
||||
Calculate the current level of nested brackets in a given string.
|
||||
"""
|
||||
level = 0
|
||||
for char in s:
|
||||
if char == opening:
|
||||
level += 1
|
||||
elif char == closing:
|
||||
level -= 1
|
||||
return level
|
||||
|
||||
@staticmethod
|
||||
def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
|
||||
# remove last '},' of the tool definition stemming from the
|
||||
# "name"/"parameters" outer object or closing ']' of the tool list
|
||||
# count occurrences of opening and closing curly braces and
|
||||
# once level 0 is reached stop outputting text
|
||||
# if 0 is reached while parsing the delta_text we know the current
|
||||
# tool will finish in this current iteration
|
||||
bracket_level = OpenAIServingChat._bracket_level(previous_text)
|
||||
updated_delta, passed_zero = "", False
|
||||
for c in delta_text:
|
||||
if c == "{":
|
||||
bracket_level += 1
|
||||
passed_zero = bracket_level == 0
|
||||
elif c == "}":
|
||||
bracket_level -= 1
|
||||
passed_zero = bracket_level == 0
|
||||
|
||||
if bracket_level != 0:
|
||||
updated_delta += c
|
||||
else:
|
||||
# if a comma is reached at level 0 we can stop
|
||||
if c == ",":
|
||||
break
|
||||
return updated_delta, passed_zero
|
||||
|
||||
def extract_tool_call_required_streaming(
|
||||
self,
|
||||
previous_text: str,
|
||||
@@ -430,97 +388,14 @@ class OpenAIServingChat(OpenAIServing):
|
||||
function_name_returned: bool,
|
||||
tool_call_idx: int | None = None,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
if current_text is None or current_text == "":
|
||||
# if the current text is empty, we cannot parse it
|
||||
return None, function_name_returned
|
||||
try:
|
||||
flags = Allow.ALL
|
||||
obj, _ = partial_json_loads(current_text, flags)
|
||||
except (
|
||||
partial_json_parser.core.exceptions.MalformedJSON,
|
||||
json.JSONDecodeError,
|
||||
):
|
||||
logger.debug("not enough tokens to parse into JSON yet")
|
||||
obj = None
|
||||
|
||||
# check if the current text is a valid array
|
||||
# containing a partial tool calling object
|
||||
# if not repeat
|
||||
if obj is None or not isinstance(obj, list) or not len(obj) > 0:
|
||||
function_name_returned = False
|
||||
delta_message = None
|
||||
else:
|
||||
_, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
|
||||
delta_text, previous_text
|
||||
)
|
||||
# take the last tool call from the generated list
|
||||
current_tool_call = obj[-1]
|
||||
|
||||
# once parameters have been generated the name is complete as well
|
||||
if not finishes_previous_tool and (
|
||||
"name" not in current_tool_call or "parameters" not in current_tool_call
|
||||
):
|
||||
function_name_returned = False
|
||||
delta_message = None
|
||||
else:
|
||||
if not function_name_returned:
|
||||
# get partly generated arguments from the latest tool call
|
||||
param_match = re.search(
|
||||
r'.*"parameters":\s*(.*)', current_text, re.DOTALL
|
||||
)
|
||||
arguments = param_match.group(1) if param_match else ""
|
||||
arguments, _ = OpenAIServingChat._filter_delta_text(
|
||||
arguments, previous_text
|
||||
)
|
||||
|
||||
# if this iteration finishes a previous tool call but a
|
||||
# new incomplete tool is already generated, take the
|
||||
# previous from the list
|
||||
if finishes_previous_tool and "parameters" not in current_tool_call:
|
||||
current_tool_call = obj[-2]
|
||||
|
||||
function_name_returned = True
|
||||
tool_call_id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=current_tool_call["name"],
|
||||
idx=tool_call_idx,
|
||||
)
|
||||
delta_message = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
id=tool_call_id,
|
||||
function=DeltaFunctionCall(
|
||||
name=current_tool_call["name"], arguments=arguments
|
||||
),
|
||||
index=len(obj) - 1,
|
||||
type="function",
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
else:
|
||||
delta_text, _ = OpenAIServingChat._filter_delta_text(
|
||||
delta_text, previous_text
|
||||
)
|
||||
|
||||
if delta_text != "":
|
||||
delta_message = DeltaMessage(
|
||||
tool_calls=[
|
||||
DeltaToolCall(
|
||||
function=DeltaFunctionCall(
|
||||
# OpenAI API returns None
|
||||
# instead of name every time
|
||||
name=None,
|
||||
arguments=delta_text,
|
||||
),
|
||||
index=len(obj) - 1,
|
||||
)
|
||||
]
|
||||
)
|
||||
else:
|
||||
delta_message = None
|
||||
|
||||
return delta_message, function_name_returned
|
||||
return extract_required_tool_call_streaming(
|
||||
previous_text=previous_text,
|
||||
current_text=current_text,
|
||||
delta_text=delta_text,
|
||||
function_name_returned=function_name_returned,
|
||||
tool_call_idx=tool_call_idx,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
)
|
||||
|
||||
async def chat_completion_stream_generator(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user