[Frontend]Responses API supports Tool/Function calling with streaming with required (#40700)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-04-27 19:36:58 -07:00
committed by GitHub
parent 0d4f714208
commit 146f44b77d
6 changed files with 931 additions and 717 deletions
@@ -9,10 +9,7 @@ from collections.abc import Sequence as GenericSequence
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Final
import partial_json_parser
import regex as re
from fastapi import Request
from partial_json_parser.core.options import Allow
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import (
@@ -73,7 +70,7 @@ from vllm.reasoning import ReasoningParser
from vllm.renderers import ChatParams
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.utils import partial_json_loads
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
@@ -383,45 +380,6 @@ class OpenAIServingChat(OpenAIServing):
return self.response_role
return request.messages[-1]["role"]
@staticmethod
def _bracket_level(s: str, opening="{", closing="}") -> int:
"""
Calculate the current level of nested brackets in a given string.
"""
level = 0
for char in s:
if char == opening:
level += 1
elif char == closing:
level -= 1
return level
@staticmethod
def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
# remove last '},' of the tool definition stemming from the
# "name"/"parameters" outer object or closing ']' of the tool list
# count occurrences of opening and closing curly braces and
# once level 0 is reached stop outputting text
# if 0 is reached while parsing the delta_text we know the current
# tool will finish in this current iteration
bracket_level = OpenAIServingChat._bracket_level(previous_text)
updated_delta, passed_zero = "", False
for c in delta_text:
if c == "{":
bracket_level += 1
passed_zero = bracket_level == 0
elif c == "}":
bracket_level -= 1
passed_zero = bracket_level == 0
if bracket_level != 0:
updated_delta += c
else:
# if a comma is reached at level 0 we can stop
if c == ",":
break
return updated_delta, passed_zero
def extract_tool_call_required_streaming(
self,
previous_text: str,
@@ -430,97 +388,14 @@ class OpenAIServingChat(OpenAIServing):
function_name_returned: bool,
tool_call_idx: int | None = None,
) -> tuple[DeltaMessage | None, bool]:
if current_text is None or current_text == "":
# if the current text is empty, we cannot parse it
return None, function_name_returned
try:
flags = Allow.ALL
obj, _ = partial_json_loads(current_text, flags)
except (
partial_json_parser.core.exceptions.MalformedJSON,
json.JSONDecodeError,
):
logger.debug("not enough tokens to parse into JSON yet")
obj = None
# check if the current text is a valid array
# containing a partial tool calling object
# if not repeat
if obj is None or not isinstance(obj, list) or not len(obj) > 0:
function_name_returned = False
delta_message = None
else:
_, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
delta_text, previous_text
)
# take the last tool call from the generated list
current_tool_call = obj[-1]
# once parameters have been generated the name is complete as well
if not finishes_previous_tool and (
"name" not in current_tool_call or "parameters" not in current_tool_call
):
function_name_returned = False
delta_message = None
else:
if not function_name_returned:
# get partly generated arguments from the latest tool call
param_match = re.search(
r'.*"parameters":\s*(.*)', current_text, re.DOTALL
)
arguments = param_match.group(1) if param_match else ""
arguments, _ = OpenAIServingChat._filter_delta_text(
arguments, previous_text
)
# if this iteration finishes a previous tool call but a
# new incomplete tool is already generated, take the
# previous from the list
if finishes_previous_tool and "parameters" not in current_tool_call:
current_tool_call = obj[-2]
function_name_returned = True
tool_call_id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=current_tool_call["name"],
idx=tool_call_idx,
)
delta_message = DeltaMessage(
tool_calls=[
DeltaToolCall(
id=tool_call_id,
function=DeltaFunctionCall(
name=current_tool_call["name"], arguments=arguments
),
index=len(obj) - 1,
type="function",
)
]
)
else:
delta_text, _ = OpenAIServingChat._filter_delta_text(
delta_text, previous_text
)
if delta_text != "":
delta_message = DeltaMessage(
tool_calls=[
DeltaToolCall(
function=DeltaFunctionCall(
# OpenAI API returns None
# instead of name every time
name=None,
arguments=delta_text,
),
index=len(obj) - 1,
)
]
)
else:
delta_message = None
return delta_message, function_name_returned
return extract_required_tool_call_streaming(
previous_text=previous_text,
current_text=current_text,
delta_text=delta_text,
function_name_returned=function_name_returned,
tool_call_idx=tool_call_idx,
tool_call_id_type=self.tool_call_id_type,
)
async def chat_completion_stream_generator(
self,