[Frontend]Responses API supports Tool/Function calling with streaming with named tool/function (#41110)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-04-29 09:11:27 +00:00
committed by GitHub
parent ef70057ca7
commit 3885d340a4
4 changed files with 57 additions and 60 deletions
@@ -323,7 +323,7 @@ async def test_function_calling_with_streaming_expected_arguments(
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"tool_choice",
["auto", "required"],
["auto", "required", {"type": "function", "name": "get_current_weather"}],
)
async def test_function_calling_with_streaming_types(
client: openai.AsyncOpenAI, model_name: str, tool_choice
@@ -462,7 +462,7 @@ async def test_function_calling_with_streaming_types(
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize(
"tool_choice",
["required", "auto"],
["required", "auto", {"type": "function", "name": "get_weather"}],
)
async def test_function_calling_with_streaming_forced_tool_choice(
client: openai.AsyncOpenAI, model_name: str, tool_choice: str
@@ -70,7 +70,10 @@ from vllm.reasoning import ReasoningParser
from vllm.renderers import ChatParams
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
from vllm.tool_parsers.streaming import (
extract_named_tool_call_streaming,
extract_required_tool_call_streaming,
)
from vllm.utils.collection_utils import as_list
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
@@ -773,43 +776,24 @@ class OpenAIServingChat(OpenAIServing):
delta_text = previous_text + delta_text
current_text = ""
if function_name_returned[i]:
delta_tool_call = DeltaToolCall(
function=DeltaFunctionCall(arguments=delta_text),
index=i,
delta_message, function_name_returned[i] = (
extract_named_tool_call_streaming(
delta_text=delta_text,
function_name=tool_choice_function_name,
function_name_returned=function_name_returned[i],
tool_call_idx=history_tool_call_cnt,
tool_call_id_type=self.tool_call_id_type,
tokenizer=tokenizer,
tool_call_array_index=i,
)
else:
# Generate ID based on tokenizer type
if is_mistral_tokenizer(tokenizer):
from vllm.tool_parsers.mistral_tool_parser import (
MistralToolCall,
)
tool_call_id = MistralToolCall.generate_random_id()
else:
tool_call_id = make_tool_call_id(
id_type=self.tool_call_id_type,
func_name=tool_choice_function_name,
idx=history_tool_call_cnt,
)
delta_tool_call = DeltaToolCall(
id=tool_call_id,
type="function",
function=DeltaFunctionCall(
name=tool_choice_function_name,
arguments=delta_text,
),
index=i,
)
function_name_returned[i] = True
history_tool_call_cnt += 1
delta_message = DeltaMessage(
tool_calls=[
delta_tool_call,
]
)
tools_streamed[i] = True
if (
delta_message
and delta_message.tool_calls
and delta_message.tool_calls[0].id is not None
):
history_tool_call_cnt += 1
tools_streamed[i] = True
# Skip when tool_choice_uses_parser so it falls through
# to the auto tool_parser branches below.
+33 -16
View File
@@ -38,7 +38,10 @@ from vllm.logger import init_logger
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import ToolParser
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
from vllm.tool_parsers.streaming import (
extract_named_tool_call_streaming,
extract_required_tool_call_streaming,
)
from vllm.tool_parsers.utils import Tool
from vllm.utils import random_uuid
@@ -423,6 +426,17 @@ class DelegatingParser(Parser):
return outputs
def _get_function_name(
self, request: ChatCompletionRequest | ResponsesRequest
) -> str:
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
return request.tool_choice.name
if request.tool_choice and isinstance(
request.tool_choice, ChatCompletionNamedToolChoiceParam
):
return request.tool_choice.function.name
raise ValueError("Invalid tool_choice for function name extraction.")
def _parse_tool_calls(
self,
request: ResponsesRequest,
@@ -440,21 +454,14 @@ class DelegatingParser(Parser):
"""
function_calls: list[FunctionCall] = []
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
# Forced Function Call (Responses API style)
assert content is not None
function_calls.append(
FunctionCall(name=request.tool_choice.name, arguments=content)
)
return function_calls, None # Clear content since tool is called.
if request.tool_choice and isinstance(
request.tool_choice, ChatCompletionNamedToolChoiceParam
request.tool_choice,
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
):
# Forced Function Call (Chat Completion API style)
# Forced Function Call
assert content is not None
function_calls.append(
FunctionCall(name=request.tool_choice.function.name, arguments=content)
FunctionCall(name=self._get_function_name(request), arguments=content)
)
return function_calls, None # Clear content since tool is called.
@@ -572,7 +579,7 @@ class DelegatingParser(Parser):
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
request: ChatCompletionRequest | ResponsesRequest,
# The following parameters are used for "required" tool choice parsing and are
# tracked in StreamState for streaming parsing.
tool_call_idx: int | None = None,
@@ -580,9 +587,19 @@ class DelegatingParser(Parser):
function_name_returned: bool = False,
) -> tuple[DeltaMessage | None, bool]:
if request.tool_choice and isinstance(
request.tool_choice, ChatCompletionNamedToolChoiceParam
request.tool_choice,
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
):
return None, False
delta_message, function_name_returned = extract_named_tool_call_streaming(
delta_text=delta_text,
function_name=self._get_function_name(request),
function_name_returned=function_name_returned,
tool_call_idx=tool_call_idx,
tool_call_id_type=tool_call_id_type,
tokenizer=self.model_tokenizer,
)
return delta_message, function_name_returned
if request.tool_choice == "required":
delta_message, function_name_returned = (
extract_required_tool_call_streaming(
@@ -602,7 +619,7 @@ class DelegatingParser(Parser):
previous_token_ids,
current_token_ids,
delta_token_ids,
request,
request, # type: ignore[arg-type]
), False
def is_reasoning_end(self, input_ids: list[int]) -> bool:
+2 -6
View File
@@ -67,10 +67,9 @@ def extract_named_tool_call_streaming(
tool_call_idx: int | None,
tool_call_id_type: str,
tokenizer: "TokenizerLike",
tool_call_array_index: int,
) -> tuple[DeltaMessage, bool, bool]:
tool_call_array_index: int = 0,
) -> tuple[DeltaMessage | None, bool]:
"""Build a streaming tool-call delta for forced named tool choice."""
created_new_tool_call = False
if function_name_returned:
delta_tool_call = DeltaToolCall(
function=DeltaFunctionCall(arguments=delta_text),
@@ -95,12 +94,9 @@ def extract_named_tool_call_streaming(
index=tool_call_array_index,
)
function_name_returned = True
created_new_tool_call = True
return (
DeltaMessage(tool_calls=[delta_tool_call]),
function_name_returned,
created_new_tool_call,
)