From 3885d340a4779c54662b10892555ae6928b3a090 Mon Sep 17 00:00:00 2001 From: Chauncey Date: Wed, 29 Apr 2026 17:11:27 +0800 Subject: [PATCH] [Frontend]Responses API supports Tool/Function calling with streaming with named tool/function (#41110) Signed-off-by: chaunceyjiang --- .../openai/responses/test_function_call.py | 4 +- .../openai/chat_completion/serving.py | 56 +++++++------------ vllm/parser/abstract_parser.py | 49 ++++++++++------ vllm/tool_parsers/streaming.py | 8 +-- 4 files changed, 57 insertions(+), 60 deletions(-) diff --git a/tests/entrypoints/openai/responses/test_function_call.py b/tests/entrypoints/openai/responses/test_function_call.py index 515f31b399..8ca43feaca 100644 --- a/tests/entrypoints/openai/responses/test_function_call.py +++ b/tests/entrypoints/openai/responses/test_function_call.py @@ -323,7 +323,7 @@ async def test_function_calling_with_streaming_expected_arguments( @pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize( "tool_choice", - ["auto", "required"], + ["auto", "required", {"type": "function", "name": "get_current_weather"}], ) async def test_function_calling_with_streaming_types( client: openai.AsyncOpenAI, model_name: str, tool_choice @@ -462,7 +462,7 @@ async def test_function_calling_with_streaming_types( @pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize( "tool_choice", - ["required", "auto"], + ["required", "auto", {"type": "function", "name": "get_weather"}], ) async def test_function_calling_with_streaming_forced_tool_choice( client: openai.AsyncOpenAI, model_name: str, tool_choice: str diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index f001aac15a..b8ad54adb5 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -70,7 +70,10 @@ from vllm.reasoning import ReasoningParser from vllm.renderers import ChatParams from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.streaming import extract_required_tool_call_streaming +from vllm.tool_parsers.streaming import ( + extract_named_tool_call_streaming, + extract_required_tool_call_streaming, +) from vllm.utils.collection_utils import as_list from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser @@ -773,43 +776,24 @@ class OpenAIServingChat(OpenAIServing): delta_text = previous_text + delta_text current_text = "" - if function_name_returned[i]: - delta_tool_call = DeltaToolCall( - function=DeltaFunctionCall(arguments=delta_text), - index=i, + delta_message, function_name_returned[i] = ( + extract_named_tool_call_streaming( + delta_text=delta_text, + function_name=tool_choice_function_name, + function_name_returned=function_name_returned[i], + tool_call_idx=history_tool_call_cnt, + tool_call_id_type=self.tool_call_id_type, + tokenizer=tokenizer, + tool_call_array_index=i, ) - else: - # Generate ID based on tokenizer type - if is_mistral_tokenizer(tokenizer): - from vllm.tool_parsers.mistral_tool_parser import ( - MistralToolCall, - ) - - tool_call_id = MistralToolCall.generate_random_id() - else: - tool_call_id = make_tool_call_id( - id_type=self.tool_call_id_type, - func_name=tool_choice_function_name, - idx=history_tool_call_cnt, - ) - delta_tool_call = DeltaToolCall( - id=tool_call_id, - type="function", - function=DeltaFunctionCall( - name=tool_choice_function_name, - arguments=delta_text, - ), - index=i, - ) - function_name_returned[i] = True - history_tool_call_cnt += 1 - - delta_message = DeltaMessage( - tool_calls=[ - delta_tool_call, - ] ) - tools_streamed[i] = True + if ( + delta_message + and delta_message.tool_calls + and delta_message.tool_calls[0].id is not None + ): + history_tool_call_cnt += 1 + tools_streamed[i] = True # Skip when tool_choice_uses_parser so it falls through # to the auto tool_parser branches below. diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 5943e5aafb..e7f83686db 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -38,7 +38,10 @@ from vllm.logger import init_logger from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import ToolParser -from vllm.tool_parsers.streaming import extract_required_tool_call_streaming +from vllm.tool_parsers.streaming import ( + extract_named_tool_call_streaming, + extract_required_tool_call_streaming, +) from vllm.tool_parsers.utils import Tool from vllm.utils import random_uuid @@ -423,6 +426,17 @@ class DelegatingParser(Parser): return outputs + def _get_function_name( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> str: + if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction): + return request.tool_choice.name + if request.tool_choice and isinstance( + request.tool_choice, ChatCompletionNamedToolChoiceParam + ): + return request.tool_choice.function.name + raise ValueError("Invalid tool_choice for function name extraction.") + def _parse_tool_calls( self, request: ResponsesRequest, @@ -440,21 +454,14 @@ class DelegatingParser(Parser): """ function_calls: list[FunctionCall] = [] - if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction): - # Forced Function Call (Responses API style) - assert content is not None - function_calls.append( - FunctionCall(name=request.tool_choice.name, arguments=content) - ) - return function_calls, None # Clear content since tool is called. - if request.tool_choice and isinstance( - request.tool_choice, ChatCompletionNamedToolChoiceParam + request.tool_choice, + (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam), ): - # Forced Function Call (Chat Completion API style) + # Forced Function Call assert content is not None function_calls.append( - FunctionCall(name=request.tool_choice.function.name, arguments=content) + FunctionCall(name=self._get_function_name(request), arguments=content) ) return function_calls, None # Clear content since tool is called. @@ -572,7 +579,7 @@ class DelegatingParser(Parser): previous_token_ids: Sequence[int], current_token_ids: Sequence[int], delta_token_ids: Sequence[int], - request: ChatCompletionRequest, + request: ChatCompletionRequest | ResponsesRequest, # The following parameters are used for "required" tool choice parsing and are # tracked in StreamState for streaming parsing. tool_call_idx: int | None = None, @@ -580,9 +587,19 @@ class DelegatingParser(Parser): function_name_returned: bool = False, ) -> tuple[DeltaMessage | None, bool]: if request.tool_choice and isinstance( - request.tool_choice, ChatCompletionNamedToolChoiceParam + request.tool_choice, + (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam), ): - return None, False + delta_message, function_name_returned = extract_named_tool_call_streaming( + delta_text=delta_text, + function_name=self._get_function_name(request), + function_name_returned=function_name_returned, + tool_call_idx=tool_call_idx, + tool_call_id_type=tool_call_id_type, + tokenizer=self.model_tokenizer, + ) + return delta_message, function_name_returned + if request.tool_choice == "required": delta_message, function_name_returned = ( extract_required_tool_call_streaming( @@ -602,7 +619,7 @@ class DelegatingParser(Parser): previous_token_ids, current_token_ids, delta_token_ids, - request, + request, # type: ignore[arg-type] ), False def is_reasoning_end(self, input_ids: list[int]) -> bool: diff --git a/vllm/tool_parsers/streaming.py b/vllm/tool_parsers/streaming.py index fc903328e3..7f6638dcb9 100644 --- a/vllm/tool_parsers/streaming.py +++ b/vllm/tool_parsers/streaming.py @@ -67,10 +67,9 @@ def extract_named_tool_call_streaming( tool_call_idx: int | None, tool_call_id_type: str, tokenizer: "TokenizerLike", - tool_call_array_index: int, -) -> tuple[DeltaMessage, bool, bool]: + tool_call_array_index: int = 0, +) -> tuple[DeltaMessage | None, bool]: """Build a streaming tool-call delta for forced named tool choice.""" - created_new_tool_call = False if function_name_returned: delta_tool_call = DeltaToolCall( function=DeltaFunctionCall(arguments=delta_text), @@ -95,12 +94,9 @@ def extract_named_tool_call_streaming( index=tool_call_array_index, ) function_name_returned = True - created_new_tool_call = True - return ( DeltaMessage(tool_calls=[delta_tool_call]), function_name_returned, - created_new_tool_call, )