[Frontend]Responses API supports Tool/Function calling with streaming with named tool/function (#41110)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -323,7 +323,7 @@ async def test_function_calling_with_streaming_expected_arguments(
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
["auto", "required"],
|
||||
["auto", "required", {"type": "function", "name": "get_current_weather"}],
|
||||
)
|
||||
async def test_function_calling_with_streaming_types(
|
||||
client: openai.AsyncOpenAI, model_name: str, tool_choice
|
||||
@@ -462,7 +462,7 @@ async def test_function_calling_with_streaming_types(
|
||||
@pytest.mark.parametrize("model_name", [MODEL_NAME])
|
||||
@pytest.mark.parametrize(
|
||||
"tool_choice",
|
||||
["required", "auto"],
|
||||
["required", "auto", {"type": "function", "name": "get_weather"}],
|
||||
)
|
||||
async def test_function_calling_with_streaming_forced_tool_choice(
|
||||
client: openai.AsyncOpenAI, model_name: str, tool_choice: str
|
||||
|
||||
@@ -70,7 +70,10 @@ from vllm.reasoning import ReasoningParser
|
||||
from vllm.renderers import ChatParams
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
||||
from vllm.tool_parsers.streaming import (
|
||||
extract_named_tool_call_streaming,
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.utils.collection_utils import as_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
||||
|
||||
@@ -773,43 +776,24 @@ class OpenAIServingChat(OpenAIServing):
|
||||
delta_text = previous_text + delta_text
|
||||
current_text = ""
|
||||
|
||||
if function_name_returned[i]:
|
||||
delta_tool_call = DeltaToolCall(
|
||||
function=DeltaFunctionCall(arguments=delta_text),
|
||||
index=i,
|
||||
delta_message, function_name_returned[i] = (
|
||||
extract_named_tool_call_streaming(
|
||||
delta_text=delta_text,
|
||||
function_name=tool_choice_function_name,
|
||||
function_name_returned=function_name_returned[i],
|
||||
tool_call_idx=history_tool_call_cnt,
|
||||
tool_call_id_type=self.tool_call_id_type,
|
||||
tokenizer=tokenizer,
|
||||
tool_call_array_index=i,
|
||||
)
|
||||
else:
|
||||
# Generate ID based on tokenizer type
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
from vllm.tool_parsers.mistral_tool_parser import (
|
||||
MistralToolCall,
|
||||
)
|
||||
|
||||
tool_call_id = MistralToolCall.generate_random_id()
|
||||
else:
|
||||
tool_call_id = make_tool_call_id(
|
||||
id_type=self.tool_call_id_type,
|
||||
func_name=tool_choice_function_name,
|
||||
idx=history_tool_call_cnt,
|
||||
)
|
||||
delta_tool_call = DeltaToolCall(
|
||||
id=tool_call_id,
|
||||
type="function",
|
||||
function=DeltaFunctionCall(
|
||||
name=tool_choice_function_name,
|
||||
arguments=delta_text,
|
||||
),
|
||||
index=i,
|
||||
)
|
||||
function_name_returned[i] = True
|
||||
history_tool_call_cnt += 1
|
||||
|
||||
delta_message = DeltaMessage(
|
||||
tool_calls=[
|
||||
delta_tool_call,
|
||||
]
|
||||
)
|
||||
tools_streamed[i] = True
|
||||
if (
|
||||
delta_message
|
||||
and delta_message.tool_calls
|
||||
and delta_message.tool_calls[0].id is not None
|
||||
):
|
||||
history_tool_call_cnt += 1
|
||||
tools_streamed[i] = True
|
||||
|
||||
# Skip when tool_choice_uses_parser so it falls through
|
||||
# to the auto tool_parser branches below.
|
||||
|
||||
@@ -38,7 +38,10 @@ from vllm.logger import init_logger
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
||||
from vllm.tool_parsers.streaming import (
|
||||
extract_named_tool_call_streaming,
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.tool_parsers.utils import Tool
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
@@ -423,6 +426,17 @@ class DelegatingParser(Parser):
|
||||
|
||||
return outputs
|
||||
|
||||
def _get_function_name(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> str:
|
||||
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
|
||||
return request.tool_choice.name
|
||||
if request.tool_choice and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
):
|
||||
return request.tool_choice.function.name
|
||||
raise ValueError("Invalid tool_choice for function name extraction.")
|
||||
|
||||
def _parse_tool_calls(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
@@ -440,21 +454,14 @@ class DelegatingParser(Parser):
|
||||
"""
|
||||
function_calls: list[FunctionCall] = []
|
||||
|
||||
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
|
||||
# Forced Function Call (Responses API style)
|
||||
assert content is not None
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.name, arguments=content)
|
||||
)
|
||||
return function_calls, None # Clear content since tool is called.
|
||||
|
||||
if request.tool_choice and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
request.tool_choice,
|
||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
||||
):
|
||||
# Forced Function Call (Chat Completion API style)
|
||||
# Forced Function Call
|
||||
assert content is not None
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.function.name, arguments=content)
|
||||
FunctionCall(name=self._get_function_name(request), arguments=content)
|
||||
)
|
||||
return function_calls, None # Clear content since tool is called.
|
||||
|
||||
@@ -572,7 +579,7 @@ class DelegatingParser(Parser):
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
# The following parameters are used for "required" tool choice parsing and are
|
||||
# tracked in StreamState for streaming parsing.
|
||||
tool_call_idx: int | None = None,
|
||||
@@ -580,9 +587,19 @@ class DelegatingParser(Parser):
|
||||
function_name_returned: bool = False,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
if request.tool_choice and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
request.tool_choice,
|
||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
||||
):
|
||||
return None, False
|
||||
delta_message, function_name_returned = extract_named_tool_call_streaming(
|
||||
delta_text=delta_text,
|
||||
function_name=self._get_function_name(request),
|
||||
function_name_returned=function_name_returned,
|
||||
tool_call_idx=tool_call_idx,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
tokenizer=self.model_tokenizer,
|
||||
)
|
||||
return delta_message, function_name_returned
|
||||
|
||||
if request.tool_choice == "required":
|
||||
delta_message, function_name_returned = (
|
||||
extract_required_tool_call_streaming(
|
||||
@@ -602,7 +619,7 @@ class DelegatingParser(Parser):
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request,
|
||||
request, # type: ignore[arg-type]
|
||||
), False
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
|
||||
@@ -67,10 +67,9 @@ def extract_named_tool_call_streaming(
|
||||
tool_call_idx: int | None,
|
||||
tool_call_id_type: str,
|
||||
tokenizer: "TokenizerLike",
|
||||
tool_call_array_index: int,
|
||||
) -> tuple[DeltaMessage, bool, bool]:
|
||||
tool_call_array_index: int = 0,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
"""Build a streaming tool-call delta for forced named tool choice."""
|
||||
created_new_tool_call = False
|
||||
if function_name_returned:
|
||||
delta_tool_call = DeltaToolCall(
|
||||
function=DeltaFunctionCall(arguments=delta_text),
|
||||
@@ -95,12 +94,9 @@ def extract_named_tool_call_streaming(
|
||||
index=tool_call_array_index,
|
||||
)
|
||||
function_name_returned = True
|
||||
created_new_tool_call = True
|
||||
|
||||
return (
|
||||
DeltaMessage(tool_calls=[delta_tool_call]),
|
||||
function_name_returned,
|
||||
created_new_tool_call,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user