[Frontend]Responses API supports Tool/Function calling with streaming with named tool/function (#41110)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -38,7 +38,10 @@ from vllm.logger import init_logger
|
||||
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tool_parsers.abstract_tool_parser import ToolParser
|
||||
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
||||
from vllm.tool_parsers.streaming import (
|
||||
extract_named_tool_call_streaming,
|
||||
extract_required_tool_call_streaming,
|
||||
)
|
||||
from vllm.tool_parsers.utils import Tool
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
@@ -423,6 +426,17 @@ class DelegatingParser(Parser):
|
||||
|
||||
return outputs
|
||||
|
||||
def _get_function_name(
|
||||
self, request: ChatCompletionRequest | ResponsesRequest
|
||||
) -> str:
|
||||
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
|
||||
return request.tool_choice.name
|
||||
if request.tool_choice and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
):
|
||||
return request.tool_choice.function.name
|
||||
raise ValueError("Invalid tool_choice for function name extraction.")
|
||||
|
||||
def _parse_tool_calls(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
@@ -440,21 +454,14 @@ class DelegatingParser(Parser):
|
||||
"""
|
||||
function_calls: list[FunctionCall] = []
|
||||
|
||||
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
|
||||
# Forced Function Call (Responses API style)
|
||||
assert content is not None
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.name, arguments=content)
|
||||
)
|
||||
return function_calls, None # Clear content since tool is called.
|
||||
|
||||
if request.tool_choice and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
request.tool_choice,
|
||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
||||
):
|
||||
# Forced Function Call (Chat Completion API style)
|
||||
# Forced Function Call
|
||||
assert content is not None
|
||||
function_calls.append(
|
||||
FunctionCall(name=request.tool_choice.function.name, arguments=content)
|
||||
FunctionCall(name=self._get_function_name(request), arguments=content)
|
||||
)
|
||||
return function_calls, None # Clear content since tool is called.
|
||||
|
||||
@@ -572,7 +579,7 @@ class DelegatingParser(Parser):
|
||||
previous_token_ids: Sequence[int],
|
||||
current_token_ids: Sequence[int],
|
||||
delta_token_ids: Sequence[int],
|
||||
request: ChatCompletionRequest,
|
||||
request: ChatCompletionRequest | ResponsesRequest,
|
||||
# The following parameters are used for "required" tool choice parsing and are
|
||||
# tracked in StreamState for streaming parsing.
|
||||
tool_call_idx: int | None = None,
|
||||
@@ -580,9 +587,19 @@ class DelegatingParser(Parser):
|
||||
function_name_returned: bool = False,
|
||||
) -> tuple[DeltaMessage | None, bool]:
|
||||
if request.tool_choice and isinstance(
|
||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||
request.tool_choice,
|
||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
||||
):
|
||||
return None, False
|
||||
delta_message, function_name_returned = extract_named_tool_call_streaming(
|
||||
delta_text=delta_text,
|
||||
function_name=self._get_function_name(request),
|
||||
function_name_returned=function_name_returned,
|
||||
tool_call_idx=tool_call_idx,
|
||||
tool_call_id_type=tool_call_id_type,
|
||||
tokenizer=self.model_tokenizer,
|
||||
)
|
||||
return delta_message, function_name_returned
|
||||
|
||||
if request.tool_choice == "required":
|
||||
delta_message, function_name_returned = (
|
||||
extract_required_tool_call_streaming(
|
||||
@@ -602,7 +619,7 @@ class DelegatingParser(Parser):
|
||||
previous_token_ids,
|
||||
current_token_ids,
|
||||
delta_token_ids,
|
||||
request,
|
||||
request, # type: ignore[arg-type]
|
||||
), False
|
||||
|
||||
def is_reasoning_end(self, input_ids: list[int]) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user