[Frontend] OpenAI Responses API supports Tool/Function calling - non-harmony (#26874)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -1098,13 +1098,13 @@ class OpenAIServing:
|
||||
)
|
||||
|
||||
if should_parse_tools:
|
||||
if not isinstance(request, ChatCompletionRequest):
|
||||
msg = "Tool usage is only supported for Chat Completions API"
|
||||
if not isinstance(request, ChatCompletionRequest | ResponsesRequest):
|
||||
msg = (
|
||||
"Tool usage is only supported for Chat Completions API "
|
||||
"or Responses API requests."
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
request = tool_parser(tokenizer).adjust_request( # type: ignore
|
||||
request=request
|
||||
)
|
||||
request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore
|
||||
|
||||
if tokenizer is None:
|
||||
assert isinstance(request_prompt, str), (
|
||||
|
||||
@@ -94,6 +94,7 @@ from vllm.entrypoints.openai.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.serving_engine import OpenAIServing
|
||||
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
|
||||
from vllm.entrypoints.responses_utils import construct_chat_message_with_tool_call
|
||||
from vllm.entrypoints.tool_server import ToolServer
|
||||
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
@@ -196,16 +197,12 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
self.default_sampling_params["stop_token_ids"].extend(
|
||||
get_stop_tokens_for_assistant_actions()
|
||||
)
|
||||
|
||||
self.enable_auto_tools = enable_auto_tools
|
||||
# set up tool use
|
||||
self.enable_auto_tools: bool = enable_auto_tools
|
||||
if self.enable_auto_tools:
|
||||
logger.info(
|
||||
'"auto" tool choice has been enabled please note that while'
|
||||
" the parallel_tool_calls client option is preset for "
|
||||
"compatibility reasons, it will be ignored."
|
||||
)
|
||||
|
||||
self.tool_parser = self._get_tool_parser(
|
||||
tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
|
||||
)
|
||||
self.exclude_tools_when_tool_choice_none = False
|
||||
# HACK(woosuk): This is a hack. We should use a better store.
|
||||
# FIXME: If enable_store=True, this may cause a memory leak since we
|
||||
# never remove responses from the store.
|
||||
@@ -511,16 +508,20 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
prev_response: ResponsesResponse | None,
|
||||
tokenizer: AnyTokenizer,
|
||||
):
|
||||
if len(request.tools) > 0:
|
||||
raise NotImplementedError(
|
||||
"Tool use is not supported in Responses API without Harmony"
|
||||
)
|
||||
if request.tools is None or (
|
||||
request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
|
||||
):
|
||||
tool_dicts = None
|
||||
else:
|
||||
tool_dicts = [tool.model_dump() for tool in request.tools]
|
||||
# Construct the input messages.
|
||||
messages = self._construct_input_messages(request, prev_response)
|
||||
_, request_prompts, engine_prompts = await self._preprocess_chat(
|
||||
request,
|
||||
tokenizer,
|
||||
messages,
|
||||
tool_dicts=tool_dicts,
|
||||
tool_parser=self.tool_parser,
|
||||
chat_template=self.chat_template,
|
||||
chat_template_content_format=self.chat_template_content_format,
|
||||
)
|
||||
@@ -802,7 +803,8 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
delta=False,
|
||||
)
|
||||
|
||||
output = []
|
||||
reasoning_item = None
|
||||
message_item = None
|
||||
if reasoning_content:
|
||||
reasoning_item = ResponseReasoningItem(
|
||||
id=f"rs_{random_uuid()}",
|
||||
@@ -815,7 +817,13 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
],
|
||||
status=None, # NOTE: Only the last output item has status.
|
||||
)
|
||||
output.append(reasoning_item)
|
||||
tool_calls, content = self._parse_tool_calls_from_content(
|
||||
request=request,
|
||||
tokenizer=tokenizer,
|
||||
content=content,
|
||||
enable_auto_tools=self.enable_auto_tools,
|
||||
tool_parser_cls=self.tool_parser,
|
||||
)
|
||||
if content:
|
||||
output_text = ResponseOutputText(
|
||||
text=content,
|
||||
@@ -832,15 +840,33 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
else None
|
||||
),
|
||||
)
|
||||
message = ResponseOutputMessage(
|
||||
message_item = ResponseOutputMessage(
|
||||
id=f"msg_{random_uuid()}",
|
||||
content=[output_text],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
)
|
||||
output.append(message)
|
||||
return output
|
||||
outputs = []
|
||||
|
||||
if reasoning_item:
|
||||
outputs.append(reasoning_item)
|
||||
if message_item:
|
||||
outputs.append(message_item)
|
||||
if tool_calls:
|
||||
tool_call_items = [
|
||||
ResponseFunctionToolCall(
|
||||
id=f"fc_{random_uuid()}",
|
||||
call_id=f"call_{random_uuid()}",
|
||||
type="function_call",
|
||||
status="completed",
|
||||
name=tool_call.name,
|
||||
arguments=tool_call.arguments,
|
||||
)
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
outputs.extend(tool_call_items)
|
||||
return outputs
|
||||
|
||||
def _make_response_output_items_with_harmony(
|
||||
self,
|
||||
@@ -893,7 +919,8 @@ class OpenAIServingResponses(OpenAIServing):
|
||||
if isinstance(request.input, str):
|
||||
messages.append({"role": "user", "content": request.input})
|
||||
else:
|
||||
messages.extend(request.input) # type: ignore
|
||||
for item in request.input:
|
||||
messages.append(construct_chat_message_with_tool_call(item))
|
||||
return messages
|
||||
|
||||
def _construct_harmony_system_input_message(
|
||||
|
||||
@@ -6,10 +6,16 @@ import os
|
||||
from collections.abc import Callable, Sequence
|
||||
from functools import cached_property
|
||||
|
||||
from openai.types.responses.response_format_text_json_schema_config import (
|
||||
ResponseFormatTextJSONSchemaConfig,
|
||||
)
|
||||
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
DeltaMessage,
|
||||
ExtractedToolCallInformation,
|
||||
ResponsesRequest,
|
||||
ResponseTextConfig,
|
||||
)
|
||||
from vllm.entrypoints.openai.tool_parsers.utils import get_json_schema_from_tools
|
||||
from vllm.logger import init_logger
|
||||
@@ -56,11 +62,21 @@ class ToolParser:
|
||||
)
|
||||
# Set structured output params for tool calling
|
||||
if json_schema_from_tool is not None:
|
||||
if request.structured_outputs is None:
|
||||
if isinstance(request, ChatCompletionRequest):
|
||||
request.structured_outputs = StructuredOutputsParams()
|
||||
# tool_choice: "Forced Function" or "required" will override
|
||||
# structured output json settings to make tool calling work correctly
|
||||
request.structured_outputs.json = json_schema_from_tool
|
||||
# tool_choice: "Forced Function" or "required" will override
|
||||
# structured output json settings to make tool calling work correctly
|
||||
request.structured_outputs.json = json_schema_from_tool
|
||||
if isinstance(request, ResponsesRequest):
|
||||
request.text = ResponseTextConfig()
|
||||
request.text.format = ResponseFormatTextJSONSchemaConfig(
|
||||
name="tool_calling_response",
|
||||
schema=json_schema_from_tool,
|
||||
type="json_schema",
|
||||
description="Response format for tool calling",
|
||||
strict=True,
|
||||
)
|
||||
|
||||
return request
|
||||
|
||||
def extract_tool_calls(
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from openai.types.chat import (
|
||||
ChatCompletionAssistantMessageParam,
|
||||
ChatCompletionMessageToolCallParam,
|
||||
ChatCompletionToolMessageParam,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message_tool_call_param import (
|
||||
Function as FunctionCallTool,
|
||||
)
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
|
||||
from vllm.entrypoints.openai.protocol import (
|
||||
ChatCompletionMessageParam,
|
||||
ResponseInputOutputItem,
|
||||
)
|
||||
|
||||
|
||||
def construct_chat_message_with_tool_call(
|
||||
item: ResponseInputOutputItem,
|
||||
) -> ChatCompletionMessageParam:
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
# Append the function call as a tool call.
|
||||
return ChatCompletionAssistantMessageParam(
|
||||
role="assistant",
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCallParam(
|
||||
id=item.call_id,
|
||||
function=FunctionCallTool(
|
||||
name=item.name,
|
||||
arguments=item.arguments,
|
||||
),
|
||||
type="function",
|
||||
)
|
||||
],
|
||||
)
|
||||
elif item.get("type") == "function_call_output":
|
||||
# Append the function call output as a tool message.
|
||||
return ChatCompletionToolMessageParam(
|
||||
role="tool",
|
||||
content=item.get("output"),
|
||||
tool_call_id=item.get("call_id"),
|
||||
)
|
||||
return item # type: ignore
|
||||
Reference in New Issue
Block a user