[Frontend] Support CustomToolCall in the non-Harmony Responses API (#57322)
Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -14,6 +14,8 @@ from openai.types.responses import (
|
||||
ResponseCodeInterpreterCallInterpretingEvent,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseContentPartDoneEvent,
|
||||
ResponseCustomToolCallInputDeltaEvent,
|
||||
ResponseCustomToolCallInputDoneEvent,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseInputItemParam,
|
||||
ResponseMcpCallArgumentsDeltaEvent,
|
||||
@@ -699,6 +701,34 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
data["input"] = processed_input
|
||||
return data
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_custom_tool_format(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
tools = data.get("tools")
|
||||
if not isinstance(tools, list):
|
||||
return data
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
tool_dict = tool
|
||||
elif hasattr(tool, "model_dump"):
|
||||
tool_dict = tool.model_dump()
|
||||
else:
|
||||
continue
|
||||
tool_format = tool_dict.get("format")
|
||||
if (
|
||||
tool_dict.get("type") == "custom"
|
||||
and isinstance(tool_format, dict)
|
||||
and tool_format.get("type") == "grammar"
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Custom tools with a grammar format are not supported; "
|
||||
"only free-form text custom tools are accepted.",
|
||||
parameter="tools",
|
||||
)
|
||||
return data
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_tool_usage(cls, data):
|
||||
@@ -708,9 +738,9 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
tools = data.get("tools")
|
||||
tool_choice = data.get("tool_choice", "auto")
|
||||
has_tools = tools is not None and len(tools) > 0
|
||||
is_named_tool_choice = (
|
||||
isinstance(tool_choice, dict) and tool_choice.get("type") == "function"
|
||||
)
|
||||
is_named_tool_choice = isinstance(tool_choice, dict) and tool_choice.get(
|
||||
"type"
|
||||
) in ("function", "custom")
|
||||
|
||||
if not has_tools:
|
||||
if tool_choice in ("auto", "none"):
|
||||
@@ -913,6 +943,8 @@ StreamingResponsesResponse: TypeAlias = (
|
||||
| ResponseOutputItemDoneEvent
|
||||
| ResponseContentPartAddedEvent
|
||||
| ResponseContentPartDoneEvent
|
||||
| ResponseCustomToolCallInputDeltaEvent
|
||||
| ResponseCustomToolCallInputDoneEvent
|
||||
| ResponseReasoningTextDeltaEvent
|
||||
| ResponseReasoningTextDoneEvent
|
||||
| ResponseCodeInterpreterCallInProgressEvent
|
||||
|
||||
@@ -29,6 +29,9 @@ from openai.types.responses import (
|
||||
ResponseCodeInterpreterToolCallParam,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseContentPartDoneEvent,
|
||||
ResponseCustomToolCall,
|
||||
ResponseCustomToolCallInputDeltaEvent,
|
||||
ResponseCustomToolCallInputDoneEvent,
|
||||
ResponseFunctionCallArgumentsDeltaEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
ResponseFunctionToolCall,
|
||||
@@ -73,6 +76,9 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_responses_tool_call_name_map,
|
||||
custom_tool_names,
|
||||
decode_custom_tool_input,
|
||||
decode_custom_tool_input_prefix,
|
||||
resolve_responses_tool_call_name,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput
|
||||
@@ -821,6 +827,8 @@ class SimpleStreamingState:
|
||||
tool_call_name: str = ""
|
||||
tool_call_namespace: str | None = None
|
||||
tool_call_index: int | None = None
|
||||
tool_call_is_custom: bool = False
|
||||
tool_call_payload: str = ""
|
||||
current_state: _StateType = field(default_factory=lambda: _StateType.NONE)
|
||||
|
||||
|
||||
@@ -1039,38 +1047,80 @@ def emit_simple_tool_call_open(
|
||||
name: str,
|
||||
index: int | None,
|
||||
namespace: str | None = None,
|
||||
is_custom: bool = False,
|
||||
call_id: str | None = None,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
state.current_state = _StateType.TOOL_CALL
|
||||
state.current_item_id = f"fc_{random_uuid()}"
|
||||
state.current_item_id = (
|
||||
f"ctc_{random_uuid()}" if is_custom else f"fc_{random_uuid()}"
|
||||
)
|
||||
state.tool_call_id = call_id or make_tool_call_id()
|
||||
state.tool_call_name = name
|
||||
state.tool_call_namespace = namespace
|
||||
state.tool_call_index = index
|
||||
state.tool_call_is_custom = is_custom
|
||||
state.tool_call_payload = ""
|
||||
state.accumulated_text = ""
|
||||
item: ResponseOutputItem
|
||||
if is_custom:
|
||||
item = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=name,
|
||||
input="",
|
||||
)
|
||||
else:
|
||||
item = ResponseFunctionToolCallItem(
|
||||
type="function_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=name,
|
||||
namespace=namespace,
|
||||
arguments="",
|
||||
status="in_progress",
|
||||
)
|
||||
return [
|
||||
ResponseOutputItemAddedEvent(
|
||||
type="response.output_item.added",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item=ResponseFunctionToolCallItem(
|
||||
type="function_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=name,
|
||||
namespace=namespace,
|
||||
arguments="",
|
||||
status="in_progress",
|
||||
),
|
||||
item=item,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _emit_custom_tool_input_delta(
|
||||
state: SimpleStreamingState,
|
||||
payload: str,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
streamed = state.tool_call_payload
|
||||
if payload == streamed:
|
||||
return []
|
||||
state.tool_call_payload = payload
|
||||
# A payload that is not an extension of what was streamed cannot be
|
||||
# retracted, so the authoritative value is resent in full.
|
||||
delta = payload[len(streamed) :] if payload.startswith(streamed) else payload
|
||||
return [
|
||||
ResponseCustomToolCallInputDeltaEvent(
|
||||
type="response.custom_tool_call_input.delta",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
delta=delta,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def emit_simple_tool_call_delta(
|
||||
state: SimpleStreamingState,
|
||||
delta: str,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
state.accumulated_text += delta
|
||||
if state.tool_call_is_custom:
|
||||
return _emit_custom_tool_input_delta(
|
||||
state, decode_custom_tool_input_prefix(state.accumulated_text)
|
||||
)
|
||||
return [
|
||||
ResponseFunctionCallArgumentsDeltaEvent(
|
||||
type="response.function_call_arguments.delta",
|
||||
@@ -1085,32 +1135,58 @@ def emit_simple_tool_call_delta(
|
||||
def emit_simple_tool_call_done(
|
||||
state: SimpleStreamingState,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
events: list[StreamingResponsesResponse] = [
|
||||
ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
arguments=state.accumulated_text,
|
||||
events: list[StreamingResponsesResponse]
|
||||
item: ResponseOutputItem
|
||||
if state.tool_call_is_custom:
|
||||
payload = decode_custom_tool_input(state.accumulated_text)
|
||||
events = _emit_custom_tool_input_delta(state, payload)
|
||||
events.append(
|
||||
ResponseCustomToolCallInputDoneEvent(
|
||||
type="response.custom_tool_call_input.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
input=payload,
|
||||
)
|
||||
)
|
||||
item = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=state.tool_call_name,
|
||||
),
|
||||
input=payload,
|
||||
)
|
||||
else:
|
||||
events = [
|
||||
ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
arguments=state.accumulated_text,
|
||||
name=state.tool_call_name,
|
||||
)
|
||||
]
|
||||
item = ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=state.tool_call_name,
|
||||
namespace=state.tool_call_namespace,
|
||||
arguments=state.accumulated_text,
|
||||
status="completed",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
)
|
||||
events.append(
|
||||
ResponseOutputItemDoneEvent(
|
||||
type="response.output_item.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item=ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=state.tool_call_name,
|
||||
namespace=state.tool_call_namespace,
|
||||
arguments=state.accumulated_text,
|
||||
status="completed",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
),
|
||||
),
|
||||
]
|
||||
item=item,
|
||||
)
|
||||
)
|
||||
state.output_index += 1
|
||||
state.tool_call_namespace = None
|
||||
state.tool_call_is_custom = False
|
||||
state.current_state = _StateType.NONE
|
||||
return events
|
||||
|
||||
@@ -1194,6 +1270,7 @@ class SimpleStreamingEventProcessor:
|
||||
) -> None:
|
||||
self.state = state or SimpleStreamingState()
|
||||
self.tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||
self.custom_tool_names = custom_tool_names(tools)
|
||||
self.output_items: list[ResponseOutputItem] = []
|
||||
|
||||
def resolve_target_state(
|
||||
@@ -1264,6 +1341,7 @@ class SimpleStreamingEventProcessor:
|
||||
call_name.name,
|
||||
tool_call.index,
|
||||
call_name.namespace,
|
||||
call_name.name in self.custom_tool_names,
|
||||
tool_call.id,
|
||||
)
|
||||
return handlers.open_fn(self.state)
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import partial_json_parser
|
||||
from openai.types.chat import (
|
||||
ChatCompletionAssistantMessageParam,
|
||||
ChatCompletionMessageToolCallParam,
|
||||
@@ -13,6 +15,7 @@ from openai.types.chat.chat_completion_message_tool_call_param import (
|
||||
Function as FunctionCallTool,
|
||||
)
|
||||
from openai.types.responses import (
|
||||
ResponseCustomToolCall,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputItem,
|
||||
ResponseOutputMessage,
|
||||
@@ -28,6 +31,7 @@ from openai.types.responses.response_reasoning_item import (
|
||||
Content as ResponseReasoningTextContent,
|
||||
)
|
||||
from openai.types.responses.tool import Tool
|
||||
from partial_json_parser.core.options import Allow
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition
|
||||
@@ -40,6 +44,7 @@ from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.chat_utils import make_tool_call_id
|
||||
from vllm.tool_parsers.utils import (
|
||||
CUSTOM_TOOL_INPUT_KEY,
|
||||
build_responses_tool_call_name_map,
|
||||
flat_namespace_tool_name,
|
||||
iter_response_function_tool_dicts,
|
||||
@@ -50,6 +55,51 @@ from vllm.utils import random_uuid
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def custom_tool_names(tools: list[Tool] | None) -> frozenset[str]:
|
||||
return frozenset(tool.name for tool in tools or [] if tool.type == "custom")
|
||||
|
||||
|
||||
def encode_custom_tool_input(payload: str) -> str:
|
||||
return json.dumps({CUSTOM_TOOL_INPUT_KEY: payload}, ensure_ascii=False)
|
||||
|
||||
|
||||
def _custom_tool_payload(parsed: Any) -> str | None:
|
||||
if not isinstance(parsed, dict):
|
||||
return None
|
||||
value = parsed.get(CUSTOM_TOOL_INPUT_KEY)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def decode_custom_tool_input(arguments: str) -> str:
|
||||
"""Payload of a completed shim call; bare text is passed through."""
|
||||
try:
|
||||
payload = _custom_tool_payload(json.loads(arguments))
|
||||
except ValueError:
|
||||
return arguments
|
||||
return arguments if payload is None else payload
|
||||
|
||||
|
||||
def decode_custom_tool_input_prefix(arguments: str) -> str:
|
||||
"""Longest decodable payload prefix of partial shim arguments, so that
|
||||
streamed ``custom_tool_call_input`` deltas stay a prefix of the final input."""
|
||||
try:
|
||||
parsed = partial_json_parser.loads(arguments, Allow.STR | Allow.OBJ)
|
||||
except (ValueError, IndexError, AssertionError):
|
||||
return ""
|
||||
payload = _custom_tool_payload(parsed) or ""
|
||||
if payload and "\ud800" <= payload[-1] <= "\udbff":
|
||||
payload = payload[:-1]
|
||||
return payload
|
||||
|
||||
|
||||
def _item_type(item: ResponseInputOutputItem) -> str | None:
|
||||
return item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
|
||||
|
||||
|
||||
def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
|
||||
return item if isinstance(item, dict) else item.model_dump()
|
||||
|
||||
|
||||
def build_response_output_items(
|
||||
reasoning: str | None,
|
||||
content: str | None,
|
||||
@@ -59,6 +109,7 @@ def build_response_output_items(
|
||||
) -> list[ResponseOutputItem]:
|
||||
outputs: list[ResponseOutputItem] = []
|
||||
tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||
custom_names = custom_tool_names(tools)
|
||||
|
||||
if reasoning:
|
||||
outputs.append(
|
||||
@@ -93,14 +144,27 @@ def build_response_output_items(
|
||||
|
||||
if tool_calls:
|
||||
for idx, tool_call in enumerate(tool_calls):
|
||||
call_id = tool_call.id or make_tool_call_id(
|
||||
func_name=tool_call.name, idx=idx
|
||||
)
|
||||
if tool_call.name in custom_names:
|
||||
outputs.append(
|
||||
ResponseCustomToolCall(
|
||||
id=f"ctc_{random_uuid()}",
|
||||
call_id=call_id,
|
||||
type="custom_tool_call",
|
||||
name=tool_call.name,
|
||||
input=decode_custom_tool_input(tool_call.arguments),
|
||||
)
|
||||
)
|
||||
continue
|
||||
call_name = resolve_responses_tool_call_name(
|
||||
tool_call.name, tool_call_name_map=tool_call_name_map
|
||||
)
|
||||
outputs.append(
|
||||
ResponseFunctionToolCall(
|
||||
id=f"fc_{random_uuid()}",
|
||||
call_id=tool_call.id
|
||||
or make_tool_call_id(func_name=tool_call.name, idx=idx),
|
||||
call_id=call_id,
|
||||
type="function_call",
|
||||
status="completed",
|
||||
name=call_name.name,
|
||||
@@ -255,6 +319,7 @@ def _construct_message_from_response_item(
|
||||
prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None
|
||||
)
|
||||
|
||||
tool_call: ChatCompletionMessageToolCallParam | None = None
|
||||
if isinstance(item, ResponseFunctionToolCall):
|
||||
tool_name = item.name
|
||||
if item.namespace:
|
||||
@@ -267,6 +332,17 @@ def _construct_message_from_response_item(
|
||||
),
|
||||
type="function",
|
||||
)
|
||||
elif _item_type(item) == "custom_tool_call":
|
||||
call = _item_dict(item)
|
||||
tool_call = ChatCompletionMessageToolCallParam(
|
||||
id=call["call_id"],
|
||||
function=FunctionCallTool(
|
||||
name=call["name"],
|
||||
arguments=encode_custom_tool_input(call.get("input") or ""),
|
||||
),
|
||||
type="function",
|
||||
)
|
||||
if tool_call is not None:
|
||||
if prev_assistant_msg:
|
||||
tool_calls = prev_assistant_msg.get("tool_calls")
|
||||
if tool_calls is None:
|
||||
@@ -285,14 +361,14 @@ def _construct_message_from_response_item(
|
||||
logger.warning(
|
||||
"Previous assistant message has unknown tool_calls format. "
|
||||
"Tool call merging is skipped and a new assistant message is created. "
|
||||
"Item %s",
|
||||
item.id,
|
||||
"Call %s",
|
||||
tool_call["id"],
|
||||
)
|
||||
return ChatCompletionAssistantMessageParam(
|
||||
role="assistant",
|
||||
tool_calls=[tool_call],
|
||||
)
|
||||
elif isinstance(item, ResponseReasoningItem):
|
||||
if isinstance(item, ResponseReasoningItem):
|
||||
reasoning = ""
|
||||
if item.encrypted_content:
|
||||
raise VLLMValidationError(
|
||||
@@ -336,12 +412,12 @@ def _construct_message_from_response_item(
|
||||
content=item.output,
|
||||
tool_call_id=item.call_id,
|
||||
)
|
||||
elif isinstance(item, dict) and item.get("type") == "function_call_output":
|
||||
# Append the function call output as a tool message.
|
||||
elif _item_type(item) in ("function_call_output", "custom_tool_call_output"):
|
||||
output = _item_dict(item)
|
||||
return ChatCompletionToolMessageParam(
|
||||
role="tool",
|
||||
content=item.get("output"),
|
||||
tool_call_id=item.get("call_id"),
|
||||
content=output.get("output"),
|
||||
tool_call_id=output.get("call_id"),
|
||||
)
|
||||
elif isinstance(item, dict) and item.get("role") == "assistant":
|
||||
content = item.get("content")
|
||||
|
||||
Reference in New Issue
Block a user