[Frontend] Support CustomToolCall in the non-Harmony Responses API (#57322)
Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -29,6 +29,9 @@ from openai.types.responses import (
|
||||
ResponseCodeInterpreterToolCallParam,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseContentPartDoneEvent,
|
||||
ResponseCustomToolCall,
|
||||
ResponseCustomToolCallInputDeltaEvent,
|
||||
ResponseCustomToolCallInputDoneEvent,
|
||||
ResponseFunctionCallArgumentsDeltaEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
ResponseFunctionToolCall,
|
||||
@@ -73,6 +76,9 @@ from vllm.entrypoints.openai.responses.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.openai.responses.utils import (
|
||||
build_responses_tool_call_name_map,
|
||||
custom_tool_names,
|
||||
decode_custom_tool_input,
|
||||
decode_custom_tool_input_prefix,
|
||||
resolve_responses_tool_call_name,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput
|
||||
@@ -821,6 +827,8 @@ class SimpleStreamingState:
|
||||
tool_call_name: str = ""
|
||||
tool_call_namespace: str | None = None
|
||||
tool_call_index: int | None = None
|
||||
tool_call_is_custom: bool = False
|
||||
tool_call_payload: str = ""
|
||||
current_state: _StateType = field(default_factory=lambda: _StateType.NONE)
|
||||
|
||||
|
||||
@@ -1039,38 +1047,80 @@ def emit_simple_tool_call_open(
|
||||
name: str,
|
||||
index: int | None,
|
||||
namespace: str | None = None,
|
||||
is_custom: bool = False,
|
||||
call_id: str | None = None,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
state.current_state = _StateType.TOOL_CALL
|
||||
state.current_item_id = f"fc_{random_uuid()}"
|
||||
state.current_item_id = (
|
||||
f"ctc_{random_uuid()}" if is_custom else f"fc_{random_uuid()}"
|
||||
)
|
||||
state.tool_call_id = call_id or make_tool_call_id()
|
||||
state.tool_call_name = name
|
||||
state.tool_call_namespace = namespace
|
||||
state.tool_call_index = index
|
||||
state.tool_call_is_custom = is_custom
|
||||
state.tool_call_payload = ""
|
||||
state.accumulated_text = ""
|
||||
item: ResponseOutputItem
|
||||
if is_custom:
|
||||
item = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=name,
|
||||
input="",
|
||||
)
|
||||
else:
|
||||
item = ResponseFunctionToolCallItem(
|
||||
type="function_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=name,
|
||||
namespace=namespace,
|
||||
arguments="",
|
||||
status="in_progress",
|
||||
)
|
||||
return [
|
||||
ResponseOutputItemAddedEvent(
|
||||
type="response.output_item.added",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item=ResponseFunctionToolCallItem(
|
||||
type="function_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=name,
|
||||
namespace=namespace,
|
||||
arguments="",
|
||||
status="in_progress",
|
||||
),
|
||||
item=item,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _emit_custom_tool_input_delta(
|
||||
state: SimpleStreamingState,
|
||||
payload: str,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
streamed = state.tool_call_payload
|
||||
if payload == streamed:
|
||||
return []
|
||||
state.tool_call_payload = payload
|
||||
# A payload that is not an extension of what was streamed cannot be
|
||||
# retracted, so the authoritative value is resent in full.
|
||||
delta = payload[len(streamed) :] if payload.startswith(streamed) else payload
|
||||
return [
|
||||
ResponseCustomToolCallInputDeltaEvent(
|
||||
type="response.custom_tool_call_input.delta",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
delta=delta,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def emit_simple_tool_call_delta(
|
||||
state: SimpleStreamingState,
|
||||
delta: str,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
state.accumulated_text += delta
|
||||
if state.tool_call_is_custom:
|
||||
return _emit_custom_tool_input_delta(
|
||||
state, decode_custom_tool_input_prefix(state.accumulated_text)
|
||||
)
|
||||
return [
|
||||
ResponseFunctionCallArgumentsDeltaEvent(
|
||||
type="response.function_call_arguments.delta",
|
||||
@@ -1085,32 +1135,58 @@ def emit_simple_tool_call_delta(
|
||||
def emit_simple_tool_call_done(
|
||||
state: SimpleStreamingState,
|
||||
) -> list[StreamingResponsesResponse]:
|
||||
events: list[StreamingResponsesResponse] = [
|
||||
ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
arguments=state.accumulated_text,
|
||||
events: list[StreamingResponsesResponse]
|
||||
item: ResponseOutputItem
|
||||
if state.tool_call_is_custom:
|
||||
payload = decode_custom_tool_input(state.accumulated_text)
|
||||
events = _emit_custom_tool_input_delta(state, payload)
|
||||
events.append(
|
||||
ResponseCustomToolCallInputDoneEvent(
|
||||
type="response.custom_tool_call_input.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
input=payload,
|
||||
)
|
||||
)
|
||||
item = ResponseCustomToolCall(
|
||||
type="custom_tool_call",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
name=state.tool_call_name,
|
||||
),
|
||||
input=payload,
|
||||
)
|
||||
else:
|
||||
events = [
|
||||
ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item_id=state.current_item_id,
|
||||
arguments=state.accumulated_text,
|
||||
name=state.tool_call_name,
|
||||
)
|
||||
]
|
||||
item = ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=state.tool_call_name,
|
||||
namespace=state.tool_call_namespace,
|
||||
arguments=state.accumulated_text,
|
||||
status="completed",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
)
|
||||
events.append(
|
||||
ResponseOutputItemDoneEvent(
|
||||
type="response.output_item.done",
|
||||
sequence_number=-1,
|
||||
output_index=state.output_index,
|
||||
item=ResponseFunctionToolCall(
|
||||
type="function_call",
|
||||
name=state.tool_call_name,
|
||||
namespace=state.tool_call_namespace,
|
||||
arguments=state.accumulated_text,
|
||||
status="completed",
|
||||
id=state.current_item_id,
|
||||
call_id=state.tool_call_id,
|
||||
),
|
||||
),
|
||||
]
|
||||
item=item,
|
||||
)
|
||||
)
|
||||
state.output_index += 1
|
||||
state.tool_call_namespace = None
|
||||
state.tool_call_is_custom = False
|
||||
state.current_state = _StateType.NONE
|
||||
return events
|
||||
|
||||
@@ -1194,6 +1270,7 @@ class SimpleStreamingEventProcessor:
|
||||
) -> None:
|
||||
self.state = state or SimpleStreamingState()
|
||||
self.tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||
self.custom_tool_names = custom_tool_names(tools)
|
||||
self.output_items: list[ResponseOutputItem] = []
|
||||
|
||||
def resolve_target_state(
|
||||
@@ -1264,6 +1341,7 @@ class SimpleStreamingEventProcessor:
|
||||
call_name.name,
|
||||
tool_call.index,
|
||||
call_name.namespace,
|
||||
call_name.name in self.custom_tool_names,
|
||||
tool_call.id,
|
||||
)
|
||||
return handlers.open_fn(self.state)
|
||||
|
||||
Reference in New Issue
Block a user