[Frontend] Support CustomToolCall in the non-Harmony Responses API (#57322)

Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Shijin Zhang
2026-10-12 17:22:55 +08:00
committed by GitHub
co-authored by Chauncey
parent a33d46b471
commit 83773ecdf3
22 changed files with 698 additions and 72 deletions
@@ -1,13 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from openai_harmony import (
Message,
)
from pydantic import ValidationError
from vllm.entrypoints.openai.responses.protocol import (
ResponsesRequest,
serialize_message,
serialize_messages,
)
from vllm.exceptions import VLLMValidationError
def test_serialize_message() -> None:
@@ -37,3 +41,34 @@ def test_serialize_messages() -> None:
}
msg = Message.from_dict(msg_value)
assert serialize_messages([msg, dict_value]) == [msg_value, dict_value]
def test_custom_tool_grammar_format_rejected() -> None:
grammar = {"type": "grammar", "syntax": "lark", "definition": 'start: "pwd"'}
with pytest.raises(VLLMValidationError) as exc_info:
ResponsesRequest(
input="hi", tools=[{"type": "custom", "name": "emit", "format": grammar}]
)
assert exc_info.value.parameter == "tools"
request = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit", "format": {"type": "text"}}],
)
assert request.tools is not None and request.tools[0].type == "custom"
@pytest.mark.parametrize(
"tools",
[
"abc",
{"type": "custom"},
[None],
[{"type": "custom", "name": "x", "format": "text"}],
],
)
def test_malformed_tools_rejected_by_field_validation(tools) -> None:
"""Malformed tools bodies fail pydantic field validation (4xx), not an
AttributeError inside the custom-format validator (500)."""
with pytest.raises(ValidationError):
ResponsesRequest(input="hi", tools=tools)
@@ -4,6 +4,7 @@
from unittest.mock import patch
import pytest
from openai.types.responses import CustomTool, FunctionTool, ResponseCustomToolCall
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_tool_call_output_item import (
ResponseFunctionToolCallOutputItem,
@@ -16,10 +17,14 @@ from openai.types.responses.response_reasoning_item import (
Summary,
)
from vllm.entrypoints.generate.base.protocol import FunctionCall
from vllm.entrypoints.openai.responses.utils import (
_construct_message_from_response_item,
build_response_output_items,
construct_chat_messages_with_tool_call,
construct_input_messages,
decode_custom_tool_input,
decode_custom_tool_input_prefix,
should_continue_final_message,
)
from vllm.exceptions import VLLMValidationError
@@ -248,6 +253,89 @@ class TestResponsesUtils:
assert formatted_item["content"] == "dongyi"
class TestCustomTools:
def test_custom_tool_call_output_item(self):
tools = [
CustomTool(type="custom", name="emit_command", format={"type": "text"}),
FunctionTool(
type="function", name="get_weather", parameters={}, strict=None
),
]
custom, function = build_response_output_items(
reasoning=None,
content=None,
tool_calls=[
FunctionCall(
id="call_1", name="emit_command", arguments='{"input": "pwd"}'
),
FunctionCall(
id="call_2", name="get_weather", arguments='{"city": "Paris"}'
),
],
tools=tools,
)
assert isinstance(custom, ResponseCustomToolCall)
assert (custom.call_id, custom.input) == ("call_1", "pwd")
assert custom.id.startswith("ctc_")
assert isinstance(function, ResponseFunctionToolCall)
assert function.arguments == '{"city": "Paris"}'
def test_custom_tool_history_replays_through_function_shim(self):
messages = construct_chat_messages_with_tool_call(
[
make_reasoning_item(content_text="Run pwd."),
ResponseCustomToolCall(
type="custom_tool_call",
id="ctc_1",
call_id="call_exec",
name="emit_command",
input='echo "hi"',
),
{
"type": "custom_tool_call_output",
"call_id": "call_exec",
"output": "hi",
},
{"role": "user", "content": "Continue"},
]
)
assert messages == [
{
"role": "assistant",
"reasoning": "Run pwd.",
"tool_calls": [
{
"id": "call_exec",
"type": "function",
"function": {
"name": "emit_command",
"arguments": '{"input": "echo \\"hi\\""}',
},
}
],
},
{"role": "tool", "content": "hi", "tool_call_id": "call_exec"},
{"role": "user", "content": "Continue"},
]
def test_decode_custom_tool_input(self):
assert decode_custom_tool_input('{"input": "pwd"}') == "pwd"
assert decode_custom_tool_input('{"cmd": "pwd"}') == '{"cmd": "pwd"}'
assert decode_custom_tool_input('{"file": "a.py"}') == '{"file": "a.py"}'
assert decode_custom_tool_input('{"input": 7}') == '{"input": 7}'
assert decode_custom_tool_input("pwd") == "pwd"
def test_decode_custom_tool_input_prefix(self):
assert decode_custom_tool_input_prefix('{"inp') == ""
assert decode_custom_tool_input_prefix('{"input": "ls \\"dir') == 'ls "dir'
assert decode_custom_tool_input_prefix('{"input": "a\\') == "a"
assert decode_custom_tool_input_prefix('{"input": "\\ud83d') == ""
assert (
decode_custom_tool_input_prefix('{"input": "\\ud83d\\ude00!"}')
== "\U0001f600!"
)
class TestReasoningItemContentPriority:
"""Tests that content is prioritized over summary for reasoning items."""
@@ -2,7 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from openai.types.responses import ResponseFunctionWebSearch, ResponseStreamEvent
from openai.types.responses import (
CustomTool,
ResponseCustomToolCall,
ResponseFunctionWebSearch,
ResponseStreamEvent,
)
from openai_harmony import Message, Role
from pydantic import TypeAdapter
@@ -20,6 +25,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
emit_reasoning_done_events,
split_delta,
)
from vllm.entrypoints.openai.responses.utils import decode_custom_tool_input
def test_browser_find_uses_responses_action_type():
@@ -190,3 +196,111 @@ class TestProcessorCompoundDeltas:
types = [e.type for e in events]
assert "response.reasoning_text.delta" in types
assert "response.output_text.delta" in types
class TestSimpleToolCallShapes:
def test_arguments_done_emitted_without_deltas(self):
processor = SimpleStreamingEventProcessor()
events = _run_through_processor(
processor, DeltaMessage(tool_calls=[_make_tool_call(0, name="noop")])
)
events += processor.close_current()
assert [e.type for e in events] == [
"response.output_item.added",
"response.function_call_arguments.done",
"response.output_item.done",
]
class TestCustomToolStreaming:
def test_custom_tool_input_events(self):
processor = SimpleStreamingEventProcessor(
tools=[CustomTool(type="custom", name="emit_command")]
)
events = _run_through_processor(
processor,
DeltaMessage(
tool_calls=[
_make_tool_call(0, name="emit_command", arguments='{"input": "ls ')
]
),
)
events += _run_through_processor(
processor,
DeltaMessage(tool_calls=[_make_tool_call(0, arguments='-la \\"x\\""}')]),
)
events += processor.close_current()
assert [e.type for e in events] == [
"response.output_item.added",
"response.custom_tool_call_input.delta",
"response.custom_tool_call_input.delta",
"response.custom_tool_call_input.done",
"response.output_item.done",
]
done = events[-1].item
assert isinstance(done, ResponseCustomToolCall)
assert done.input == 'ls -la "x"'
assert "".join(e.delta for e in events[1:3]) == done.input
assert events[3].input == done.input
assert events[0].item.type == "custom_tool_call"
assert events[0].item.id == done.id
assert done.id.startswith("ctc_")
assert events[0].item.call_id == done.call_id
def test_unrelated_keys_are_not_streamed_as_payload(self):
"""Review repro: chunks of an object whose input key arrives last must
not leak fragments of other keys into the payload."""
processor = SimpleStreamingEventProcessor(
tools=[CustomTool(type="custom", name="emit_command")]
)
events = _run_through_processor(
processor,
DeltaMessage(
tool_calls=[
_make_tool_call(0, name="emit_command", arguments='{"a": "xy"')
]
),
)
events += _run_through_processor(
processor,
DeltaMessage(
tool_calls=[_make_tool_call(0, arguments=', "input": "hello"}')]
),
)
events += processor.close_current()
deltas = [
e for e in events if e.type == "response.custom_tool_call_input.delta"
]
done_input = next(
e for e in events if e.type == "response.custom_tool_call_input.done"
).input
item = events[-1].item
assert "".join(e.delta for e in deltas) == "hello" # "xy" never leaks
assert done_input == "hello"
assert item.input == "hello"
assert decode_custom_tool_input('{"a": "xy", "input": "hello"}') == "hello"
def test_done_reported_authoritative_payload_on_mismatch(self):
"""If the final decode is not an extension of what was streamed, the
done event and item carry the authoritative value and a correcting
delta is emitted rather than dropping the divergence."""
processor = SimpleStreamingEventProcessor(
tools=[CustomTool(type="custom", name="emit_command")]
)
processor.state.current_state = _StateType.TOOL_CALL
processor.state.tool_call_is_custom = True
processor.state.tool_call_payload = "xy" # stale fragment already streamed
processor.state.accumulated_text = '{"a": "xy", "input": "hello"}'
events = processor.close_current()
delta = next(
e for e in events if e.type == "response.custom_tool_call_input.delta"
)
done_input = next(
e for e in events if e.type == "response.custom_tool_call_input.done"
).input
assert delta.delta == "hello" # full authoritative resend, not a dropped suffix
assert done_input == "hello"
assert events[-1].item.input == "hello"
@@ -8,6 +8,7 @@ from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.gemma4 import (
TOOL_CALL_END,
TOOL_CALL_START,
@@ -117,6 +118,21 @@ def mock_request():
return request
def test_named_custom_tool_choice_skips_structured_outputs(mock_tokenizer):
# Responses custom tools ride the function shim, so a named custom choice
# must route like a named function choice, not into guided JSON.
tools = [{"type": "custom", "name": "emit_command"}]
request = ResponsesRequest(
input="hi",
tools=tools,
tool_choice={"type": "custom", "name": "emit_command"},
)
Gemma4EngineToolParser(mock_tokenizer, tools=tools).adjust_request(request)
assert request.skip_special_tokens is False
assert request.structured_outputs is None
# ---------------------------------------------------------------------------
# Unit tests for _parse_gemma4_args (shared parser logic)
# ---------------------------------------------------------------------------
@@ -8,6 +8,7 @@ import pytest
from vllm.entrypoints.generate.base.protocol import DeltaMessage
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.parser.parser_manager import ParserManager
from vllm.tool_parsers import ToolParserManager
from vllm.tool_parsers.k2_horizon_tool_parser import K2HorizonToolParser
@@ -316,3 +317,14 @@ def test_whitespace_only_surrounding_content_is_not_preserved(tokenizer):
def test_tool_parser_registered():
assert ToolParserManager.get_tool_parser("k2_horizon") is K2HorizonToolParser
assert K2HorizonToolParser.supports_required_and_named is False
def test_named_tool_resolves_custom_choice():
# Custom tools ride the function shim; a ToolChoiceCustom must resolve to
# its name like ToolChoiceFunction does.
request = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
assert K2HorizonToolParser._named_tool(request) == "emit_command"
@@ -8,6 +8,7 @@ import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import StructuredOutputsParams
@@ -60,3 +61,26 @@ def test_named_choice_allowed_with_structural_tag():
def test_named_choice_rejected_without_structural_tag():
with pytest.raises(VLLMValidationError):
_parser().adjust_request(_request(with_tag=False))
def _custom_request(with_tag: bool) -> ResponsesRequest:
req = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
if with_tag:
req.structured_outputs = StructuredOutputsParams(
structural_tag='{"type": "structural_tag", "format": {}}'
)
return req
def test_named_custom_choice_allowed_with_structural_tag():
req = _parser().adjust_request(_custom_request(with_tag=True))
assert req.skip_special_tokens is False
def test_named_custom_choice_rejected_without_structural_tag():
with pytest.raises(VLLMValidationError):
_parser().adjust_request(_custom_request(with_tag=False))
@@ -78,12 +78,15 @@ def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionR
def _build_responses_request(
*, tool_choice: str | dict[str, Any], include: list[str] | None = None
*,
tool_choice: str | dict[str, Any],
include: list[str] | None = None,
tools: list[dict[str, Any]] | None = None,
) -> ResponsesRequest:
return ResponsesRequest(
model="poolside-test",
input=[{"role": "user", "content": "write the file"}],
tools=[_responses_write_file_tool()],
tools=tools if tools is not None else [_responses_write_file_tool()],
tool_choice=tool_choice,
stream=True,
max_output_tokens=200,
@@ -145,6 +148,19 @@ def test_named_skips_structured_outputs_responses() -> None:
assert request.skip_special_tokens is False
def test_named_custom_tool_choice_skips_structured_outputs_responses() -> None:
# A named custom choice parses to ToolChoiceCustom; custom tools ride the
# function shim, so it must route like ToolChoiceFunction.
request = _build_responses_request(
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
assert request.text is None
assert request.skip_special_tokens is False
def test_auto_still_keeps_special_tokens() -> None:
request = _build_chat_request(tool_choice="auto")
_make_parser(request).adjust_request(request)
@@ -7,7 +7,7 @@ from typing import Any
from unittest.mock import MagicMock
import pytest
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import CustomTool, ToolChoiceFunction
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
from xgrammar import Grammar, StructuralTag
from xgrammar import get_model_structural_tag as get_xgrammar_builtin_structural_tag
@@ -787,6 +787,19 @@ def test_get_model_structural_tag_supports_named_tool_choice(
assert isinstance(tag, StructuralTag)
def test_glm_4_7_forced_custom_tool_uses_function_shim():
tools = [CustomTool(type="custom", name="emit_command", description="Emit.")]
tag = get_model_structural_tag(
model="glm_4_7",
tools=tools,
tool_choice="required",
reasoning=False,
)
assert isinstance(tag, StructuralTag)
assert "emit_command" in tag.model_dump_json()
@pytest.mark.parametrize(
("parser_cls", "model"),
[
+13
View File
@@ -29,6 +29,7 @@ from typing import cast
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
@@ -400,3 +401,15 @@ def test_exact_match_kept():
T, _call("get_weather"), _req("get_weather")
)
assert out.tool_calls[0].function.name == "get_weather"
def test_named_custom_tool_choice_skips_structured_outputs():
# Custom tools ride the function shim; a named custom choice must skip
# the base adjust_request like a named function choice does.
request = ResponsesRequest(
input="hi",
tools=[{"type": "custom", "name": "emit_command"}],
tool_choice={"type": "custom", "name": "emit_command"},
)
T.adjust_request(request)
assert request.structured_outputs is None
@@ -6,7 +6,9 @@ from copy import deepcopy
import pytest
import regex as re
from openai.types.responses import (
CustomTool,
FunctionTool,
ToolChoiceCustom,
ToolChoiceFunction,
WebSearchTool,
)
@@ -21,8 +23,10 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
from vllm.tool_parsers.utils import (
custom_tool_parameters,
find_tool_properties,
get_json_schema_from_tools,
iter_response_function_tool_dicts,
)
pytestmark = pytest.mark.cpu_test
@@ -483,6 +487,37 @@ class TestNonFunctionToolsSkipped:
get_json_schema_from_tools(tools=[WEB_SEARCH_TOOL], tool_choice="required")
class TestCustomToolShim:
"""``custom`` tools are shown to the model as a single-string function."""
def test_rendered_as_single_string_function(self):
tool = CustomTool(
type="custom", name="emit_command", description="Emit a command."
)
[tool_dict] = iter_response_function_tool_dicts([tool])
assert tool_dict["type"] == "function"
assert tool_dict["name"] == "emit_command"
assert tool_dict["description"] == "Emit a command."
assert tool_dict["parameters"] == custom_tool_parameters()
assert tool_dict["parameters"]["required"] == ["input"]
def test_forced_tool_choice_sees_custom_tool(self):
tools = [CustomTool(type="custom", name="emit_command"), FUNCTION_TOOL]
schema = get_json_schema_from_tools(tools=tools, tool_choice="required")
assert isinstance(schema, dict)
names = [e["properties"]["name"]["enum"] for e in schema["items"]["anyOf"]]
assert names == [["emit_command"], ["get_weather"]]
named = get_json_schema_from_tools(
tools=tools,
tool_choice=ToolChoiceCustom(type="custom", name="emit_command"),
)
assert named == custom_tool_parameters()
assert (
find_tool_properties(tools, "emit_command")
== custom_tool_parameters()["properties"]
)
class TestMalformedToolSchemaDefs:
"""Malformed `$defs` in caller-supplied tool parameters is a 400, not a 500."""
+35 -3
View File
@@ -14,6 +14,8 @@ from openai.types.responses import (
ResponseCodeInterpreterCallInterpretingEvent,
ResponseContentPartAddedEvent,
ResponseContentPartDoneEvent,
ResponseCustomToolCallInputDeltaEvent,
ResponseCustomToolCallInputDoneEvent,
ResponseFunctionToolCall,
ResponseInputItemParam,
ResponseMcpCallArgumentsDeltaEvent,
@@ -699,6 +701,34 @@ class ResponsesRequest(OpenAIBaseModel):
data["input"] = processed_input
return data
@model_validator(mode="before")
@classmethod
def check_custom_tool_format(cls, data):
if not isinstance(data, dict):
return data
tools = data.get("tools")
if not isinstance(tools, list):
return data
for tool in tools:
if isinstance(tool, dict):
tool_dict = tool
elif hasattr(tool, "model_dump"):
tool_dict = tool.model_dump()
else:
continue
tool_format = tool_dict.get("format")
if (
tool_dict.get("type") == "custom"
and isinstance(tool_format, dict)
and tool_format.get("type") == "grammar"
):
raise VLLMValidationError(
"Custom tools with a grammar format are not supported; "
"only free-form text custom tools are accepted.",
parameter="tools",
)
return data
@model_validator(mode="before")
@classmethod
def check_tool_usage(cls, data):
@@ -708,9 +738,9 @@ class ResponsesRequest(OpenAIBaseModel):
tools = data.get("tools")
tool_choice = data.get("tool_choice", "auto")
has_tools = tools is not None and len(tools) > 0
is_named_tool_choice = (
isinstance(tool_choice, dict) and tool_choice.get("type") == "function"
)
is_named_tool_choice = isinstance(tool_choice, dict) and tool_choice.get(
"type"
) in ("function", "custom")
if not has_tools:
if tool_choice in ("auto", "none"):
@@ -913,6 +943,8 @@ StreamingResponsesResponse: TypeAlias = (
| ResponseOutputItemDoneEvent
| ResponseContentPartAddedEvent
| ResponseContentPartDoneEvent
| ResponseCustomToolCallInputDeltaEvent
| ResponseCustomToolCallInputDoneEvent
| ResponseReasoningTextDeltaEvent
| ResponseReasoningTextDoneEvent
| ResponseCodeInterpreterCallInProgressEvent
@@ -29,6 +29,9 @@ from openai.types.responses import (
ResponseCodeInterpreterToolCallParam,
ResponseContentPartAddedEvent,
ResponseContentPartDoneEvent,
ResponseCustomToolCall,
ResponseCustomToolCallInputDeltaEvent,
ResponseCustomToolCallInputDoneEvent,
ResponseFunctionCallArgumentsDeltaEvent,
ResponseFunctionCallArgumentsDoneEvent,
ResponseFunctionToolCall,
@@ -73,6 +76,9 @@ from vllm.entrypoints.openai.responses.protocol import (
)
from vllm.entrypoints.openai.responses.utils import (
build_responses_tool_call_name_map,
custom_tool_names,
decode_custom_tool_input,
decode_custom_tool_input_prefix,
resolve_responses_tool_call_name,
)
from vllm.outputs import CompletionOutput
@@ -821,6 +827,8 @@ class SimpleStreamingState:
tool_call_name: str = ""
tool_call_namespace: str | None = None
tool_call_index: int | None = None
tool_call_is_custom: bool = False
tool_call_payload: str = ""
current_state: _StateType = field(default_factory=lambda: _StateType.NONE)
@@ -1039,38 +1047,80 @@ def emit_simple_tool_call_open(
name: str,
index: int | None,
namespace: str | None = None,
is_custom: bool = False,
call_id: str | None = None,
) -> list[StreamingResponsesResponse]:
state.current_state = _StateType.TOOL_CALL
state.current_item_id = f"fc_{random_uuid()}"
state.current_item_id = (
f"ctc_{random_uuid()}" if is_custom else f"fc_{random_uuid()}"
)
state.tool_call_id = call_id or make_tool_call_id()
state.tool_call_name = name
state.tool_call_namespace = namespace
state.tool_call_index = index
state.tool_call_is_custom = is_custom
state.tool_call_payload = ""
state.accumulated_text = ""
item: ResponseOutputItem
if is_custom:
item = ResponseCustomToolCall(
type="custom_tool_call",
id=state.current_item_id,
call_id=state.tool_call_id,
name=name,
input="",
)
else:
item = ResponseFunctionToolCallItem(
type="function_call",
id=state.current_item_id,
call_id=state.tool_call_id,
name=name,
namespace=namespace,
arguments="",
status="in_progress",
)
return [
ResponseOutputItemAddedEvent(
type="response.output_item.added",
sequence_number=-1,
output_index=state.output_index,
item=ResponseFunctionToolCallItem(
type="function_call",
id=state.current_item_id,
call_id=state.tool_call_id,
name=name,
namespace=namespace,
arguments="",
status="in_progress",
),
item=item,
),
]
def _emit_custom_tool_input_delta(
state: SimpleStreamingState,
payload: str,
) -> list[StreamingResponsesResponse]:
streamed = state.tool_call_payload
if payload == streamed:
return []
state.tool_call_payload = payload
# A payload that is not an extension of what was streamed cannot be
# retracted, so the authoritative value is resent in full.
delta = payload[len(streamed) :] if payload.startswith(streamed) else payload
return [
ResponseCustomToolCallInputDeltaEvent(
type="response.custom_tool_call_input.delta",
sequence_number=-1,
output_index=state.output_index,
item_id=state.current_item_id,
delta=delta,
)
]
def emit_simple_tool_call_delta(
state: SimpleStreamingState,
delta: str,
) -> list[StreamingResponsesResponse]:
state.accumulated_text += delta
if state.tool_call_is_custom:
return _emit_custom_tool_input_delta(
state, decode_custom_tool_input_prefix(state.accumulated_text)
)
return [
ResponseFunctionCallArgumentsDeltaEvent(
type="response.function_call_arguments.delta",
@@ -1085,32 +1135,58 @@ def emit_simple_tool_call_delta(
def emit_simple_tool_call_done(
state: SimpleStreamingState,
) -> list[StreamingResponsesResponse]:
events: list[StreamingResponsesResponse] = [
ResponseFunctionCallArgumentsDoneEvent(
type="response.function_call_arguments.done",
sequence_number=-1,
output_index=state.output_index,
item_id=state.current_item_id,
arguments=state.accumulated_text,
events: list[StreamingResponsesResponse]
item: ResponseOutputItem
if state.tool_call_is_custom:
payload = decode_custom_tool_input(state.accumulated_text)
events = _emit_custom_tool_input_delta(state, payload)
events.append(
ResponseCustomToolCallInputDoneEvent(
type="response.custom_tool_call_input.done",
sequence_number=-1,
output_index=state.output_index,
item_id=state.current_item_id,
input=payload,
)
)
item = ResponseCustomToolCall(
type="custom_tool_call",
id=state.current_item_id,
call_id=state.tool_call_id,
name=state.tool_call_name,
),
input=payload,
)
else:
events = [
ResponseFunctionCallArgumentsDoneEvent(
type="response.function_call_arguments.done",
sequence_number=-1,
output_index=state.output_index,
item_id=state.current_item_id,
arguments=state.accumulated_text,
name=state.tool_call_name,
)
]
item = ResponseFunctionToolCall(
type="function_call",
name=state.tool_call_name,
namespace=state.tool_call_namespace,
arguments=state.accumulated_text,
status="completed",
id=state.current_item_id,
call_id=state.tool_call_id,
)
events.append(
ResponseOutputItemDoneEvent(
type="response.output_item.done",
sequence_number=-1,
output_index=state.output_index,
item=ResponseFunctionToolCall(
type="function_call",
name=state.tool_call_name,
namespace=state.tool_call_namespace,
arguments=state.accumulated_text,
status="completed",
id=state.current_item_id,
call_id=state.tool_call_id,
),
),
]
item=item,
)
)
state.output_index += 1
state.tool_call_namespace = None
state.tool_call_is_custom = False
state.current_state = _StateType.NONE
return events
@@ -1194,6 +1270,7 @@ class SimpleStreamingEventProcessor:
) -> None:
self.state = state or SimpleStreamingState()
self.tool_call_name_map = build_responses_tool_call_name_map(tools)
self.custom_tool_names = custom_tool_names(tools)
self.output_items: list[ResponseOutputItem] = []
def resolve_target_state(
@@ -1264,6 +1341,7 @@ class SimpleStreamingEventProcessor:
call_name.name,
tool_call.index,
call_name.namespace,
call_name.name in self.custom_tool_names,
tool_call.id,
)
return handlers.open_fn(self.state)
+85 -9
View File
@@ -1,9 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Iterable
from typing import Any
import partial_json_parser
from openai.types.chat import (
ChatCompletionAssistantMessageParam,
ChatCompletionMessageToolCallParam,
@@ -13,6 +15,7 @@ from openai.types.chat.chat_completion_message_tool_call_param import (
Function as FunctionCallTool,
)
from openai.types.responses import (
ResponseCustomToolCall,
ResponseFunctionToolCall,
ResponseOutputItem,
ResponseOutputMessage,
@@ -28,6 +31,7 @@ from openai.types.responses.response_reasoning_item import (
Content as ResponseReasoningTextContent,
)
from openai.types.responses.tool import Tool
from partial_json_parser.core.options import Allow
from vllm import envs
from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition
@@ -40,6 +44,7 @@ from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.renderers.chat_utils import make_tool_call_id
from vllm.tool_parsers.utils import (
CUSTOM_TOOL_INPUT_KEY,
build_responses_tool_call_name_map,
flat_namespace_tool_name,
iter_response_function_tool_dicts,
@@ -50,6 +55,51 @@ from vllm.utils import random_uuid
logger = init_logger(__name__)
def custom_tool_names(tools: list[Tool] | None) -> frozenset[str]:
return frozenset(tool.name for tool in tools or [] if tool.type == "custom")
def encode_custom_tool_input(payload: str) -> str:
return json.dumps({CUSTOM_TOOL_INPUT_KEY: payload}, ensure_ascii=False)
def _custom_tool_payload(parsed: Any) -> str | None:
if not isinstance(parsed, dict):
return None
value = parsed.get(CUSTOM_TOOL_INPUT_KEY)
return value if isinstance(value, str) else None
def decode_custom_tool_input(arguments: str) -> str:
"""Payload of a completed shim call; bare text is passed through."""
try:
payload = _custom_tool_payload(json.loads(arguments))
except ValueError:
return arguments
return arguments if payload is None else payload
def decode_custom_tool_input_prefix(arguments: str) -> str:
"""Longest decodable payload prefix of partial shim arguments, so that
streamed ``custom_tool_call_input`` deltas stay a prefix of the final input."""
try:
parsed = partial_json_parser.loads(arguments, Allow.STR | Allow.OBJ)
except (ValueError, IndexError, AssertionError):
return ""
payload = _custom_tool_payload(parsed) or ""
if payload and "\ud800" <= payload[-1] <= "\udbff":
payload = payload[:-1]
return payload
def _item_type(item: ResponseInputOutputItem) -> str | None:
return item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
return item if isinstance(item, dict) else item.model_dump()
def build_response_output_items(
reasoning: str | None,
content: str | None,
@@ -59,6 +109,7 @@ def build_response_output_items(
) -> list[ResponseOutputItem]:
outputs: list[ResponseOutputItem] = []
tool_call_name_map = build_responses_tool_call_name_map(tools)
custom_names = custom_tool_names(tools)
if reasoning:
outputs.append(
@@ -93,14 +144,27 @@ def build_response_output_items(
if tool_calls:
for idx, tool_call in enumerate(tool_calls):
call_id = tool_call.id or make_tool_call_id(
func_name=tool_call.name, idx=idx
)
if tool_call.name in custom_names:
outputs.append(
ResponseCustomToolCall(
id=f"ctc_{random_uuid()}",
call_id=call_id,
type="custom_tool_call",
name=tool_call.name,
input=decode_custom_tool_input(tool_call.arguments),
)
)
continue
call_name = resolve_responses_tool_call_name(
tool_call.name, tool_call_name_map=tool_call_name_map
)
outputs.append(
ResponseFunctionToolCall(
id=f"fc_{random_uuid()}",
call_id=tool_call.id
or make_tool_call_id(func_name=tool_call.name, idx=idx),
call_id=call_id,
type="function_call",
status="completed",
name=call_name.name,
@@ -255,6 +319,7 @@ def _construct_message_from_response_item(
prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None
)
tool_call: ChatCompletionMessageToolCallParam | None = None
if isinstance(item, ResponseFunctionToolCall):
tool_name = item.name
if item.namespace:
@@ -267,6 +332,17 @@ def _construct_message_from_response_item(
),
type="function",
)
elif _item_type(item) == "custom_tool_call":
call = _item_dict(item)
tool_call = ChatCompletionMessageToolCallParam(
id=call["call_id"],
function=FunctionCallTool(
name=call["name"],
arguments=encode_custom_tool_input(call.get("input") or ""),
),
type="function",
)
if tool_call is not None:
if prev_assistant_msg:
tool_calls = prev_assistant_msg.get("tool_calls")
if tool_calls is None:
@@ -285,14 +361,14 @@ def _construct_message_from_response_item(
logger.warning(
"Previous assistant message has unknown tool_calls format. "
"Tool call merging is skipped and a new assistant message is created. "
"Item %s",
item.id,
"Call %s",
tool_call["id"],
)
return ChatCompletionAssistantMessageParam(
role="assistant",
tool_calls=[tool_call],
)
elif isinstance(item, ResponseReasoningItem):
if isinstance(item, ResponseReasoningItem):
reasoning = ""
if item.encrypted_content:
raise VLLMValidationError(
@@ -336,12 +412,12 @@ def _construct_message_from_response_item(
content=item.output,
tool_call_id=item.call_id,
)
elif isinstance(item, dict) and item.get("type") == "function_call_output":
# Append the function call output as a tool message.
elif _item_type(item) in ("function_call_output", "custom_tool_call_output"):
output = _item_dict(item)
return ChatCompletionToolMessageParam(
role="tool",
content=item.get("output"),
tool_call_id=item.get("call_id"),
content=output.get("output"),
tool_call_id=output.get("call_id"),
)
elif isinstance(item, dict) and item.get("role") == "assistant":
content = item.get("content")
+15 -5
View File
@@ -7,7 +7,7 @@ from abc import abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass, field, replace
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from pydantic import TypeAdapter, ValidationError
from xgrammar import Grammar, StructuralTag
from xgrammar.structural_tag import (
@@ -359,7 +359,9 @@ class DelegatingParser(Parser):
def _get_function_name(
self, request: ChatCompletionRequest | ResponsesRequest
) -> str:
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
if request.tool_choice and isinstance(
request.tool_choice, (ToolChoiceFunction, ToolChoiceCustom)
):
return request.tool_choice.name
if request.tool_choice and isinstance(
request.tool_choice, ChatCompletionNamedToolChoiceParam
@@ -398,7 +400,7 @@ class DelegatingParser(Parser):
supports_required_and_named = tool_parser.supports_required_and_named
is_named_tool_choice = request.tool_choice and isinstance(
request.tool_choice,
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
(ToolChoiceFunction, ToolChoiceCustom, ChatCompletionNamedToolChoiceParam),
)
is_required_tool_choice = request.tool_choice == "required"
is_auto_tool_choice = enable_auto_tools and (
@@ -503,7 +505,11 @@ class DelegatingParser(Parser):
or request.tool_choice == "required"
or isinstance(
request.tool_choice,
(ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
(
ChatCompletionNamedToolChoiceParam,
ToolChoiceFunction,
ToolChoiceCustom,
),
)
)
if not need_tool_calling:
@@ -723,7 +729,11 @@ class DelegatingParser(Parser):
and request.tool_choice
and isinstance(
request.tool_choice,
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
(
ToolChoiceFunction,
ToolChoiceCustom,
ChatCompletionNamedToolChoiceParam,
),
)
):
delta_message, function_name_returned = extract_named_tool_call_streaming(
+3 -2
View File
@@ -9,7 +9,7 @@ import json
from collections.abc import Callable, Iterable, Sequence
from typing import TYPE_CHECKING, Any, cast
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
@@ -173,7 +173,8 @@ def _streaming_template(template: dict[str, Any]) -> ResponseTemplate:
def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool:
return isinstance(
request.tool_choice, (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam)
request.tool_choice,
(ToolChoiceFunction, ToolChoiceCustom, ChatCompletionNamedToolChoiceParam),
)
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionNamedToolChoiceParam,
@@ -29,7 +29,12 @@ class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-typ
if request.tools:
tc = request.tool_choice
if tc == "required" or isinstance(
tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
tc,
(
ChatCompletionNamedToolChoiceParam,
ToolChoiceFunction,
ToolChoiceCustom,
),
):
request.skip_special_tokens = False
return request
+2 -2
View File
@@ -6,7 +6,7 @@ from collections.abc import Sequence
from typing import Any
import regex as re
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
@@ -184,7 +184,7 @@ class K2HorizonToolParser(ToolParser):
tool_choice = request.tool_choice
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
return tool_choice.function.name
if isinstance(tool_choice, ToolChoiceFunction):
if isinstance(tool_choice, (ToolChoiceFunction, ToolChoiceCustom)):
return tool_choice.name
return None
+6 -2
View File
@@ -35,7 +35,7 @@ import json
from collections.abc import Sequence
import regex as re
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
@@ -149,7 +149,11 @@ class KimiK3ToolParser(ToolParser):
) -> ChatCompletionRequest | ResponsesRequest:
named = isinstance(
request.tool_choice,
(ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
(
ChatCompletionNamedToolChoiceParam,
ToolChoiceFunction,
ToolChoiceCustom,
),
)
structured_outputs = getattr(request, "structured_outputs", None)
has_structural_tag = (
@@ -38,7 +38,7 @@ import json
from collections.abc import Iterator, Sequence
import regex as re
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from transformers import PreTrainedTokenizerBase
from vllm.entrypoints.generate.base.protocol import (
@@ -228,7 +228,12 @@ class MuseGlimmerToolParser(ToolParser):
if request.tools and (
tool_choice == "required"
or isinstance(
tool_choice, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
tool_choice,
(
ChatCompletionNamedToolChoiceParam,
ToolChoiceFunction,
ToolChoiceCustom,
),
)
):
return request
+7 -2
View File
@@ -16,7 +16,7 @@ from typing import Any
import partial_json_parser.core.complete
import regex as re
from openai.types.responses import ToolChoiceFunction
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
from partial_json_parser.core.options import Allow
from vllm.entrypoints.generate.base.protocol import (
@@ -171,7 +171,12 @@ class PoolsideV1ToolParser(ToolParser):
if request.tools:
tc = request.tool_choice
if tc == "required" or isinstance(
tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
tc,
(
ChatCompletionNamedToolChoiceParam,
ToolChoiceFunction,
ToolChoiceCustom,
),
):
request.skip_special_tokens = False
return request
+12 -3
View File
@@ -6,7 +6,7 @@ import json
from collections.abc import Callable, Sequence
from typing import Any, Literal, TypeAlias
from openai.types.responses import FunctionTool
from openai.types.responses import CustomTool, FunctionTool, ToolChoiceCustom
from openai.types.responses.response import ToolChoice as ResponsesToolChoice
from openai.types.responses.tool import Tool as ResponsesTool
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
@@ -39,7 +39,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionToolsParam,
)
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
from vllm.tool_parsers.utils import get_function_tools, require_function_tools
from vllm.tool_parsers.utils import (
custom_tool_parameters,
get_function_tools,
require_function_tools,
)
ToolChoice: TypeAlias = (
Literal["none", "auto", "required"]
@@ -287,6 +291,11 @@ def _dump_tool_for_xgrammar(
if tool.strict is not None:
function["strict"] = tool.strict
return {"type": "function", "function": function}
if isinstance(tool, CustomTool):
function = {"name": tool.name, "parameters": custom_tool_parameters()}
if tool.description is not None:
function["description"] = tool.description
return {"type": "function", "function": function}
dumped_tool = tool.model_dump(mode="json", exclude_none=True)
if isinstance(tool, ChatCompletionToolsParam):
return dumped_tool
@@ -306,7 +315,7 @@ def _dump_tool_choice_for_xgrammar(
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
return tool_choice.model_dump(mode="json", exclude_none=True)
if isinstance(tool_choice, ToolChoiceFunction):
if isinstance(tool_choice, (ToolChoiceFunction, ToolChoiceCustom)):
return {
"type": "function",
"function": {"name": tool_choice.name},
+42 -7
View File
@@ -13,8 +13,10 @@ from typing import Any, TypeAlias
import partial_json_parser
from openai.types.responses import (
CustomTool,
FunctionTool,
NamespaceTool,
ToolChoiceCustom,
ToolChoiceFunction,
)
from openai.types.responses.tool import Tool as ResponsesTool
@@ -37,6 +39,32 @@ Tool: TypeAlias = ChatCompletionToolsParam | ResponsesTool
logger = init_logger(__name__)
CUSTOM_TOOL_INPUT_KEY = "input"
def custom_tool_parameters() -> dict[str, Any]:
"""Schema of the single-string function shim a ``custom`` tool is shown as."""
return {
"type": "object",
"properties": {
CUSTOM_TOOL_INPUT_KEY: {
"type": "string",
"description": "The freeform text payload for this tool.",
}
},
"required": [CUSTOM_TOOL_INPUT_KEY],
"additionalProperties": False,
}
def custom_tool_as_function_dict(tool: CustomTool) -> dict[str, Any]:
return {
"type": "function",
"name": tool.name,
"description": tool.description,
"parameters": custom_tool_parameters(),
}
def safe_literal_eval(text: str):
with warnings.catch_warnings():
@@ -188,8 +216,9 @@ def get_function_tools(
This is the single source for both the chat template's tool list and the
tools a tool-call grammar constrains, so the two cannot drift apart.
Namespace functions are flattened to ``<namespace>__<name>``; other
Responses tool types are never rendered for the model and are dropped.
Namespace functions are flattened to ``<namespace>__<name>``; custom tools
become a single-string function shim. Other Responses tool types are never
rendered for the model and are dropped.
"""
function_tools: list[ChatCompletionToolsParam | FunctionTool] = []
for tool in tools:
@@ -206,6 +235,10 @@ def get_function_tools(
for namespaced_tool in tool.tools
if namespaced_tool.type == "function"
)
elif isinstance(tool, CustomTool):
function_tools.append(
FunctionTool.model_validate(custom_tool_as_function_dict(tool))
)
elif isinstance(tool, (FunctionTool, ChatCompletionToolsParam)):
function_tools.append(tool)
return function_tools
@@ -307,7 +340,7 @@ def find_tool_properties(
if not tools:
return {}
for tool in tools:
if isinstance(tool, (FunctionTool, NamespaceTool)):
if isinstance(tool, (FunctionTool, NamespaceTool, CustomTool)):
for name, params in iter_response_function_tool_info(tool):
if name == tool_name:
return (params or {}).get("properties", {})
@@ -328,7 +361,7 @@ def find_tool_name(
if not tools:
return False
for tool in tools:
if isinstance(tool, (FunctionTool, NamespaceTool)):
if isinstance(tool, (FunctionTool, NamespaceTool, CustomTool)):
for name, _ in iter_response_function_tool_info(tool):
if name == tool_name:
return True
@@ -414,7 +447,9 @@ def _get_json_schema_from_tools(
def get_json_schema_from_tools(
tool_choice: str | ToolChoiceFunction | ChatCompletionNamedToolChoiceParam,
tool_choice: (
str | ToolChoiceFunction | ToolChoiceCustom | ChatCompletionNamedToolChoiceParam
),
tools: list[Tool] | None,
parallel_tool_calls: bool | None = None,
) -> str | dict | None:
@@ -423,12 +458,12 @@ def get_json_schema_from_tools(
return None
# tool_choice: Forced Function (Responses)
if (not isinstance(tool_choice, str)) and isinstance(
tool_choice, ToolChoiceFunction
tool_choice, (ToolChoiceFunction, ToolChoiceCustom)
):
tool_name = tool_choice.name
responses_tool_map: dict[str, dict[str, Any] | None] = {}
for tool in tools:
if not isinstance(tool, (FunctionTool, NamespaceTool)):
if not isinstance(tool, (FunctionTool, NamespaceTool, CustomTool)):
continue
for name, params in iter_response_function_tool_info(tool):
responses_tool_map[name] = params