[Frontend] Support CustomToolCall in the non-Harmony Responses API (#57322)
Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -1,13 +1,17 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
import pytest
|
||||||
from openai_harmony import (
|
from openai_harmony import (
|
||||||
Message,
|
Message,
|
||||||
)
|
)
|
||||||
|
from pydantic import ValidationError
|
||||||
|
|
||||||
from vllm.entrypoints.openai.responses.protocol import (
|
from vllm.entrypoints.openai.responses.protocol import (
|
||||||
|
ResponsesRequest,
|
||||||
serialize_message,
|
serialize_message,
|
||||||
serialize_messages,
|
serialize_messages,
|
||||||
)
|
)
|
||||||
|
from vllm.exceptions import VLLMValidationError
|
||||||
|
|
||||||
|
|
||||||
def test_serialize_message() -> None:
|
def test_serialize_message() -> None:
|
||||||
@@ -37,3 +41,34 @@ def test_serialize_messages() -> None:
|
|||||||
}
|
}
|
||||||
msg = Message.from_dict(msg_value)
|
msg = Message.from_dict(msg_value)
|
||||||
assert serialize_messages([msg, dict_value]) == [msg_value, dict_value]
|
assert serialize_messages([msg, dict_value]) == [msg_value, dict_value]
|
||||||
|
|
||||||
|
|
||||||
|
def test_custom_tool_grammar_format_rejected() -> None:
|
||||||
|
grammar = {"type": "grammar", "syntax": "lark", "definition": 'start: "pwd"'}
|
||||||
|
with pytest.raises(VLLMValidationError) as exc_info:
|
||||||
|
ResponsesRequest(
|
||||||
|
input="hi", tools=[{"type": "custom", "name": "emit", "format": grammar}]
|
||||||
|
)
|
||||||
|
assert exc_info.value.parameter == "tools"
|
||||||
|
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hi",
|
||||||
|
tools=[{"type": "custom", "name": "emit", "format": {"type": "text"}}],
|
||||||
|
)
|
||||||
|
assert request.tools is not None and request.tools[0].type == "custom"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"tools",
|
||||||
|
[
|
||||||
|
"abc",
|
||||||
|
{"type": "custom"},
|
||||||
|
[None],
|
||||||
|
[{"type": "custom", "name": "x", "format": "text"}],
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_malformed_tools_rejected_by_field_validation(tools) -> None:
|
||||||
|
"""Malformed tools bodies fail pydantic field validation (4xx), not an
|
||||||
|
AttributeError inside the custom-format validator (500)."""
|
||||||
|
with pytest.raises(ValidationError):
|
||||||
|
ResponsesRequest(input="hi", tools=tools)
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from openai.types.responses import CustomTool, FunctionTool, ResponseCustomToolCall
|
||||||
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
|
||||||
from openai.types.responses.response_function_tool_call_output_item import (
|
from openai.types.responses.response_function_tool_call_output_item import (
|
||||||
ResponseFunctionToolCallOutputItem,
|
ResponseFunctionToolCallOutputItem,
|
||||||
@@ -16,10 +17,14 @@ from openai.types.responses.response_reasoning_item import (
|
|||||||
Summary,
|
Summary,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from vllm.entrypoints.generate.base.protocol import FunctionCall
|
||||||
from vllm.entrypoints.openai.responses.utils import (
|
from vllm.entrypoints.openai.responses.utils import (
|
||||||
_construct_message_from_response_item,
|
_construct_message_from_response_item,
|
||||||
|
build_response_output_items,
|
||||||
construct_chat_messages_with_tool_call,
|
construct_chat_messages_with_tool_call,
|
||||||
construct_input_messages,
|
construct_input_messages,
|
||||||
|
decode_custom_tool_input,
|
||||||
|
decode_custom_tool_input_prefix,
|
||||||
should_continue_final_message,
|
should_continue_final_message,
|
||||||
)
|
)
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
@@ -248,6 +253,89 @@ class TestResponsesUtils:
|
|||||||
assert formatted_item["content"] == "dongyi"
|
assert formatted_item["content"] == "dongyi"
|
||||||
|
|
||||||
|
|
||||||
|
class TestCustomTools:
|
||||||
|
def test_custom_tool_call_output_item(self):
|
||||||
|
tools = [
|
||||||
|
CustomTool(type="custom", name="emit_command", format={"type": "text"}),
|
||||||
|
FunctionTool(
|
||||||
|
type="function", name="get_weather", parameters={}, strict=None
|
||||||
|
),
|
||||||
|
]
|
||||||
|
custom, function = build_response_output_items(
|
||||||
|
reasoning=None,
|
||||||
|
content=None,
|
||||||
|
tool_calls=[
|
||||||
|
FunctionCall(
|
||||||
|
id="call_1", name="emit_command", arguments='{"input": "pwd"}'
|
||||||
|
),
|
||||||
|
FunctionCall(
|
||||||
|
id="call_2", name="get_weather", arguments='{"city": "Paris"}'
|
||||||
|
),
|
||||||
|
],
|
||||||
|
tools=tools,
|
||||||
|
)
|
||||||
|
assert isinstance(custom, ResponseCustomToolCall)
|
||||||
|
assert (custom.call_id, custom.input) == ("call_1", "pwd")
|
||||||
|
assert custom.id.startswith("ctc_")
|
||||||
|
assert isinstance(function, ResponseFunctionToolCall)
|
||||||
|
assert function.arguments == '{"city": "Paris"}'
|
||||||
|
|
||||||
|
def test_custom_tool_history_replays_through_function_shim(self):
|
||||||
|
messages = construct_chat_messages_with_tool_call(
|
||||||
|
[
|
||||||
|
make_reasoning_item(content_text="Run pwd."),
|
||||||
|
ResponseCustomToolCall(
|
||||||
|
type="custom_tool_call",
|
||||||
|
id="ctc_1",
|
||||||
|
call_id="call_exec",
|
||||||
|
name="emit_command",
|
||||||
|
input='echo "hi"',
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"type": "custom_tool_call_output",
|
||||||
|
"call_id": "call_exec",
|
||||||
|
"output": "hi",
|
||||||
|
},
|
||||||
|
{"role": "user", "content": "Continue"},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
assert messages == [
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"reasoning": "Run pwd.",
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"id": "call_exec",
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "emit_command",
|
||||||
|
"arguments": '{"input": "echo \\"hi\\""}',
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{"role": "tool", "content": "hi", "tool_call_id": "call_exec"},
|
||||||
|
{"role": "user", "content": "Continue"},
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_decode_custom_tool_input(self):
|
||||||
|
assert decode_custom_tool_input('{"input": "pwd"}') == "pwd"
|
||||||
|
assert decode_custom_tool_input('{"cmd": "pwd"}') == '{"cmd": "pwd"}'
|
||||||
|
assert decode_custom_tool_input('{"file": "a.py"}') == '{"file": "a.py"}'
|
||||||
|
assert decode_custom_tool_input('{"input": 7}') == '{"input": 7}'
|
||||||
|
assert decode_custom_tool_input("pwd") == "pwd"
|
||||||
|
|
||||||
|
def test_decode_custom_tool_input_prefix(self):
|
||||||
|
assert decode_custom_tool_input_prefix('{"inp') == ""
|
||||||
|
assert decode_custom_tool_input_prefix('{"input": "ls \\"dir') == 'ls "dir'
|
||||||
|
assert decode_custom_tool_input_prefix('{"input": "a\\') == "a"
|
||||||
|
assert decode_custom_tool_input_prefix('{"input": "\\ud83d') == ""
|
||||||
|
assert (
|
||||||
|
decode_custom_tool_input_prefix('{"input": "\\ud83d\\ude00!"}')
|
||||||
|
== "\U0001f600!"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestReasoningItemContentPriority:
|
class TestReasoningItemContentPriority:
|
||||||
"""Tests that content is prioritized over summary for reasoning items."""
|
"""Tests that content is prioritized over summary for reasoning items."""
|
||||||
|
|
||||||
|
|||||||
@@ -2,7 +2,12 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from openai.types.responses import ResponseFunctionWebSearch, ResponseStreamEvent
|
from openai.types.responses import (
|
||||||
|
CustomTool,
|
||||||
|
ResponseCustomToolCall,
|
||||||
|
ResponseFunctionWebSearch,
|
||||||
|
ResponseStreamEvent,
|
||||||
|
)
|
||||||
from openai_harmony import Message, Role
|
from openai_harmony import Message, Role
|
||||||
from pydantic import TypeAdapter
|
from pydantic import TypeAdapter
|
||||||
|
|
||||||
@@ -20,6 +25,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
|
|||||||
emit_reasoning_done_events,
|
emit_reasoning_done_events,
|
||||||
split_delta,
|
split_delta,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.responses.utils import decode_custom_tool_input
|
||||||
|
|
||||||
|
|
||||||
def test_browser_find_uses_responses_action_type():
|
def test_browser_find_uses_responses_action_type():
|
||||||
@@ -190,3 +196,111 @@ class TestProcessorCompoundDeltas:
|
|||||||
types = [e.type for e in events]
|
types = [e.type for e in events]
|
||||||
assert "response.reasoning_text.delta" in types
|
assert "response.reasoning_text.delta" in types
|
||||||
assert "response.output_text.delta" in types
|
assert "response.output_text.delta" in types
|
||||||
|
|
||||||
|
|
||||||
|
class TestSimpleToolCallShapes:
|
||||||
|
def test_arguments_done_emitted_without_deltas(self):
|
||||||
|
processor = SimpleStreamingEventProcessor()
|
||||||
|
events = _run_through_processor(
|
||||||
|
processor, DeltaMessage(tool_calls=[_make_tool_call(0, name="noop")])
|
||||||
|
)
|
||||||
|
events += processor.close_current()
|
||||||
|
assert [e.type for e in events] == [
|
||||||
|
"response.output_item.added",
|
||||||
|
"response.function_call_arguments.done",
|
||||||
|
"response.output_item.done",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class TestCustomToolStreaming:
|
||||||
|
def test_custom_tool_input_events(self):
|
||||||
|
processor = SimpleStreamingEventProcessor(
|
||||||
|
tools=[CustomTool(type="custom", name="emit_command")]
|
||||||
|
)
|
||||||
|
events = _run_through_processor(
|
||||||
|
processor,
|
||||||
|
DeltaMessage(
|
||||||
|
tool_calls=[
|
||||||
|
_make_tool_call(0, name="emit_command", arguments='{"input": "ls ')
|
||||||
|
]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
events += _run_through_processor(
|
||||||
|
processor,
|
||||||
|
DeltaMessage(tool_calls=[_make_tool_call(0, arguments='-la \\"x\\""}')]),
|
||||||
|
)
|
||||||
|
events += processor.close_current()
|
||||||
|
|
||||||
|
assert [e.type for e in events] == [
|
||||||
|
"response.output_item.added",
|
||||||
|
"response.custom_tool_call_input.delta",
|
||||||
|
"response.custom_tool_call_input.delta",
|
||||||
|
"response.custom_tool_call_input.done",
|
||||||
|
"response.output_item.done",
|
||||||
|
]
|
||||||
|
done = events[-1].item
|
||||||
|
assert isinstance(done, ResponseCustomToolCall)
|
||||||
|
assert done.input == 'ls -la "x"'
|
||||||
|
assert "".join(e.delta for e in events[1:3]) == done.input
|
||||||
|
assert events[3].input == done.input
|
||||||
|
assert events[0].item.type == "custom_tool_call"
|
||||||
|
assert events[0].item.id == done.id
|
||||||
|
assert done.id.startswith("ctc_")
|
||||||
|
assert events[0].item.call_id == done.call_id
|
||||||
|
|
||||||
|
def test_unrelated_keys_are_not_streamed_as_payload(self):
|
||||||
|
"""Review repro: chunks of an object whose input key arrives last must
|
||||||
|
not leak fragments of other keys into the payload."""
|
||||||
|
processor = SimpleStreamingEventProcessor(
|
||||||
|
tools=[CustomTool(type="custom", name="emit_command")]
|
||||||
|
)
|
||||||
|
events = _run_through_processor(
|
||||||
|
processor,
|
||||||
|
DeltaMessage(
|
||||||
|
tool_calls=[
|
||||||
|
_make_tool_call(0, name="emit_command", arguments='{"a": "xy"')
|
||||||
|
]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
events += _run_through_processor(
|
||||||
|
processor,
|
||||||
|
DeltaMessage(
|
||||||
|
tool_calls=[_make_tool_call(0, arguments=', "input": "hello"}')]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
events += processor.close_current()
|
||||||
|
|
||||||
|
deltas = [
|
||||||
|
e for e in events if e.type == "response.custom_tool_call_input.delta"
|
||||||
|
]
|
||||||
|
done_input = next(
|
||||||
|
e for e in events if e.type == "response.custom_tool_call_input.done"
|
||||||
|
).input
|
||||||
|
item = events[-1].item
|
||||||
|
assert "".join(e.delta for e in deltas) == "hello" # "xy" never leaks
|
||||||
|
assert done_input == "hello"
|
||||||
|
assert item.input == "hello"
|
||||||
|
assert decode_custom_tool_input('{"a": "xy", "input": "hello"}') == "hello"
|
||||||
|
|
||||||
|
def test_done_reported_authoritative_payload_on_mismatch(self):
|
||||||
|
"""If the final decode is not an extension of what was streamed, the
|
||||||
|
done event and item carry the authoritative value and a correcting
|
||||||
|
delta is emitted rather than dropping the divergence."""
|
||||||
|
processor = SimpleStreamingEventProcessor(
|
||||||
|
tools=[CustomTool(type="custom", name="emit_command")]
|
||||||
|
)
|
||||||
|
processor.state.current_state = _StateType.TOOL_CALL
|
||||||
|
processor.state.tool_call_is_custom = True
|
||||||
|
processor.state.tool_call_payload = "xy" # stale fragment already streamed
|
||||||
|
processor.state.accumulated_text = '{"a": "xy", "input": "hello"}'
|
||||||
|
|
||||||
|
events = processor.close_current()
|
||||||
|
delta = next(
|
||||||
|
e for e in events if e.type == "response.custom_tool_call_input.delta"
|
||||||
|
)
|
||||||
|
done_input = next(
|
||||||
|
e for e in events if e.type == "response.custom_tool_call_input.done"
|
||||||
|
).input
|
||||||
|
assert delta.delta == "hello" # full authoritative resend, not a dropped suffix
|
||||||
|
assert done_input == "hello"
|
||||||
|
assert events[-1].item.input == "hello"
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from unittest.mock import MagicMock
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.parser.gemma4 import (
|
from vllm.parser.gemma4 import (
|
||||||
TOOL_CALL_END,
|
TOOL_CALL_END,
|
||||||
TOOL_CALL_START,
|
TOOL_CALL_START,
|
||||||
@@ -117,6 +118,21 @@ def mock_request():
|
|||||||
return request
|
return request
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_custom_tool_choice_skips_structured_outputs(mock_tokenizer):
|
||||||
|
# Responses custom tools ride the function shim, so a named custom choice
|
||||||
|
# must route like a named function choice, not into guided JSON.
|
||||||
|
tools = [{"type": "custom", "name": "emit_command"}]
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hi",
|
||||||
|
tools=tools,
|
||||||
|
tool_choice={"type": "custom", "name": "emit_command"},
|
||||||
|
)
|
||||||
|
Gemma4EngineToolParser(mock_tokenizer, tools=tools).adjust_request(request)
|
||||||
|
|
||||||
|
assert request.skip_special_tokens is False
|
||||||
|
assert request.structured_outputs is None
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Unit tests for _parse_gemma4_args (shared parser logic)
|
# Unit tests for _parse_gemma4_args (shared parser logic)
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import pytest
|
|||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
from vllm.entrypoints.generate.base.protocol import DeltaMessage
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.parser.parser_manager import ParserManager
|
from vllm.parser.parser_manager import ParserManager
|
||||||
from vllm.tool_parsers import ToolParserManager
|
from vllm.tool_parsers import ToolParserManager
|
||||||
from vllm.tool_parsers.k2_horizon_tool_parser import K2HorizonToolParser
|
from vllm.tool_parsers.k2_horizon_tool_parser import K2HorizonToolParser
|
||||||
@@ -316,3 +317,14 @@ def test_whitespace_only_surrounding_content_is_not_preserved(tokenizer):
|
|||||||
def test_tool_parser_registered():
|
def test_tool_parser_registered():
|
||||||
assert ToolParserManager.get_tool_parser("k2_horizon") is K2HorizonToolParser
|
assert ToolParserManager.get_tool_parser("k2_horizon") is K2HorizonToolParser
|
||||||
assert K2HorizonToolParser.supports_required_and_named is False
|
assert K2HorizonToolParser.supports_required_and_named is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_tool_resolves_custom_choice():
|
||||||
|
# Custom tools ride the function shim; a ToolChoiceCustom must resolve to
|
||||||
|
# its name like ToolChoiceFunction does.
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hi",
|
||||||
|
tools=[{"type": "custom", "name": "emit_command"}],
|
||||||
|
tool_choice={"type": "custom", "name": "emit_command"},
|
||||||
|
)
|
||||||
|
assert K2HorizonToolParser._named_tool(request) == "emit_command"
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import pytest
|
|||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
)
|
)
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.sampling_params import StructuredOutputsParams
|
from vllm.sampling_params import StructuredOutputsParams
|
||||||
|
|
||||||
@@ -60,3 +61,26 @@ def test_named_choice_allowed_with_structural_tag():
|
|||||||
def test_named_choice_rejected_without_structural_tag():
|
def test_named_choice_rejected_without_structural_tag():
|
||||||
with pytest.raises(VLLMValidationError):
|
with pytest.raises(VLLMValidationError):
|
||||||
_parser().adjust_request(_request(with_tag=False))
|
_parser().adjust_request(_request(with_tag=False))
|
||||||
|
|
||||||
|
|
||||||
|
def _custom_request(with_tag: bool) -> ResponsesRequest:
|
||||||
|
req = ResponsesRequest(
|
||||||
|
input="hi",
|
||||||
|
tools=[{"type": "custom", "name": "emit_command"}],
|
||||||
|
tool_choice={"type": "custom", "name": "emit_command"},
|
||||||
|
)
|
||||||
|
if with_tag:
|
||||||
|
req.structured_outputs = StructuredOutputsParams(
|
||||||
|
structural_tag='{"type": "structural_tag", "format": {}}'
|
||||||
|
)
|
||||||
|
return req
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_custom_choice_allowed_with_structural_tag():
|
||||||
|
req = _parser().adjust_request(_custom_request(with_tag=True))
|
||||||
|
assert req.skip_special_tokens is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_custom_choice_rejected_without_structural_tag():
|
||||||
|
with pytest.raises(VLLMValidationError):
|
||||||
|
_parser().adjust_request(_custom_request(with_tag=False))
|
||||||
|
|||||||
@@ -78,12 +78,15 @@ def _build_chat_request(*, tool_choice: str | dict[str, Any]) -> ChatCompletionR
|
|||||||
|
|
||||||
|
|
||||||
def _build_responses_request(
|
def _build_responses_request(
|
||||||
*, tool_choice: str | dict[str, Any], include: list[str] | None = None
|
*,
|
||||||
|
tool_choice: str | dict[str, Any],
|
||||||
|
include: list[str] | None = None,
|
||||||
|
tools: list[dict[str, Any]] | None = None,
|
||||||
) -> ResponsesRequest:
|
) -> ResponsesRequest:
|
||||||
return ResponsesRequest(
|
return ResponsesRequest(
|
||||||
model="poolside-test",
|
model="poolside-test",
|
||||||
input=[{"role": "user", "content": "write the file"}],
|
input=[{"role": "user", "content": "write the file"}],
|
||||||
tools=[_responses_write_file_tool()],
|
tools=tools if tools is not None else [_responses_write_file_tool()],
|
||||||
tool_choice=tool_choice,
|
tool_choice=tool_choice,
|
||||||
stream=True,
|
stream=True,
|
||||||
max_output_tokens=200,
|
max_output_tokens=200,
|
||||||
@@ -145,6 +148,19 @@ def test_named_skips_structured_outputs_responses() -> None:
|
|||||||
assert request.skip_special_tokens is False
|
assert request.skip_special_tokens is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_custom_tool_choice_skips_structured_outputs_responses() -> None:
|
||||||
|
# A named custom choice parses to ToolChoiceCustom; custom tools ride the
|
||||||
|
# function shim, so it must route like ToolChoiceFunction.
|
||||||
|
request = _build_responses_request(
|
||||||
|
tools=[{"type": "custom", "name": "emit_command"}],
|
||||||
|
tool_choice={"type": "custom", "name": "emit_command"},
|
||||||
|
)
|
||||||
|
PoolsideV1ToolParser(_StubTokenizer()).adjust_request(request)
|
||||||
|
|
||||||
|
assert request.text is None
|
||||||
|
assert request.skip_special_tokens is False
|
||||||
|
|
||||||
|
|
||||||
def test_auto_still_keeps_special_tokens() -> None:
|
def test_auto_still_keeps_special_tokens() -> None:
|
||||||
request = _build_chat_request(tool_choice="auto")
|
request = _build_chat_request(tool_choice="auto")
|
||||||
_make_parser(request).adjust_request(request)
|
_make_parser(request).adjust_request(request)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from typing import Any
|
|||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import CustomTool, ToolChoiceFunction
|
||||||
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
||||||
from xgrammar import Grammar, StructuralTag
|
from xgrammar import Grammar, StructuralTag
|
||||||
from xgrammar import get_model_structural_tag as get_xgrammar_builtin_structural_tag
|
from xgrammar import get_model_structural_tag as get_xgrammar_builtin_structural_tag
|
||||||
@@ -787,6 +787,19 @@ def test_get_model_structural_tag_supports_named_tool_choice(
|
|||||||
assert isinstance(tag, StructuralTag)
|
assert isinstance(tag, StructuralTag)
|
||||||
|
|
||||||
|
|
||||||
|
def test_glm_4_7_forced_custom_tool_uses_function_shim():
|
||||||
|
tools = [CustomTool(type="custom", name="emit_command", description="Emit.")]
|
||||||
|
tag = get_model_structural_tag(
|
||||||
|
model="glm_4_7",
|
||||||
|
tools=tools,
|
||||||
|
tool_choice="required",
|
||||||
|
reasoning=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(tag, StructuralTag)
|
||||||
|
assert "emit_command" in tag.model_dump_json()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("parser_cls", "model"),
|
("parser_cls", "model"),
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ from typing import cast
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||||
|
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||||
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
|
from vllm.reasoning.muse_glimmer_reasoning_parser import MuseGlimmerReasoningParser
|
||||||
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
|
from vllm.tool_parsers.muse_glimmer_tool_parser import MuseGlimmerToolParser
|
||||||
|
|
||||||
@@ -400,3 +401,15 @@ def test_exact_match_kept():
|
|||||||
T, _call("get_weather"), _req("get_weather")
|
T, _call("get_weather"), _req("get_weather")
|
||||||
)
|
)
|
||||||
assert out.tool_calls[0].function.name == "get_weather"
|
assert out.tool_calls[0].function.name == "get_weather"
|
||||||
|
|
||||||
|
|
||||||
|
def test_named_custom_tool_choice_skips_structured_outputs():
|
||||||
|
# Custom tools ride the function shim; a named custom choice must skip
|
||||||
|
# the base adjust_request like a named function choice does.
|
||||||
|
request = ResponsesRequest(
|
||||||
|
input="hi",
|
||||||
|
tools=[{"type": "custom", "name": "emit_command"}],
|
||||||
|
tool_choice={"type": "custom", "name": "emit_command"},
|
||||||
|
)
|
||||||
|
T.adjust_request(request)
|
||||||
|
assert request.structured_outputs is None
|
||||||
|
|||||||
@@ -6,7 +6,9 @@ from copy import deepcopy
|
|||||||
import pytest
|
import pytest
|
||||||
import regex as re
|
import regex as re
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
|
CustomTool,
|
||||||
FunctionTool,
|
FunctionTool,
|
||||||
|
ToolChoiceCustom,
|
||||||
ToolChoiceFunction,
|
ToolChoiceFunction,
|
||||||
WebSearchTool,
|
WebSearchTool,
|
||||||
)
|
)
|
||||||
@@ -21,8 +23,10 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
|||||||
from vllm.exceptions import VLLMValidationError
|
from vllm.exceptions import VLLMValidationError
|
||||||
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
from vllm.tool_parsers.streaming import extract_required_tool_call_streaming
|
||||||
from vllm.tool_parsers.utils import (
|
from vllm.tool_parsers.utils import (
|
||||||
|
custom_tool_parameters,
|
||||||
find_tool_properties,
|
find_tool_properties,
|
||||||
get_json_schema_from_tools,
|
get_json_schema_from_tools,
|
||||||
|
iter_response_function_tool_dicts,
|
||||||
)
|
)
|
||||||
|
|
||||||
pytestmark = pytest.mark.cpu_test
|
pytestmark = pytest.mark.cpu_test
|
||||||
@@ -483,6 +487,37 @@ class TestNonFunctionToolsSkipped:
|
|||||||
get_json_schema_from_tools(tools=[WEB_SEARCH_TOOL], tool_choice="required")
|
get_json_schema_from_tools(tools=[WEB_SEARCH_TOOL], tool_choice="required")
|
||||||
|
|
||||||
|
|
||||||
|
class TestCustomToolShim:
|
||||||
|
"""``custom`` tools are shown to the model as a single-string function."""
|
||||||
|
|
||||||
|
def test_rendered_as_single_string_function(self):
|
||||||
|
tool = CustomTool(
|
||||||
|
type="custom", name="emit_command", description="Emit a command."
|
||||||
|
)
|
||||||
|
[tool_dict] = iter_response_function_tool_dicts([tool])
|
||||||
|
assert tool_dict["type"] == "function"
|
||||||
|
assert tool_dict["name"] == "emit_command"
|
||||||
|
assert tool_dict["description"] == "Emit a command."
|
||||||
|
assert tool_dict["parameters"] == custom_tool_parameters()
|
||||||
|
assert tool_dict["parameters"]["required"] == ["input"]
|
||||||
|
|
||||||
|
def test_forced_tool_choice_sees_custom_tool(self):
|
||||||
|
tools = [CustomTool(type="custom", name="emit_command"), FUNCTION_TOOL]
|
||||||
|
schema = get_json_schema_from_tools(tools=tools, tool_choice="required")
|
||||||
|
assert isinstance(schema, dict)
|
||||||
|
names = [e["properties"]["name"]["enum"] for e in schema["items"]["anyOf"]]
|
||||||
|
assert names == [["emit_command"], ["get_weather"]]
|
||||||
|
named = get_json_schema_from_tools(
|
||||||
|
tools=tools,
|
||||||
|
tool_choice=ToolChoiceCustom(type="custom", name="emit_command"),
|
||||||
|
)
|
||||||
|
assert named == custom_tool_parameters()
|
||||||
|
assert (
|
||||||
|
find_tool_properties(tools, "emit_command")
|
||||||
|
== custom_tool_parameters()["properties"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestMalformedToolSchemaDefs:
|
class TestMalformedToolSchemaDefs:
|
||||||
"""Malformed `$defs` in caller-supplied tool parameters is a 400, not a 500."""
|
"""Malformed `$defs` in caller-supplied tool parameters is a 400, not a 500."""
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ from openai.types.responses import (
|
|||||||
ResponseCodeInterpreterCallInterpretingEvent,
|
ResponseCodeInterpreterCallInterpretingEvent,
|
||||||
ResponseContentPartAddedEvent,
|
ResponseContentPartAddedEvent,
|
||||||
ResponseContentPartDoneEvent,
|
ResponseContentPartDoneEvent,
|
||||||
|
ResponseCustomToolCallInputDeltaEvent,
|
||||||
|
ResponseCustomToolCallInputDoneEvent,
|
||||||
ResponseFunctionToolCall,
|
ResponseFunctionToolCall,
|
||||||
ResponseInputItemParam,
|
ResponseInputItemParam,
|
||||||
ResponseMcpCallArgumentsDeltaEvent,
|
ResponseMcpCallArgumentsDeltaEvent,
|
||||||
@@ -699,6 +701,34 @@ class ResponsesRequest(OpenAIBaseModel):
|
|||||||
data["input"] = processed_input
|
data["input"] = processed_input
|
||||||
return data
|
return data
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def check_custom_tool_format(cls, data):
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return data
|
||||||
|
tools = data.get("tools")
|
||||||
|
if not isinstance(tools, list):
|
||||||
|
return data
|
||||||
|
for tool in tools:
|
||||||
|
if isinstance(tool, dict):
|
||||||
|
tool_dict = tool
|
||||||
|
elif hasattr(tool, "model_dump"):
|
||||||
|
tool_dict = tool.model_dump()
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
tool_format = tool_dict.get("format")
|
||||||
|
if (
|
||||||
|
tool_dict.get("type") == "custom"
|
||||||
|
and isinstance(tool_format, dict)
|
||||||
|
and tool_format.get("type") == "grammar"
|
||||||
|
):
|
||||||
|
raise VLLMValidationError(
|
||||||
|
"Custom tools with a grammar format are not supported; "
|
||||||
|
"only free-form text custom tools are accepted.",
|
||||||
|
parameter="tools",
|
||||||
|
)
|
||||||
|
return data
|
||||||
|
|
||||||
@model_validator(mode="before")
|
@model_validator(mode="before")
|
||||||
@classmethod
|
@classmethod
|
||||||
def check_tool_usage(cls, data):
|
def check_tool_usage(cls, data):
|
||||||
@@ -708,9 +738,9 @@ class ResponsesRequest(OpenAIBaseModel):
|
|||||||
tools = data.get("tools")
|
tools = data.get("tools")
|
||||||
tool_choice = data.get("tool_choice", "auto")
|
tool_choice = data.get("tool_choice", "auto")
|
||||||
has_tools = tools is not None and len(tools) > 0
|
has_tools = tools is not None and len(tools) > 0
|
||||||
is_named_tool_choice = (
|
is_named_tool_choice = isinstance(tool_choice, dict) and tool_choice.get(
|
||||||
isinstance(tool_choice, dict) and tool_choice.get("type") == "function"
|
"type"
|
||||||
)
|
) in ("function", "custom")
|
||||||
|
|
||||||
if not has_tools:
|
if not has_tools:
|
||||||
if tool_choice in ("auto", "none"):
|
if tool_choice in ("auto", "none"):
|
||||||
@@ -913,6 +943,8 @@ StreamingResponsesResponse: TypeAlias = (
|
|||||||
| ResponseOutputItemDoneEvent
|
| ResponseOutputItemDoneEvent
|
||||||
| ResponseContentPartAddedEvent
|
| ResponseContentPartAddedEvent
|
||||||
| ResponseContentPartDoneEvent
|
| ResponseContentPartDoneEvent
|
||||||
|
| ResponseCustomToolCallInputDeltaEvent
|
||||||
|
| ResponseCustomToolCallInputDoneEvent
|
||||||
| ResponseReasoningTextDeltaEvent
|
| ResponseReasoningTextDeltaEvent
|
||||||
| ResponseReasoningTextDoneEvent
|
| ResponseReasoningTextDoneEvent
|
||||||
| ResponseCodeInterpreterCallInProgressEvent
|
| ResponseCodeInterpreterCallInProgressEvent
|
||||||
|
|||||||
@@ -29,6 +29,9 @@ from openai.types.responses import (
|
|||||||
ResponseCodeInterpreterToolCallParam,
|
ResponseCodeInterpreterToolCallParam,
|
||||||
ResponseContentPartAddedEvent,
|
ResponseContentPartAddedEvent,
|
||||||
ResponseContentPartDoneEvent,
|
ResponseContentPartDoneEvent,
|
||||||
|
ResponseCustomToolCall,
|
||||||
|
ResponseCustomToolCallInputDeltaEvent,
|
||||||
|
ResponseCustomToolCallInputDoneEvent,
|
||||||
ResponseFunctionCallArgumentsDeltaEvent,
|
ResponseFunctionCallArgumentsDeltaEvent,
|
||||||
ResponseFunctionCallArgumentsDoneEvent,
|
ResponseFunctionCallArgumentsDoneEvent,
|
||||||
ResponseFunctionToolCall,
|
ResponseFunctionToolCall,
|
||||||
@@ -73,6 +76,9 @@ from vllm.entrypoints.openai.responses.protocol import (
|
|||||||
)
|
)
|
||||||
from vllm.entrypoints.openai.responses.utils import (
|
from vllm.entrypoints.openai.responses.utils import (
|
||||||
build_responses_tool_call_name_map,
|
build_responses_tool_call_name_map,
|
||||||
|
custom_tool_names,
|
||||||
|
decode_custom_tool_input,
|
||||||
|
decode_custom_tool_input_prefix,
|
||||||
resolve_responses_tool_call_name,
|
resolve_responses_tool_call_name,
|
||||||
)
|
)
|
||||||
from vllm.outputs import CompletionOutput
|
from vllm.outputs import CompletionOutput
|
||||||
@@ -821,6 +827,8 @@ class SimpleStreamingState:
|
|||||||
tool_call_name: str = ""
|
tool_call_name: str = ""
|
||||||
tool_call_namespace: str | None = None
|
tool_call_namespace: str | None = None
|
||||||
tool_call_index: int | None = None
|
tool_call_index: int | None = None
|
||||||
|
tool_call_is_custom: bool = False
|
||||||
|
tool_call_payload: str = ""
|
||||||
current_state: _StateType = field(default_factory=lambda: _StateType.NONE)
|
current_state: _StateType = field(default_factory=lambda: _StateType.NONE)
|
||||||
|
|
||||||
|
|
||||||
@@ -1039,38 +1047,80 @@ def emit_simple_tool_call_open(
|
|||||||
name: str,
|
name: str,
|
||||||
index: int | None,
|
index: int | None,
|
||||||
namespace: str | None = None,
|
namespace: str | None = None,
|
||||||
|
is_custom: bool = False,
|
||||||
call_id: str | None = None,
|
call_id: str | None = None,
|
||||||
) -> list[StreamingResponsesResponse]:
|
) -> list[StreamingResponsesResponse]:
|
||||||
state.current_state = _StateType.TOOL_CALL
|
state.current_state = _StateType.TOOL_CALL
|
||||||
state.current_item_id = f"fc_{random_uuid()}"
|
state.current_item_id = (
|
||||||
|
f"ctc_{random_uuid()}" if is_custom else f"fc_{random_uuid()}"
|
||||||
|
)
|
||||||
state.tool_call_id = call_id or make_tool_call_id()
|
state.tool_call_id = call_id or make_tool_call_id()
|
||||||
state.tool_call_name = name
|
state.tool_call_name = name
|
||||||
state.tool_call_namespace = namespace
|
state.tool_call_namespace = namespace
|
||||||
state.tool_call_index = index
|
state.tool_call_index = index
|
||||||
|
state.tool_call_is_custom = is_custom
|
||||||
|
state.tool_call_payload = ""
|
||||||
state.accumulated_text = ""
|
state.accumulated_text = ""
|
||||||
|
item: ResponseOutputItem
|
||||||
|
if is_custom:
|
||||||
|
item = ResponseCustomToolCall(
|
||||||
|
type="custom_tool_call",
|
||||||
|
id=state.current_item_id,
|
||||||
|
call_id=state.tool_call_id,
|
||||||
|
name=name,
|
||||||
|
input="",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
item = ResponseFunctionToolCallItem(
|
||||||
|
type="function_call",
|
||||||
|
id=state.current_item_id,
|
||||||
|
call_id=state.tool_call_id,
|
||||||
|
name=name,
|
||||||
|
namespace=namespace,
|
||||||
|
arguments="",
|
||||||
|
status="in_progress",
|
||||||
|
)
|
||||||
return [
|
return [
|
||||||
ResponseOutputItemAddedEvent(
|
ResponseOutputItemAddedEvent(
|
||||||
type="response.output_item.added",
|
type="response.output_item.added",
|
||||||
sequence_number=-1,
|
sequence_number=-1,
|
||||||
output_index=state.output_index,
|
output_index=state.output_index,
|
||||||
item=ResponseFunctionToolCallItem(
|
item=item,
|
||||||
type="function_call",
|
|
||||||
id=state.current_item_id,
|
|
||||||
call_id=state.tool_call_id,
|
|
||||||
name=name,
|
|
||||||
namespace=namespace,
|
|
||||||
arguments="",
|
|
||||||
status="in_progress",
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _emit_custom_tool_input_delta(
|
||||||
|
state: SimpleStreamingState,
|
||||||
|
payload: str,
|
||||||
|
) -> list[StreamingResponsesResponse]:
|
||||||
|
streamed = state.tool_call_payload
|
||||||
|
if payload == streamed:
|
||||||
|
return []
|
||||||
|
state.tool_call_payload = payload
|
||||||
|
# A payload that is not an extension of what was streamed cannot be
|
||||||
|
# retracted, so the authoritative value is resent in full.
|
||||||
|
delta = payload[len(streamed) :] if payload.startswith(streamed) else payload
|
||||||
|
return [
|
||||||
|
ResponseCustomToolCallInputDeltaEvent(
|
||||||
|
type="response.custom_tool_call_input.delta",
|
||||||
|
sequence_number=-1,
|
||||||
|
output_index=state.output_index,
|
||||||
|
item_id=state.current_item_id,
|
||||||
|
delta=delta,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def emit_simple_tool_call_delta(
|
def emit_simple_tool_call_delta(
|
||||||
state: SimpleStreamingState,
|
state: SimpleStreamingState,
|
||||||
delta: str,
|
delta: str,
|
||||||
) -> list[StreamingResponsesResponse]:
|
) -> list[StreamingResponsesResponse]:
|
||||||
state.accumulated_text += delta
|
state.accumulated_text += delta
|
||||||
|
if state.tool_call_is_custom:
|
||||||
|
return _emit_custom_tool_input_delta(
|
||||||
|
state, decode_custom_tool_input_prefix(state.accumulated_text)
|
||||||
|
)
|
||||||
return [
|
return [
|
||||||
ResponseFunctionCallArgumentsDeltaEvent(
|
ResponseFunctionCallArgumentsDeltaEvent(
|
||||||
type="response.function_call_arguments.delta",
|
type="response.function_call_arguments.delta",
|
||||||
@@ -1085,32 +1135,58 @@ def emit_simple_tool_call_delta(
|
|||||||
def emit_simple_tool_call_done(
|
def emit_simple_tool_call_done(
|
||||||
state: SimpleStreamingState,
|
state: SimpleStreamingState,
|
||||||
) -> list[StreamingResponsesResponse]:
|
) -> list[StreamingResponsesResponse]:
|
||||||
events: list[StreamingResponsesResponse] = [
|
events: list[StreamingResponsesResponse]
|
||||||
ResponseFunctionCallArgumentsDoneEvent(
|
item: ResponseOutputItem
|
||||||
type="response.function_call_arguments.done",
|
if state.tool_call_is_custom:
|
||||||
sequence_number=-1,
|
payload = decode_custom_tool_input(state.accumulated_text)
|
||||||
output_index=state.output_index,
|
events = _emit_custom_tool_input_delta(state, payload)
|
||||||
item_id=state.current_item_id,
|
events.append(
|
||||||
arguments=state.accumulated_text,
|
ResponseCustomToolCallInputDoneEvent(
|
||||||
|
type="response.custom_tool_call_input.done",
|
||||||
|
sequence_number=-1,
|
||||||
|
output_index=state.output_index,
|
||||||
|
item_id=state.current_item_id,
|
||||||
|
input=payload,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
item = ResponseCustomToolCall(
|
||||||
|
type="custom_tool_call",
|
||||||
|
id=state.current_item_id,
|
||||||
|
call_id=state.tool_call_id,
|
||||||
name=state.tool_call_name,
|
name=state.tool_call_name,
|
||||||
),
|
input=payload,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
events = [
|
||||||
|
ResponseFunctionCallArgumentsDoneEvent(
|
||||||
|
type="response.function_call_arguments.done",
|
||||||
|
sequence_number=-1,
|
||||||
|
output_index=state.output_index,
|
||||||
|
item_id=state.current_item_id,
|
||||||
|
arguments=state.accumulated_text,
|
||||||
|
name=state.tool_call_name,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
item = ResponseFunctionToolCall(
|
||||||
|
type="function_call",
|
||||||
|
name=state.tool_call_name,
|
||||||
|
namespace=state.tool_call_namespace,
|
||||||
|
arguments=state.accumulated_text,
|
||||||
|
status="completed",
|
||||||
|
id=state.current_item_id,
|
||||||
|
call_id=state.tool_call_id,
|
||||||
|
)
|
||||||
|
events.append(
|
||||||
ResponseOutputItemDoneEvent(
|
ResponseOutputItemDoneEvent(
|
||||||
type="response.output_item.done",
|
type="response.output_item.done",
|
||||||
sequence_number=-1,
|
sequence_number=-1,
|
||||||
output_index=state.output_index,
|
output_index=state.output_index,
|
||||||
item=ResponseFunctionToolCall(
|
item=item,
|
||||||
type="function_call",
|
)
|
||||||
name=state.tool_call_name,
|
)
|
||||||
namespace=state.tool_call_namespace,
|
|
||||||
arguments=state.accumulated_text,
|
|
||||||
status="completed",
|
|
||||||
id=state.current_item_id,
|
|
||||||
call_id=state.tool_call_id,
|
|
||||||
),
|
|
||||||
),
|
|
||||||
]
|
|
||||||
state.output_index += 1
|
state.output_index += 1
|
||||||
state.tool_call_namespace = None
|
state.tool_call_namespace = None
|
||||||
|
state.tool_call_is_custom = False
|
||||||
state.current_state = _StateType.NONE
|
state.current_state = _StateType.NONE
|
||||||
return events
|
return events
|
||||||
|
|
||||||
@@ -1194,6 +1270,7 @@ class SimpleStreamingEventProcessor:
|
|||||||
) -> None:
|
) -> None:
|
||||||
self.state = state or SimpleStreamingState()
|
self.state = state or SimpleStreamingState()
|
||||||
self.tool_call_name_map = build_responses_tool_call_name_map(tools)
|
self.tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||||
|
self.custom_tool_names = custom_tool_names(tools)
|
||||||
self.output_items: list[ResponseOutputItem] = []
|
self.output_items: list[ResponseOutputItem] = []
|
||||||
|
|
||||||
def resolve_target_state(
|
def resolve_target_state(
|
||||||
@@ -1264,6 +1341,7 @@ class SimpleStreamingEventProcessor:
|
|||||||
call_name.name,
|
call_name.name,
|
||||||
tool_call.index,
|
tool_call.index,
|
||||||
call_name.namespace,
|
call_name.namespace,
|
||||||
|
call_name.name in self.custom_tool_names,
|
||||||
tool_call.id,
|
tool_call.id,
|
||||||
)
|
)
|
||||||
return handlers.open_fn(self.state)
|
return handlers.open_fn(self.state)
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
|
import json
|
||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
import partial_json_parser
|
||||||
from openai.types.chat import (
|
from openai.types.chat import (
|
||||||
ChatCompletionAssistantMessageParam,
|
ChatCompletionAssistantMessageParam,
|
||||||
ChatCompletionMessageToolCallParam,
|
ChatCompletionMessageToolCallParam,
|
||||||
@@ -13,6 +15,7 @@ from openai.types.chat.chat_completion_message_tool_call_param import (
|
|||||||
Function as FunctionCallTool,
|
Function as FunctionCallTool,
|
||||||
)
|
)
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
|
ResponseCustomToolCall,
|
||||||
ResponseFunctionToolCall,
|
ResponseFunctionToolCall,
|
||||||
ResponseOutputItem,
|
ResponseOutputItem,
|
||||||
ResponseOutputMessage,
|
ResponseOutputMessage,
|
||||||
@@ -28,6 +31,7 @@ from openai.types.responses.response_reasoning_item import (
|
|||||||
Content as ResponseReasoningTextContent,
|
Content as ResponseReasoningTextContent,
|
||||||
)
|
)
|
||||||
from openai.types.responses.tool import Tool
|
from openai.types.responses.tool import Tool
|
||||||
|
from partial_json_parser.core.options import Allow
|
||||||
|
|
||||||
from vllm import envs
|
from vllm import envs
|
||||||
from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition
|
from vllm.entrypoints.generate.base.protocol import FunctionCall, FunctionDefinition
|
||||||
@@ -40,6 +44,7 @@ from vllm.exceptions import VLLMValidationError
|
|||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.renderers.chat_utils import make_tool_call_id
|
from vllm.renderers.chat_utils import make_tool_call_id
|
||||||
from vllm.tool_parsers.utils import (
|
from vllm.tool_parsers.utils import (
|
||||||
|
CUSTOM_TOOL_INPUT_KEY,
|
||||||
build_responses_tool_call_name_map,
|
build_responses_tool_call_name_map,
|
||||||
flat_namespace_tool_name,
|
flat_namespace_tool_name,
|
||||||
iter_response_function_tool_dicts,
|
iter_response_function_tool_dicts,
|
||||||
@@ -50,6 +55,51 @@ from vllm.utils import random_uuid
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def custom_tool_names(tools: list[Tool] | None) -> frozenset[str]:
|
||||||
|
return frozenset(tool.name for tool in tools or [] if tool.type == "custom")
|
||||||
|
|
||||||
|
|
||||||
|
def encode_custom_tool_input(payload: str) -> str:
|
||||||
|
return json.dumps({CUSTOM_TOOL_INPUT_KEY: payload}, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def _custom_tool_payload(parsed: Any) -> str | None:
|
||||||
|
if not isinstance(parsed, dict):
|
||||||
|
return None
|
||||||
|
value = parsed.get(CUSTOM_TOOL_INPUT_KEY)
|
||||||
|
return value if isinstance(value, str) else None
|
||||||
|
|
||||||
|
|
||||||
|
def decode_custom_tool_input(arguments: str) -> str:
|
||||||
|
"""Payload of a completed shim call; bare text is passed through."""
|
||||||
|
try:
|
||||||
|
payload = _custom_tool_payload(json.loads(arguments))
|
||||||
|
except ValueError:
|
||||||
|
return arguments
|
||||||
|
return arguments if payload is None else payload
|
||||||
|
|
||||||
|
|
||||||
|
def decode_custom_tool_input_prefix(arguments: str) -> str:
|
||||||
|
"""Longest decodable payload prefix of partial shim arguments, so that
|
||||||
|
streamed ``custom_tool_call_input`` deltas stay a prefix of the final input."""
|
||||||
|
try:
|
||||||
|
parsed = partial_json_parser.loads(arguments, Allow.STR | Allow.OBJ)
|
||||||
|
except (ValueError, IndexError, AssertionError):
|
||||||
|
return ""
|
||||||
|
payload = _custom_tool_payload(parsed) or ""
|
||||||
|
if payload and "\ud800" <= payload[-1] <= "\udbff":
|
||||||
|
payload = payload[:-1]
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def _item_type(item: ResponseInputOutputItem) -> str | None:
|
||||||
|
return item.get("type") if isinstance(item, dict) else getattr(item, "type", None)
|
||||||
|
|
||||||
|
|
||||||
|
def _item_dict(item: ResponseInputOutputItem) -> dict[str, Any]:
|
||||||
|
return item if isinstance(item, dict) else item.model_dump()
|
||||||
|
|
||||||
|
|
||||||
def build_response_output_items(
|
def build_response_output_items(
|
||||||
reasoning: str | None,
|
reasoning: str | None,
|
||||||
content: str | None,
|
content: str | None,
|
||||||
@@ -59,6 +109,7 @@ def build_response_output_items(
|
|||||||
) -> list[ResponseOutputItem]:
|
) -> list[ResponseOutputItem]:
|
||||||
outputs: list[ResponseOutputItem] = []
|
outputs: list[ResponseOutputItem] = []
|
||||||
tool_call_name_map = build_responses_tool_call_name_map(tools)
|
tool_call_name_map = build_responses_tool_call_name_map(tools)
|
||||||
|
custom_names = custom_tool_names(tools)
|
||||||
|
|
||||||
if reasoning:
|
if reasoning:
|
||||||
outputs.append(
|
outputs.append(
|
||||||
@@ -93,14 +144,27 @@ def build_response_output_items(
|
|||||||
|
|
||||||
if tool_calls:
|
if tool_calls:
|
||||||
for idx, tool_call in enumerate(tool_calls):
|
for idx, tool_call in enumerate(tool_calls):
|
||||||
|
call_id = tool_call.id or make_tool_call_id(
|
||||||
|
func_name=tool_call.name, idx=idx
|
||||||
|
)
|
||||||
|
if tool_call.name in custom_names:
|
||||||
|
outputs.append(
|
||||||
|
ResponseCustomToolCall(
|
||||||
|
id=f"ctc_{random_uuid()}",
|
||||||
|
call_id=call_id,
|
||||||
|
type="custom_tool_call",
|
||||||
|
name=tool_call.name,
|
||||||
|
input=decode_custom_tool_input(tool_call.arguments),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
call_name = resolve_responses_tool_call_name(
|
call_name = resolve_responses_tool_call_name(
|
||||||
tool_call.name, tool_call_name_map=tool_call_name_map
|
tool_call.name, tool_call_name_map=tool_call_name_map
|
||||||
)
|
)
|
||||||
outputs.append(
|
outputs.append(
|
||||||
ResponseFunctionToolCall(
|
ResponseFunctionToolCall(
|
||||||
id=f"fc_{random_uuid()}",
|
id=f"fc_{random_uuid()}",
|
||||||
call_id=tool_call.id
|
call_id=call_id,
|
||||||
or make_tool_call_id(func_name=tool_call.name, idx=idx),
|
|
||||||
type="function_call",
|
type="function_call",
|
||||||
status="completed",
|
status="completed",
|
||||||
name=call_name.name,
|
name=call_name.name,
|
||||||
@@ -255,6 +319,7 @@ def _construct_message_from_response_item(
|
|||||||
prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None
|
prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None
|
||||||
)
|
)
|
||||||
|
|
||||||
|
tool_call: ChatCompletionMessageToolCallParam | None = None
|
||||||
if isinstance(item, ResponseFunctionToolCall):
|
if isinstance(item, ResponseFunctionToolCall):
|
||||||
tool_name = item.name
|
tool_name = item.name
|
||||||
if item.namespace:
|
if item.namespace:
|
||||||
@@ -267,6 +332,17 @@ def _construct_message_from_response_item(
|
|||||||
),
|
),
|
||||||
type="function",
|
type="function",
|
||||||
)
|
)
|
||||||
|
elif _item_type(item) == "custom_tool_call":
|
||||||
|
call = _item_dict(item)
|
||||||
|
tool_call = ChatCompletionMessageToolCallParam(
|
||||||
|
id=call["call_id"],
|
||||||
|
function=FunctionCallTool(
|
||||||
|
name=call["name"],
|
||||||
|
arguments=encode_custom_tool_input(call.get("input") or ""),
|
||||||
|
),
|
||||||
|
type="function",
|
||||||
|
)
|
||||||
|
if tool_call is not None:
|
||||||
if prev_assistant_msg:
|
if prev_assistant_msg:
|
||||||
tool_calls = prev_assistant_msg.get("tool_calls")
|
tool_calls = prev_assistant_msg.get("tool_calls")
|
||||||
if tool_calls is None:
|
if tool_calls is None:
|
||||||
@@ -285,14 +361,14 @@ def _construct_message_from_response_item(
|
|||||||
logger.warning(
|
logger.warning(
|
||||||
"Previous assistant message has unknown tool_calls format. "
|
"Previous assistant message has unknown tool_calls format. "
|
||||||
"Tool call merging is skipped and a new assistant message is created. "
|
"Tool call merging is skipped and a new assistant message is created. "
|
||||||
"Item %s",
|
"Call %s",
|
||||||
item.id,
|
tool_call["id"],
|
||||||
)
|
)
|
||||||
return ChatCompletionAssistantMessageParam(
|
return ChatCompletionAssistantMessageParam(
|
||||||
role="assistant",
|
role="assistant",
|
||||||
tool_calls=[tool_call],
|
tool_calls=[tool_call],
|
||||||
)
|
)
|
||||||
elif isinstance(item, ResponseReasoningItem):
|
if isinstance(item, ResponseReasoningItem):
|
||||||
reasoning = ""
|
reasoning = ""
|
||||||
if item.encrypted_content:
|
if item.encrypted_content:
|
||||||
raise VLLMValidationError(
|
raise VLLMValidationError(
|
||||||
@@ -336,12 +412,12 @@ def _construct_message_from_response_item(
|
|||||||
content=item.output,
|
content=item.output,
|
||||||
tool_call_id=item.call_id,
|
tool_call_id=item.call_id,
|
||||||
)
|
)
|
||||||
elif isinstance(item, dict) and item.get("type") == "function_call_output":
|
elif _item_type(item) in ("function_call_output", "custom_tool_call_output"):
|
||||||
# Append the function call output as a tool message.
|
output = _item_dict(item)
|
||||||
return ChatCompletionToolMessageParam(
|
return ChatCompletionToolMessageParam(
|
||||||
role="tool",
|
role="tool",
|
||||||
content=item.get("output"),
|
content=output.get("output"),
|
||||||
tool_call_id=item.get("call_id"),
|
tool_call_id=output.get("call_id"),
|
||||||
)
|
)
|
||||||
elif isinstance(item, dict) and item.get("role") == "assistant":
|
elif isinstance(item, dict) and item.get("role") == "assistant":
|
||||||
content = item.get("content")
|
content = item.get("content")
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from abc import abstractmethod
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from dataclasses import dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
|
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
from pydantic import TypeAdapter, ValidationError
|
from pydantic import TypeAdapter, ValidationError
|
||||||
from xgrammar import Grammar, StructuralTag
|
from xgrammar import Grammar, StructuralTag
|
||||||
from xgrammar.structural_tag import (
|
from xgrammar.structural_tag import (
|
||||||
@@ -359,7 +359,9 @@ class DelegatingParser(Parser):
|
|||||||
def _get_function_name(
|
def _get_function_name(
|
||||||
self, request: ChatCompletionRequest | ResponsesRequest
|
self, request: ChatCompletionRequest | ResponsesRequest
|
||||||
) -> str:
|
) -> str:
|
||||||
if request.tool_choice and isinstance(request.tool_choice, ToolChoiceFunction):
|
if request.tool_choice and isinstance(
|
||||||
|
request.tool_choice, (ToolChoiceFunction, ToolChoiceCustom)
|
||||||
|
):
|
||||||
return request.tool_choice.name
|
return request.tool_choice.name
|
||||||
if request.tool_choice and isinstance(
|
if request.tool_choice and isinstance(
|
||||||
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
request.tool_choice, ChatCompletionNamedToolChoiceParam
|
||||||
@@ -398,7 +400,7 @@ class DelegatingParser(Parser):
|
|||||||
supports_required_and_named = tool_parser.supports_required_and_named
|
supports_required_and_named = tool_parser.supports_required_and_named
|
||||||
is_named_tool_choice = request.tool_choice and isinstance(
|
is_named_tool_choice = request.tool_choice and isinstance(
|
||||||
request.tool_choice,
|
request.tool_choice,
|
||||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
(ToolChoiceFunction, ToolChoiceCustom, ChatCompletionNamedToolChoiceParam),
|
||||||
)
|
)
|
||||||
is_required_tool_choice = request.tool_choice == "required"
|
is_required_tool_choice = request.tool_choice == "required"
|
||||||
is_auto_tool_choice = enable_auto_tools and (
|
is_auto_tool_choice = enable_auto_tools and (
|
||||||
@@ -503,7 +505,11 @@ class DelegatingParser(Parser):
|
|||||||
or request.tool_choice == "required"
|
or request.tool_choice == "required"
|
||||||
or isinstance(
|
or isinstance(
|
||||||
request.tool_choice,
|
request.tool_choice,
|
||||||
(ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
|
(
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
|
ToolChoiceFunction,
|
||||||
|
ToolChoiceCustom,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if not need_tool_calling:
|
if not need_tool_calling:
|
||||||
@@ -723,7 +729,11 @@ class DelegatingParser(Parser):
|
|||||||
and request.tool_choice
|
and request.tool_choice
|
||||||
and isinstance(
|
and isinstance(
|
||||||
request.tool_choice,
|
request.tool_choice,
|
||||||
(ToolChoiceFunction, ChatCompletionNamedToolChoiceParam),
|
(
|
||||||
|
ToolChoiceFunction,
|
||||||
|
ToolChoiceCustom,
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
delta_message, function_name_returned = extract_named_tool_call_streaming(
|
delta_message, function_name_returned = extract_named_tool_call_streaming(
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import json
|
|||||||
from collections.abc import Callable, Iterable, Sequence
|
from collections.abc import Callable, Iterable, Sequence
|
||||||
from typing import TYPE_CHECKING, Any, cast
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
|
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
@@ -173,7 +173,8 @@ def _streaming_template(template: dict[str, Any]) -> ResponseTemplate:
|
|||||||
|
|
||||||
def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool:
|
def _is_named_tool_choice(request: ChatCompletionRequest | ResponsesRequest) -> bool:
|
||||||
return isinstance(
|
return isinstance(
|
||||||
request.tool_choice, (ToolChoiceFunction, ChatCompletionNamedToolChoiceParam)
|
request.tool_choice,
|
||||||
|
(ToolChoiceFunction, ToolChoiceCustom, ChatCompletionNamedToolChoiceParam),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
|
|
||||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||||
ChatCompletionNamedToolChoiceParam,
|
ChatCompletionNamedToolChoiceParam,
|
||||||
@@ -29,7 +29,12 @@ class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-typ
|
|||||||
if request.tools:
|
if request.tools:
|
||||||
tc = request.tool_choice
|
tc = request.tool_choice
|
||||||
if tc == "required" or isinstance(
|
if tc == "required" or isinstance(
|
||||||
tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
|
tc,
|
||||||
|
(
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
|
ToolChoiceFunction,
|
||||||
|
ToolChoiceCustom,
|
||||||
|
),
|
||||||
):
|
):
|
||||||
request.skip_special_tokens = False
|
request.skip_special_tokens = False
|
||||||
return request
|
return request
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ from collections.abc import Sequence
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
@@ -184,7 +184,7 @@ class K2HorizonToolParser(ToolParser):
|
|||||||
tool_choice = request.tool_choice
|
tool_choice = request.tool_choice
|
||||||
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
|
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||||
return tool_choice.function.name
|
return tool_choice.function.name
|
||||||
if isinstance(tool_choice, ToolChoiceFunction):
|
if isinstance(tool_choice, (ToolChoiceFunction, ToolChoiceCustom)):
|
||||||
return tool_choice.name
|
return tool_choice.name
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ import json
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
DeltaFunctionCall,
|
DeltaFunctionCall,
|
||||||
@@ -149,7 +149,11 @@ class KimiK3ToolParser(ToolParser):
|
|||||||
) -> ChatCompletionRequest | ResponsesRequest:
|
) -> ChatCompletionRequest | ResponsesRequest:
|
||||||
named = isinstance(
|
named = isinstance(
|
||||||
request.tool_choice,
|
request.tool_choice,
|
||||||
(ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
|
(
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
|
ToolChoiceFunction,
|
||||||
|
ToolChoiceCustom,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
structured_outputs = getattr(request, "structured_outputs", None)
|
structured_outputs = getattr(request, "structured_outputs", None)
|
||||||
has_structural_tag = (
|
has_structural_tag = (
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ import json
|
|||||||
from collections.abc import Iterator, Sequence
|
from collections.abc import Iterator, Sequence
|
||||||
|
|
||||||
import regex as re
|
import regex as re
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
@@ -228,7 +228,12 @@ class MuseGlimmerToolParser(ToolParser):
|
|||||||
if request.tools and (
|
if request.tools and (
|
||||||
tool_choice == "required"
|
tool_choice == "required"
|
||||||
or isinstance(
|
or isinstance(
|
||||||
tool_choice, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
|
tool_choice,
|
||||||
|
(
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
|
ToolChoiceFunction,
|
||||||
|
ToolChoiceCustom,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
):
|
):
|
||||||
return request
|
return request
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ from typing import Any
|
|||||||
|
|
||||||
import partial_json_parser.core.complete
|
import partial_json_parser.core.complete
|
||||||
import regex as re
|
import regex as re
|
||||||
from openai.types.responses import ToolChoiceFunction
|
from openai.types.responses import ToolChoiceCustom, ToolChoiceFunction
|
||||||
from partial_json_parser.core.options import Allow
|
from partial_json_parser.core.options import Allow
|
||||||
|
|
||||||
from vllm.entrypoints.generate.base.protocol import (
|
from vllm.entrypoints.generate.base.protocol import (
|
||||||
@@ -171,7 +171,12 @@ class PoolsideV1ToolParser(ToolParser):
|
|||||||
if request.tools:
|
if request.tools:
|
||||||
tc = request.tool_choice
|
tc = request.tool_choice
|
||||||
if tc == "required" or isinstance(
|
if tc == "required" or isinstance(
|
||||||
tc, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction)
|
tc,
|
||||||
|
(
|
||||||
|
ChatCompletionNamedToolChoiceParam,
|
||||||
|
ToolChoiceFunction,
|
||||||
|
ToolChoiceCustom,
|
||||||
|
),
|
||||||
):
|
):
|
||||||
request.skip_special_tokens = False
|
request.skip_special_tokens = False
|
||||||
return request
|
return request
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import json
|
|||||||
from collections.abc import Callable, Sequence
|
from collections.abc import Callable, Sequence
|
||||||
from typing import Any, Literal, TypeAlias
|
from typing import Any, Literal, TypeAlias
|
||||||
|
|
||||||
from openai.types.responses import FunctionTool
|
from openai.types.responses import CustomTool, FunctionTool, ToolChoiceCustom
|
||||||
from openai.types.responses.response import ToolChoice as ResponsesToolChoice
|
from openai.types.responses.response import ToolChoice as ResponsesToolChoice
|
||||||
from openai.types.responses.tool import Tool as ResponsesTool
|
from openai.types.responses.tool import Tool as ResponsesTool
|
||||||
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
||||||
@@ -39,7 +39,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
from vllm.tool_parsers.tool_strict_level import ToolStrictLevel
|
||||||
from vllm.tool_parsers.utils import get_function_tools, require_function_tools
|
from vllm.tool_parsers.utils import (
|
||||||
|
custom_tool_parameters,
|
||||||
|
get_function_tools,
|
||||||
|
require_function_tools,
|
||||||
|
)
|
||||||
|
|
||||||
ToolChoice: TypeAlias = (
|
ToolChoice: TypeAlias = (
|
||||||
Literal["none", "auto", "required"]
|
Literal["none", "auto", "required"]
|
||||||
@@ -287,6 +291,11 @@ def _dump_tool_for_xgrammar(
|
|||||||
if tool.strict is not None:
|
if tool.strict is not None:
|
||||||
function["strict"] = tool.strict
|
function["strict"] = tool.strict
|
||||||
return {"type": "function", "function": function}
|
return {"type": "function", "function": function}
|
||||||
|
if isinstance(tool, CustomTool):
|
||||||
|
function = {"name": tool.name, "parameters": custom_tool_parameters()}
|
||||||
|
if tool.description is not None:
|
||||||
|
function["description"] = tool.description
|
||||||
|
return {"type": "function", "function": function}
|
||||||
dumped_tool = tool.model_dump(mode="json", exclude_none=True)
|
dumped_tool = tool.model_dump(mode="json", exclude_none=True)
|
||||||
if isinstance(tool, ChatCompletionToolsParam):
|
if isinstance(tool, ChatCompletionToolsParam):
|
||||||
return dumped_tool
|
return dumped_tool
|
||||||
@@ -306,7 +315,7 @@ def _dump_tool_choice_for_xgrammar(
|
|||||||
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
|
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||||
return tool_choice.model_dump(mode="json", exclude_none=True)
|
return tool_choice.model_dump(mode="json", exclude_none=True)
|
||||||
|
|
||||||
if isinstance(tool_choice, ToolChoiceFunction):
|
if isinstance(tool_choice, (ToolChoiceFunction, ToolChoiceCustom)):
|
||||||
return {
|
return {
|
||||||
"type": "function",
|
"type": "function",
|
||||||
"function": {"name": tool_choice.name},
|
"function": {"name": tool_choice.name},
|
||||||
|
|||||||
@@ -13,8 +13,10 @@ from typing import Any, TypeAlias
|
|||||||
|
|
||||||
import partial_json_parser
|
import partial_json_parser
|
||||||
from openai.types.responses import (
|
from openai.types.responses import (
|
||||||
|
CustomTool,
|
||||||
FunctionTool,
|
FunctionTool,
|
||||||
NamespaceTool,
|
NamespaceTool,
|
||||||
|
ToolChoiceCustom,
|
||||||
ToolChoiceFunction,
|
ToolChoiceFunction,
|
||||||
)
|
)
|
||||||
from openai.types.responses.tool import Tool as ResponsesTool
|
from openai.types.responses.tool import Tool as ResponsesTool
|
||||||
@@ -37,6 +39,32 @@ Tool: TypeAlias = ChatCompletionToolsParam | ResponsesTool
|
|||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
CUSTOM_TOOL_INPUT_KEY = "input"
|
||||||
|
|
||||||
|
|
||||||
|
def custom_tool_parameters() -> dict[str, Any]:
|
||||||
|
"""Schema of the single-string function shim a ``custom`` tool is shown as."""
|
||||||
|
return {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
CUSTOM_TOOL_INPUT_KEY: {
|
||||||
|
"type": "string",
|
||||||
|
"description": "The freeform text payload for this tool.",
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [CUSTOM_TOOL_INPUT_KEY],
|
||||||
|
"additionalProperties": False,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def custom_tool_as_function_dict(tool: CustomTool) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"type": "function",
|
||||||
|
"name": tool.name,
|
||||||
|
"description": tool.description,
|
||||||
|
"parameters": custom_tool_parameters(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def safe_literal_eval(text: str):
|
def safe_literal_eval(text: str):
|
||||||
with warnings.catch_warnings():
|
with warnings.catch_warnings():
|
||||||
@@ -188,8 +216,9 @@ def get_function_tools(
|
|||||||
|
|
||||||
This is the single source for both the chat template's tool list and the
|
This is the single source for both the chat template's tool list and the
|
||||||
tools a tool-call grammar constrains, so the two cannot drift apart.
|
tools a tool-call grammar constrains, so the two cannot drift apart.
|
||||||
Namespace functions are flattened to ``<namespace>__<name>``; other
|
Namespace functions are flattened to ``<namespace>__<name>``; custom tools
|
||||||
Responses tool types are never rendered for the model and are dropped.
|
become a single-string function shim. Other Responses tool types are never
|
||||||
|
rendered for the model and are dropped.
|
||||||
"""
|
"""
|
||||||
function_tools: list[ChatCompletionToolsParam | FunctionTool] = []
|
function_tools: list[ChatCompletionToolsParam | FunctionTool] = []
|
||||||
for tool in tools:
|
for tool in tools:
|
||||||
@@ -206,6 +235,10 @@ def get_function_tools(
|
|||||||
for namespaced_tool in tool.tools
|
for namespaced_tool in tool.tools
|
||||||
if namespaced_tool.type == "function"
|
if namespaced_tool.type == "function"
|
||||||
)
|
)
|
||||||
|
elif isinstance(tool, CustomTool):
|
||||||
|
function_tools.append(
|
||||||
|
FunctionTool.model_validate(custom_tool_as_function_dict(tool))
|
||||||
|
)
|
||||||
elif isinstance(tool, (FunctionTool, ChatCompletionToolsParam)):
|
elif isinstance(tool, (FunctionTool, ChatCompletionToolsParam)):
|
||||||
function_tools.append(tool)
|
function_tools.append(tool)
|
||||||
return function_tools
|
return function_tools
|
||||||
@@ -307,7 +340,7 @@ def find_tool_properties(
|
|||||||
if not tools:
|
if not tools:
|
||||||
return {}
|
return {}
|
||||||
for tool in tools:
|
for tool in tools:
|
||||||
if isinstance(tool, (FunctionTool, NamespaceTool)):
|
if isinstance(tool, (FunctionTool, NamespaceTool, CustomTool)):
|
||||||
for name, params in iter_response_function_tool_info(tool):
|
for name, params in iter_response_function_tool_info(tool):
|
||||||
if name == tool_name:
|
if name == tool_name:
|
||||||
return (params or {}).get("properties", {})
|
return (params or {}).get("properties", {})
|
||||||
@@ -328,7 +361,7 @@ def find_tool_name(
|
|||||||
if not tools:
|
if not tools:
|
||||||
return False
|
return False
|
||||||
for tool in tools:
|
for tool in tools:
|
||||||
if isinstance(tool, (FunctionTool, NamespaceTool)):
|
if isinstance(tool, (FunctionTool, NamespaceTool, CustomTool)):
|
||||||
for name, _ in iter_response_function_tool_info(tool):
|
for name, _ in iter_response_function_tool_info(tool):
|
||||||
if name == tool_name:
|
if name == tool_name:
|
||||||
return True
|
return True
|
||||||
@@ -414,7 +447,9 @@ def _get_json_schema_from_tools(
|
|||||||
|
|
||||||
|
|
||||||
def get_json_schema_from_tools(
|
def get_json_schema_from_tools(
|
||||||
tool_choice: str | ToolChoiceFunction | ChatCompletionNamedToolChoiceParam,
|
tool_choice: (
|
||||||
|
str | ToolChoiceFunction | ToolChoiceCustom | ChatCompletionNamedToolChoiceParam
|
||||||
|
),
|
||||||
tools: list[Tool] | None,
|
tools: list[Tool] | None,
|
||||||
parallel_tool_calls: bool | None = None,
|
parallel_tool_calls: bool | None = None,
|
||||||
) -> str | dict | None:
|
) -> str | dict | None:
|
||||||
@@ -423,12 +458,12 @@ def get_json_schema_from_tools(
|
|||||||
return None
|
return None
|
||||||
# tool_choice: Forced Function (Responses)
|
# tool_choice: Forced Function (Responses)
|
||||||
if (not isinstance(tool_choice, str)) and isinstance(
|
if (not isinstance(tool_choice, str)) and isinstance(
|
||||||
tool_choice, ToolChoiceFunction
|
tool_choice, (ToolChoiceFunction, ToolChoiceCustom)
|
||||||
):
|
):
|
||||||
tool_name = tool_choice.name
|
tool_name = tool_choice.name
|
||||||
responses_tool_map: dict[str, dict[str, Any] | None] = {}
|
responses_tool_map: dict[str, dict[str, Any] | None] = {}
|
||||||
for tool in tools:
|
for tool in tools:
|
||||||
if not isinstance(tool, (FunctionTool, NamespaceTool)):
|
if not isinstance(tool, (FunctionTool, NamespaceTool, CustomTool)):
|
||||||
continue
|
continue
|
||||||
for name, params in iter_response_function_tool_info(tool):
|
for name, params in iter_response_function_tool_info(tool):
|
||||||
responses_tool_map[name] = params
|
responses_tool_map[name] = params
|
||||||
|
|||||||
Reference in New Issue
Block a user