[Frontend] Support strict mode for tool calling with ResponsesAPI (#45396)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -390,7 +390,6 @@ def server_with_store(default_server_args):
|
|||||||
env_dict={
|
env_dict={
|
||||||
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
|
||||||
"VLLM_SERVER_DEV_MODE": "1",
|
"VLLM_SERVER_DEV_MODE": "1",
|
||||||
"VLLM_ENFORCE_STRICT_TOOL_CALLING": "0",
|
|
||||||
},
|
},
|
||||||
) as remote_server:
|
) as remote_server:
|
||||||
yield remote_server
|
yield remote_server
|
||||||
|
|||||||
@@ -438,8 +438,7 @@ class DelegatingParser(Parser):
|
|||||||
self, request: ChatCompletionRequest | ResponsesRequest
|
self, request: ChatCompletionRequest | ResponsesRequest
|
||||||
) -> ChatCompletionRequest | ResponsesRequest:
|
) -> ChatCompletionRequest | ResponsesRequest:
|
||||||
if (
|
if (
|
||||||
not isinstance(request, ChatCompletionRequest)
|
self._tool_parser is None
|
||||||
or self._tool_parser is None
|
|
||||||
or self._tool_parser.structural_tag_model is None
|
or self._tool_parser.structural_tag_model is None
|
||||||
or not request.tools
|
or not request.tools
|
||||||
):
|
):
|
||||||
@@ -448,7 +447,10 @@ class DelegatingParser(Parser):
|
|||||||
need_tool_calling = (
|
need_tool_calling = (
|
||||||
request.tool_choice == "auto"
|
request.tool_choice == "auto"
|
||||||
or request.tool_choice == "required"
|
or request.tool_choice == "required"
|
||||||
or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
or isinstance(
|
||||||
|
request.tool_choice,
|
||||||
|
(ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
if not need_tool_calling:
|
if not need_tool_calling:
|
||||||
return request
|
return request
|
||||||
@@ -464,7 +466,10 @@ class DelegatingParser(Parser):
|
|||||||
request.structured_outputs = StructuredOutputsParams(
|
request.structured_outputs = StructuredOutputsParams(
|
||||||
structural_tag=structural_tag,
|
structural_tag=structural_tag,
|
||||||
)
|
)
|
||||||
request.response_format = None
|
if isinstance(request, ResponsesRequest):
|
||||||
|
request.text = None
|
||||||
|
else:
|
||||||
|
request.response_format = None
|
||||||
return request
|
return request
|
||||||
|
|
||||||
def extract_reasoning_streaming(
|
def extract_reasoning_streaming(
|
||||||
|
|||||||
@@ -181,9 +181,8 @@ class ReasoningParser:
|
|||||||
) -> str | None:
|
) -> str | None:
|
||||||
"""
|
"""
|
||||||
Instance method that is implemented for preparing the structured tag
|
Instance method that is implemented for preparing the structured tag
|
||||||
Otherwise, None is returned
|
|
||||||
"""
|
"""
|
||||||
return None
|
return original_tag
|
||||||
|
|
||||||
|
|
||||||
class ReasoningParserManager:
|
class ReasoningParserManager:
|
||||||
|
|||||||
@@ -165,7 +165,10 @@ class ToolParser:
|
|||||||
return request
|
return request
|
||||||
|
|
||||||
def get_structural_tag(
|
def get_structural_tag(
|
||||||
self, request: ChatCompletionRequest, *, reasoning: bool = False
|
self,
|
||||||
|
request: ChatCompletionRequest | ResponsesRequest,
|
||||||
|
*,
|
||||||
|
reasoning: bool = False,
|
||||||
):
|
):
|
||||||
if self.structural_tag_model is None:
|
if self.structural_tag_model is None:
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -1,9 +1,14 @@
|
|||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable, Sequence
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal, TypeAlias
|
||||||
|
|
||||||
|
from openai.types.responses import FunctionTool
|
||||||
|
from openai.types.responses.response import ToolChoice as ResponsesToolChoice
|
||||||
|
from openai.types.responses.tool import Tool as ResponsesTool
|
||||||
|
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
|
||||||
|
from openai.types.responses.tool_choice_function import ToolChoiceFunction
|
||||||
from xgrammar import StructuralTag, normalize_tool_choice
|
from xgrammar import StructuralTag, normalize_tool_choice
|
||||||
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
|
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
|
||||||
from xgrammar.openai_tool_call_schema import (
|
from xgrammar.openai_tool_call_schema import (
|
||||||
@@ -25,11 +30,15 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
|||||||
ChatCompletionToolsParam,
|
ChatCompletionToolsParam,
|
||||||
)
|
)
|
||||||
|
|
||||||
ToolChoice = (
|
ToolChoice: TypeAlias = (
|
||||||
Literal["none", "auto", "required"] | ChatCompletionNamedToolChoiceParam | None
|
Literal["none", "auto", "required"]
|
||||||
|
| ChatCompletionNamedToolChoiceParam
|
||||||
|
| ResponsesToolChoice
|
||||||
|
| None
|
||||||
)
|
)
|
||||||
SimplifiedToolChoice = Literal["auto", "required", "forced"]
|
AllowedToolRef: TypeAlias = dict[str, object]
|
||||||
StructuralTagBuilder = Callable[
|
SimplifiedToolChoice: TypeAlias = Literal["auto", "required", "forced"]
|
||||||
|
StructuralTagBuilder: TypeAlias = Callable[
|
||||||
[
|
[
|
||||||
list[FunctionToolParam],
|
list[FunctionToolParam],
|
||||||
list[BuiltinToolParam],
|
list[BuiltinToolParam],
|
||||||
@@ -77,7 +86,7 @@ def register_vllm_structural_tag(model: str):
|
|||||||
|
|
||||||
def get_model_structural_tag(
|
def get_model_structural_tag(
|
||||||
model: str,
|
model: str,
|
||||||
tools: list[ChatCompletionToolsParam] | None,
|
tools: Sequence[ChatCompletionToolsParam | ResponsesTool] | None,
|
||||||
tool_choice: ToolChoice,
|
tool_choice: ToolChoice,
|
||||||
reasoning: bool,
|
reasoning: bool,
|
||||||
) -> StructuralTag | None:
|
) -> StructuralTag | None:
|
||||||
@@ -86,8 +95,8 @@ def get_model_structural_tag(
|
|||||||
if not tools or tool_choice == "none":
|
if not tools or tool_choice == "none":
|
||||||
return None
|
return None
|
||||||
|
|
||||||
dumped_tools = [_model_dump(tool) for tool in tools]
|
dumped_tools = [_dump_tool_for_xgrammar(tool) for tool in tools]
|
||||||
dumped_tool_choice = _model_dump(tool_choice)
|
dumped_tool_choice = _dump_tool_choice_for_xgrammar(tool_choice)
|
||||||
|
|
||||||
if model in _VLLM_STRUCTURAL_TAG_REGISTRY:
|
if model in _VLLM_STRUCTURAL_TAG_REGISTRY:
|
||||||
function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice(
|
function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice(
|
||||||
@@ -113,12 +122,72 @@ def get_model_structural_tag(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _model_dump(value: Any) -> Any:
|
def _dump_tool_for_xgrammar(
|
||||||
"""Convert vLLM/Pydantic request objects to xgrammar's dict protocol."""
|
tool: ChatCompletionToolsParam | ResponsesTool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Convert tool objects to xgrammar's Chat Completions tool protocol."""
|
||||||
|
|
||||||
if hasattr(value, "model_dump"):
|
if isinstance(tool, FunctionTool):
|
||||||
return value.model_dump(exclude_none=True)
|
function: dict[str, Any] = {"name": tool.name}
|
||||||
return value
|
if tool.description is not None:
|
||||||
|
function["description"] = tool.description
|
||||||
|
if tool.parameters is not None:
|
||||||
|
function["parameters"] = tool.parameters
|
||||||
|
if tool.strict is not None:
|
||||||
|
function["strict"] = tool.strict
|
||||||
|
return {"type": "function", "function": function}
|
||||||
|
dumped_tool = tool.model_dump(mode="json", exclude_none=True)
|
||||||
|
if isinstance(tool, ChatCompletionToolsParam):
|
||||||
|
return dumped_tool
|
||||||
|
return dict(dumped_tool)
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_tool_choice_for_xgrammar(
|
||||||
|
tool_choice: ToolChoice,
|
||||||
|
) -> dict[str, Any] | str | None:
|
||||||
|
"""Convert tool_choice objects to xgrammar's expected protocol."""
|
||||||
|
|
||||||
|
if tool_choice is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
if isinstance(tool_choice, str):
|
||||||
|
return tool_choice
|
||||||
|
|
||||||
|
if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
|
||||||
|
return tool_choice.model_dump(mode="json", exclude_none=True)
|
||||||
|
|
||||||
|
if isinstance(tool_choice, ToolChoiceFunction):
|
||||||
|
return {
|
||||||
|
"type": "function",
|
||||||
|
"function": {"name": tool_choice.name},
|
||||||
|
}
|
||||||
|
|
||||||
|
if isinstance(tool_choice, ToolChoiceAllowed):
|
||||||
|
return {
|
||||||
|
"type": "allowed_tools",
|
||||||
|
"allowed_tools": {
|
||||||
|
"mode": tool_choice.mode,
|
||||||
|
"tools": [
|
||||||
|
_dump_allowed_tool_ref_for_xgrammar(tool)
|
||||||
|
for tool in tool_choice.tools
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
return tool_choice.model_dump(mode="json", exclude_none=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedToolRef:
|
||||||
|
if (
|
||||||
|
tool_ref.get("type") == "function"
|
||||||
|
and "function" not in tool_ref
|
||||||
|
and "name" in tool_ref
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
"type": "function",
|
||||||
|
"function": {"name": tool_ref["name"]},
|
||||||
|
}
|
||||||
|
return tool_ref
|
||||||
|
|
||||||
|
|
||||||
def _get_function_parameters(function) -> dict[str, Any] | bool:
|
def _get_function_parameters(function) -> dict[str, Any] | bool:
|
||||||
|
|||||||
Reference in New Issue
Block a user