[Frontend] Add defer_loading and tool_reference support for Anthropic and OpenAI APIs (#40190)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
sfeng33
Chauncey
parent
33f36d4260
commit
11b69129e2
@@ -39,6 +39,7 @@ class AnthropicContentBlock(BaseModel):
|
||||
"image",
|
||||
"tool_use",
|
||||
"tool_result",
|
||||
"tool_reference",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
]
|
||||
@@ -52,6 +53,8 @@ class AnthropicContentBlock(BaseModel):
|
||||
input: dict[str, Any] | None = None
|
||||
content: str | list[dict[str, Any]] | None = None
|
||||
is_error: bool | None = None
|
||||
# For tool_reference content
|
||||
tool_name: str | None = None
|
||||
# For thinking content
|
||||
thinking: str | None = None
|
||||
signature: str | None = None
|
||||
@@ -72,6 +75,7 @@ class AnthropicTool(BaseModel):
|
||||
name: str
|
||||
description: str | None = None
|
||||
input_schema: dict[str, Any]
|
||||
defer_loading: bool | None = None
|
||||
|
||||
@field_validator("input_schema")
|
||||
@classmethod
|
||||
|
||||
@@ -237,6 +237,10 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
cls._convert_tool_use_block(block, tool_calls)
|
||||
elif block.type == "tool_result":
|
||||
cls._convert_tool_result_block(block, role, openai_messages, content_parts)
|
||||
elif block.type == "tool_reference":
|
||||
# Tool references are expanded during tool_result processing
|
||||
# when they appear inside tool_result content.
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None:
|
||||
@@ -275,6 +279,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
"""Convert user tool_result with text and image support"""
|
||||
tool_text = ""
|
||||
tool_image_urls: list[str] = []
|
||||
tool_reference: list[dict[str, Any]] = []
|
||||
|
||||
if isinstance(block.content, str):
|
||||
tool_text = block.content
|
||||
@@ -291,6 +296,12 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
url = cls._convert_image_source_to_url(source)
|
||||
if url:
|
||||
tool_image_urls.append(url)
|
||||
elif item_type == "tool_reference":
|
||||
ref_name = item.get("tool_name") or item.get("name")
|
||||
if ref_name:
|
||||
tool_reference.append(
|
||||
{"type": "tool_reference", "name": ref_name}
|
||||
)
|
||||
tool_text = "\n".join(text_parts)
|
||||
|
||||
openai_messages.append(
|
||||
@@ -312,6 +323,15 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
}
|
||||
)
|
||||
|
||||
if tool_reference:
|
||||
openai_messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": block.tool_use_id or "",
|
||||
"content": tool_reference, # type: ignore[dict-item]
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _build_base_request(
|
||||
cls,
|
||||
@@ -400,6 +420,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
"name": tool.name,
|
||||
"description": tool.description,
|
||||
"parameters": tool.input_schema,
|
||||
"defer_loading": tool.defer_loading,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -255,6 +255,23 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
|
||||
"""The thinking type."""
|
||||
|
||||
|
||||
class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False):
|
||||
"""A tool reference content param that only accepts a plain tool name.
|
||||
|
||||
Example:
|
||||
{
|
||||
"name": "get_weather",
|
||||
"type": "tool_reference"
|
||||
}
|
||||
"""
|
||||
|
||||
name: str
|
||||
"""The name of the tool being referenced."""
|
||||
|
||||
type: Literal["tool_reference"]
|
||||
"""The content type."""
|
||||
|
||||
|
||||
ChatCompletionContentPartParam: TypeAlias = (
|
||||
OpenAIChatCompletionContentPartParam
|
||||
| ChatCompletionContentPartAudioParam
|
||||
@@ -267,6 +284,7 @@ ChatCompletionContentPartParam: TypeAlias = (
|
||||
| ChatCompletionContentPartAudioEmbedsParam
|
||||
| CustomChatCompletionContentSimpleAudioParam
|
||||
| CustomChatCompletionContentSimpleVideoParam
|
||||
| CustomChatCompletionContentToolReferenceParam
|
||||
| str
|
||||
| CustomThinkCompletionContentParam
|
||||
)
|
||||
@@ -1281,6 +1299,9 @@ MM_PARSER_MAP: dict[
|
||||
"input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
|
||||
"refusal": lambda part: _RefusalParser(part).get("refusal", None),
|
||||
"video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
|
||||
"tool_reference": lambda part: cast(
|
||||
CustomChatCompletionContentToolReferenceParam, part
|
||||
).get("name", None),
|
||||
}
|
||||
|
||||
|
||||
@@ -1372,6 +1393,12 @@ def _parse_chat_message_content_mm_part(
|
||||
# with url as a dict of {"url": url}
|
||||
video_url = video_url.get("url", None)
|
||||
return "video_url", video_url
|
||||
if "tool_reference" in part:
|
||||
tool_reference_params = cast(
|
||||
CustomChatCompletionContentToolReferenceParam, part
|
||||
)
|
||||
tool_reference = tool_reference_params.get("name", None)
|
||||
return "tool_reference", tool_reference
|
||||
# Raise an error if no 'type' or direct URL is found.
|
||||
raise ValueError("Missing 'type' field in multimodal part.")
|
||||
|
||||
@@ -1501,6 +1528,12 @@ def _parse_chat_message_content_part(
|
||||
str_content = cast(str, content)
|
||||
mm_parser.parse_video(str_content, uuid)
|
||||
modality = "video"
|
||||
elif part_type == "tool_reference":
|
||||
# Tool references are not multimodal data — they reference deferred
|
||||
# tools and are passed through as-is for the chat template to expand.
|
||||
if wrap_dicts:
|
||||
return {"type": "tool_reference", "name": cast(str, content)}
|
||||
return cast(str, content)
|
||||
else:
|
||||
supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT))
|
||||
raise VLLMValidationError(
|
||||
@@ -1567,14 +1600,24 @@ def _parse_chat_message_content(
|
||||
# string. Clients like Claude Code / Cursor send tool results as
|
||||
# [{"type": "text", "text": "..."}], but most chat templates only
|
||||
# handle string content for tool messages.
|
||||
# However, tool_reference items must be preserved as structured
|
||||
# dicts for the chat template to expand them.
|
||||
msg_content = result_msg.get("content")
|
||||
if isinstance(msg_content, list):
|
||||
texts = [
|
||||
item.get("text", "")
|
||||
has_non_text = any(
|
||||
isinstance(item, dict) and item.get("type") != "text"
|
||||
for item in msg_content
|
||||
if isinstance(item, dict) and item.get("type") == "text"
|
||||
]
|
||||
result_msg["content"] = "\n".join(texts) if texts else ""
|
||||
)
|
||||
if has_non_text:
|
||||
# Keep structured content (e.g., tool_reference)
|
||||
result_msg["content"] = msg_content
|
||||
else:
|
||||
texts = [
|
||||
item.get("text", "")
|
||||
for item in msg_content
|
||||
if isinstance(item, dict) and item.get("type") == "text"
|
||||
]
|
||||
result_msg["content"] = "\n".join(texts) if texts else ""
|
||||
|
||||
if "name" in message and isinstance(message["name"], str):
|
||||
result_msg["name"] = message["name"]
|
||||
|
||||
@@ -11,7 +11,7 @@ from openai.types.chat.chat_completion_audio import (
|
||||
ChatCompletionAudio as OpenAIChatCompletionAudio,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
|
||||
from pydantic import Field, PrivateAttr, model_validator
|
||||
from pydantic import Field, PrivateAttr, model_serializer, model_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.utils import replace
|
||||
@@ -139,6 +139,20 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||
class ChatCompletionToolsParam(OpenAIBaseModel):
|
||||
type: Literal["function"] = "function"
|
||||
function: FunctionDefinition
|
||||
defer_loading: bool | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _propagate_defer_loading(self) -> "ChatCompletionToolsParam":
|
||||
if self.defer_loading is not None and self.function.defer_loading is None:
|
||||
self.function.defer_loading = self.defer_loading
|
||||
return self
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
data = handler(self)
|
||||
if self.defer_loading is None:
|
||||
data.pop("defer_loading", None)
|
||||
return data
|
||||
|
||||
|
||||
class ChatCompletionNamedFunction(OpenAIBaseModel):
|
||||
|
||||
@@ -12,6 +12,7 @@ from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
model_serializer,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
@@ -166,6 +167,14 @@ class FunctionDefinition(OpenAIBaseModel):
|
||||
name: str
|
||||
description: str | None = None
|
||||
parameters: dict[str, Any] | None = None
|
||||
defer_loading: bool | None = None
|
||||
|
||||
@model_serializer(mode="wrap")
|
||||
def _serialize(self, handler):
|
||||
data = handler(self)
|
||||
if self.defer_loading is None:
|
||||
data.pop("defer_loading", None)
|
||||
return data
|
||||
|
||||
|
||||
# extra="forbid" is a workaround to have kwargs as a field,
|
||||
|
||||
Reference in New Issue
Block a user