[Frontend] Add defer_loading and tool_reference support for Anthropic and OpenAI APIs (#40190)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Jared Wen
2026-04-29 04:35:50 -07:00
committed by GitHub
co-authored by sfeng33 Chauncey
parent 33f36d4260
commit 11b69129e2
5 changed files with 97 additions and 6 deletions
+4
View File
@@ -39,6 +39,7 @@ class AnthropicContentBlock(BaseModel):
"image",
"tool_use",
"tool_result",
"tool_reference",
"thinking",
"redacted_thinking",
]
@@ -52,6 +53,8 @@ class AnthropicContentBlock(BaseModel):
input: dict[str, Any] | None = None
content: str | list[dict[str, Any]] | None = None
is_error: bool | None = None
# For tool_reference content
tool_name: str | None = None
# For thinking content
thinking: str | None = None
signature: str | None = None
@@ -72,6 +75,7 @@ class AnthropicTool(BaseModel):
name: str
description: str | None = None
input_schema: dict[str, Any]
defer_loading: bool | None = None
@field_validator("input_schema")
@classmethod
+21
View File
@@ -237,6 +237,10 @@ class AnthropicServingMessages(OpenAIServingChat):
cls._convert_tool_use_block(block, tool_calls)
elif block.type == "tool_result":
cls._convert_tool_result_block(block, role, openai_messages, content_parts)
elif block.type == "tool_reference":
# Tool references are expanded during tool_result processing
# when they appear inside tool_result content.
pass
@classmethod
def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None:
@@ -275,6 +279,7 @@ class AnthropicServingMessages(OpenAIServingChat):
"""Convert user tool_result with text and image support"""
tool_text = ""
tool_image_urls: list[str] = []
tool_reference: list[dict[str, Any]] = []
if isinstance(block.content, str):
tool_text = block.content
@@ -291,6 +296,12 @@ class AnthropicServingMessages(OpenAIServingChat):
url = cls._convert_image_source_to_url(source)
if url:
tool_image_urls.append(url)
elif item_type == "tool_reference":
ref_name = item.get("tool_name") or item.get("name")
if ref_name:
tool_reference.append(
{"type": "tool_reference", "name": ref_name}
)
tool_text = "\n".join(text_parts)
openai_messages.append(
@@ -312,6 +323,15 @@ class AnthropicServingMessages(OpenAIServingChat):
}
)
if tool_reference:
openai_messages.append(
{
"role": "tool",
"tool_call_id": block.tool_use_id or "",
"content": tool_reference, # type: ignore[dict-item]
}
)
@classmethod
def _build_base_request(
cls,
@@ -400,6 +420,7 @@ class AnthropicServingMessages(OpenAIServingChat):
"name": tool.name,
"description": tool.description,
"parameters": tool.input_schema,
"defer_loading": tool.defer_loading,
},
}
)
+48 -5
View File
@@ -255,6 +255,23 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
"""The thinking type."""
class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False):
"""A tool reference content param that only accepts a plain tool name.
Example:
{
"name": "get_weather",
"type": "tool_reference"
}
"""
name: str
"""The name of the tool being referenced."""
type: Literal["tool_reference"]
"""The content type."""
ChatCompletionContentPartParam: TypeAlias = (
OpenAIChatCompletionContentPartParam
| ChatCompletionContentPartAudioParam
@@ -267,6 +284,7 @@ ChatCompletionContentPartParam: TypeAlias = (
| ChatCompletionContentPartAudioEmbedsParam
| CustomChatCompletionContentSimpleAudioParam
| CustomChatCompletionContentSimpleVideoParam
| CustomChatCompletionContentToolReferenceParam
| str
| CustomThinkCompletionContentParam
)
@@ -1281,6 +1299,9 @@ MM_PARSER_MAP: dict[
"input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
"refusal": lambda part: _RefusalParser(part).get("refusal", None),
"video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
"tool_reference": lambda part: cast(
CustomChatCompletionContentToolReferenceParam, part
).get("name", None),
}
@@ -1372,6 +1393,12 @@ def _parse_chat_message_content_mm_part(
# with url as a dict of {"url": url}
video_url = video_url.get("url", None)
return "video_url", video_url
if "tool_reference" in part:
tool_reference_params = cast(
CustomChatCompletionContentToolReferenceParam, part
)
tool_reference = tool_reference_params.get("name", None)
return "tool_reference", tool_reference
# Raise an error if no 'type' or direct URL is found.
raise ValueError("Missing 'type' field in multimodal part.")
@@ -1501,6 +1528,12 @@ def _parse_chat_message_content_part(
str_content = cast(str, content)
mm_parser.parse_video(str_content, uuid)
modality = "video"
elif part_type == "tool_reference":
# Tool references are not multimodal data — they reference deferred
# tools and are passed through as-is for the chat template to expand.
if wrap_dicts:
return {"type": "tool_reference", "name": cast(str, content)}
return cast(str, content)
else:
supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT))
raise VLLMValidationError(
@@ -1567,14 +1600,24 @@ def _parse_chat_message_content(
# string. Clients like Claude Code / Cursor send tool results as
# [{"type": "text", "text": "..."}], but most chat templates only
# handle string content for tool messages.
# However, tool_reference items must be preserved as structured
# dicts for the chat template to expand them.
msg_content = result_msg.get("content")
if isinstance(msg_content, list):
texts = [
item.get("text", "")
has_non_text = any(
isinstance(item, dict) and item.get("type") != "text"
for item in msg_content
if isinstance(item, dict) and item.get("type") == "text"
]
result_msg["content"] = "\n".join(texts) if texts else ""
)
if has_non_text:
# Keep structured content (e.g., tool_reference)
result_msg["content"] = msg_content
else:
texts = [
item.get("text", "")
for item in msg_content
if isinstance(item, dict) and item.get("type") == "text"
]
result_msg["content"] = "\n".join(texts) if texts else ""
if "name" in message and isinstance(message["name"], str):
result_msg["name"] = message["name"]
@@ -11,7 +11,7 @@ from openai.types.chat.chat_completion_audio import (
ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
from pydantic import Field, PrivateAttr, model_validator
from pydantic import Field, PrivateAttr, model_serializer, model_validator
from vllm.config import ModelConfig
from vllm.config.utils import replace
@@ -139,6 +139,20 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
class ChatCompletionToolsParam(OpenAIBaseModel):
type: Literal["function"] = "function"
function: FunctionDefinition
defer_loading: bool | None = None
@model_validator(mode="after")
def _propagate_defer_loading(self) -> "ChatCompletionToolsParam":
if self.defer_loading is not None and self.function.defer_loading is None:
self.function.defer_loading = self.defer_loading
return self
@model_serializer(mode="wrap")
def _serialize(self, handler):
data = handler(self)
if self.defer_loading is None:
data.pop("defer_loading", None)
return data
class ChatCompletionNamedFunction(OpenAIBaseModel):
@@ -12,6 +12,7 @@ from pydantic import (
BaseModel,
ConfigDict,
Field,
model_serializer,
model_validator,
)
@@ -166,6 +167,14 @@ class FunctionDefinition(OpenAIBaseModel):
name: str
description: str | None = None
parameters: dict[str, Any] | None = None
defer_loading: bool | None = None
@model_serializer(mode="wrap")
def _serialize(self, handler):
data = handler(self)
if self.defer_loading is None:
data.pop("defer_loading", None)
return data
# extra="forbid" is a workaround to have kwargs as a field,