diff --git a/vllm/entrypoints/anthropic/protocol.py b/vllm/entrypoints/anthropic/protocol.py index 52eb77b511..f3c4dd7f3e 100644 --- a/vllm/entrypoints/anthropic/protocol.py +++ b/vllm/entrypoints/anthropic/protocol.py @@ -39,6 +39,7 @@ class AnthropicContentBlock(BaseModel): "image", "tool_use", "tool_result", + "tool_reference", "thinking", "redacted_thinking", ] @@ -52,6 +53,8 @@ class AnthropicContentBlock(BaseModel): input: dict[str, Any] | None = None content: str | list[dict[str, Any]] | None = None is_error: bool | None = None + # For tool_reference content + tool_name: str | None = None # For thinking content thinking: str | None = None signature: str | None = None @@ -72,6 +75,7 @@ class AnthropicTool(BaseModel): name: str description: str | None = None input_schema: dict[str, Any] + defer_loading: bool | None = None @field_validator("input_schema") @classmethod diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 939f5a7ed4..867ee73948 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -237,6 +237,10 @@ class AnthropicServingMessages(OpenAIServingChat): cls._convert_tool_use_block(block, tool_calls) elif block.type == "tool_result": cls._convert_tool_result_block(block, role, openai_messages, content_parts) + elif block.type == "tool_reference": + # Tool references are expanded during tool_result processing + # when they appear inside tool_result content. + pass @classmethod def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None: @@ -275,6 +279,7 @@ class AnthropicServingMessages(OpenAIServingChat): """Convert user tool_result with text and image support""" tool_text = "" tool_image_urls: list[str] = [] + tool_reference: list[dict[str, Any]] = [] if isinstance(block.content, str): tool_text = block.content @@ -291,6 +296,12 @@ class AnthropicServingMessages(OpenAIServingChat): url = cls._convert_image_source_to_url(source) if url: tool_image_urls.append(url) + elif item_type == "tool_reference": + ref_name = item.get("tool_name") or item.get("name") + if ref_name: + tool_reference.append( + {"type": "tool_reference", "name": ref_name} + ) tool_text = "\n".join(text_parts) openai_messages.append( @@ -312,6 +323,15 @@ class AnthropicServingMessages(OpenAIServingChat): } ) + if tool_reference: + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.tool_use_id or "", + "content": tool_reference, # type: ignore[dict-item] + } + ) + @classmethod def _build_base_request( cls, @@ -400,6 +420,7 @@ class AnthropicServingMessages(OpenAIServingChat): "name": tool.name, "description": tool.description, "parameters": tool.input_schema, + "defer_loading": tool.defer_loading, }, } ) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 73a32033aa..c6bf6f0e6e 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -255,6 +255,23 @@ class CustomThinkCompletionContentParam(TypedDict, total=False): """The thinking type.""" +class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False): + """A tool reference content param that only accepts a plain tool name. + + Example: + { + "name": "get_weather", + "type": "tool_reference" + } + """ + + name: str + """The name of the tool being referenced.""" + + type: Literal["tool_reference"] + """The content type.""" + + ChatCompletionContentPartParam: TypeAlias = ( OpenAIChatCompletionContentPartParam | ChatCompletionContentPartAudioParam @@ -267,6 +284,7 @@ ChatCompletionContentPartParam: TypeAlias = ( | ChatCompletionContentPartAudioEmbedsParam | CustomChatCompletionContentSimpleAudioParam | CustomChatCompletionContentSimpleVideoParam + | CustomChatCompletionContentToolReferenceParam | str | CustomThinkCompletionContentParam ) @@ -1281,6 +1299,9 @@ MM_PARSER_MAP: dict[ "input_audio": lambda part: _InputAudioParser(part).get("input_audio", None), "refusal": lambda part: _RefusalParser(part).get("refusal", None), "video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None), + "tool_reference": lambda part: cast( + CustomChatCompletionContentToolReferenceParam, part + ).get("name", None), } @@ -1372,6 +1393,12 @@ def _parse_chat_message_content_mm_part( # with url as a dict of {"url": url} video_url = video_url.get("url", None) return "video_url", video_url + if "tool_reference" in part: + tool_reference_params = cast( + CustomChatCompletionContentToolReferenceParam, part + ) + tool_reference = tool_reference_params.get("name", None) + return "tool_reference", tool_reference # Raise an error if no 'type' or direct URL is found. raise ValueError("Missing 'type' field in multimodal part.") @@ -1501,6 +1528,12 @@ def _parse_chat_message_content_part( str_content = cast(str, content) mm_parser.parse_video(str_content, uuid) modality = "video" + elif part_type == "tool_reference": + # Tool references are not multimodal data — they reference deferred + # tools and are passed through as-is for the chat template to expand. + if wrap_dicts: + return {"type": "tool_reference", "name": cast(str, content)} + return cast(str, content) else: supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT)) raise VLLMValidationError( @@ -1567,14 +1600,24 @@ def _parse_chat_message_content( # string. Clients like Claude Code / Cursor send tool results as # [{"type": "text", "text": "..."}], but most chat templates only # handle string content for tool messages. + # However, tool_reference items must be preserved as structured + # dicts for the chat template to expand them. msg_content = result_msg.get("content") if isinstance(msg_content, list): - texts = [ - item.get("text", "") + has_non_text = any( + isinstance(item, dict) and item.get("type") != "text" for item in msg_content - if isinstance(item, dict) and item.get("type") == "text" - ] - result_msg["content"] = "\n".join(texts) if texts else "" + ) + if has_non_text: + # Keep structured content (e.g., tool_reference) + result_msg["content"] = msg_content + else: + texts = [ + item.get("text", "") + for item in msg_content + if isinstance(item, dict) and item.get("type") == "text" + ] + result_msg["content"] = "\n".join(texts) if texts else "" if "name" in message and isinstance(message["name"], str): result_msg["name"] = message["name"] diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 9f36199235..c92cc13da0 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -11,7 +11,7 @@ from openai.types.chat.chat_completion_audio import ( ChatCompletionAudio as OpenAIChatCompletionAudio, ) from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation -from pydantic import Field, PrivateAttr, model_validator +from pydantic import Field, PrivateAttr, model_serializer, model_validator from vllm.config import ModelConfig from vllm.config.utils import replace @@ -139,6 +139,20 @@ class ChatCompletionStreamResponse(OpenAIBaseModel): class ChatCompletionToolsParam(OpenAIBaseModel): type: Literal["function"] = "function" function: FunctionDefinition + defer_loading: bool | None = None + + @model_validator(mode="after") + def _propagate_defer_loading(self) -> "ChatCompletionToolsParam": + if self.defer_loading is not None and self.function.defer_loading is None: + self.function.defer_loading = self.defer_loading + return self + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self.defer_loading is None: + data.pop("defer_loading", None) + return data class ChatCompletionNamedFunction(OpenAIBaseModel): diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index 8f6cdb3e62..890af0300e 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -12,6 +12,7 @@ from pydantic import ( BaseModel, ConfigDict, Field, + model_serializer, model_validator, ) @@ -166,6 +167,14 @@ class FunctionDefinition(OpenAIBaseModel): name: str description: str | None = None parameters: dict[str, Any] | None = None + defer_loading: bool | None = None + + @model_serializer(mode="wrap") + def _serialize(self, handler): + data = handler(self) + if self.defer_loading is None: + data.pop("defer_loading", None) + return data # extra="forbid" is a workaround to have kwargs as a field,