[Bugfix][Frontend] Accept Anthropic tool_addition and tool_removal content blocks (#57693)
Signed-off-by: Anurag Singh <dr.anuragkr@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Antigravity <antigravity@google.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Antigravity
sfeng33
parent
e6674262f1
commit
28c57456db
@@ -27,6 +27,7 @@ from pydantic import BaseModel, Field, ValidationError
|
||||
|
||||
from vllm.entrypoints.anthropic.api_router import attach_router
|
||||
from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicCountTokensRequest,
|
||||
AnthropicMessagesRequest,
|
||||
)
|
||||
from vllm.entrypoints.anthropic.serving import (
|
||||
@@ -2032,3 +2033,87 @@ class TestProbeDisabledThinkingEffort:
|
||||
@pytest.mark.asyncio
|
||||
async def test_renderer_rejects_none(self):
|
||||
assert await self._probe(self._reject_none) == "low"
|
||||
|
||||
|
||||
class TestMidConversationToolChanges:
|
||||
"""``tool_addition``/``tool_removal`` blocks decide which tools the chat
|
||||
template is given, since it has no other way to see them."""
|
||||
|
||||
@staticmethod
|
||||
def _convert_tools(*changes: dict) -> dict[str, bool | None]:
|
||||
request = _make_request(
|
||||
[
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "system",
|
||||
"content": [{"type": "text", "text": "Tools changed."}, *changes],
|
||||
},
|
||||
],
|
||||
tools=[
|
||||
{"name": "bash", "input_schema": {}},
|
||||
{"name": "search", "input_schema": {}, "defer_loading": True},
|
||||
{"name": "fetch", "input_schema": {}, "defer_loading": True},
|
||||
],
|
||||
)
|
||||
result = _convert(request)
|
||||
assert result.messages[-1] == {"role": "system", "content": "Tools changed."}
|
||||
assert result.tools is not None
|
||||
return {t.function.name: t.function.defer_loading for t in result.tools}
|
||||
|
||||
@staticmethod
|
||||
def _change(block_type: str, name: str) -> dict:
|
||||
return {"type": block_type, "tool": {"type": "tool_reference", "name": name}}
|
||||
|
||||
def test_addition_loads_deferred_tool(self):
|
||||
tools = self._convert_tools(self._change("tool_addition", "search"))
|
||||
assert tools == {"bash": None, "search": None, "fetch": True}
|
||||
|
||||
def test_removal_withdraws_tool_until_added_again(self):
|
||||
removal = self._change("tool_removal", "bash")
|
||||
assert "bash" not in self._convert_tools(removal)
|
||||
readded = self._convert_tools(removal, self._change("tool_addition", "bash"))
|
||||
assert list(readded) == ["bash", "search", "fetch"]
|
||||
|
||||
def test_addition_defines_tool_by_value(self):
|
||||
definition = {"name": "db_query", "input_schema": {}}
|
||||
tools = self._convert_tools(
|
||||
{
|
||||
"type": "tool_addition",
|
||||
"tool": {"type": "tool_definition", "definition": definition},
|
||||
}
|
||||
)
|
||||
assert list(tools) == ["bash", "search", "fetch", "db_query"]
|
||||
|
||||
def test_unknown_tool_reference_is_rejected(self):
|
||||
with pytest.raises(ValueError, match="not declared in tools"):
|
||||
self._convert_tools(self._change("tool_addition", "missing"))
|
||||
|
||||
def test_removal_by_value_is_rejected(self):
|
||||
definition = {"name": "bash", "input_schema": {}}
|
||||
with pytest.raises(ValidationError, match="only accepts a tool_reference"):
|
||||
self._convert_tools(
|
||||
{
|
||||
"type": "tool_removal",
|
||||
"tool": {"type": "tool_definition", "definition": definition},
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_cls", [AnthropicMessagesRequest, AnthropicCountTokensRequest]
|
||||
)
|
||||
@pytest.mark.parametrize("in_top_level_system", [False, True])
|
||||
def test_tool_change_outside_system_message_is_rejected(
|
||||
self, request_cls, in_top_level_system
|
||||
):
|
||||
change = self._change("tool_addition", "search")
|
||||
if in_top_level_system:
|
||||
fields = {
|
||||
"messages": [{"role": "user", "content": "Hi"}],
|
||||
"system": [change],
|
||||
}
|
||||
else:
|
||||
fields = {"messages": [{"role": "user", "content": [change]}]}
|
||||
with pytest.raises(
|
||||
ValidationError, match='only allowed in messages with role "system"'
|
||||
):
|
||||
request_cls(model="test-model", max_tokens=128, **fields)
|
||||
|
||||
@@ -44,6 +44,8 @@ class AnthropicContentBlock(BaseModel):
|
||||
"tool_reference",
|
||||
"thinking",
|
||||
"redacted_thinking",
|
||||
"tool_addition",
|
||||
"tool_removal",
|
||||
]
|
||||
text: str | None = None
|
||||
# For image content
|
||||
@@ -62,6 +64,28 @@ class AnthropicContentBlock(BaseModel):
|
||||
signature: str | None = None
|
||||
# For redacted thinking content (safety-filtered by the API)
|
||||
data: str | None = None
|
||||
# For tool_addition/tool_removal content
|
||||
tool: "AnthropicToolChange | None" = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_tool_change(self) -> "AnthropicContentBlock":
|
||||
if self.type not in ("tool_addition", "tool_removal"):
|
||||
return self
|
||||
if self.tool is None:
|
||||
raise ValueError(f"tool is required for {self.type} blocks")
|
||||
if self.type == "tool_removal" and self.tool.type != "tool_reference":
|
||||
raise ValueError("tool_removal only accepts a tool_reference")
|
||||
return self
|
||||
|
||||
|
||||
def _reject_tool_changes(content: str | list[AnthropicContentBlock] | None) -> None:
|
||||
if content is None or isinstance(content, str):
|
||||
return
|
||||
for block in content:
|
||||
if block.type in ("tool_addition", "tool_removal"):
|
||||
raise ValueError(
|
||||
f'{block.type} blocks are only allowed in messages with role "system"'
|
||||
)
|
||||
|
||||
|
||||
class AnthropicMessage(BaseModel):
|
||||
@@ -70,6 +94,12 @@ class AnthropicMessage(BaseModel):
|
||||
role: Literal["user", "assistant", "system"]
|
||||
content: str | list[AnthropicContentBlock]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_tool_change_role(self) -> "AnthropicMessage":
|
||||
if self.role != "system":
|
||||
_reject_tool_changes(self.content)
|
||||
return self
|
||||
|
||||
|
||||
class AnthropicTool(BaseModel):
|
||||
"""Tool definition."""
|
||||
@@ -90,6 +120,26 @@ class AnthropicTool(BaseModel):
|
||||
return v
|
||||
|
||||
|
||||
class AnthropicToolChangeReference(BaseModel):
|
||||
"""A tool named by a tool_addition or tool_removal block."""
|
||||
|
||||
type: Literal["tool_reference"]
|
||||
name: str
|
||||
|
||||
|
||||
class AnthropicToolChangeDefinition(BaseModel):
|
||||
"""A tool defined by value in a tool_addition block."""
|
||||
|
||||
type: Literal["tool_definition"]
|
||||
definition: AnthropicTool
|
||||
|
||||
|
||||
AnthropicToolChange = Annotated[
|
||||
AnthropicToolChangeReference | AnthropicToolChangeDefinition,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class AnthropicToolChoice(BaseModel):
|
||||
"""Tool Choice definition."""
|
||||
|
||||
@@ -231,6 +281,12 @@ class AnthropicMessagesRequest(BaseModel):
|
||||
raise ValueError("max_tokens must be positive")
|
||||
return v
|
||||
|
||||
@field_validator("system")
|
||||
@classmethod
|
||||
def validate_system(cls, v):
|
||||
_reject_tool_changes(v)
|
||||
return v
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_thinking_budget(self) -> "AnthropicMessagesRequest":
|
||||
# P/D prefill legs are sent with max_tokens=1 and never decode; the
|
||||
@@ -343,6 +399,12 @@ class AnthropicCountTokensRequest(BaseModel):
|
||||
raise ValueError("Model is required")
|
||||
return v
|
||||
|
||||
@field_validator("system")
|
||||
@classmethod
|
||||
def validate_system(cls, v):
|
||||
_reject_tool_changes(v)
|
||||
return v
|
||||
|
||||
|
||||
class AnthropicCountTokensResponse(BaseModel):
|
||||
"""Anthropic messages.count_tokens response."""
|
||||
|
||||
@@ -31,6 +31,8 @@ from vllm.entrypoints.anthropic.protocol import (
|
||||
AnthropicOutputConfig,
|
||||
AnthropicStreamEvent,
|
||||
AnthropicThinkingConfig,
|
||||
AnthropicTool,
|
||||
AnthropicToolChangeDefinition,
|
||||
AnthropicUsage,
|
||||
)
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
@@ -710,6 +712,51 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _resolve_tools(
|
||||
cls,
|
||||
anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
|
||||
) -> list[AnthropicTool] | None:
|
||||
"""Apply tool_addition/tool_removal blocks in ``messages`` to ``tools``.
|
||||
|
||||
Chat templates see a single tool list, so this resolves the tools in
|
||||
effect at the end of the conversation: a tool added by reference is no
|
||||
longer deferred, a tool added by value is declared, and a removed tool
|
||||
is dropped.
|
||||
"""
|
||||
changes = [
|
||||
block
|
||||
for msg in anthropic_request.messages
|
||||
if not isinstance(msg.content, str)
|
||||
for block in msg.content
|
||||
if block.type in ("tool_addition", "tool_removal")
|
||||
]
|
||||
if not changes:
|
||||
return anthropic_request.tools
|
||||
|
||||
tools = {tool.name: tool for tool in anthropic_request.tools or []}
|
||||
removed: set[str] = set()
|
||||
for block in changes:
|
||||
change = block.tool
|
||||
assert change is not None
|
||||
if isinstance(change, AnthropicToolChangeDefinition):
|
||||
name = change.definition.name
|
||||
tools[name] = change.definition
|
||||
else:
|
||||
name = change.name
|
||||
if name not in tools:
|
||||
raise ValueError(
|
||||
f"{block.type} references tool {name!r}, "
|
||||
"which is not declared in tools"
|
||||
)
|
||||
if block.type == "tool_addition":
|
||||
tools[name] = tools[name].model_copy(update={"defer_loading": None})
|
||||
if block.type == "tool_removal":
|
||||
removed.add(name)
|
||||
else:
|
||||
removed.discard(name)
|
||||
return [tool for name, tool in tools.items() if name not in removed]
|
||||
|
||||
@classmethod
|
||||
def _convert_tools(
|
||||
cls,
|
||||
@@ -717,11 +764,12 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
req: ChatCompletionRequest,
|
||||
) -> None:
|
||||
"""Convert Anthropic tools to OpenAI format."""
|
||||
if anthropic_request.tools is None:
|
||||
anthropic_tools = cls._resolve_tools(anthropic_request)
|
||||
if anthropic_tools is None:
|
||||
return
|
||||
|
||||
tools = []
|
||||
for tool in anthropic_request.tools:
|
||||
for tool in anthropic_tools:
|
||||
tools.append(
|
||||
ChatCompletionToolsParam.model_validate(
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user