[Bugfix][Frontend] Report named Anthropic tool calls as tool_use (#47598)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
@@ -37,6 +37,8 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaFunctionCall,
|
||||
DeltaMessage,
|
||||
DeltaToolCall,
|
||||
FunctionCall,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionResponse,
|
||||
@@ -999,11 +1001,6 @@ class TestInlineSystemMessageInMessagesArray:
|
||||
|
||||
def _make_stream_converter():
|
||||
obj = MagicMock(spec=AnthropicServingMessages)
|
||||
obj.stop_reason_map = {
|
||||
"stop": "end_turn",
|
||||
"length": "max_tokens",
|
||||
"tool_calls": "tool_use",
|
||||
}
|
||||
obj.message_stream_converter = (
|
||||
AnthropicServingMessages.message_stream_converter.__get__(obj)
|
||||
)
|
||||
@@ -1671,6 +1668,78 @@ class TestStopSequenceReason:
|
||||
assert msg_deltas[0]["delta"]["stop_sequence"] is None
|
||||
|
||||
|
||||
class TestToolUseStopReason:
|
||||
"""A response carrying a tool_use block reports ``stop_reason="tool_use"``
|
||||
even when the chat layer says ``finish_reason="stop"``, as it does for a
|
||||
forced (named) tool. A call cut short by max_tokens or a stop string does not.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("finish_reason", "stop_reason", "expected"),
|
||||
[
|
||||
("stop", None, "tool_use"),
|
||||
("tool_calls", "</x>", "tool_use"),
|
||||
],
|
||||
)
|
||||
def test_non_streaming(self, finish_reason, stop_reason, expected):
|
||||
tool_call = ToolCall(
|
||||
function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}')
|
||||
)
|
||||
response = ChatCompletionResponse(
|
||||
id="chatcmpl-test",
|
||||
model="test-model",
|
||||
choices=[
|
||||
ChatCompletionResponseChoice(
|
||||
index=0,
|
||||
message=ChatMessage(role="assistant", tool_calls=[tool_call]),
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason,
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
|
||||
result = _make_full_converter().messages_full_converter(response)
|
||||
|
||||
assert result.content[0].type == "tool_use"
|
||||
assert result.stop_reason == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("args", "finish_reason", "stop_reason", "expected"),
|
||||
[
|
||||
('{"city": "Paris"}', "stop", None, "tool_use"),
|
||||
('{"city": "Pa', "length", None, "max_tokens"),
|
||||
('{"city": "Paris"}', "tool_calls", "</x>", "tool_use"),
|
||||
('{"city": "Pa', "tool_calls", "</x>", "stop_sequence"),
|
||||
],
|
||||
)
|
||||
async def test_streaming(self, args, finish_reason, stop_reason, expected):
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(
|
||||
tool_calls=[_tc(id="call_1", name="get_weather", args=args)]
|
||||
)
|
||||
)
|
||||
yield _make_stream_chunk(
|
||||
finish_reason=finish_reason, stop_reason=stop_reason
|
||||
)
|
||||
yield _make_stream_chunk(
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
events = _parse_sse_events(
|
||||
[e async for e in converter.message_stream_converter(sse_input())]
|
||||
)
|
||||
|
||||
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
|
||||
assert msg_deltas[0]["delta"]["stop_reason"] == expected
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Client-caused errors are 4xx, not 500 (Issue #52088)
|
||||
# ======================================================================
|
||||
|
||||
@@ -10,7 +10,7 @@ import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any, get_args
|
||||
from typing import Any, Literal, get_args
|
||||
|
||||
import jinja2
|
||||
from fastapi import Request
|
||||
@@ -97,6 +97,46 @@ def _build_anthropic_usage(
|
||||
return AnthropicUsage(**kwargs)
|
||||
|
||||
|
||||
def _to_anthropic_stop_reason(
|
||||
finish_reason: str | None,
|
||||
stop_reason: int | str | None,
|
||||
tool_used: bool,
|
||||
tool_complete: bool,
|
||||
) -> tuple[
|
||||
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None,
|
||||
str | None,
|
||||
]:
|
||||
"""Map an OpenAI finish reason to Anthropic ``(stop_reason, stop_sequence)``.
|
||||
|
||||
Anthropic reports ``tool_use`` whenever the model invoked a tool. The chat
|
||||
layer reports ``finish_reason="stop"`` for a named (forced) tool call, so
|
||||
``tool_used`` records whether a ``tool_use`` block was produced.
|
||||
``max_tokens`` always wins. A matched stop string wins unless the tool
|
||||
calls are complete (``tool_complete``): a call it cut short must not run.
|
||||
"""
|
||||
if finish_reason == "length":
|
||||
return "max_tokens", None
|
||||
if finish_reason not in ("stop", "tool_calls"):
|
||||
return None, None
|
||||
if tool_complete:
|
||||
return "tool_use", None
|
||||
# vLLM reports the matched stop string in stop_reason (a str);
|
||||
# an int stop-token-id or None (natural EOS) is not a stop sequence.
|
||||
if isinstance(stop_reason, str):
|
||||
return "stop_sequence", stop_reason
|
||||
if finish_reason == "tool_calls" or tool_used:
|
||||
return "tool_use", None
|
||||
return "end_turn", None
|
||||
|
||||
|
||||
def _is_json(text: str) -> bool:
|
||||
try:
|
||||
json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def wrap_data_with_event(data: str, event: str):
|
||||
return f"event: {event}\ndata: {data}\n\n"
|
||||
|
||||
@@ -141,11 +181,6 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
enable_force_include_usage=enable_force_include_usage,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
self.stop_reason_map = {
|
||||
"stop": "end_turn",
|
||||
"length": "max_tokens",
|
||||
"tool_calls": "tool_use",
|
||||
}
|
||||
self._merge_inline_system = self._should_merge_inline_system(online_renderer)
|
||||
# Resolved lazily from the renderer when "auto".
|
||||
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
|
||||
@@ -751,18 +786,13 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
ec_transfer_params=generator.ec_transfer_params,
|
||||
)
|
||||
choice = generator.choices[0]
|
||||
if choice.finish_reason == "stop":
|
||||
# vLLM reports the matched stop string in stop_reason (a str);
|
||||
# an int stop-token-id or None (natural EOS) maps to end_turn.
|
||||
if isinstance(choice.stop_reason, str):
|
||||
result.stop_reason = "stop_sequence"
|
||||
result.stop_sequence = choice.stop_reason
|
||||
else:
|
||||
result.stop_reason = "end_turn"
|
||||
elif choice.finish_reason == "length":
|
||||
result.stop_reason = "max_tokens"
|
||||
elif choice.finish_reason == "tool_calls":
|
||||
result.stop_reason = "tool_use"
|
||||
result.stop_reason, result.stop_sequence = _to_anthropic_stop_reason(
|
||||
choice.finish_reason,
|
||||
choice.stop_reason,
|
||||
tool_used=bool(choice.message.tool_calls),
|
||||
# Their arguments are json-decoded below, so parsed calls are complete.
|
||||
tool_complete=bool(choice.message.tool_calls),
|
||||
)
|
||||
|
||||
content: list[AnthropicContentBlock] = []
|
||||
if choice.message.reasoning:
|
||||
@@ -845,6 +875,10 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
# Matched stop string, when generation stopped on one (a str);
|
||||
# int stop-token-id / None are not stop sequences.
|
||||
stop_sequence: int | str | None = None
|
||||
tool_used = False
|
||||
# Arguments of the latest tool_use block; only the last call can be
|
||||
# cut short by a stop string.
|
||||
last_tool_args: list[str] = []
|
||||
state = _ActiveBlockState()
|
||||
# Map from tool call index to tool_use_id
|
||||
tool_index_to_id: dict[int, str] = {}
|
||||
@@ -954,19 +988,21 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
if len(origin_chunk.choices) == 0:
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
if isinstance(stop_sequence, str):
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason="stop_sequence",
|
||||
stop_sequence=stop_sequence,
|
||||
)
|
||||
else:
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason=self.stop_reason_map.get(
|
||||
finish_reason or "stop"
|
||||
anthropic_stop_reason, anthropic_stop_sequence = (
|
||||
_to_anthropic_stop_reason(
|
||||
finish_reason or "stop",
|
||||
stop_sequence,
|
||||
tool_used=tool_used,
|
||||
tool_complete=(
|
||||
tool_used and _is_json("".join(last_tool_args))
|
||||
),
|
||||
# Set explicitly so exclude_unset=True keeps it.
|
||||
stop_sequence=None,
|
||||
)
|
||||
)
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason=anthropic_stop_reason,
|
||||
# Set explicitly so exclude_unset=True keeps it.
|
||||
stop_sequence=anthropic_stop_sequence,
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_delta",
|
||||
delta=stop_delta,
|
||||
@@ -1060,6 +1096,8 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
):
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
tool_used = True
|
||||
last_tool_args = []
|
||||
start_event = start_block(
|
||||
AnthropicContentBlock(
|
||||
type="tool_use",
|
||||
@@ -1075,6 +1113,9 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
and tool_call.function.arguments
|
||||
and state.tool_use_id == tool_call.id
|
||||
):
|
||||
last_tool_args.append(
|
||||
tool_call.function.arguments
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
@@ -1100,6 +1141,9 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
and tool_call.function.arguments
|
||||
and state.tool_use_id == tool_use_id
|
||||
):
|
||||
last_tool_args.append(
|
||||
tool_call.function.arguments
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
index=(
|
||||
state.block_index
|
||||
|
||||
Reference in New Issue
Block a user