[Bugfix][Frontend] Report named Anthropic tool calls as tool_use (#47598)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Ting SUN
2026-09-30 05:32:38 +00:00
committed by GitHub
co-authored by Flora Feng
parent ac68c30872
commit 329e6eee3f
2 changed files with 147 additions and 34 deletions
@@ -37,6 +37,8 @@ from vllm.entrypoints.generate.base.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
FunctionCall,
ToolCall,
)
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionResponse,
@@ -999,11 +1001,6 @@ class TestInlineSystemMessageInMessagesArray:
def _make_stream_converter():
obj = MagicMock(spec=AnthropicServingMessages)
obj.stop_reason_map = {
"stop": "end_turn",
"length": "max_tokens",
"tool_calls": "tool_use",
}
obj.message_stream_converter = (
AnthropicServingMessages.message_stream_converter.__get__(obj)
)
@@ -1671,6 +1668,78 @@ class TestStopSequenceReason:
assert msg_deltas[0]["delta"]["stop_sequence"] is None
class TestToolUseStopReason:
"""A response carrying a tool_use block reports ``stop_reason="tool_use"``
even when the chat layer says ``finish_reason="stop"``, as it does for a
forced (named) tool. A call cut short by max_tokens or a stop string does not.
"""
@pytest.mark.parametrize(
("finish_reason", "stop_reason", "expected"),
[
("stop", None, "tool_use"),
("tool_calls", "</x>", "tool_use"),
],
)
def test_non_streaming(self, finish_reason, stop_reason, expected):
tool_call = ToolCall(
function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}')
)
response = ChatCompletionResponse(
id="chatcmpl-test",
model="test-model",
choices=[
ChatCompletionResponseChoice(
index=0,
message=ChatMessage(role="assistant", tool_calls=[tool_call]),
finish_reason=finish_reason,
stop_reason=stop_reason,
)
],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
result = _make_full_converter().messages_full_converter(response)
assert result.content[0].type == "tool_use"
assert result.stop_reason == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
("args", "finish_reason", "stop_reason", "expected"),
[
('{"city": "Paris"}', "stop", None, "tool_use"),
('{"city": "Pa', "length", None, "max_tokens"),
('{"city": "Paris"}', "tool_calls", "</x>", "tool_use"),
('{"city": "Pa', "tool_calls", "</x>", "stop_sequence"),
],
)
async def test_streaming(self, args, finish_reason, stop_reason, expected):
async def sse_input():
yield _make_stream_chunk(delta=DeltaMessage(role="assistant"))
yield _make_stream_chunk(
delta=DeltaMessage(
tool_calls=[_tc(id="call_1", name="get_weather", args=args)]
)
)
yield _make_stream_chunk(
finish_reason=finish_reason, stop_reason=stop_reason
)
yield _make_stream_chunk(
choices=[],
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
)
yield "data: [DONE]"
converter = _make_stream_converter()
events = _parse_sse_events(
[e async for e in converter.message_stream_converter(sse_input())]
)
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
assert msg_deltas[0]["delta"]["stop_reason"] == expected
# ======================================================================
# Client-caused errors are 4xx, not 500 (Issue #52088)
# ======================================================================
+73 -29
View File
@@ -10,7 +10,7 @@ import logging
import time
import uuid
from collections.abc import AsyncGenerator
from typing import Any, get_args
from typing import Any, Literal, get_args
import jinja2
from fastapi import Request
@@ -97,6 +97,46 @@ def _build_anthropic_usage(
return AnthropicUsage(**kwargs)
def _to_anthropic_stop_reason(
finish_reason: str | None,
stop_reason: int | str | None,
tool_used: bool,
tool_complete: bool,
) -> tuple[
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None,
str | None,
]:
"""Map an OpenAI finish reason to Anthropic ``(stop_reason, stop_sequence)``.
Anthropic reports ``tool_use`` whenever the model invoked a tool. The chat
layer reports ``finish_reason="stop"`` for a named (forced) tool call, so
``tool_used`` records whether a ``tool_use`` block was produced.
``max_tokens`` always wins. A matched stop string wins unless the tool
calls are complete (``tool_complete``): a call it cut short must not run.
"""
if finish_reason == "length":
return "max_tokens", None
if finish_reason not in ("stop", "tool_calls"):
return None, None
if tool_complete:
return "tool_use", None
# vLLM reports the matched stop string in stop_reason (a str);
# an int stop-token-id or None (natural EOS) is not a stop sequence.
if isinstance(stop_reason, str):
return "stop_sequence", stop_reason
if finish_reason == "tool_calls" or tool_used:
return "tool_use", None
return "end_turn", None
def _is_json(text: str) -> bool:
try:
json.loads(text)
except json.JSONDecodeError:
return False
return True
def wrap_data_with_event(data: str, event: str):
return f"event: {event}\ndata: {data}\n\n"
@@ -141,11 +181,6 @@ class AnthropicServingMessages(OpenAIServingChat):
enable_force_include_usage=enable_force_include_usage,
default_chat_template_kwargs=default_chat_template_kwargs,
)
self.stop_reason_map = {
"stop": "end_turn",
"length": "max_tokens",
"tool_calls": "tool_use",
}
self._merge_inline_system = self._should_merge_inline_system(online_renderer)
# Resolved lazily from the renderer when "auto".
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
@@ -751,18 +786,13 @@ class AnthropicServingMessages(OpenAIServingChat):
ec_transfer_params=generator.ec_transfer_params,
)
choice = generator.choices[0]
if choice.finish_reason == "stop":
# vLLM reports the matched stop string in stop_reason (a str);
# an int stop-token-id or None (natural EOS) maps to end_turn.
if isinstance(choice.stop_reason, str):
result.stop_reason = "stop_sequence"
result.stop_sequence = choice.stop_reason
else:
result.stop_reason = "end_turn"
elif choice.finish_reason == "length":
result.stop_reason = "max_tokens"
elif choice.finish_reason == "tool_calls":
result.stop_reason = "tool_use"
result.stop_reason, result.stop_sequence = _to_anthropic_stop_reason(
choice.finish_reason,
choice.stop_reason,
tool_used=bool(choice.message.tool_calls),
# Their arguments are json-decoded below, so parsed calls are complete.
tool_complete=bool(choice.message.tool_calls),
)
content: list[AnthropicContentBlock] = []
if choice.message.reasoning:
@@ -845,6 +875,10 @@ class AnthropicServingMessages(OpenAIServingChat):
# Matched stop string, when generation stopped on one (a str);
# int stop-token-id / None are not stop sequences.
stop_sequence: int | str | None = None
tool_used = False
# Arguments of the latest tool_use block; only the last call can be
# cut short by a stop string.
last_tool_args: list[str] = []
state = _ActiveBlockState()
# Map from tool call index to tool_use_id
tool_index_to_id: dict[int, str] = {}
@@ -954,19 +988,21 @@ class AnthropicServingMessages(OpenAIServingChat):
if len(origin_chunk.choices) == 0:
for event in stop_and_flush():
yield event
if isinstance(stop_sequence, str):
stop_delta = AnthropicDelta(
stop_reason="stop_sequence",
stop_sequence=stop_sequence,
)
else:
stop_delta = AnthropicDelta(
stop_reason=self.stop_reason_map.get(
finish_reason or "stop"
anthropic_stop_reason, anthropic_stop_sequence = (
_to_anthropic_stop_reason(
finish_reason or "stop",
stop_sequence,
tool_used=tool_used,
tool_complete=(
tool_used and _is_json("".join(last_tool_args))
),
# Set explicitly so exclude_unset=True keeps it.
stop_sequence=None,
)
)
stop_delta = AnthropicDelta(
stop_reason=anthropic_stop_reason,
# Set explicitly so exclude_unset=True keeps it.
stop_sequence=anthropic_stop_sequence,
)
chunk = AnthropicStreamEvent(
type="message_delta",
delta=stop_delta,
@@ -1060,6 +1096,8 @@ class AnthropicServingMessages(OpenAIServingChat):
):
for event in stop_and_flush():
yield event
tool_used = True
last_tool_args = []
start_event = start_block(
AnthropicContentBlock(
type="tool_use",
@@ -1075,6 +1113,9 @@ class AnthropicServingMessages(OpenAIServingChat):
and tool_call.function.arguments
and state.tool_use_id == tool_call.id
):
last_tool_args.append(
tool_call.function.arguments
)
chunk = AnthropicStreamEvent(
index=(
state.block_index
@@ -1100,6 +1141,9 @@ class AnthropicServingMessages(OpenAIServingChat):
and tool_call.function.arguments
and state.tool_use_id == tool_use_id
):
last_tool_args.append(
tool_call.function.arguments
)
chunk = AnthropicStreamEvent(
index=(
state.block_index