diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 06efefa099..ccd88c462b 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -37,6 +37,8 @@ from vllm.entrypoints.generate.base.protocol import ( DeltaFunctionCall, DeltaMessage, DeltaToolCall, + FunctionCall, + ToolCall, ) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionResponse, @@ -999,11 +1001,6 @@ class TestInlineSystemMessageInMessagesArray: def _make_stream_converter(): obj = MagicMock(spec=AnthropicServingMessages) - obj.stop_reason_map = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - } obj.message_stream_converter = ( AnthropicServingMessages.message_stream_converter.__get__(obj) ) @@ -1671,6 +1668,78 @@ class TestStopSequenceReason: assert msg_deltas[0]["delta"]["stop_sequence"] is None +class TestToolUseStopReason: + """A response carrying a tool_use block reports ``stop_reason="tool_use"`` + even when the chat layer says ``finish_reason="stop"``, as it does for a + forced (named) tool. A call cut short by max_tokens or a stop string does not. + """ + + @pytest.mark.parametrize( + ("finish_reason", "stop_reason", "expected"), + [ + ("stop", None, "tool_use"), + ("tool_calls", "", "tool_use"), + ], + ) + def test_non_streaming(self, finish_reason, stop_reason, expected): + tool_call = ToolCall( + function=FunctionCall(name="get_weather", arguments='{"city": "Paris"}') + ) + response = ChatCompletionResponse( + id="chatcmpl-test", + model="test-model", + choices=[ + ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role="assistant", tool_calls=[tool_call]), + finish_reason=finish_reason, + stop_reason=stop_reason, + ) + ], + usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), + ) + + result = _make_full_converter().messages_full_converter(response) + + assert result.content[0].type == "tool_use" + assert result.stop_reason == expected + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("args", "finish_reason", "stop_reason", "expected"), + [ + ('{"city": "Paris"}', "stop", None, "tool_use"), + ('{"city": "Pa', "length", None, "max_tokens"), + ('{"city": "Paris"}', "tool_calls", "", "tool_use"), + ('{"city": "Pa', "tool_calls", "", "stop_sequence"), + ], + ) + async def test_streaming(self, args, finish_reason, stop_reason, expected): + async def sse_input(): + yield _make_stream_chunk(delta=DeltaMessage(role="assistant")) + yield _make_stream_chunk( + delta=DeltaMessage( + tool_calls=[_tc(id="call_1", name="get_weather", args=args)] + ) + ) + yield _make_stream_chunk( + finish_reason=finish_reason, stop_reason=stop_reason + ) + yield _make_stream_chunk( + choices=[], + usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + events = _parse_sse_events( + [e async for e in converter.message_stream_converter(sse_input())] + ) + + msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"] + assert msg_deltas[0]["delta"]["stop_reason"] == expected + + # ====================================================================== # Client-caused errors are 4xx, not 500 (Issue #52088) # ====================================================================== diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index cd31908508..91820de2fd 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -10,7 +10,7 @@ import logging import time import uuid from collections.abc import AsyncGenerator -from typing import Any, get_args +from typing import Any, Literal, get_args import jinja2 from fastapi import Request @@ -97,6 +97,46 @@ def _build_anthropic_usage( return AnthropicUsage(**kwargs) +def _to_anthropic_stop_reason( + finish_reason: str | None, + stop_reason: int | str | None, + tool_used: bool, + tool_complete: bool, +) -> tuple[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None, + str | None, +]: + """Map an OpenAI finish reason to Anthropic ``(stop_reason, stop_sequence)``. + + Anthropic reports ``tool_use`` whenever the model invoked a tool. The chat + layer reports ``finish_reason="stop"`` for a named (forced) tool call, so + ``tool_used`` records whether a ``tool_use`` block was produced. + ``max_tokens`` always wins. A matched stop string wins unless the tool + calls are complete (``tool_complete``): a call it cut short must not run. + """ + if finish_reason == "length": + return "max_tokens", None + if finish_reason not in ("stop", "tool_calls"): + return None, None + if tool_complete: + return "tool_use", None + # vLLM reports the matched stop string in stop_reason (a str); + # an int stop-token-id or None (natural EOS) is not a stop sequence. + if isinstance(stop_reason, str): + return "stop_sequence", stop_reason + if finish_reason == "tool_calls" or tool_used: + return "tool_use", None + return "end_turn", None + + +def _is_json(text: str) -> bool: + try: + json.loads(text) + except json.JSONDecodeError: + return False + return True + + def wrap_data_with_event(data: str, event: str): return f"event: {event}\ndata: {data}\n\n" @@ -141,11 +181,6 @@ class AnthropicServingMessages(OpenAIServingChat): enable_force_include_usage=enable_force_include_usage, default_chat_template_kwargs=default_chat_template_kwargs, ) - self.stop_reason_map = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - } self._merge_inline_system = self._should_merge_inline_system(online_renderer) # Resolved lazily from the renderer when "auto". self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = ( @@ -751,18 +786,13 @@ class AnthropicServingMessages(OpenAIServingChat): ec_transfer_params=generator.ec_transfer_params, ) choice = generator.choices[0] - if choice.finish_reason == "stop": - # vLLM reports the matched stop string in stop_reason (a str); - # an int stop-token-id or None (natural EOS) maps to end_turn. - if isinstance(choice.stop_reason, str): - result.stop_reason = "stop_sequence" - result.stop_sequence = choice.stop_reason - else: - result.stop_reason = "end_turn" - elif choice.finish_reason == "length": - result.stop_reason = "max_tokens" - elif choice.finish_reason == "tool_calls": - result.stop_reason = "tool_use" + result.stop_reason, result.stop_sequence = _to_anthropic_stop_reason( + choice.finish_reason, + choice.stop_reason, + tool_used=bool(choice.message.tool_calls), + # Their arguments are json-decoded below, so parsed calls are complete. + tool_complete=bool(choice.message.tool_calls), + ) content: list[AnthropicContentBlock] = [] if choice.message.reasoning: @@ -845,6 +875,10 @@ class AnthropicServingMessages(OpenAIServingChat): # Matched stop string, when generation stopped on one (a str); # int stop-token-id / None are not stop sequences. stop_sequence: int | str | None = None + tool_used = False + # Arguments of the latest tool_use block; only the last call can be + # cut short by a stop string. + last_tool_args: list[str] = [] state = _ActiveBlockState() # Map from tool call index to tool_use_id tool_index_to_id: dict[int, str] = {} @@ -954,19 +988,21 @@ class AnthropicServingMessages(OpenAIServingChat): if len(origin_chunk.choices) == 0: for event in stop_and_flush(): yield event - if isinstance(stop_sequence, str): - stop_delta = AnthropicDelta( - stop_reason="stop_sequence", - stop_sequence=stop_sequence, - ) - else: - stop_delta = AnthropicDelta( - stop_reason=self.stop_reason_map.get( - finish_reason or "stop" + anthropic_stop_reason, anthropic_stop_sequence = ( + _to_anthropic_stop_reason( + finish_reason or "stop", + stop_sequence, + tool_used=tool_used, + tool_complete=( + tool_used and _is_json("".join(last_tool_args)) ), - # Set explicitly so exclude_unset=True keeps it. - stop_sequence=None, ) + ) + stop_delta = AnthropicDelta( + stop_reason=anthropic_stop_reason, + # Set explicitly so exclude_unset=True keeps it. + stop_sequence=anthropic_stop_sequence, + ) chunk = AnthropicStreamEvent( type="message_delta", delta=stop_delta, @@ -1060,6 +1096,8 @@ class AnthropicServingMessages(OpenAIServingChat): ): for event in stop_and_flush(): yield event + tool_used = True + last_tool_args = [] start_event = start_block( AnthropicContentBlock( type="tool_use", @@ -1075,6 +1113,9 @@ class AnthropicServingMessages(OpenAIServingChat): and tool_call.function.arguments and state.tool_use_id == tool_call.id ): + last_tool_args.append( + tool_call.function.arguments + ) chunk = AnthropicStreamEvent( index=( state.block_index @@ -1100,6 +1141,9 @@ class AnthropicServingMessages(OpenAIServingChat): and tool_call.function.arguments and state.tool_use_id == tool_use_id ): + last_tool_args.append( + tool_call.function.arguments + ) chunk = AnthropicStreamEvent( index=( state.block_index