fix: report stop_sequence stop_reason in Anthropic Messages API (#45807)
Signed-off-by: Yufeng He <40085740+he-yufeng@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -975,6 +975,7 @@ def _make_stream_chunk(
|
||||
*,
|
||||
delta: DeltaMessage | None = None,
|
||||
finish_reason: str | None = None,
|
||||
stop_reason: int | str | None = None,
|
||||
choices: list[ChatCompletionResponseStreamChoice] | None = None,
|
||||
usage: UsageInfo | None = None,
|
||||
) -> str:
|
||||
@@ -984,6 +985,7 @@ def _make_stream_chunk(
|
||||
index=0,
|
||||
delta=delta or DeltaMessage(),
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason,
|
||||
)
|
||||
]
|
||||
chunk = ChatCompletionStreamResponse(
|
||||
@@ -1463,6 +1465,103 @@ class TestCacheSalt:
|
||||
handler.create_messages.assert_not_awaited()
|
||||
|
||||
|
||||
class TestStopSequenceReason:
|
||||
"""When generation stops because a configured stop string matched, the
|
||||
Anthropic Messages API must report ``stop_reason="stop_sequence"`` and echo
|
||||
the matched string in ``stop_sequence``. vLLM surfaces the matched string in
|
||||
the OpenAI choice's ``stop_reason`` field (a str) while ``finish_reason``
|
||||
stays ``"stop"``. A natural EOS (stop_reason None) or a stop token id (int)
|
||||
must still map to ``end_turn``.
|
||||
"""
|
||||
|
||||
def test_non_streaming_stop_string_maps_to_stop_sequence(self):
|
||||
converter = _make_full_converter()
|
||||
response = ChatCompletionResponse(
|
||||
id="chatcmpl-test",
|
||||
model="test-model",
|
||||
choices=[
|
||||
ChatCompletionResponseChoice(
|
||||
index=0,
|
||||
message=ChatMessage(role="assistant", content="hello"),
|
||||
finish_reason="stop",
|
||||
stop_reason="</tool>",
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
|
||||
result = converter.messages_full_converter(response)
|
||||
|
||||
assert result.stop_reason == "stop_sequence"
|
||||
assert result.stop_sequence == "</tool>"
|
||||
|
||||
def test_non_streaming_natural_eos_maps_to_end_turn(self):
|
||||
converter = _make_full_converter()
|
||||
response = ChatCompletionResponse(
|
||||
id="chatcmpl-test",
|
||||
model="test-model",
|
||||
choices=[
|
||||
ChatCompletionResponseChoice(
|
||||
index=0,
|
||||
message=ChatMessage(role="assistant", content="hello"),
|
||||
finish_reason="stop",
|
||||
stop_reason=None,
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
|
||||
result = converter.messages_full_converter(response)
|
||||
|
||||
assert result.stop_reason == "end_turn"
|
||||
assert result.stop_sequence is None
|
||||
|
||||
def test_non_streaming_stop_token_id_maps_to_end_turn(self):
|
||||
converter = _make_full_converter()
|
||||
response = ChatCompletionResponse(
|
||||
id="chatcmpl-test",
|
||||
model="test-model",
|
||||
choices=[
|
||||
ChatCompletionResponseChoice(
|
||||
index=0,
|
||||
message=ChatMessage(role="assistant", content="hello"),
|
||||
finish_reason="stop",
|
||||
stop_reason=128009,
|
||||
)
|
||||
],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
|
||||
result = converter.messages_full_converter(response)
|
||||
|
||||
assert result.stop_reason == "end_turn"
|
||||
assert result.stop_sequence is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_stop_string_maps_to_stop_sequence(self):
|
||||
async def sse_input():
|
||||
yield _make_stream_chunk(
|
||||
delta=DeltaMessage(content="hi"),
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=5, completion_tokens=0),
|
||||
)
|
||||
yield _make_stream_chunk(finish_reason="stop", stop_reason="</tool>")
|
||||
yield _make_stream_chunk(
|
||||
choices=[],
|
||||
usage=UsageInfo(prompt_tokens=5, total_tokens=8, completion_tokens=3),
|
||||
)
|
||||
yield "data: [DONE]"
|
||||
|
||||
converter = _make_stream_converter()
|
||||
output = []
|
||||
async for event in converter.message_stream_converter(sse_input()):
|
||||
output.append(event)
|
||||
|
||||
events = _parse_sse_events(output)
|
||||
msg_deltas = [data for ev_type, data in events if ev_type == "message_delta"]
|
||||
assert msg_deltas[0]["delta"]["stop_reason"] == "stop_sequence"
|
||||
assert msg_deltas[0]["delta"]["stop_sequence"] == "</tool>"
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Client-caused errors are 4xx, not 500 (Issue #52088)
|
||||
# ======================================================================
|
||||
|
||||
@@ -631,7 +631,13 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
)
|
||||
choice = generator.choices[0]
|
||||
if choice.finish_reason == "stop":
|
||||
result.stop_reason = "end_turn"
|
||||
# vLLM reports the matched stop string in stop_reason (a str);
|
||||
# an int stop-token-id or None (natural EOS) maps to end_turn.
|
||||
if isinstance(choice.stop_reason, str):
|
||||
result.stop_reason = "stop_sequence"
|
||||
result.stop_sequence = choice.stop_reason
|
||||
else:
|
||||
result.stop_reason = "end_turn"
|
||||
elif choice.finish_reason == "length":
|
||||
result.stop_reason = "max_tokens"
|
||||
elif choice.finish_reason == "tool_calls":
|
||||
@@ -715,6 +721,9 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
|
||||
first_item = True
|
||||
finish_reason = None
|
||||
# Matched stop string, when generation stopped on one (a str);
|
||||
# int stop-token-id / None are not stop sequences.
|
||||
stop_sequence: int | str | None = None
|
||||
state = _ActiveBlockState()
|
||||
# Map from tool call index to tool_use_id
|
||||
tool_index_to_id: dict[int, str] = {}
|
||||
@@ -824,12 +833,20 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
if len(origin_chunk.choices) == 0:
|
||||
for event in stop_and_flush():
|
||||
yield event
|
||||
stop_reason = self.stop_reason_map.get(
|
||||
finish_reason or "stop"
|
||||
)
|
||||
if isinstance(stop_sequence, str):
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason="stop_sequence",
|
||||
stop_sequence=stop_sequence,
|
||||
)
|
||||
else:
|
||||
stop_delta = AnthropicDelta(
|
||||
stop_reason=self.stop_reason_map.get(
|
||||
finish_reason or "stop"
|
||||
)
|
||||
)
|
||||
chunk = AnthropicStreamEvent(
|
||||
type="message_delta",
|
||||
delta=AnthropicDelta(stop_reason=stop_reason),
|
||||
delta=stop_delta,
|
||||
usage=_build_anthropic_usage(origin_chunk.usage),
|
||||
)
|
||||
data = chunk.model_dump_json(exclude_unset=True)
|
||||
@@ -838,6 +855,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
|
||||
if origin_chunk.choices[0].finish_reason is not None:
|
||||
finish_reason = origin_chunk.choices[0].finish_reason
|
||||
stop_sequence = origin_chunk.choices[0].stop_reason
|
||||
# continue
|
||||
|
||||
# thinking / text content
|
||||
|
||||
Reference in New Issue
Block a user