[Bugfix][Frontend] Report Cohere stop sequences correctly (#51556)
Signed-off-by: cherry77-cloud <1615405@qq.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
dc5101fb1b
commit
65b7662d3f
@@ -75,6 +75,7 @@ def _build_chat_completion_response(
|
||||
reasoning: str | None = None,
|
||||
tool_calls: list[dict[str, Any]] | None = None,
|
||||
finish_reason: str | None = "stop",
|
||||
stop_reason: int | str | None = None,
|
||||
citations: list[Any] | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
@@ -93,7 +94,14 @@ def _build_chat_completion_response(
|
||||
object="chat.completion",
|
||||
created=0,
|
||||
model="m",
|
||||
choices=[{"index": 0, "message": message, "finish_reason": finish_reason}],
|
||||
choices=[
|
||||
{
|
||||
"index": 0,
|
||||
"message": message,
|
||||
"finish_reason": finish_reason,
|
||||
"stop_reason": stop_reason,
|
||||
}
|
||||
],
|
||||
# ``usage`` is a required field on ChatCompletionResponse, but the
|
||||
# production code defensively handles ``None`` -> no usage block;
|
||||
# we round-trip that behavior by post-setting the attribute below.
|
||||
@@ -137,6 +145,11 @@ class TestMapFinishReason:
|
||||
def test_unknown_reason_defaults_to_complete(self):
|
||||
assert _map_finish_reason("not_a_real_reason") == "COMPLETE"
|
||||
|
||||
def test_only_string_stop_reasons_are_stop_sequences(self):
|
||||
assert _map_finish_reason("stop", "<END>") == "STOP_SEQUENCE"
|
||||
assert _map_finish_reason("stop", 42) == "COMPLETE"
|
||||
assert _map_finish_reason("stop", None) == "COMPLETE"
|
||||
|
||||
def test_finish_reason_map_is_complete(self):
|
||||
# Sanity check that the lookup table covers all documented states.
|
||||
assert set(_FINISH_REASON_MAP) == {
|
||||
@@ -1488,6 +1501,14 @@ class TestChatCompletionToV2:
|
||||
assert v2.message.tool_plan is None
|
||||
assert v2.usage is None
|
||||
|
||||
def test_stop_sequence_finish_reason(self):
|
||||
serving = _serving()
|
||||
resp = _build_chat_completion_response(
|
||||
finish_reason="stop", stop_reason="<END>"
|
||||
)
|
||||
v2 = serving._chat_completion_to_v2(resp, _make_request())
|
||||
assert v2.finish_reason == "STOP_SEQUENCE"
|
||||
|
||||
def test_reasoning_model_keeps_thinking_with_tool_calls(self):
|
||||
# Reasoning Command models: ``thinking`` block stays in
|
||||
# ``message.content`` and ``tool_plan`` is left unset, even when
|
||||
|
||||
@@ -81,6 +81,7 @@ def _make_chunk(
|
||||
reasoning: str | None = None,
|
||||
tool_calls: list[dict[str, Any]] | None = None,
|
||||
finish_reason: str | None = None,
|
||||
stop_reason: int | str | None = None,
|
||||
usage: dict[str, Any] | None = None,
|
||||
omit_choices: bool = False,
|
||||
citations: list[Any] | None = None,
|
||||
@@ -107,7 +108,12 @@ def _make_chunk(
|
||||
}
|
||||
if not omit_choices:
|
||||
payload["choices"] = [
|
||||
{"index": 0, "delta": delta, "finish_reason": finish_reason}
|
||||
{
|
||||
"index": 0,
|
||||
"delta": delta,
|
||||
"finish_reason": finish_reason,
|
||||
"stop_reason": stop_reason,
|
||||
}
|
||||
]
|
||||
else:
|
||||
payload["choices"] = []
|
||||
@@ -652,6 +658,23 @@ class TestChatCompletionStreamToV2:
|
||||
assert end_payload["delta"]["finish_reason"] == "COMPLETE"
|
||||
assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_sequence_finish_reason(self):
|
||||
serving = _serving()
|
||||
frames = await _drain(
|
||||
serving,
|
||||
[
|
||||
_make_chunk(role="assistant"),
|
||||
_make_chunk(
|
||||
finish_reason="stop",
|
||||
stop_reason="<END>",
|
||||
),
|
||||
_make_chunk(omit_choices=True),
|
||||
],
|
||||
)
|
||||
end_payload = _parse_event(frames[-2])
|
||||
assert end_payload["delta"]["finish_reason"] == "STOP_SEQUENCE"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_then_tool_call_closes_text_first(self):
|
||||
serving = _serving(is_reasoning_model=True)
|
||||
|
||||
@@ -155,7 +155,13 @@ _FINISH_REASON_MAP: dict[str | None, CohereFinishReason] = {
|
||||
}
|
||||
|
||||
|
||||
def _map_finish_reason(reason: str | None) -> CohereFinishReason:
|
||||
def _map_finish_reason(
|
||||
reason: str | None, stop_reason: int | str | None = None
|
||||
) -> CohereFinishReason:
|
||||
# vLLM reports both EOS and matched stop strings as finish_reason="stop".
|
||||
# The latter is distinguished by the matched string in stop_reason.
|
||||
if reason == "stop" and isinstance(stop_reason, str):
|
||||
return "STOP_SEQUENCE"
|
||||
return _FINISH_REASON_MAP.get(reason, "COMPLETE")
|
||||
|
||||
|
||||
@@ -1006,7 +1012,10 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
|
||||
return CohereChatV2Response(
|
||||
id=response.id or f"chat_{int(time.time() * 1000)}",
|
||||
finish_reason=_map_finish_reason(choice.finish_reason),
|
||||
finish_reason=_map_finish_reason(
|
||||
choice.finish_reason,
|
||||
choice.stop_reason,
|
||||
),
|
||||
message=assistant_msg,
|
||||
usage=usage,
|
||||
kv_transfer_params=response.kv_transfer_params,
|
||||
@@ -1328,6 +1337,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
yield self._build_message_end_event(
|
||||
chunk_id=chunk.id,
|
||||
finish_reason=state.finish_reason,
|
||||
stop_reason=state.stop_reason,
|
||||
usage_chunk=chunk,
|
||||
)
|
||||
state.ended = True
|
||||
@@ -1336,6 +1346,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
choice = chunk.choices[0]
|
||||
if choice.finish_reason is not None:
|
||||
state.finish_reason = choice.finish_reason
|
||||
state.stop_reason = choice.stop_reason
|
||||
|
||||
delta = choice.delta
|
||||
|
||||
@@ -1391,6 +1402,7 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
yield self._build_message_end_event(
|
||||
chunk_id=state.last_chunk_id,
|
||||
finish_reason=state.finish_reason,
|
||||
stop_reason=state.stop_reason,
|
||||
usage_chunk=None,
|
||||
)
|
||||
state.ended = True
|
||||
@@ -1594,10 +1606,11 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
self,
|
||||
chunk_id: str,
|
||||
finish_reason: str | None,
|
||||
stop_reason: int | str | None = None,
|
||||
usage_chunk: ChatCompletionStreamResponse | None = None,
|
||||
) -> str:
|
||||
delta: dict[str, Any] = {
|
||||
"finish_reason": _map_finish_reason(finish_reason),
|
||||
"finish_reason": _map_finish_reason(finish_reason, stop_reason),
|
||||
}
|
||||
if usage_chunk is not None and usage_chunk.usage is not None:
|
||||
prompt = usage_chunk.usage.prompt_tokens
|
||||
@@ -1659,6 +1672,7 @@ class _StreamState:
|
||||
self.started: bool = False
|
||||
self.ended: bool = False
|
||||
self.finish_reason: str | None = None
|
||||
self.stop_reason: int | str | None = None
|
||||
self.last_chunk_id: str = ""
|
||||
self.active_block: ContentBlockType | None = None
|
||||
self.active_block_index: int | None = None
|
||||
|
||||
Reference in New Issue
Block a user