From 65b7662d3fcb773afaf751ab29ac6960a0cf011d Mon Sep 17 00:00:00 2001 From: cherry77-cloud <1615405@qq.com> Date: Tue, 11 Aug 2026 12:49:49 +0800 Subject: [PATCH] [Bugfix][Frontend] Report Cohere stop sequences correctly (#51556) Signed-off-by: cherry77-cloud <1615405@qq.com> Co-authored-by: OpenAI Codex --- .../cohere/test_serving_conversion.py | 23 ++++++++++++++++- .../cohere/test_serving_streaming.py | 25 ++++++++++++++++++- vllm/entrypoints/cohere/serving.py | 20 ++++++++++++--- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/tests/entrypoints/cohere/test_serving_conversion.py b/tests/entrypoints/cohere/test_serving_conversion.py index 6ee6ffef28..a456d113ac 100644 --- a/tests/entrypoints/cohere/test_serving_conversion.py +++ b/tests/entrypoints/cohere/test_serving_conversion.py @@ -75,6 +75,7 @@ def _build_chat_completion_response( reasoning: str | None = None, tool_calls: list[dict[str, Any]] | None = None, finish_reason: str | None = "stop", + stop_reason: int | str | None = None, citations: list[Any] | None = None, usage: dict[str, Any] | None = None, kv_transfer_params: dict[str, Any] | None = None, @@ -93,7 +94,14 @@ def _build_chat_completion_response( object="chat.completion", created=0, model="m", - choices=[{"index": 0, "message": message, "finish_reason": finish_reason}], + choices=[ + { + "index": 0, + "message": message, + "finish_reason": finish_reason, + "stop_reason": stop_reason, + } + ], # ``usage`` is a required field on ChatCompletionResponse, but the # production code defensively handles ``None`` -> no usage block; # we round-trip that behavior by post-setting the attribute below. @@ -137,6 +145,11 @@ class TestMapFinishReason: def test_unknown_reason_defaults_to_complete(self): assert _map_finish_reason("not_a_real_reason") == "COMPLETE" + def test_only_string_stop_reasons_are_stop_sequences(self): + assert _map_finish_reason("stop", "") == "STOP_SEQUENCE" + assert _map_finish_reason("stop", 42) == "COMPLETE" + assert _map_finish_reason("stop", None) == "COMPLETE" + def test_finish_reason_map_is_complete(self): # Sanity check that the lookup table covers all documented states. assert set(_FINISH_REASON_MAP) == { @@ -1488,6 +1501,14 @@ class TestChatCompletionToV2: assert v2.message.tool_plan is None assert v2.usage is None + def test_stop_sequence_finish_reason(self): + serving = _serving() + resp = _build_chat_completion_response( + finish_reason="stop", stop_reason="" + ) + v2 = serving._chat_completion_to_v2(resp, _make_request()) + assert v2.finish_reason == "STOP_SEQUENCE" + def test_reasoning_model_keeps_thinking_with_tool_calls(self): # Reasoning Command models: ``thinking`` block stays in # ``message.content`` and ``tool_plan`` is left unset, even when diff --git a/tests/entrypoints/cohere/test_serving_streaming.py b/tests/entrypoints/cohere/test_serving_streaming.py index 34cd2eaf07..8c2e8623ad 100644 --- a/tests/entrypoints/cohere/test_serving_streaming.py +++ b/tests/entrypoints/cohere/test_serving_streaming.py @@ -81,6 +81,7 @@ def _make_chunk( reasoning: str | None = None, tool_calls: list[dict[str, Any]] | None = None, finish_reason: str | None = None, + stop_reason: int | str | None = None, usage: dict[str, Any] | None = None, omit_choices: bool = False, citations: list[Any] | None = None, @@ -107,7 +108,12 @@ def _make_chunk( } if not omit_choices: payload["choices"] = [ - {"index": 0, "delta": delta, "finish_reason": finish_reason} + { + "index": 0, + "delta": delta, + "finish_reason": finish_reason, + "stop_reason": stop_reason, + } ] else: payload["choices"] = [] @@ -652,6 +658,23 @@ class TestChatCompletionStreamToV2: assert end_payload["delta"]["finish_reason"] == "COMPLETE" assert end_payload["delta"]["usage"]["billed_units"]["input_tokens"] == 4 + @pytest.mark.asyncio + async def test_stop_sequence_finish_reason(self): + serving = _serving() + frames = await _drain( + serving, + [ + _make_chunk(role="assistant"), + _make_chunk( + finish_reason="stop", + stop_reason="", + ), + _make_chunk(omit_choices=True), + ], + ) + end_payload = _parse_event(frames[-2]) + assert end_payload["delta"]["finish_reason"] == "STOP_SEQUENCE" + @pytest.mark.asyncio async def test_text_then_tool_call_closes_text_first(self): serving = _serving(is_reasoning_model=True) diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 04180b5cfa..3873f70433 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -155,7 +155,13 @@ _FINISH_REASON_MAP: dict[str | None, CohereFinishReason] = { } -def _map_finish_reason(reason: str | None) -> CohereFinishReason: +def _map_finish_reason( + reason: str | None, stop_reason: int | str | None = None +) -> CohereFinishReason: + # vLLM reports both EOS and matched stop strings as finish_reason="stop". + # The latter is distinguished by the matched string in stop_reason. + if reason == "stop" and isinstance(stop_reason, str): + return "STOP_SEQUENCE" return _FINISH_REASON_MAP.get(reason, "COMPLETE") @@ -1006,7 +1012,10 @@ class CohereServingChatV2(OpenAIServingChat): return CohereChatV2Response( id=response.id or f"chat_{int(time.time() * 1000)}", - finish_reason=_map_finish_reason(choice.finish_reason), + finish_reason=_map_finish_reason( + choice.finish_reason, + choice.stop_reason, + ), message=assistant_msg, usage=usage, kv_transfer_params=response.kv_transfer_params, @@ -1328,6 +1337,7 @@ class CohereServingChatV2(OpenAIServingChat): yield self._build_message_end_event( chunk_id=chunk.id, finish_reason=state.finish_reason, + stop_reason=state.stop_reason, usage_chunk=chunk, ) state.ended = True @@ -1336,6 +1346,7 @@ class CohereServingChatV2(OpenAIServingChat): choice = chunk.choices[0] if choice.finish_reason is not None: state.finish_reason = choice.finish_reason + state.stop_reason = choice.stop_reason delta = choice.delta @@ -1391,6 +1402,7 @@ class CohereServingChatV2(OpenAIServingChat): yield self._build_message_end_event( chunk_id=state.last_chunk_id, finish_reason=state.finish_reason, + stop_reason=state.stop_reason, usage_chunk=None, ) state.ended = True @@ -1594,10 +1606,11 @@ class CohereServingChatV2(OpenAIServingChat): self, chunk_id: str, finish_reason: str | None, + stop_reason: int | str | None = None, usage_chunk: ChatCompletionStreamResponse | None = None, ) -> str: delta: dict[str, Any] = { - "finish_reason": _map_finish_reason(finish_reason), + "finish_reason": _map_finish_reason(finish_reason, stop_reason), } if usage_chunk is not None and usage_chunk.usage is not None: prompt = usage_chunk.usage.prompt_tokens @@ -1659,6 +1672,7 @@ class _StreamState: self.started: bool = False self.ended: bool = False self.finish_reason: str | None = None + self.stop_reason: int | str | None = None self.last_chunk_id: str = "" self.active_block: ContentBlockType | None = None self.active_block_index: int | None = None