[Frontend] Report stop_reason on /inference/v1/generate and pass it through derender (#60251)

Signed-off-by: Hyeonki Hong <hhk7734@gmail.com>
Signed-off-by: Shimi Bandiel <shimib@google.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Hyeonki Hong <hhk7734@gmail.com>
Co-authored-by: Mimir <mimir@nousresearch.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>

rust/src/server/src/routes/inference/generate.rs

@@ -226,6 +226,8 @@ async fn generate_chunk_stream(
                     None
                 };
 
+                let stop_reason =
+                    finish_reason.as_ref().and_then(|reason| reason.as_stop_reason().cloned());
                 let prompt_token_ids = prompt_token_ids.take();
                 y.yield_ok(GenerateStreamResponse {
                     request_id: request_id.clone(),
@@ -233,6 +235,7 @@ async fn generate_chunk_stream(
                         index: 0,
                         logprobs,
                         finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
+                        stop_reason,
                         token_ids,
                         sampling_mask: output.sampling_mask.map(|mask| mask.rows),
                         weight_versions: output.weight_versions,
@@ -319,6 +322,7 @@ fn collect_generate(
         None
     };
     let finish_reason = collected.finish_reason.as_str().to_string();
+    let stop_reason = collected.finish_reason.as_stop_reason().cloned();
 
     if enable_log_requests {
         info!(
@@ -335,6 +339,7 @@ fn collect_generate(
             index: 0,
             logprobs,
             finish_reason: Some(finish_reason),
+            stop_reason,
             token_ids: collected.token_ids,
             sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
             weight_versions: collected.weight_versions,
@@ -537,7 +542,7 @@ mod tests {
 
     use futures::{TryStreamExt as _, stream};
     use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
-    use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
+    use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason};
     use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
     use vllm_llm::GeneratePromptInfo;
 
@@ -1125,4 +1130,69 @@ mod tests {
         )
         .expect_err("multi-token prompt without payload is an engine failure");
     }
+
+    #[tokio::test]
+    async fn generate_chunk_stream_reports_stop_reason() {
+        let chunks = collect_chunks(
+            vec![
+                stream_output(Some(&[11, 22]), vec![33], None),
+                stream_output(
+                    Some(&[11, 22]),
+                    vec![44],
+                    Some(FinishReason::Stop(Some(StopReason::Text(
+                        "wor".to_string(),
+                    )))),
+                ),
+            ],
+            false,
+            None,
+        )
+        .await;
+
+        assert_eq!(chunks.len(), 2);
+        assert_eq!(chunks[0].choices[0].stop_reason, None);
+        assert_eq!(chunks[1].choices[0].finish_reason.as_deref(), Some("stop"));
+        assert_eq!(
+            chunks[1].choices[0].stop_reason,
+            Some(StopReason::Text("wor".to_string()))
+        );
+    }
+
+    #[test]
+    fn collect_generate_reports_stop_token_id() {
+        let output = CollectedGenerateOutput {
+            request_id: "raw-stop".to_string(),
+            prompt_logprobs: None,
+            prompt_token_id_logprobs: None,
+            token_ids: vec![30, 7],
+            logprobs: None,
+            finish_reason: FinishReason::Stop(Some(StopReason::TokenId(7))),
+            usage: TokenUsage {
+                prompt_token_count: 2,
+                output_token_count: 2,
+                cached_token_count: 0,
+            },
+            kv_transfer_params: None,
+            ec_transfer_params: None,
+            prompt_token_ids: vec![10, 20],
+            sampling_mask: None,
+            spec_decode_metrics: None,
+            weight_versions: None,
+        };
+
+        let response = collect_generate(
+            output,
+            "raw-stop".to_string(),
+            ApiServerOptions::default(),
+            ResponseOptions::default(),
+            None,
+        )
+        .expect("response");
+
+        assert_eq!(response.choices[0].finish_reason.as_deref(), Some("stop"));
+        assert_eq!(
+            response.choices[0].stop_reason,
+            Some(StopReason::TokenId(7))
+        );
+    }
 }

rust/src/server/src/routes/inference/generate/types.rs

@@ -7,7 +7,7 @@ use llm_multimodal::MediaContentPart;
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 use validator::Validate;
-use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
+use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason};
 use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
 use vllm_llm::WeightVersionSpan;
 use vllm_text::SamplingParams;
@@ -72,6 +72,7 @@ pub(super) struct GenerateResponseChoice {
     pub index: u32,
     pub logprobs: Option<GenerateLogProbs>,
     pub finish_reason: Option<String>,
+    pub stop_reason: Option<StopReason>,
     pub token_ids: Vec<u32>,
     pub sampling_mask: Option<Vec<Vec<u32>>>,
     pub weight_versions: Option<Vec<WeightVersionSpan>>,
@@ -84,6 +85,7 @@ pub(super) struct GenerateResponseStreamChoice {
     pub index: u32,
     pub logprobs: Option<GenerateLogProbs>,
     pub finish_reason: Option<String>,
+    pub stop_reason: Option<StopReason>,
     pub token_ids: Vec<u32>,
     pub sampling_mask: Option<Vec<Vec<u32>>>,
     /// Set on the final chunk only; covers the whole output.

tests/entrypoints/scale_out/derender/test_derender.py

@@ -112,6 +112,38 @@ async def test_derender_chat_roundtrip(client):
     assert data["choices"][0]["message"]["role"] == "assistant"
 
 
+@pytest.mark.asyncio
+async def test_derender_chat_passes_stop_reason_through(client):
+    """The coupled server reports the stop string or token id that ended
+    generation; derender must carry the generate value through unchanged."""
+    gen_req = await _render_chat(client)
+    generate_response = _make_generate_response(gen_req["token_ids"][:3])
+    generate_response["choices"][0]["stop_reason"] = "\n\n"
+
+    response = await client.post(
+        "/v1/chat/completions/derender",
+        json={"model": MODEL_NAME, "generate_response": generate_response},
+    )
+    assert response.status_code == 200
+    assert response.json()["choices"][0]["stop_reason"] == "\n\n"
+
+
+@pytest.mark.asyncio
+async def test_derender_completion_passes_stop_reason_through(client):
+    gen_req = await _render_completion(client, "Hello world")
+    generate_response = _make_completion_generate_response(
+        gen_req["token_ids"][:3], "gen-stop"
+    )
+    generate_response["choices"][0]["stop_reason"] = 151645
+
+    response = await client.post(
+        "/v1/completions/derender",
+        json={"model": MODEL_NAME, "generate_responses": [generate_response]},
+    )
+    assert response.status_code == 200
+    assert response.json()["choices"][0]["stop_reason"] == 151645
+
+
 @pytest.mark.asyncio
 async def test_derender_chat_usage(client):
     """Supplied prompt_tokens flows through into usage correctly."""

tests/entrypoints/scale_out/derender/test_derender_stream.py

@@ -195,6 +195,7 @@ def _make_stream_chunk(
     metrics: PerRequestMetrics | None = None,
     logprobs: dict | None = None,
     prompt_token_ids: list[int] | None = None,
+    stop_reason: int | str | None = None,
 ) -> GenerateStreamResponse:
     """Build a GenerateStreamResponse SSE chunk."""
     return GenerateTokensStreamResponse(
@@ -204,6 +205,7 @@ def _make_stream_chunk(
                 index=index,
                 token_ids=token_ids,
                 finish_reason=finish_reason,
+                stop_reason=stop_reason,
                 logprobs=logprobs,
             )
         ],
@@ -886,6 +888,19 @@ class TestDerenderCompletionStream:
         )
         assert chunk.choices[0].finish_reason == "length"
 
+    @pytest.mark.asyncio
+    async def test_stop_reason_forwarded(self, derenderer, tokenizer):
+        """stop_reason from the final generate chunk reaches the derendered
+        choice, matching what the coupled server reports."""
+        token_ids = tokenizer.encode("done")[:2]
+        chunk, _ = await derenderer.derender_completion_stream(
+            model=MODEL_NAME,
+            generate_chunk=_make_stream_chunk(
+                token_ids, finish_reason="stop", stop_reason="\n\n"
+            ),
+        )
+        assert chunk.choices[0].stop_reason == "\n\n"
+
 
 class TestStreamLogprobs:
     """Streaming derender must carry per-chunk logprobs with token ids

tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py

@@ -139,6 +139,7 @@ def _make_request_output(
     request_id: str,
     token_ids: list[int],
     finish_reason: str | None = None,
+    stop_reason: int | str | None = None,
     finished: bool = False,
     prompt_token_ids: list[int] | None = None,
     logprobs: list[dict[int, Any] | None] | None = None,
@@ -160,6 +161,7 @@ def _make_request_output(
                 cumulative_logprob=None,
                 logprobs=logprobs,
                 finish_reason=finish_reason,
+                stop_reason=stop_reason,
                 spec_decode_metrics=spec_decode_metrics,
             )
         ],
@@ -378,6 +380,71 @@ async def test_stream_basic():
     assert data_chunks[2]["choices"][0]["finish_reason"] == "stop"
 
 
+@pytest.mark.asyncio
+async def test_non_stream_reports_engine_stop_reason():
+    """The matched stop token id from the engine lands on the choice."""
+    engine = _mock_engine()
+
+    async def mock_generate(*args, **kwargs):
+        yield _make_request_output(
+            "req-1",
+            token_ids=[10, 42],
+            finish_reason="stop",
+            stop_reason=42,
+            finished=True,
+        )
+
+    engine.generate = MagicMock(side_effect=mock_generate)
+    serving = _build_serving_tokens(engine)
+
+    request = GenerateRequest(
+        token_ids=[1, 2, 3],
+        sampling_params=SamplingParams(max_tokens=10),
+        model=MODEL_NAME,
+        stream=False,
+    )
+
+    response = await serving.serve_tokens(request)
+    choice = response.model_dump()["choices"][0]
+    assert choice["finish_reason"] == "stop"
+    assert choice["stop_reason"] == 42
+
+
+@pytest.mark.asyncio
+async def test_stream_reports_engine_stop_reason_on_final_chunk():
+    """A matched stop string is reported with finish_reason and null before."""
+    engine = _mock_engine()
+
+    async def mock_generate(*args, **kwargs):
+        yield _make_request_output("req-1", token_ids=[10])
+        yield _make_request_output(
+            "req-1",
+            token_ids=[20],
+            finish_reason="stop",
+            stop_reason="done",
+            finished=True,
+        )
+
+    engine.generate = MagicMock(side_effect=mock_generate)
+    serving = _build_serving_tokens(engine)
+
+    request = GenerateRequest(
+        token_ids=[1, 2, 3],
+        sampling_params=SamplingParams(max_tokens=10),
+        model=MODEL_NAME,
+        stream=True,
+    )
+
+    response = await serving.serve_tokens(request)
+    parsed = _parse_sse_chunks([chunk async for chunk in response])
+    data_chunks = [c for c in parsed if c != "[DONE]"]
+    assert len(data_chunks) == 2
+
+    assert data_chunks[0]["choices"][0]["stop_reason"] is None
+    assert data_chunks[1]["choices"][0]["finish_reason"] == "stop"
+    assert data_chunks[1]["choices"][0]["stop_reason"] == "done"
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("stream", [False, True])
 async def test_serve_tokens_returns_spec_decode_metrics(stream: bool):

vllm/entrypoints/scale_out/token_in_token_out/protocol.py

@@ -400,6 +400,9 @@ class GenerateChoiceBase(BaseModel):
     index: int
     # per OpenAI spec this is the default
     finish_reason: str | None = "stop"
+    # Stop string or token id that ended generation (repetition detection
+    # sets it too); None on EOS, length and abort.
+    stop_reason: int | str | None = None
     token_ids: list[int] | None = None
     # Per-token expert routing decisions, base64-encoded `.npy` bytes
     # (numpy serialization). Shape after decode:
@@ -443,6 +446,9 @@ class GenerateStreamChoiceBase(BaseModel):
 
     index: int
     finish_reason: str | None = None
+    # Stop string or token id that ended generation (repetition detection
+    # sets it too); None on EOS, length and abort.
+    stop_reason: int | str | None = None
     token_ids: list[int] | None = None
     routed_experts: str | None = None
     sampling_mask: list[list[int]] | None = None

vllm/entrypoints/scale_out/token_in_token_out/serving.py

@@ -450,6 +450,7 @@ class ServingTokens(GenerateBaseServing):
                 index=output.index,
                 logprobs=logprobs,
                 finish_reason=output.finish_reason if output.finish_reason else "stop",
+                stop_reason=output.stop_reason,
                 token_ids=as_list(output.token_ids),
                 routed_experts=routed_experts_b64,
                 sampling_mask=sampling_mask,
@@ -623,6 +624,7 @@ class ServingTokens(GenerateBaseServing):
                         index=i,
                         logprobs=logprobs,
                         finish_reason=finish_reason,
+                        stop_reason=output.stop_reason,
                         token_ids=as_list(delta_token_ids),
                         routed_experts=routed_experts_b64,
                         sampling_mask=sampling_mask,

vllm/renderers/online_derenderer.py

@@ -206,6 +206,7 @@ class OnlineDerenderer:
                     message=message,
                     logprobs=resolved_logprobs,
                     finish_reason=choice.finish_reason,
+                    stop_reason=choice.stop_reason,
                     token_ids=choice.token_ids if return_token_ids else None,
                 )
             )
@@ -484,6 +485,7 @@ class OnlineDerenderer:
                     delta=delta,
                     logprobs=resolved_logprobs,
                     finish_reason=choice.finish_reason,
+                    stop_reason=choice.stop_reason,
                     token_ids=choice.token_ids if return_token_ids else None,
                 )
             )
@@ -672,6 +674,7 @@ class OnlineDerenderer:
                 index=choice.index,
                 delta=delta_message,
                 finish_reason=finish_reason,
+                stop_reason=choice.stop_reason,
                 token_ids=choice.token_ids if return_token_ids else None,
             )
             stream_choices.append(
@@ -777,6 +780,7 @@ class OnlineDerenderer:
                         index=index,
                         text=decoded_text,
                         finish_reason=choice.finish_reason,
+                        stop_reason=choice.stop_reason,
                         logprobs=completion_logprobs,
                     )
                 )
@@ -878,6 +882,7 @@ class OnlineDerenderer:
                     text=new_text,
                     logprobs=completion_logprobs,
                     finish_reason=choice.finish_reason,
+                    stop_reason=choice.stop_reason,
                 )
             )