diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index a30cd0c85a..930c698c30 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -226,6 +226,8 @@ async fn generate_chunk_stream( None }; + let stop_reason = + finish_reason.as_ref().and_then(|reason| reason.as_stop_reason().cloned()); let prompt_token_ids = prompt_token_ids.take(); y.yield_ok(GenerateStreamResponse { request_id: request_id.clone(), @@ -233,6 +235,7 @@ async fn generate_chunk_stream( index: 0, logprobs, finish_reason: finish_reason.map(|reason| reason.as_str().to_string()), + stop_reason, token_ids, sampling_mask: output.sampling_mask.map(|mask| mask.rows), weight_versions: output.weight_versions, @@ -319,6 +322,7 @@ fn collect_generate( None }; let finish_reason = collected.finish_reason.as_str().to_string(); + let stop_reason = collected.finish_reason.as_stop_reason().cloned(); if enable_log_requests { info!( @@ -335,6 +339,7 @@ fn collect_generate( index: 0, logprobs, finish_reason: Some(finish_reason), + stop_reason, token_ids: collected.token_ids, sampling_mask: collected.sampling_mask.map(|mask| mask.rows), weight_versions: collected.weight_versions, @@ -537,7 +542,7 @@ mod tests { use futures::{TryStreamExt as _, stream}; use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange}; - use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; + use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason}; use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_llm::GeneratePromptInfo; @@ -1125,4 +1130,69 @@ mod tests { ) .expect_err("multi-token prompt without payload is an engine failure"); } + + #[tokio::test] + async fn generate_chunk_stream_reports_stop_reason() { + let chunks = collect_chunks( + vec![ + stream_output(Some(&[11, 22]), vec![33], None), + stream_output( + Some(&[11, 22]), + vec![44], + Some(FinishReason::Stop(Some(StopReason::Text( + "wor".to_string(), + )))), + ), + ], + false, + None, + ) + .await; + + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].choices[0].stop_reason, None); + assert_eq!(chunks[1].choices[0].finish_reason.as_deref(), Some("stop")); + assert_eq!( + chunks[1].choices[0].stop_reason, + Some(StopReason::Text("wor".to_string())) + ); + } + + #[test] + fn collect_generate_reports_stop_token_id() { + let output = CollectedGenerateOutput { + request_id: "raw-stop".to_string(), + prompt_logprobs: None, + prompt_token_id_logprobs: None, + token_ids: vec![30, 7], + logprobs: None, + finish_reason: FinishReason::Stop(Some(StopReason::TokenId(7))), + usage: TokenUsage { + prompt_token_count: 2, + output_token_count: 2, + cached_token_count: 0, + }, + kv_transfer_params: None, + ec_transfer_params: None, + prompt_token_ids: vec![10, 20], + sampling_mask: None, + spec_decode_metrics: None, + weight_versions: None, + }; + + let response = collect_generate( + output, + "raw-stop".to_string(), + ApiServerOptions::default(), + ResponseOptions::default(), + None, + ) + .expect("response"); + + assert_eq!(response.choices[0].finish_reason.as_deref(), Some("stop")); + assert_eq!( + response.choices[0].stop_reason, + Some(StopReason::TokenId(7)) + ); + } } diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 6352b18725..d2737b5bce 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -7,7 +7,7 @@ use llm_multimodal::MediaContentPart; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use validator::Validate; -use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; +use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason}; use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; use vllm_llm::WeightVersionSpan; use vllm_text::SamplingParams; @@ -72,6 +72,7 @@ pub(super) struct GenerateResponseChoice { pub index: u32, pub logprobs: Option, pub finish_reason: Option, + pub stop_reason: Option, pub token_ids: Vec, pub sampling_mask: Option>>, pub weight_versions: Option>, @@ -84,6 +85,7 @@ pub(super) struct GenerateResponseStreamChoice { pub index: u32, pub logprobs: Option, pub finish_reason: Option, + pub stop_reason: Option, pub token_ids: Vec, pub sampling_mask: Option>>, /// Set on the final chunk only; covers the whole output. diff --git a/tests/entrypoints/scale_out/derender/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py index d29aa3f53f..30d64ed39b 100644 --- a/tests/entrypoints/scale_out/derender/test_derender.py +++ b/tests/entrypoints/scale_out/derender/test_derender.py @@ -112,6 +112,38 @@ async def test_derender_chat_roundtrip(client): assert data["choices"][0]["message"]["role"] == "assistant" +@pytest.mark.asyncio +async def test_derender_chat_passes_stop_reason_through(client): + """The coupled server reports the stop string or token id that ended + generation; derender must carry the generate value through unchanged.""" + gen_req = await _render_chat(client) + generate_response = _make_generate_response(gen_req["token_ids"][:3]) + generate_response["choices"][0]["stop_reason"] = "\n\n" + + response = await client.post( + "/v1/chat/completions/derender", + json={"model": MODEL_NAME, "generate_response": generate_response}, + ) + assert response.status_code == 200 + assert response.json()["choices"][0]["stop_reason"] == "\n\n" + + +@pytest.mark.asyncio +async def test_derender_completion_passes_stop_reason_through(client): + gen_req = await _render_completion(client, "Hello world") + generate_response = _make_completion_generate_response( + gen_req["token_ids"][:3], "gen-stop" + ) + generate_response["choices"][0]["stop_reason"] = 151645 + + response = await client.post( + "/v1/completions/derender", + json={"model": MODEL_NAME, "generate_responses": [generate_response]}, + ) + assert response.status_code == 200 + assert response.json()["choices"][0]["stop_reason"] == 151645 + + @pytest.mark.asyncio async def test_derender_chat_usage(client): """Supplied prompt_tokens flows through into usage correctly.""" diff --git a/tests/entrypoints/scale_out/derender/test_derender_stream.py b/tests/entrypoints/scale_out/derender/test_derender_stream.py index d97a2f992a..55b0789843 100644 --- a/tests/entrypoints/scale_out/derender/test_derender_stream.py +++ b/tests/entrypoints/scale_out/derender/test_derender_stream.py @@ -195,6 +195,7 @@ def _make_stream_chunk( metrics: PerRequestMetrics | None = None, logprobs: dict | None = None, prompt_token_ids: list[int] | None = None, + stop_reason: int | str | None = None, ) -> GenerateStreamResponse: """Build a GenerateStreamResponse SSE chunk.""" return GenerateTokensStreamResponse( @@ -204,6 +205,7 @@ def _make_stream_chunk( index=index, token_ids=token_ids, finish_reason=finish_reason, + stop_reason=stop_reason, logprobs=logprobs, ) ], @@ -886,6 +888,19 @@ class TestDerenderCompletionStream: ) assert chunk.choices[0].finish_reason == "length" + @pytest.mark.asyncio + async def test_stop_reason_forwarded(self, derenderer, tokenizer): + """stop_reason from the final generate chunk reaches the derendered + choice, matching what the coupled server reports.""" + token_ids = tokenizer.encode("done")[:2] + chunk, _ = await derenderer.derender_completion_stream( + model=MODEL_NAME, + generate_chunk=_make_stream_chunk( + token_ids, finish_reason="stop", stop_reason="\n\n" + ), + ) + assert chunk.choices[0].stop_reason == "\n\n" + class TestStreamLogprobs: """Streaming derender must carry per-chunk logprobs with token ids diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index 7f0cd12860..c30732f315 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -139,6 +139,7 @@ def _make_request_output( request_id: str, token_ids: list[int], finish_reason: str | None = None, + stop_reason: int | str | None = None, finished: bool = False, prompt_token_ids: list[int] | None = None, logprobs: list[dict[int, Any] | None] | None = None, @@ -160,6 +161,7 @@ def _make_request_output( cumulative_logprob=None, logprobs=logprobs, finish_reason=finish_reason, + stop_reason=stop_reason, spec_decode_metrics=spec_decode_metrics, ) ], @@ -378,6 +380,71 @@ async def test_stream_basic(): assert data_chunks[2]["choices"][0]["finish_reason"] == "stop" +@pytest.mark.asyncio +async def test_non_stream_reports_engine_stop_reason(): + """The matched stop token id from the engine lands on the choice.""" + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output( + "req-1", + token_ids=[10, 42], + finish_reason="stop", + stop_reason=42, + finished=True, + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=10), + model=MODEL_NAME, + stream=False, + ) + + response = await serving.serve_tokens(request) + choice = response.model_dump()["choices"][0] + assert choice["finish_reason"] == "stop" + assert choice["stop_reason"] == 42 + + +@pytest.mark.asyncio +async def test_stream_reports_engine_stop_reason_on_final_chunk(): + """A matched stop string is reported with finish_reason and null before.""" + engine = _mock_engine() + + async def mock_generate(*args, **kwargs): + yield _make_request_output("req-1", token_ids=[10]) + yield _make_request_output( + "req-1", + token_ids=[20], + finish_reason="stop", + stop_reason="done", + finished=True, + ) + + engine.generate = MagicMock(side_effect=mock_generate) + serving = _build_serving_tokens(engine) + + request = GenerateRequest( + token_ids=[1, 2, 3], + sampling_params=SamplingParams(max_tokens=10), + model=MODEL_NAME, + stream=True, + ) + + response = await serving.serve_tokens(request) + parsed = _parse_sse_chunks([chunk async for chunk in response]) + data_chunks = [c for c in parsed if c != "[DONE]"] + assert len(data_chunks) == 2 + + assert data_chunks[0]["choices"][0]["stop_reason"] is None + assert data_chunks[1]["choices"][0]["finish_reason"] == "stop" + assert data_chunks[1]["choices"][0]["stop_reason"] == "done" + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) async def test_serve_tokens_returns_spec_decode_metrics(stream: bool): diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index bccc8d8eff..c8d82eeb5a 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -400,6 +400,9 @@ class GenerateChoiceBase(BaseModel): index: int # per OpenAI spec this is the default finish_reason: str | None = "stop" + # Stop string or token id that ended generation (repetition detection + # sets it too); None on EOS, length and abort. + stop_reason: int | str | None = None token_ids: list[int] | None = None # Per-token expert routing decisions, base64-encoded `.npy` bytes # (numpy serialization). Shape after decode: @@ -443,6 +446,9 @@ class GenerateStreamChoiceBase(BaseModel): index: int finish_reason: str | None = None + # Stop string or token id that ended generation (repetition detection + # sets it too); None on EOS, length and abort. + stop_reason: int | str | None = None token_ids: list[int] | None = None routed_experts: str | None = None sampling_mask: list[list[int]] | None = None diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 085ae97e16..f09ee6fad5 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -450,6 +450,7 @@ class ServingTokens(GenerateBaseServing): index=output.index, logprobs=logprobs, finish_reason=output.finish_reason if output.finish_reason else "stop", + stop_reason=output.stop_reason, token_ids=as_list(output.token_ids), routed_experts=routed_experts_b64, sampling_mask=sampling_mask, @@ -623,6 +624,7 @@ class ServingTokens(GenerateBaseServing): index=i, logprobs=logprobs, finish_reason=finish_reason, + stop_reason=output.stop_reason, token_ids=as_list(delta_token_ids), routed_experts=routed_experts_b64, sampling_mask=sampling_mask, diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 3fe50f16c3..345e5443ea 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -206,6 +206,7 @@ class OnlineDerenderer: message=message, logprobs=resolved_logprobs, finish_reason=choice.finish_reason, + stop_reason=choice.stop_reason, token_ids=choice.token_ids if return_token_ids else None, ) ) @@ -484,6 +485,7 @@ class OnlineDerenderer: delta=delta, logprobs=resolved_logprobs, finish_reason=choice.finish_reason, + stop_reason=choice.stop_reason, token_ids=choice.token_ids if return_token_ids else None, ) ) @@ -672,6 +674,7 @@ class OnlineDerenderer: index=choice.index, delta=delta_message, finish_reason=finish_reason, + stop_reason=choice.stop_reason, token_ids=choice.token_ids if return_token_ids else None, ) stream_choices.append( @@ -777,6 +780,7 @@ class OnlineDerenderer: index=index, text=decoded_text, finish_reason=choice.finish_reason, + stop_reason=choice.stop_reason, logprobs=completion_logprobs, ) ) @@ -878,6 +882,7 @@ class OnlineDerenderer: text=new_text, logprobs=completion_logprobs, finish_reason=choice.finish_reason, + stop_reason=choice.stop_reason, ) )