[Frontend] Report stop_reason on /inference/v1/generate and pass it through derender (#60251)

Signed-off-by: Hyeonki Hong <hhk7734@gmail.com>
Signed-off-by: Shimi Bandiel <shimib@google.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Hyeonki Hong <hhk7734@gmail.com>
Co-authored-by: Mimir <mimir@nousresearch.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Shimi Bandiel
2026-10-12 19:37:53 +08:00
committed by GitHub
co-authored by Hyeonki Hong Mimir Claude Bugen Zhao
parent 32b05addc1
commit 8d7b93fec2
8 changed files with 201 additions and 2 deletions
@@ -226,6 +226,8 @@ async fn generate_chunk_stream(
None
};
let stop_reason =
finish_reason.as_ref().and_then(|reason| reason.as_stop_reason().cloned());
let prompt_token_ids = prompt_token_ids.take();
y.yield_ok(GenerateStreamResponse {
request_id: request_id.clone(),
@@ -233,6 +235,7 @@ async fn generate_chunk_stream(
index: 0,
logprobs,
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
stop_reason,
token_ids,
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
weight_versions: output.weight_versions,
@@ -319,6 +322,7 @@ fn collect_generate(
None
};
let finish_reason = collected.finish_reason.as_str().to_string();
let stop_reason = collected.finish_reason.as_stop_reason().cloned();
if enable_log_requests {
info!(
@@ -335,6 +339,7 @@ fn collect_generate(
index: 0,
logprobs,
finish_reason: Some(finish_reason),
stop_reason,
token_ids: collected.token_ids,
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
weight_versions: collected.weight_versions,
@@ -537,7 +542,7 @@ mod tests {
use futures::{TryStreamExt as _, stream};
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason};
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_llm::GeneratePromptInfo;
@@ -1125,4 +1130,69 @@ mod tests {
)
.expect_err("multi-token prompt without payload is an engine failure");
}
#[tokio::test]
async fn generate_chunk_stream_reports_stop_reason() {
let chunks = collect_chunks(
vec![
stream_output(Some(&[11, 22]), vec![33], None),
stream_output(
Some(&[11, 22]),
vec![44],
Some(FinishReason::Stop(Some(StopReason::Text(
"wor".to_string(),
)))),
),
],
false,
None,
)
.await;
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].choices[0].stop_reason, None);
assert_eq!(chunks[1].choices[0].finish_reason.as_deref(), Some("stop"));
assert_eq!(
chunks[1].choices[0].stop_reason,
Some(StopReason::Text("wor".to_string()))
);
}
#[test]
fn collect_generate_reports_stop_token_id() {
let output = CollectedGenerateOutput {
request_id: "raw-stop".to_string(),
prompt_logprobs: None,
prompt_token_id_logprobs: None,
token_ids: vec![30, 7],
logprobs: None,
finish_reason: FinishReason::Stop(Some(StopReason::TokenId(7))),
usage: TokenUsage {
prompt_token_count: 2,
output_token_count: 2,
cached_token_count: 0,
},
kv_transfer_params: None,
ec_transfer_params: None,
prompt_token_ids: vec![10, 20],
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
};
let response = collect_generate(
output,
"raw-stop".to_string(),
ApiServerOptions::default(),
ResponseOptions::default(),
None,
)
.expect("response");
assert_eq!(response.choices[0].finish_reason.as_deref(), Some("stop"));
assert_eq!(
response.choices[0].stop_reason,
Some(StopReason::TokenId(7))
);
}
}
@@ -7,7 +7,7 @@ use llm_multimodal::MediaContentPart;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use validator::Validate;
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason};
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::WeightVersionSpan;
use vllm_text::SamplingParams;
@@ -72,6 +72,7 @@ pub(super) struct GenerateResponseChoice {
pub index: u32,
pub logprobs: Option<GenerateLogProbs>,
pub finish_reason: Option<String>,
pub stop_reason: Option<StopReason>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
pub weight_versions: Option<Vec<WeightVersionSpan>>,
@@ -84,6 +85,7 @@ pub(super) struct GenerateResponseStreamChoice {
pub index: u32,
pub logprobs: Option<GenerateLogProbs>,
pub finish_reason: Option<String>,
pub stop_reason: Option<StopReason>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
/// Set on the final chunk only; covers the whole output.