[Frontend] Report stop_reason on /inference/v1/generate and pass it through derender (#60251)
Signed-off-by: Hyeonki Hong <hhk7734@gmail.com> Signed-off-by: Shimi Bandiel <shimib@google.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Hyeonki Hong <hhk7734@gmail.com> Co-authored-by: Mimir <mimir@nousresearch.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Hyeonki Hong
Mimir
Claude
Bugen Zhao
parent
32b05addc1
commit
8d7b93fec2
@@ -226,6 +226,8 @@ async fn generate_chunk_stream(
|
||||
None
|
||||
};
|
||||
|
||||
let stop_reason =
|
||||
finish_reason.as_ref().and_then(|reason| reason.as_stop_reason().cloned());
|
||||
let prompt_token_ids = prompt_token_ids.take();
|
||||
y.yield_ok(GenerateStreamResponse {
|
||||
request_id: request_id.clone(),
|
||||
@@ -233,6 +235,7 @@ async fn generate_chunk_stream(
|
||||
index: 0,
|
||||
logprobs,
|
||||
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
|
||||
stop_reason,
|
||||
token_ids,
|
||||
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
|
||||
weight_versions: output.weight_versions,
|
||||
@@ -319,6 +322,7 @@ fn collect_generate(
|
||||
None
|
||||
};
|
||||
let finish_reason = collected.finish_reason.as_str().to_string();
|
||||
let stop_reason = collected.finish_reason.as_stop_reason().cloned();
|
||||
|
||||
if enable_log_requests {
|
||||
info!(
|
||||
@@ -335,6 +339,7 @@ fn collect_generate(
|
||||
index: 0,
|
||||
logprobs,
|
||||
finish_reason: Some(finish_reason),
|
||||
stop_reason,
|
||||
token_ids: collected.token_ids,
|
||||
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
|
||||
weight_versions: collected.weight_versions,
|
||||
@@ -537,7 +542,7 @@ mod tests {
|
||||
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason};
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_llm::GeneratePromptInfo;
|
||||
|
||||
@@ -1125,4 +1130,69 @@ mod tests {
|
||||
)
|
||||
.expect_err("multi-token prompt without payload is an engine failure");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_reports_stop_reason() {
|
||||
let chunks = collect_chunks(
|
||||
vec![
|
||||
stream_output(Some(&[11, 22]), vec![33], None),
|
||||
stream_output(
|
||||
Some(&[11, 22]),
|
||||
vec![44],
|
||||
Some(FinishReason::Stop(Some(StopReason::Text(
|
||||
"wor".to_string(),
|
||||
)))),
|
||||
),
|
||||
],
|
||||
false,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert_eq!(chunks.len(), 2);
|
||||
assert_eq!(chunks[0].choices[0].stop_reason, None);
|
||||
assert_eq!(chunks[1].choices[0].finish_reason.as_deref(), Some("stop"));
|
||||
assert_eq!(
|
||||
chunks[1].choices[0].stop_reason,
|
||||
Some(StopReason::Text("wor".to_string()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_generate_reports_stop_token_id() {
|
||||
let output = CollectedGenerateOutput {
|
||||
request_id: "raw-stop".to_string(),
|
||||
prompt_logprobs: None,
|
||||
prompt_token_id_logprobs: None,
|
||||
token_ids: vec![30, 7],
|
||||
logprobs: None,
|
||||
finish_reason: FinishReason::Stop(Some(StopReason::TokenId(7))),
|
||||
usage: TokenUsage {
|
||||
prompt_token_count: 2,
|
||||
output_token_count: 2,
|
||||
cached_token_count: 0,
|
||||
},
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
output,
|
||||
"raw-stop".to_string(),
|
||||
ApiServerOptions::default(),
|
||||
ResponseOptions::default(),
|
||||
None,
|
||||
)
|
||||
.expect("response");
|
||||
|
||||
assert_eq!(response.choices[0].finish_reason.as_deref(), Some("stop"));
|
||||
assert_eq!(
|
||||
response.choices[0].stop_reason,
|
||||
Some(StopReason::TokenId(7))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ use llm_multimodal::MediaContentPart;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::output::{RequestSpecDecodeMetrics, StopReason};
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_llm::WeightVersionSpan;
|
||||
use vllm_text::SamplingParams;
|
||||
@@ -72,6 +72,7 @@ pub(super) struct GenerateResponseChoice {
|
||||
pub index: u32,
|
||||
pub logprobs: Option<GenerateLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<StopReason>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
@@ -84,6 +85,7 @@ pub(super) struct GenerateResponseStreamChoice {
|
||||
pub index: u32,
|
||||
pub logprobs: Option<GenerateLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub stop_reason: Option<StopReason>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
/// Set on the final chunk only; covers the whole output.
|
||||
|
||||
Reference in New Issue
Block a user