[Rust Frontend] Return weight-version spans on final outputs (#61070)

Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Shuolei Wang
2026-10-12 15:56:59 +08:00
committed by GitHub
co-authored by aoshen02 Bugen Zhao Claude Opus 5.5
parent f4114e068e
commit 4163a80142
12 changed files with 127 additions and 12 deletions
@@ -235,6 +235,7 @@ async fn generate_chunk_stream(
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
token_ids,
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
weight_versions: output.weight_versions,
}],
usage: include_continuous_usage
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
@@ -336,6 +337,7 @@ fn collect_generate(
finish_reason: Some(finish_reason),
token_ids: collected.token_ids,
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
weight_versions: collected.weight_versions,
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
@@ -629,6 +631,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}),
Ok(GenerateOutput {
request_id: String::new(),
@@ -645,6 +648,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}),
]);
@@ -731,6 +735,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}
}
@@ -905,6 +910,7 @@ mod tests {
rows: vec![vec![30, 40]],
}),
spec_decode_metrics: None,
weight_versions: None,
};
let response = collect_generate(
@@ -959,6 +965,7 @@ mod tests {
prompt_token_ids: vec![10],
sampling_mask: None,
spec_decode_metrics: Some(metrics.clone()),
weight_versions: None,
};
let response = collect_generate(
@@ -1051,6 +1058,7 @@ mod tests {
prompt_token_ids: vec![10, 20],
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
};
let response = collect_generate(
@@ -1086,6 +1094,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
prompt_token_ids,
};
@@ -9,6 +9,7 @@ use serde_json::{Map, Value};
use validator::Validate;
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::WeightVersionSpan;
use vllm_text::SamplingParams;
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
@@ -73,6 +74,7 @@ pub(super) struct GenerateResponseChoice {
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
pub weight_versions: Option<Vec<WeightVersionSpan>>,
}
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
@@ -84,6 +86,8 @@ pub(super) struct GenerateResponseStreamChoice {
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
/// Set on the final chunk only; covers the whole output.
pub weight_versions: Option<Vec<WeightVersionSpan>>,
}
/// Mirrors the Python vLLM `GenerateStreamResponse` class.