[Rust Frontend] Return weight-version spans on final outputs (#61070)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Bugen Zhao
Claude Opus 5.5
parent
f4114e068e
commit
4163a80142
@@ -235,6 +235,7 @@ async fn generate_chunk_stream(
|
||||
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
|
||||
token_ids,
|
||||
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
|
||||
weight_versions: output.weight_versions,
|
||||
}],
|
||||
usage: include_continuous_usage
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
@@ -336,6 +337,7 @@ fn collect_generate(
|
||||
finish_reason: Some(finish_reason),
|
||||
token_ids: collected.token_ids,
|
||||
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
|
||||
weight_versions: collected.weight_versions,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
@@ -629,6 +631,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}),
|
||||
Ok(GenerateOutput {
|
||||
request_id: String::new(),
|
||||
@@ -645,6 +648,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -731,6 +735,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -905,6 +910,7 @@ mod tests {
|
||||
rows: vec![vec![30, 40]],
|
||||
}),
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -959,6 +965,7 @@ mod tests {
|
||||
prompt_token_ids: vec![10],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: Some(metrics.clone()),
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -1051,6 +1058,7 @@ mod tests {
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -1086,6 +1094,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
prompt_token_ids,
|
||||
};
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_llm::WeightVersionSpan;
|
||||
use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
|
||||
@@ -73,6 +74,7 @@ pub(super) struct GenerateResponseChoice {
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
|
||||
@@ -84,6 +86,8 @@ pub(super) struct GenerateResponseStreamChoice {
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
/// Set on the final chunk only; covers the whole output.
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateStreamResponse` class.
|
||||
|
||||
Reference in New Issue
Block a user