[Rust Frontend] Return weight-version spans on final outputs (#61070)

Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Shuolei Wang
2026-10-12 15:56:59 +08:00
committed by GitHub
co-authored by aoshen02 Bugen Zhao Claude Opus 5.5
parent f4114e068e
commit 4163a80142
12 changed files with 127 additions and 12 deletions
@@ -453,10 +453,12 @@ pub(crate) async fn run_output_dispatcher_loop(
let has_outputs = !batch.outputs.is_empty();
let mut iteration_tokens: u64 =
batch.outputs.iter().map(output_iteration_tokens).sum();
let (engine_index, timestamp) = (batch.engine_index, batch.timestamp);
let (engine_index, timestamp, weight_version) =
(batch.engine_index, batch.timestamp, batch.weight_version);
let outputs = batch.outputs.into_iter().map(|output| EngineCoreStreamOutput {
engine_index,
timestamp,
weight_version: weight_version.clone(),
output,
});
for output in inner.send_outputs(outputs) {
@@ -130,6 +130,7 @@ mod tests {
EngineCoreStreamOutput {
engine_index: 0,
timestamp: 0.0,
weight_version: None,
output: EngineCoreOutput {
request_id: "req".to_string(),
new_token_ids: token_ids.to_vec(),
@@ -344,6 +344,7 @@ impl RequestRegistry {
let output = EngineCoreStreamOutput {
engine_index: engine_id.engine_index().unwrap_or(0),
timestamp,
weight_version: None,
output: EngineCoreOutput {
request_id: request_id.clone(),
finish_reason: Some(EngineCoreFinishReason::Abort),
@@ -527,6 +528,7 @@ mod tests {
EngineCoreStreamOutput {
engine_index: 0,
timestamp: 0.0,
weight_version: None,
output,
}
}
@@ -3,6 +3,7 @@
use std::ops::Deref;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use futures::stream::FusedStream;
@@ -30,6 +31,7 @@ enum State {
pub struct EngineCoreStreamOutput {
pub engine_index: u32,
pub timestamp: f64,
pub weight_version: Option<Arc<str>>,
pub output: EngineCoreOutput,
}
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeSet;
use std::sync::Arc;
use bytes::Bytes;
use enum_as_inner::EnumAsInner;
@@ -228,7 +229,8 @@ struct WireEngineCoreOutputs {
/// wave needs to start in other engines.
#[serde(default)]
start_wave: Option<u32>,
/// Decoded for Python wire compatibility; not exposed by the Rust frontend.
/// Label of the weights used for this step, as set by the
/// `set_weight_version` utility; `"default"` until set.
#[serde(default)]
weight_version: Option<String>,
}
@@ -247,6 +249,9 @@ pub struct RequestBatchOutputs {
pub scheduler_stats: Option<Box<SchedulerStats>>,
pub timestamp: f64,
pub finished_requests: Option<BTreeSet<String>>,
/// Label of the weights used for this step. It changes only while
/// generation is paused, so it applies to every output in the batch.
pub weight_version: Option<Arc<str>>,
}
#[derive(Debug, Clone, PartialEq, Default)]
@@ -327,6 +332,7 @@ impl TryFrom<WireEngineCoreOutputs> for EngineCoreOutputs {
scheduler_stats: value.scheduler_stats,
timestamp: value.timestamp,
finished_requests: value.finished_requests,
weight_version: value.weight_version.map(Arc::from),
}
.into()),
(false, Some(_), None, None) => Ok(UtilityCallOutput {
@@ -365,6 +371,7 @@ impl From<EngineCoreOutputs> for WireEngineCoreOutputs {
scheduler_stats: batch.scheduler_stats,
timestamp: batch.timestamp,
finished_requests: batch.finished_requests,
weight_version: batch.weight_version.map(|version| version.to_string()),
..Default::default()
},
EngineCoreOutputs::Utility(utility) => Self {
@@ -558,6 +565,7 @@ mod tests {
"req-1",
},
),
weight_version: None,
},
)
"#]]
@@ -2884,6 +2884,9 @@ fn python_msgpack_fixtures_match_rust_encoding() {
"req-1",
},
),
weight_version: Some(
"v7",
),
},
)
"#]]