[Rust Frontend] Return weight-version spans on final outputs (#61070)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Bugen Zhao
Claude Opus 5.5
parent
f4114e068e
commit
4163a80142
@@ -453,10 +453,12 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
let has_outputs = !batch.outputs.is_empty();
|
||||
let mut iteration_tokens: u64 =
|
||||
batch.outputs.iter().map(output_iteration_tokens).sum();
|
||||
let (engine_index, timestamp) = (batch.engine_index, batch.timestamp);
|
||||
let (engine_index, timestamp, weight_version) =
|
||||
(batch.engine_index, batch.timestamp, batch.weight_version);
|
||||
let outputs = batch.outputs.into_iter().map(|output| EngineCoreStreamOutput {
|
||||
engine_index,
|
||||
timestamp,
|
||||
weight_version: weight_version.clone(),
|
||||
output,
|
||||
});
|
||||
for output in inner.send_outputs(outputs) {
|
||||
|
||||
@@ -130,6 +130,7 @@ mod tests {
|
||||
EngineCoreStreamOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
weight_version: None,
|
||||
output: EngineCoreOutput {
|
||||
request_id: "req".to_string(),
|
||||
new_token_ids: token_ids.to_vec(),
|
||||
|
||||
@@ -344,6 +344,7 @@ impl RequestRegistry {
|
||||
let output = EngineCoreStreamOutput {
|
||||
engine_index: engine_id.engine_index().unwrap_or(0),
|
||||
timestamp,
|
||||
weight_version: None,
|
||||
output: EngineCoreOutput {
|
||||
request_id: request_id.clone(),
|
||||
finish_reason: Some(EngineCoreFinishReason::Abort),
|
||||
@@ -527,6 +528,7 @@ mod tests {
|
||||
EngineCoreStreamOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
weight_version: None,
|
||||
output,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::ops::Deref;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::stream::FusedStream;
|
||||
@@ -30,6 +31,7 @@ enum State {
|
||||
pub struct EngineCoreStreamOutput {
|
||||
pub engine_index: u32,
|
||||
pub timestamp: f64,
|
||||
pub weight_version: Option<Arc<str>>,
|
||||
pub output: EngineCoreOutput,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
@@ -228,7 +229,8 @@ struct WireEngineCoreOutputs {
|
||||
/// wave needs to start in other engines.
|
||||
#[serde(default)]
|
||||
start_wave: Option<u32>,
|
||||
/// Decoded for Python wire compatibility; not exposed by the Rust frontend.
|
||||
/// Label of the weights used for this step, as set by the
|
||||
/// `set_weight_version` utility; `"default"` until set.
|
||||
#[serde(default)]
|
||||
weight_version: Option<String>,
|
||||
}
|
||||
@@ -247,6 +249,9 @@ pub struct RequestBatchOutputs {
|
||||
pub scheduler_stats: Option<Box<SchedulerStats>>,
|
||||
pub timestamp: f64,
|
||||
pub finished_requests: Option<BTreeSet<String>>,
|
||||
/// Label of the weights used for this step. It changes only while
|
||||
/// generation is paused, so it applies to every output in the batch.
|
||||
pub weight_version: Option<Arc<str>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
@@ -327,6 +332,7 @@ impl TryFrom<WireEngineCoreOutputs> for EngineCoreOutputs {
|
||||
scheduler_stats: value.scheduler_stats,
|
||||
timestamp: value.timestamp,
|
||||
finished_requests: value.finished_requests,
|
||||
weight_version: value.weight_version.map(Arc::from),
|
||||
}
|
||||
.into()),
|
||||
(false, Some(_), None, None) => Ok(UtilityCallOutput {
|
||||
@@ -365,6 +371,7 @@ impl From<EngineCoreOutputs> for WireEngineCoreOutputs {
|
||||
scheduler_stats: batch.scheduler_stats,
|
||||
timestamp: batch.timestamp,
|
||||
finished_requests: batch.finished_requests,
|
||||
weight_version: batch.weight_version.map(|version| version.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
EngineCoreOutputs::Utility(utility) => Self {
|
||||
@@ -558,6 +565,7 @@ mod tests {
|
||||
"req-1",
|
||||
},
|
||||
),
|
||||
weight_version: None,
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
|
||||
@@ -2884,6 +2884,9 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
"req-1",
|
||||
},
|
||||
),
|
||||
weight_version: Some(
|
||||
"v7",
|
||||
),
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
|
||||
Reference in New Issue
Block a user