[Rust Frontend] Return weight-version spans on final outputs (#61070)

Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Shuolei Wang
2026-10-12 15:56:59 +08:00
committed by GitHub
co-authored by aoshen02 Bugen Zhao Claude Opus 5.5
parent f4114e068e
commit 4163a80142
12 changed files with 127 additions and 12 deletions
@@ -66,7 +66,7 @@ The final output of every choice carries `weight_versions`, the label of the wei
]
```
- With `stream: true`, only the last chunk of each choice carries the list, including the last chunk after an abort. Its offsets cover the whole output. Earlier chunks carry `null`.
- With `stream: true`, only the last chunk of each choice carries the list, including the last chunk after an abort. Its offsets cover the whole output. Earlier chunks carry `null` or omit the field.
- An output with no tokens, such as a request aborted before its first token, gets `[]`.
- The labels are the ones you set with `/update_weight_version` or `/finish_weight_update`, see [Async Reinforcement Learning](../../training/async_rl.md#weight-versions-in-outputs). vLLM doesn't order or check them, and the label is `"default"` until you set one.
@@ -453,10 +453,12 @@ pub(crate) async fn run_output_dispatcher_loop(
let has_outputs = !batch.outputs.is_empty();
let mut iteration_tokens: u64 =
batch.outputs.iter().map(output_iteration_tokens).sum();
let (engine_index, timestamp) = (batch.engine_index, batch.timestamp);
let (engine_index, timestamp, weight_version) =
(batch.engine_index, batch.timestamp, batch.weight_version);
let outputs = batch.outputs.into_iter().map(|output| EngineCoreStreamOutput {
engine_index,
timestamp,
weight_version: weight_version.clone(),
output,
});
for output in inner.send_outputs(outputs) {
@@ -130,6 +130,7 @@ mod tests {
EngineCoreStreamOutput {
engine_index: 0,
timestamp: 0.0,
weight_version: None,
output: EngineCoreOutput {
request_id: "req".to_string(),
new_token_ids: token_ids.to_vec(),
@@ -344,6 +344,7 @@ impl RequestRegistry {
let output = EngineCoreStreamOutput {
engine_index: engine_id.engine_index().unwrap_or(0),
timestamp,
weight_version: None,
output: EngineCoreOutput {
request_id: request_id.clone(),
finish_reason: Some(EngineCoreFinishReason::Abort),
@@ -527,6 +528,7 @@ mod tests {
EngineCoreStreamOutput {
engine_index: 0,
timestamp: 0.0,
weight_version: None,
output,
}
}
@@ -3,6 +3,7 @@
use std::ops::Deref;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use futures::stream::FusedStream;
@@ -30,6 +31,7 @@ enum State {
pub struct EngineCoreStreamOutput {
pub engine_index: u32,
pub timestamp: f64,
pub weight_version: Option<Arc<str>>,
pub output: EngineCoreOutput,
}
@@ -2,6 +2,7 @@
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
use std::collections::BTreeSet;
use std::sync::Arc;
use bytes::Bytes;
use enum_as_inner::EnumAsInner;
@@ -228,7 +229,8 @@ struct WireEngineCoreOutputs {
/// wave needs to start in other engines.
#[serde(default)]
start_wave: Option<u32>,
/// Decoded for Python wire compatibility; not exposed by the Rust frontend.
/// Label of the weights used for this step, as set by the
/// `set_weight_version` utility; `"default"` until set.
#[serde(default)]
weight_version: Option<String>,
}
@@ -247,6 +249,9 @@ pub struct RequestBatchOutputs {
pub scheduler_stats: Option<Box<SchedulerStats>>,
pub timestamp: f64,
pub finished_requests: Option<BTreeSet<String>>,
/// Label of the weights used for this step. It changes only while
/// generation is paused, so it applies to every output in the batch.
pub weight_version: Option<Arc<str>>,
}
#[derive(Debug, Clone, PartialEq, Default)]
@@ -327,6 +332,7 @@ impl TryFrom<WireEngineCoreOutputs> for EngineCoreOutputs {
scheduler_stats: value.scheduler_stats,
timestamp: value.timestamp,
finished_requests: value.finished_requests,
weight_version: value.weight_version.map(Arc::from),
}
.into()),
(false, Some(_), None, None) => Ok(UtilityCallOutput {
@@ -365,6 +371,7 @@ impl From<EngineCoreOutputs> for WireEngineCoreOutputs {
scheduler_stats: batch.scheduler_stats,
timestamp: batch.timestamp,
finished_requests: batch.finished_requests,
weight_version: batch.weight_version.map(|version| version.to_string()),
..Default::default()
},
EngineCoreOutputs::Utility(utility) => Self {
@@ -558,6 +565,7 @@ mod tests {
"req-1",
},
),
weight_version: None,
},
)
"#]]
@@ -2884,6 +2884,9 @@ fn python_msgpack_fixtures_match_rust_encoding() {
"req-1",
},
),
weight_version: Some(
"v7",
),
},
)
"#]]
+1 -1
View File
@@ -16,7 +16,7 @@ mod request_metrics;
pub use error::{Error, Result};
pub use output::{
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStream,
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage,
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage, WeightVersionSpan,
};
pub use request::GenerateRequest;
pub use request_metrics::current_unix_timestamp_secs;
+55
View File
@@ -24,6 +24,14 @@ use crate::error::Result;
use crate::inflight::RequestGuard;
use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs};
/// Output tokens `[start, end)` were sampled by weights labelled `version`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct WeightVersionSpan {
pub version: Arc<str>,
pub start: usize,
pub end: usize,
}
/// Token usage metadata for one request.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct TokenUsage {
@@ -55,6 +63,7 @@ pub struct CollectedGenerateOutput {
pub sampling_mask: Option<SamplingMask>,
/// Per-request speculative-decoding metrics from the terminal output.
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
pub weight_versions: Option<Vec<WeightVersionSpan>>,
}
/// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for
@@ -172,6 +181,8 @@ pub struct GenerateOutput {
pub sampling_mask: Option<SamplingMask>,
/// Per-request speculative-decoding metrics, present on terminal outputs.
pub spec_decode_metrics: Option<Box<RequestSpecDecodeMetrics>>,
/// Weight-version spans over all output tokens, set on terminal outputs only.
pub weight_versions: Option<Vec<WeightVersionSpan>>,
}
impl GenerateOutput {
@@ -225,6 +236,7 @@ impl GenerateOutput {
self.kv_transfer_params = next.kv_transfer_params;
self.ec_transfer_params = next.ec_transfer_params;
self.spec_decode_metrics = next.spec_decode_metrics;
self.weight_versions = next.weight_versions;
}
}
@@ -253,6 +265,7 @@ impl GenerateOutput {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}
}
}
@@ -267,6 +280,9 @@ pub struct GenerateOutputStream {
pending_prompt_info: Option<Box<GeneratePromptInfo>>,
raw_stream: EngineCoreOutputStream,
request_metrics: RequestMetricsTracker,
/// (weight version, first output-token index) for each span.
weight_version_starts: Vec<(Arc<str>, usize)>,
num_output_tokens: usize,
/// Removes this request's external→internal tracking edge on drop. Held for
/// its `Drop` side effect only; never read directly.
_request_guard: RequestGuard,
@@ -289,6 +305,8 @@ impl GenerateOutputStream {
})),
raw_stream,
request_metrics,
weight_version_starts: Vec::new(),
num_output_tokens: 0,
_request_guard: request_guard,
}
}
@@ -320,6 +338,15 @@ impl GenerateOutputStream {
fn convert_output(&mut self, raw: EngineCoreStreamOutput) -> Result<GenerateOutput> {
let received_at = current_unix_timestamp_secs();
self.request_metrics.observe_output(raw.timestamp, received_at, &raw.output);
let num_new_tokens = raw.output.new_token_ids.len();
// Start a span when tokens arrive under a new label.
if num_new_tokens > 0
&& let Some(version) = raw.weight_version
&& self.weight_version_starts.last().is_none_or(|(last, _)| *last != version)
{
self.weight_version_starts.push((version, self.num_output_tokens));
}
self.num_output_tokens += num_new_tokens;
let raw = raw.output;
@@ -357,6 +384,11 @@ impl GenerateOutputStream {
if let Some(finish_reason) = finish_reason.as_ref() {
self.request_metrics.record_finished(received_at, finish_reason.clone());
}
let weight_versions = if finish_reason.is_some() {
self.weight_version_spans()
} else {
None
};
let output = GenerateOutput {
request_id: raw.request_id,
@@ -369,10 +401,31 @@ impl GenerateOutputStream {
ec_transfer_params: raw.ec_transfer_params,
sampling_mask,
spec_decode_metrics: raw.spec_decode_metrics,
weight_versions,
};
Ok(output)
}
/// Spans over the output so far; `[]` for no tokens, `None` if no label was reported.
fn weight_version_spans(&self) -> Option<Vec<WeightVersionSpan>> {
let starts = &self.weight_version_starts;
if starts.is_empty() && self.num_output_tokens > 0 {
return None;
}
let ends = starts.iter().skip(1).map(|(_, start)| *start).chain([self.num_output_tokens]);
Some(
starts
.iter()
.zip(ends)
.map(|((version, start), end)| WeightVersionSpan {
version: version.clone(),
start: *start,
end,
})
.collect(),
)
}
}
impl Stream for GenerateOutputStream {
@@ -476,6 +529,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
ec_transfer_params: None,
sampling_mask,
spec_decode_metrics: None,
weight_versions: None,
});
}
@@ -490,6 +544,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
collected.kv_transfer_params = output.kv_transfer_params.map(|value| *value);
collected.ec_transfer_params = output.ec_transfer_params.map(|value| *value);
collected.spec_decode_metrics = output.spec_decode_metrics.map(|value| *value);
collected.weight_versions = output.weight_versions;
if let Some(mask) = collected.sampling_mask.as_ref()
&& mask.rows.len() != collected.token_ids.len()
{
+37 -8
View File
@@ -31,6 +31,7 @@ use vllm_engine_core_client::test_utils::{
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::{
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
WeightVersionSpan,
};
use vllm_metrics::METRICS;
use zeromq::prelude::{SocketRecv, SocketSend};
@@ -342,15 +343,16 @@ async fn generate_merges_outputs_by_stream_interval() {
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
// One engine step per batch.
let steps: [(&[u32], Option<EngineCoreFinishReason>); 6] = [
(&[1], None),
(&[2], None),
(&[3], None),
(&[4, 5], None),
(&[6], None),
(&[7], Some(EngineCoreFinishReason::Length)),
// The label changes within the merged output [2, 3, 4, 5].
let steps: [(&[u32], &str, Option<EngineCoreFinishReason>); 6] = [
(&[1], "v7", None),
(&[2], "v7", None),
(&[3], "v8", None),
(&[4, 5], "v8", None),
(&[6], "v7", None),
(&[7], "v7", Some(EngineCoreFinishReason::Length)),
];
for (token_ids, finish_reason) in steps {
for (token_ids, weight_version, finish_reason) in steps {
send_outputs(
push,
RequestBatchOutputs {
@@ -361,6 +363,7 @@ async fn generate_merges_outputs_by_stream_interval() {
Some(token_logprobs(token_ids)),
None,
)],
weight_version: Some(weight_version.into()),
..Default::default()
}
.into(),
@@ -412,6 +415,21 @@ async fn generate_merges_outputs_by_stream_interval() {
for output in &outputs {
assert_eq!(output.logprobs, Some(token_logprobs(&output.token_ids)));
}
let span = |version: &str, start, end| WeightVersionSpan {
version: version.into(),
start,
end,
};
let weight_versions: Vec<_> =
outputs.iter().map(|output| output.weight_versions.clone()).collect();
assert_eq!(
weight_versions,
[
None,
None,
Some(vec![span("v7", 0, 2), span("v8", 2, 5), span("v7", 5, 7)])
]
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -530,6 +548,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
rows: vec![vec![1, 33, 99]],
}),
spec_decode_metrics: None,
weight_versions: None,
};
let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err();
@@ -731,6 +750,7 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine()
push,
RequestBatchOutputs {
outputs: vec![request_output(&request.request_id, vec![7], None)],
weight_version: Some("v7".into()),
..Default::default()
}
.into(),
@@ -764,6 +784,15 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine()
let terminal = stream.next().await.unwrap().unwrap();
assert_eq!(terminal.finish_reason, Some(FinishReason::Abort));
assert!(terminal.token_ids.is_empty());
// The synthesized abort output reports the spans received so far.
assert_eq!(
terminal.weight_versions,
Some(vec![WeightVersionSpan {
version: "v7".into(),
start: 0,
end: 1
}])
);
assert!(stream.next().await.is_none());
let _ = shutdown_tx.send(());
@@ -235,6 +235,7 @@ async fn generate_chunk_stream(
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
token_ids,
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
weight_versions: output.weight_versions,
}],
usage: include_continuous_usage
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
@@ -336,6 +337,7 @@ fn collect_generate(
finish_reason: Some(finish_reason),
token_ids: collected.token_ids,
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
weight_versions: collected.weight_versions,
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
@@ -629,6 +631,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}),
Ok(GenerateOutput {
request_id: String::new(),
@@ -645,6 +648,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}),
]);
@@ -731,6 +735,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
}
}
@@ -905,6 +910,7 @@ mod tests {
rows: vec![vec![30, 40]],
}),
spec_decode_metrics: None,
weight_versions: None,
};
let response = collect_generate(
@@ -959,6 +965,7 @@ mod tests {
prompt_token_ids: vec![10],
sampling_mask: None,
spec_decode_metrics: Some(metrics.clone()),
weight_versions: None,
};
let response = collect_generate(
@@ -1051,6 +1058,7 @@ mod tests {
prompt_token_ids: vec![10, 20],
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
};
let response = collect_generate(
@@ -1086,6 +1094,7 @@ mod tests {
ec_transfer_params: None,
sampling_mask: None,
spec_decode_metrics: None,
weight_versions: None,
prompt_token_ids,
};
@@ -9,6 +9,7 @@ use serde_json::{Map, Value};
use validator::Validate;
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
use vllm_llm::WeightVersionSpan;
use vllm_text::SamplingParams;
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
@@ -73,6 +74,7 @@ pub(super) struct GenerateResponseChoice {
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
pub weight_versions: Option<Vec<WeightVersionSpan>>,
}
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
@@ -84,6 +86,8 @@ pub(super) struct GenerateResponseStreamChoice {
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
/// Set on the final chunk only; covers the whole output.
pub weight_versions: Option<Vec<WeightVersionSpan>>,
}
/// Mirrors the Python vLLM `GenerateStreamResponse` class.