[Rust Frontend] Return weight-version spans on final outputs (#61070)
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Bugen Zhao
Claude Opus 5.5
parent
f4114e068e
commit
4163a80142
@@ -66,7 +66,7 @@ The final output of every choice carries `weight_versions`, the label of the wei
|
||||
]
|
||||
```
|
||||
|
||||
- With `stream: true`, only the last chunk of each choice carries the list, including the last chunk after an abort. Its offsets cover the whole output. Earlier chunks carry `null`.
|
||||
- With `stream: true`, only the last chunk of each choice carries the list, including the last chunk after an abort. Its offsets cover the whole output. Earlier chunks carry `null` or omit the field.
|
||||
- An output with no tokens, such as a request aborted before its first token, gets `[]`.
|
||||
- The labels are the ones you set with `/update_weight_version` or `/finish_weight_update`, see [Async Reinforcement Learning](../../training/async_rl.md#weight-versions-in-outputs). vLLM doesn't order or check them, and the label is `"default"` until you set one.
|
||||
|
||||
|
||||
@@ -453,10 +453,12 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
let has_outputs = !batch.outputs.is_empty();
|
||||
let mut iteration_tokens: u64 =
|
||||
batch.outputs.iter().map(output_iteration_tokens).sum();
|
||||
let (engine_index, timestamp) = (batch.engine_index, batch.timestamp);
|
||||
let (engine_index, timestamp, weight_version) =
|
||||
(batch.engine_index, batch.timestamp, batch.weight_version);
|
||||
let outputs = batch.outputs.into_iter().map(|output| EngineCoreStreamOutput {
|
||||
engine_index,
|
||||
timestamp,
|
||||
weight_version: weight_version.clone(),
|
||||
output,
|
||||
});
|
||||
for output in inner.send_outputs(outputs) {
|
||||
|
||||
@@ -130,6 +130,7 @@ mod tests {
|
||||
EngineCoreStreamOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
weight_version: None,
|
||||
output: EngineCoreOutput {
|
||||
request_id: "req".to_string(),
|
||||
new_token_ids: token_ids.to_vec(),
|
||||
|
||||
@@ -344,6 +344,7 @@ impl RequestRegistry {
|
||||
let output = EngineCoreStreamOutput {
|
||||
engine_index: engine_id.engine_index().unwrap_or(0),
|
||||
timestamp,
|
||||
weight_version: None,
|
||||
output: EngineCoreOutput {
|
||||
request_id: request_id.clone(),
|
||||
finish_reason: Some(EngineCoreFinishReason::Abort),
|
||||
@@ -527,6 +528,7 @@ mod tests {
|
||||
EngineCoreStreamOutput {
|
||||
engine_index: 0,
|
||||
timestamp: 0.0,
|
||||
weight_version: None,
|
||||
output,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use std::ops::Deref;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
use futures::stream::FusedStream;
|
||||
@@ -30,6 +31,7 @@ enum State {
|
||||
pub struct EngineCoreStreamOutput {
|
||||
pub engine_index: u32,
|
||||
pub timestamp: f64,
|
||||
pub weight_version: Option<Arc<str>>,
|
||||
pub output: EngineCoreOutput,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::sync::Arc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use enum_as_inner::EnumAsInner;
|
||||
@@ -228,7 +229,8 @@ struct WireEngineCoreOutputs {
|
||||
/// wave needs to start in other engines.
|
||||
#[serde(default)]
|
||||
start_wave: Option<u32>,
|
||||
/// Decoded for Python wire compatibility; not exposed by the Rust frontend.
|
||||
/// Label of the weights used for this step, as set by the
|
||||
/// `set_weight_version` utility; `"default"` until set.
|
||||
#[serde(default)]
|
||||
weight_version: Option<String>,
|
||||
}
|
||||
@@ -247,6 +249,9 @@ pub struct RequestBatchOutputs {
|
||||
pub scheduler_stats: Option<Box<SchedulerStats>>,
|
||||
pub timestamp: f64,
|
||||
pub finished_requests: Option<BTreeSet<String>>,
|
||||
/// Label of the weights used for this step. It changes only while
|
||||
/// generation is paused, so it applies to every output in the batch.
|
||||
pub weight_version: Option<Arc<str>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
@@ -327,6 +332,7 @@ impl TryFrom<WireEngineCoreOutputs> for EngineCoreOutputs {
|
||||
scheduler_stats: value.scheduler_stats,
|
||||
timestamp: value.timestamp,
|
||||
finished_requests: value.finished_requests,
|
||||
weight_version: value.weight_version.map(Arc::from),
|
||||
}
|
||||
.into()),
|
||||
(false, Some(_), None, None) => Ok(UtilityCallOutput {
|
||||
@@ -365,6 +371,7 @@ impl From<EngineCoreOutputs> for WireEngineCoreOutputs {
|
||||
scheduler_stats: batch.scheduler_stats,
|
||||
timestamp: batch.timestamp,
|
||||
finished_requests: batch.finished_requests,
|
||||
weight_version: batch.weight_version.map(|version| version.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
EngineCoreOutputs::Utility(utility) => Self {
|
||||
@@ -558,6 +565,7 @@ mod tests {
|
||||
"req-1",
|
||||
},
|
||||
),
|
||||
weight_version: None,
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
|
||||
@@ -2884,6 +2884,9 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
"req-1",
|
||||
},
|
||||
),
|
||||
weight_version: Some(
|
||||
"v7",
|
||||
),
|
||||
},
|
||||
)
|
||||
"#]]
|
||||
|
||||
@@ -16,7 +16,7 @@ mod request_metrics;
|
||||
pub use error::{Error, Result};
|
||||
pub use output::{
|
||||
CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStream,
|
||||
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage,
|
||||
GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage, WeightVersionSpan,
|
||||
};
|
||||
pub use request::GenerateRequest;
|
||||
pub use request_metrics::current_unix_timestamp_secs;
|
||||
|
||||
@@ -24,6 +24,14 @@ use crate::error::Result;
|
||||
use crate::inflight::RequestGuard;
|
||||
use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs};
|
||||
|
||||
/// Output tokens `[start, end)` were sampled by weights labelled `version`.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
pub struct WeightVersionSpan {
|
||||
pub version: Arc<str>,
|
||||
pub start: usize,
|
||||
pub end: usize,
|
||||
}
|
||||
|
||||
/// Token usage metadata for one request.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct TokenUsage {
|
||||
@@ -55,6 +63,7 @@ pub struct CollectedGenerateOutput {
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
/// Per-request speculative-decoding metrics from the terminal output.
|
||||
pub spec_decode_metrics: Option<RequestSpecDecodeMetrics>,
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
}
|
||||
|
||||
/// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for
|
||||
@@ -172,6 +181,8 @@ pub struct GenerateOutput {
|
||||
pub sampling_mask: Option<SamplingMask>,
|
||||
/// Per-request speculative-decoding metrics, present on terminal outputs.
|
||||
pub spec_decode_metrics: Option<Box<RequestSpecDecodeMetrics>>,
|
||||
/// Weight-version spans over all output tokens, set on terminal outputs only.
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
}
|
||||
|
||||
impl GenerateOutput {
|
||||
@@ -225,6 +236,7 @@ impl GenerateOutput {
|
||||
self.kv_transfer_params = next.kv_transfer_params;
|
||||
self.ec_transfer_params = next.ec_transfer_params;
|
||||
self.spec_decode_metrics = next.spec_decode_metrics;
|
||||
self.weight_versions = next.weight_versions;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -253,6 +265,7 @@ impl GenerateOutput {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -267,6 +280,9 @@ pub struct GenerateOutputStream {
|
||||
pending_prompt_info: Option<Box<GeneratePromptInfo>>,
|
||||
raw_stream: EngineCoreOutputStream,
|
||||
request_metrics: RequestMetricsTracker,
|
||||
/// (weight version, first output-token index) for each span.
|
||||
weight_version_starts: Vec<(Arc<str>, usize)>,
|
||||
num_output_tokens: usize,
|
||||
/// Removes this request's external→internal tracking edge on drop. Held for
|
||||
/// its `Drop` side effect only; never read directly.
|
||||
_request_guard: RequestGuard,
|
||||
@@ -289,6 +305,8 @@ impl GenerateOutputStream {
|
||||
})),
|
||||
raw_stream,
|
||||
request_metrics,
|
||||
weight_version_starts: Vec::new(),
|
||||
num_output_tokens: 0,
|
||||
_request_guard: request_guard,
|
||||
}
|
||||
}
|
||||
@@ -320,6 +338,15 @@ impl GenerateOutputStream {
|
||||
fn convert_output(&mut self, raw: EngineCoreStreamOutput) -> Result<GenerateOutput> {
|
||||
let received_at = current_unix_timestamp_secs();
|
||||
self.request_metrics.observe_output(raw.timestamp, received_at, &raw.output);
|
||||
let num_new_tokens = raw.output.new_token_ids.len();
|
||||
// Start a span when tokens arrive under a new label.
|
||||
if num_new_tokens > 0
|
||||
&& let Some(version) = raw.weight_version
|
||||
&& self.weight_version_starts.last().is_none_or(|(last, _)| *last != version)
|
||||
{
|
||||
self.weight_version_starts.push((version, self.num_output_tokens));
|
||||
}
|
||||
self.num_output_tokens += num_new_tokens;
|
||||
|
||||
let raw = raw.output;
|
||||
|
||||
@@ -357,6 +384,11 @@ impl GenerateOutputStream {
|
||||
if let Some(finish_reason) = finish_reason.as_ref() {
|
||||
self.request_metrics.record_finished(received_at, finish_reason.clone());
|
||||
}
|
||||
let weight_versions = if finish_reason.is_some() {
|
||||
self.weight_version_spans()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let output = GenerateOutput {
|
||||
request_id: raw.request_id,
|
||||
@@ -369,10 +401,31 @@ impl GenerateOutputStream {
|
||||
ec_transfer_params: raw.ec_transfer_params,
|
||||
sampling_mask,
|
||||
spec_decode_metrics: raw.spec_decode_metrics,
|
||||
weight_versions,
|
||||
};
|
||||
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Spans over the output so far; `[]` for no tokens, `None` if no label was reported.
|
||||
fn weight_version_spans(&self) -> Option<Vec<WeightVersionSpan>> {
|
||||
let starts = &self.weight_version_starts;
|
||||
if starts.is_empty() && self.num_output_tokens > 0 {
|
||||
return None;
|
||||
}
|
||||
let ends = starts.iter().skip(1).map(|(_, start)| *start).chain([self.num_output_tokens]);
|
||||
Some(
|
||||
starts
|
||||
.iter()
|
||||
.zip(ends)
|
||||
.map(|((version, start), end)| WeightVersionSpan {
|
||||
version: version.clone(),
|
||||
start: *start,
|
||||
end,
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for GenerateOutputStream {
|
||||
@@ -476,6 +529,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -490,6 +544,7 @@ impl<T: Stream<Item = Result<GenerateOutput>> + Send> T {
|
||||
collected.kv_transfer_params = output.kv_transfer_params.map(|value| *value);
|
||||
collected.ec_transfer_params = output.ec_transfer_params.map(|value| *value);
|
||||
collected.spec_decode_metrics = output.spec_decode_metrics.map(|value| *value);
|
||||
collected.weight_versions = output.weight_versions;
|
||||
if let Some(mask) = collected.sampling_mask.as_ref()
|
||||
&& mask.rows.len() != collected.token_ids.len()
|
||||
{
|
||||
|
||||
@@ -31,6 +31,7 @@ use vllm_engine_core_client::test_utils::{
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::{
|
||||
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
|
||||
WeightVersionSpan,
|
||||
};
|
||||
use vllm_metrics::METRICS;
|
||||
use zeromq::prelude::{SocketRecv, SocketSend};
|
||||
@@ -342,15 +343,16 @@ async fn generate_merges_outputs_by_stream_interval() {
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
|
||||
// One engine step per batch.
|
||||
let steps: [(&[u32], Option<EngineCoreFinishReason>); 6] = [
|
||||
(&[1], None),
|
||||
(&[2], None),
|
||||
(&[3], None),
|
||||
(&[4, 5], None),
|
||||
(&[6], None),
|
||||
(&[7], Some(EngineCoreFinishReason::Length)),
|
||||
// The label changes within the merged output [2, 3, 4, 5].
|
||||
let steps: [(&[u32], &str, Option<EngineCoreFinishReason>); 6] = [
|
||||
(&[1], "v7", None),
|
||||
(&[2], "v7", None),
|
||||
(&[3], "v8", None),
|
||||
(&[4, 5], "v8", None),
|
||||
(&[6], "v7", None),
|
||||
(&[7], "v7", Some(EngineCoreFinishReason::Length)),
|
||||
];
|
||||
for (token_ids, finish_reason) in steps {
|
||||
for (token_ids, weight_version, finish_reason) in steps {
|
||||
send_outputs(
|
||||
push,
|
||||
RequestBatchOutputs {
|
||||
@@ -361,6 +363,7 @@ async fn generate_merges_outputs_by_stream_interval() {
|
||||
Some(token_logprobs(token_ids)),
|
||||
None,
|
||||
)],
|
||||
weight_version: Some(weight_version.into()),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
@@ -412,6 +415,21 @@ async fn generate_merges_outputs_by_stream_interval() {
|
||||
for output in &outputs {
|
||||
assert_eq!(output.logprobs, Some(token_logprobs(&output.token_ids)));
|
||||
}
|
||||
let span = |version: &str, start, end| WeightVersionSpan {
|
||||
version: version.into(),
|
||||
start,
|
||||
end,
|
||||
};
|
||||
let weight_versions: Vec<_> =
|
||||
outputs.iter().map(|output| output.weight_versions.clone()).collect();
|
||||
assert_eq!(
|
||||
weight_versions,
|
||||
[
|
||||
None,
|
||||
None,
|
||||
Some(vec![span("v7", 0, 2), span("v8", 2, 5), span("v7", 5, 7)])
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
@@ -530,6 +548,7 @@ async fn collect_output_rejects_partial_sampling_mask() {
|
||||
rows: vec![vec![1, 33, 99]],
|
||||
}),
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err();
|
||||
@@ -731,6 +750,7 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine()
|
||||
push,
|
||||
RequestBatchOutputs {
|
||||
outputs: vec![request_output(&request.request_id, vec![7], None)],
|
||||
weight_version: Some("v7".into()),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
@@ -764,6 +784,15 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine()
|
||||
let terminal = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(terminal.finish_reason, Some(FinishReason::Abort));
|
||||
assert!(terminal.token_ids.is_empty());
|
||||
// The synthesized abort output reports the spans received so far.
|
||||
assert_eq!(
|
||||
terminal.weight_versions,
|
||||
Some(vec![WeightVersionSpan {
|
||||
version: "v7".into(),
|
||||
start: 0,
|
||||
end: 1
|
||||
}])
|
||||
);
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
|
||||
@@ -235,6 +235,7 @@ async fn generate_chunk_stream(
|
||||
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
|
||||
token_ids,
|
||||
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
|
||||
weight_versions: output.weight_versions,
|
||||
}],
|
||||
usage: include_continuous_usage
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
@@ -336,6 +337,7 @@ fn collect_generate(
|
||||
finish_reason: Some(finish_reason),
|
||||
token_ids: collected.token_ids,
|
||||
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
|
||||
weight_versions: collected.weight_versions,
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
@@ -629,6 +631,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}),
|
||||
Ok(GenerateOutput {
|
||||
request_id: String::new(),
|
||||
@@ -645,6 +648,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}),
|
||||
]);
|
||||
|
||||
@@ -731,6 +735,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -905,6 +910,7 @@ mod tests {
|
||||
rows: vec![vec![30, 40]],
|
||||
}),
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -959,6 +965,7 @@ mod tests {
|
||||
prompt_token_ids: vec![10],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: Some(metrics.clone()),
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -1051,6 +1058,7 @@ mod tests {
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
};
|
||||
|
||||
let response = collect_generate(
|
||||
@@ -1086,6 +1094,7 @@ mod tests {
|
||||
ec_transfer_params: None,
|
||||
sampling_mask: None,
|
||||
spec_decode_metrics: None,
|
||||
weight_versions: None,
|
||||
prompt_token_ids,
|
||||
};
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ use serde_json::{Map, Value};
|
||||
use validator::Validate;
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
|
||||
use vllm_llm::WeightVersionSpan;
|
||||
use vllm_text::SamplingParams;
|
||||
|
||||
use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage};
|
||||
@@ -73,6 +74,7 @@ pub(super) struct GenerateResponseChoice {
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
|
||||
@@ -84,6 +86,8 @@ pub(super) struct GenerateResponseStreamChoice {
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
/// Set on the final chunk only; covers the whole output.
|
||||
pub weight_versions: Option<Vec<WeightVersionSpan>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateStreamResponse` class.
|
||||
|
||||
Reference in New Issue
Block a user