diff --git a/docs/serving/online_serving/token_in_token_out.md b/docs/serving/online_serving/token_in_token_out.md index b3e15ae616..eade608bc4 100644 --- a/docs/serving/online_serving/token_in_token_out.md +++ b/docs/serving/online_serving/token_in_token_out.md @@ -66,7 +66,7 @@ The final output of every choice carries `weight_versions`, the label of the wei ] ``` -- With `stream: true`, only the last chunk of each choice carries the list, including the last chunk after an abort. Its offsets cover the whole output. Earlier chunks carry `null`. +- With `stream: true`, only the last chunk of each choice carries the list, including the last chunk after an abort. Its offsets cover the whole output. Earlier chunks carry `null` or omit the field. - An output with no tokens, such as a request aborted before its first token, gets `[]`. - The labels are the ones you set with `/update_weight_version` or `/finish_weight_update`, see [Async Reinforcement Learning](../../training/async_rl.md#weight-versions-in-outputs). vLLM doesn't order or check them, and the label is `"default"` until you set one. diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index 0a4af1a5ee..fa82d9591a 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -453,10 +453,12 @@ pub(crate) async fn run_output_dispatcher_loop( let has_outputs = !batch.outputs.is_empty(); let mut iteration_tokens: u64 = batch.outputs.iter().map(output_iteration_tokens).sum(); - let (engine_index, timestamp) = (batch.engine_index, batch.timestamp); + let (engine_index, timestamp, weight_version) = + (batch.engine_index, batch.timestamp, batch.weight_version); let outputs = batch.outputs.into_iter().map(|output| EngineCoreStreamOutput { engine_index, timestamp, + weight_version: weight_version.clone(), output, }); for output in inner.send_outputs(outputs) { diff --git a/rust/src/engine-core-client/src/client/output_channel.rs b/rust/src/engine-core-client/src/client/output_channel.rs index 7b58d3dc2f..3de4da366d 100644 --- a/rust/src/engine-core-client/src/client/output_channel.rs +++ b/rust/src/engine-core-client/src/client/output_channel.rs @@ -130,6 +130,7 @@ mod tests { EngineCoreStreamOutput { engine_index: 0, timestamp: 0.0, + weight_version: None, output: EngineCoreOutput { request_id: "req".to_string(), new_token_ids: token_ids.to_vec(), diff --git a/rust/src/engine-core-client/src/client/state.rs b/rust/src/engine-core-client/src/client/state.rs index 9c183b87e4..9f6f8bbf5c 100644 --- a/rust/src/engine-core-client/src/client/state.rs +++ b/rust/src/engine-core-client/src/client/state.rs @@ -344,6 +344,7 @@ impl RequestRegistry { let output = EngineCoreStreamOutput { engine_index: engine_id.engine_index().unwrap_or(0), timestamp, + weight_version: None, output: EngineCoreOutput { request_id: request_id.clone(), finish_reason: Some(EngineCoreFinishReason::Abort), @@ -527,6 +528,7 @@ mod tests { EngineCoreStreamOutput { engine_index: 0, timestamp: 0.0, + weight_version: None, output, } } diff --git a/rust/src/engine-core-client/src/client/stream.rs b/rust/src/engine-core-client/src/client/stream.rs index 6f363f6101..0f54da98c9 100644 --- a/rust/src/engine-core-client/src/client/stream.rs +++ b/rust/src/engine-core-client/src/client/stream.rs @@ -3,6 +3,7 @@ use std::ops::Deref; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use futures::stream::FusedStream; @@ -30,6 +31,7 @@ enum State { pub struct EngineCoreStreamOutput { pub engine_index: u32, pub timestamp: f64, + pub weight_version: Option>, pub output: EngineCoreOutput, } diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index 8945f8b110..d269f24911 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -2,6 +2,7 @@ // SPDX-FileCopyrightText: Copyright contributors to the vLLM project use std::collections::BTreeSet; +use std::sync::Arc; use bytes::Bytes; use enum_as_inner::EnumAsInner; @@ -228,7 +229,8 @@ struct WireEngineCoreOutputs { /// wave needs to start in other engines. #[serde(default)] start_wave: Option, - /// Decoded for Python wire compatibility; not exposed by the Rust frontend. + /// Label of the weights used for this step, as set by the + /// `set_weight_version` utility; `"default"` until set. #[serde(default)] weight_version: Option, } @@ -247,6 +249,9 @@ pub struct RequestBatchOutputs { pub scheduler_stats: Option>, pub timestamp: f64, pub finished_requests: Option>, + /// Label of the weights used for this step. It changes only while + /// generation is paused, so it applies to every output in the batch. + pub weight_version: Option>, } #[derive(Debug, Clone, PartialEq, Default)] @@ -327,6 +332,7 @@ impl TryFrom for EngineCoreOutputs { scheduler_stats: value.scheduler_stats, timestamp: value.timestamp, finished_requests: value.finished_requests, + weight_version: value.weight_version.map(Arc::from), } .into()), (false, Some(_), None, None) => Ok(UtilityCallOutput { @@ -365,6 +371,7 @@ impl From for WireEngineCoreOutputs { scheduler_stats: batch.scheduler_stats, timestamp: batch.timestamp, finished_requests: batch.finished_requests, + weight_version: batch.weight_version.map(|version| version.to_string()), ..Default::default() }, EngineCoreOutputs::Utility(utility) => Self { @@ -558,6 +565,7 @@ mod tests { "req-1", }, ), + weight_version: None, }, ) "#]] diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index 0ad58b0271..43eb88db79 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2884,6 +2884,9 @@ fn python_msgpack_fixtures_match_rust_encoding() { "req-1", }, ), + weight_version: Some( + "v7", + ), }, ) "#]] diff --git a/rust/src/llm/src/lib.rs b/rust/src/llm/src/lib.rs index 7e9b5e0ba5..c94c1d80f7 100644 --- a/rust/src/llm/src/lib.rs +++ b/rust/src/llm/src/lib.rs @@ -16,7 +16,7 @@ mod request_metrics; pub use error::{Error, Result}; pub use output::{ CollectedGenerateOutput, FinishReason, GenerateOutput, GenerateOutputStream, - GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage, + GenerateOutputStreamExt, GeneratePromptInfo, TokenUsage, WeightVersionSpan, }; pub use request::GenerateRequest; pub use request_metrics::current_unix_timestamp_secs; diff --git a/rust/src/llm/src/output.rs b/rust/src/llm/src/output.rs index b6ccb77175..81dbb3fda5 100644 --- a/rust/src/llm/src/output.rs +++ b/rust/src/llm/src/output.rs @@ -24,6 +24,14 @@ use crate::error::Result; use crate::inflight::RequestGuard; use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs}; +/// Output tokens `[start, end)` were sampled by weights labelled `version`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WeightVersionSpan { + pub version: Arc, + pub start: usize, + pub end: usize, +} + /// Token usage metadata for one request. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct TokenUsage { @@ -55,6 +63,7 @@ pub struct CollectedGenerateOutput { pub sampling_mask: Option, /// Per-request speculative-decoding metrics from the terminal output. pub spec_decode_metrics: Option, + pub weight_versions: Option>, } /// Prompt-scoped metadata emitted only once on the first [`GenerateOutput`] for @@ -172,6 +181,8 @@ pub struct GenerateOutput { pub sampling_mask: Option, /// Per-request speculative-decoding metrics, present on terminal outputs. pub spec_decode_metrics: Option>, + /// Weight-version spans over all output tokens, set on terminal outputs only. + pub weight_versions: Option>, } impl GenerateOutput { @@ -225,6 +236,7 @@ impl GenerateOutput { self.kv_transfer_params = next.kv_transfer_params; self.ec_transfer_params = next.ec_transfer_params; self.spec_decode_metrics = next.spec_decode_metrics; + self.weight_versions = next.weight_versions; } } @@ -253,6 +265,7 @@ impl GenerateOutput { ec_transfer_params: None, sampling_mask: None, spec_decode_metrics: None, + weight_versions: None, } } } @@ -267,6 +280,9 @@ pub struct GenerateOutputStream { pending_prompt_info: Option>, raw_stream: EngineCoreOutputStream, request_metrics: RequestMetricsTracker, + /// (weight version, first output-token index) for each span. + weight_version_starts: Vec<(Arc, usize)>, + num_output_tokens: usize, /// Removes this request's external→internal tracking edge on drop. Held for /// its `Drop` side effect only; never read directly. _request_guard: RequestGuard, @@ -289,6 +305,8 @@ impl GenerateOutputStream { })), raw_stream, request_metrics, + weight_version_starts: Vec::new(), + num_output_tokens: 0, _request_guard: request_guard, } } @@ -320,6 +338,15 @@ impl GenerateOutputStream { fn convert_output(&mut self, raw: EngineCoreStreamOutput) -> Result { let received_at = current_unix_timestamp_secs(); self.request_metrics.observe_output(raw.timestamp, received_at, &raw.output); + let num_new_tokens = raw.output.new_token_ids.len(); + // Start a span when tokens arrive under a new label. + if num_new_tokens > 0 + && let Some(version) = raw.weight_version + && self.weight_version_starts.last().is_none_or(|(last, _)| *last != version) + { + self.weight_version_starts.push((version, self.num_output_tokens)); + } + self.num_output_tokens += num_new_tokens; let raw = raw.output; @@ -357,6 +384,11 @@ impl GenerateOutputStream { if let Some(finish_reason) = finish_reason.as_ref() { self.request_metrics.record_finished(received_at, finish_reason.clone()); } + let weight_versions = if finish_reason.is_some() { + self.weight_version_spans() + } else { + None + }; let output = GenerateOutput { request_id: raw.request_id, @@ -369,10 +401,31 @@ impl GenerateOutputStream { ec_transfer_params: raw.ec_transfer_params, sampling_mask, spec_decode_metrics: raw.spec_decode_metrics, + weight_versions, }; Ok(output) } + + /// Spans over the output so far; `[]` for no tokens, `None` if no label was reported. + fn weight_version_spans(&self) -> Option> { + let starts = &self.weight_version_starts; + if starts.is_empty() && self.num_output_tokens > 0 { + return None; + } + let ends = starts.iter().skip(1).map(|(_, start)| *start).chain([self.num_output_tokens]); + Some( + starts + .iter() + .zip(ends) + .map(|((version, start), end)| WeightVersionSpan { + version: version.clone(), + start: *start, + end, + }) + .collect(), + ) + } } impl Stream for GenerateOutputStream { @@ -476,6 +529,7 @@ impl> + Send> T { ec_transfer_params: None, sampling_mask, spec_decode_metrics: None, + weight_versions: None, }); } @@ -490,6 +544,7 @@ impl> + Send> T { collected.kv_transfer_params = output.kv_transfer_params.map(|value| *value); collected.ec_transfer_params = output.ec_transfer_params.map(|value| *value); collected.spec_decode_metrics = output.spec_decode_metrics.map(|value| *value); + collected.weight_versions = output.weight_versions; if let Some(mask) = collected.sampling_mask.as_ref() && mask.rows.len() != collected.token_ids.len() { diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index e1205ede99..8a150d07a4 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -31,6 +31,7 @@ use vllm_engine_core_client::test_utils::{ use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::{ Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm, + WeightVersionSpan, }; use vllm_metrics::METRICS; use zeromq::prelude::{SocketRecv, SocketSend}; @@ -342,15 +343,16 @@ async fn generate_merges_outputs_by_stream_interval() { let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap(); // One engine step per batch. - let steps: [(&[u32], Option); 6] = [ - (&[1], None), - (&[2], None), - (&[3], None), - (&[4, 5], None), - (&[6], None), - (&[7], Some(EngineCoreFinishReason::Length)), + // The label changes within the merged output [2, 3, 4, 5]. + let steps: [(&[u32], &str, Option); 6] = [ + (&[1], "v7", None), + (&[2], "v7", None), + (&[3], "v8", None), + (&[4, 5], "v8", None), + (&[6], "v7", None), + (&[7], "v7", Some(EngineCoreFinishReason::Length)), ]; - for (token_ids, finish_reason) in steps { + for (token_ids, weight_version, finish_reason) in steps { send_outputs( push, RequestBatchOutputs { @@ -361,6 +363,7 @@ async fn generate_merges_outputs_by_stream_interval() { Some(token_logprobs(token_ids)), None, )], + weight_version: Some(weight_version.into()), ..Default::default() } .into(), @@ -412,6 +415,21 @@ async fn generate_merges_outputs_by_stream_interval() { for output in &outputs { assert_eq!(output.logprobs, Some(token_logprobs(&output.token_ids))); } + let span = |version: &str, start, end| WeightVersionSpan { + version: version.into(), + start, + end, + }; + let weight_versions: Vec<_> = + outputs.iter().map(|output| output.weight_versions.clone()).collect(); + assert_eq!( + weight_versions, + [ + None, + None, + Some(vec![span("v7", 0, 2), span("v8", 2, 5), span("v7", 5, 7)]) + ] + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -530,6 +548,7 @@ async fn collect_output_rejects_partial_sampling_mask() { rows: vec![vec![1, 33, 99]], }), spec_decode_metrics: None, + weight_versions: None, }; let error = futures::stream::iter([Ok(output)]).collect_output().await.unwrap_err(); @@ -731,6 +750,7 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine() push, RequestBatchOutputs { outputs: vec![request_output(&request.request_id, vec![7], None)], + weight_version: Some("v7".into()), ..Default::default() } .into(), @@ -764,6 +784,15 @@ async fn abort_resolves_external_request_id_to_internal_before_reaching_engine() let terminal = stream.next().await.unwrap().unwrap(); assert_eq!(terminal.finish_reason, Some(FinishReason::Abort)); assert!(terminal.token_ids.is_empty()); + // The synthesized abort output reports the spans received so far. + assert_eq!( + terminal.weight_versions, + Some(vec![WeightVersionSpan { + version: "v7".into(), + start: 0, + end: 1 + }]) + ); assert!(stream.next().await.is_none()); let _ = shutdown_tx.send(()); diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 985fe44fb1..a30cd0c85a 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -235,6 +235,7 @@ async fn generate_chunk_stream( finish_reason: finish_reason.map(|reason| reason.as_str().to_string()), token_ids, sampling_mask: output.sampling_mask.map(|mask| mask.rows), + weight_versions: output.weight_versions, }], usage: include_continuous_usage .then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)), @@ -336,6 +337,7 @@ fn collect_generate( finish_reason: Some(finish_reason), token_ids: collected.token_ids, sampling_mask: collected.sampling_mask.map(|mask| mask.rows), + weight_versions: collected.weight_versions, }], prompt_logprobs, prompt_token_id_logprobs: collected @@ -629,6 +631,7 @@ mod tests { ec_transfer_params: None, sampling_mask: None, spec_decode_metrics: None, + weight_versions: None, }), Ok(GenerateOutput { request_id: String::new(), @@ -645,6 +648,7 @@ mod tests { ec_transfer_params: None, sampling_mask: None, spec_decode_metrics: None, + weight_versions: None, }), ]); @@ -731,6 +735,7 @@ mod tests { ec_transfer_params: None, sampling_mask: None, spec_decode_metrics: None, + weight_versions: None, } } @@ -905,6 +910,7 @@ mod tests { rows: vec![vec![30, 40]], }), spec_decode_metrics: None, + weight_versions: None, }; let response = collect_generate( @@ -959,6 +965,7 @@ mod tests { prompt_token_ids: vec![10], sampling_mask: None, spec_decode_metrics: Some(metrics.clone()), + weight_versions: None, }; let response = collect_generate( @@ -1051,6 +1058,7 @@ mod tests { prompt_token_ids: vec![10, 20], sampling_mask: None, spec_decode_metrics: None, + weight_versions: None, }; let response = collect_generate( @@ -1086,6 +1094,7 @@ mod tests { ec_transfer_params: None, sampling_mask: None, spec_decode_metrics: None, + weight_versions: None, prompt_token_ids, }; diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 0a37640d63..6352b18725 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -9,6 +9,7 @@ use serde_json::{Map, Value}; use validator::Validate; use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; use vllm_engine_core_client::protocol::request::ReasoningParserKwargs; +use vllm_llm::WeightVersionSpan; use vllm_text::SamplingParams; use crate::routes::openai::utils::types::{Normalizable, StreamOptions, Usage}; @@ -73,6 +74,7 @@ pub(super) struct GenerateResponseChoice { pub finish_reason: Option, pub token_ids: Vec, pub sampling_mask: Option>>, + pub weight_versions: Option>, } /// Mirrors the Python vLLM `GenerateResponseStreamChoice` class. @@ -84,6 +86,8 @@ pub(super) struct GenerateResponseStreamChoice { pub finish_reason: Option, pub token_ids: Vec, pub sampling_mask: Option>>, + /// Set on the final chunk only; covers the whole output. + pub weight_versions: Option>, } /// Mirrors the Python vLLM `GenerateStreamResponse` class.