From 78eaaad2c7a7f8c7864d9c9c72772e3f9f2669d0 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 15 Sep 2026 20:12:15 +0800 Subject: [PATCH] [Rust Frontend] Add iteration token histogram (#56990) Co-authored-by: Codex Signed-off-by: Bugen Zhao --- rust/src/engine-core-client/src/client/imp.rs | 26 +++++++++++++++++- rust/src/engine-core-client/src/metrics.rs | 21 +++++++++++++-- rust/src/metrics/src/request.rs | 27 ++++++++++++++----- rust/src/mock-engine/src/engine.rs | 19 +++++++++---- rust/src/mock-engine/src/tests.rs | 11 ++++++++ 5 files changed, 90 insertions(+), 14 deletions(-) diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index 117473ce47..7318fa01b8 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -19,7 +19,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti use crate::client::stream::EngineCoreStreamOutput; use crate::client::{AbortCause, AbortRequest}; use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output}; -use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder}; +use crate::metrics::{IterationMetricHandles, LoraInfoExporter, SchedulerStatsRecorder}; use crate::protocol::encode_msgpack; use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs}; use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType}; @@ -47,6 +47,7 @@ pub(crate) struct ClientInner { /// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created. msgpack_zero_copy_threshold: usize, scheduler_stats_recorder: SchedulerStatsRecorder, + iteration_metrics: BTreeMap, request_reg: Mutex, utility_reg: Mutex, health_error: ArcSwapOption, @@ -64,12 +65,21 @@ impl ClientInner { ) -> Self { let scheduler_stats_recorder = SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines); + let iteration_metrics = engines + .iter() + .filter_map(|engine| { + let engine = engine.engine_id.engine_index()?; + let handles = IterationMetricHandles::new(&METRICS.request, &model_name, engine); + Some((engine, handles)) + }) + .collect(); Self { input_send, handle, model_name, msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(), scheduler_stats_recorder, + iteration_metrics, request_reg: Mutex::new(RequestRegistry::new(engines)), utility_reg: Mutex::new(UtilityRegistry::default()), health_error: ArcSwapOption::empty(), @@ -415,6 +425,8 @@ pub(crate) async fn run_output_dispatcher_loop( match outputs { EngineCoreOutputs::RequestBatch(batch) => { + let has_outputs = !batch.outputs.is_empty(); + let mut iteration_tokens = 0_u64; let senders = inner.take_senders_for_outputs(&batch.outputs); for (output, sender) in batch.outputs.into_iter().zip(senders) { let request_id = output.request_id.clone(); @@ -423,6 +435,12 @@ pub(crate) async fn run_output_dispatcher_loop( continue; }; + iteration_tokens += output.new_token_ids.len() as u64; + if let Some(prefill) = &output.prefill_stats { + // The engine emits prefill_stats once, on the first output. + iteration_tokens += u64::from(prefill.num_computed_tokens); + } + let wrapped_output = EngineCoreStreamOutput { engine_index: batch.engine_index, timestamp: batch.timestamp, @@ -433,6 +451,12 @@ pub(crate) async fn run_output_dispatcher_loop( } } + if has_outputs + && let Some(handles) = inner.iteration_metrics.get(&batch.engine_index) + { + handles.iteration_tokens_total.observe(iteration_tokens as f64); + } + // The sender for normally-finished requests should have already been removed // from the registry when their final output was dispatched // above. This serves as a safety net to capture any diff --git a/rust/src/engine-core-client/src/metrics.rs b/rust/src/engine-core-client/src/metrics.rs index e15258d608..b392d52601 100644 --- a/rust/src/engine-core-client/src/metrics.rs +++ b/rust/src/engine-core-client/src/metrics.rs @@ -8,8 +8,8 @@ use std::time::{SystemTime, UNIX_EPOCH}; use vllm_metrics::{ EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames, LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily, - MooncakeOperationLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge, - WaitingReasonLabels, + MooncakeOperationLabels, RequestMetrics, SchedulerLogStatsAccumulator, SchedulerMetrics, + U64Counter, U64Gauge, WaitingReasonLabels, }; use crate::protocol::stats::{ @@ -20,6 +20,23 @@ use crate::transport::ConnectedEngine; const WAITING_REASON_CAPACITY: &str = "capacity"; const WAITING_REASON_DEFERRED: &str = "deferred"; +/// Cached output-batch metric handles for one model and engine index. +pub(crate) struct IterationMetricHandles { + pub iteration_tokens_total: HistogramMetric, +} + +impl IterationMetricHandles { + pub(crate) fn new(metrics: &RequestMetrics, model_name: &str, engine: u32) -> Self { + let labels = EngineLabels { + model_name: model_name.to_string(), + engine, + }; + Self { + iteration_tokens_total: metrics.iteration_tokens_total.get_or_create_owned(&labels), + } + } +} + /// Cached scheduler-stats metric handles for all engines connected to one /// frontend client. pub(crate) struct SchedulerStatsRecorder { diff --git a/rust/src/metrics/src/request.rs b/rust/src/metrics/src/request.rs index c1135781b2..691c789936 100644 --- a/rust/src/metrics/src/request.rs +++ b/rust/src/metrics/src/request.rs @@ -21,6 +21,9 @@ const REQUEST_LATENCY_BUCKETS: [f64; 21] = [ 480.0, 960.0, 1920.0, 7680.0, ]; const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0]; +const ITERATION_TOKENS_BUCKETS: [f64; 13] = [ + 1.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0, 16384.0, +]; fn build_1_2_5_buckets(max_value: u32) -> Vec { let mut buckets = Vec::new(); @@ -66,6 +69,10 @@ fn request_params_n_histogram() -> Histogram { Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied()) } +fn iteration_tokens_histogram() -> Histogram { + Histogram::new(ITERATION_TOKENS_BUCKETS.iter().copied()) +} + #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)] pub struct FinishedReasonLabels { pub model_name: String, @@ -83,8 +90,12 @@ pub struct PromptTokenSourceLabels { pub(crate) type FinishedReasonCounterFamily = Family; pub(crate) type PromptTokenSourceCounterFamily = Family; -/// Request-lifecycle Prometheus families exported from the `llm` layer. +/// Request and output-batch Prometheus families. pub struct RequestMetrics { + /// Computed prompt and generated tokens per nonempty engine output batch, + /// recorded by the engine-core client for requests tracked by this frontend. + pub iteration_tokens_total: HistogramFamily, + // Request-derived counters. pub num_preemptions: Family, pub prompt_tokens: Family, @@ -92,11 +103,6 @@ pub struct RequestMetrics { pub prompt_tokens_cached: Family, pub generation_tokens: Family, - // We intentionally don't support iteration-level histograms for now, since it seems to make - // more sense if the engine maintains these metrics and frontend simply forwards. - // - // pub iteration_tokens_total: HistogramFamily, - // Request lifecycle counters and histograms. pub request_success: FinishedReasonCounterFamily, pub request_prompt_tokens: HistogramFamily, @@ -118,6 +124,14 @@ pub struct RequestMetrics { impl RequestMetrics { /// Register the request-oriented metric families into the shared registry. pub(crate) fn register(registry: &mut Registry) -> Self { + let iteration_tokens_total = + Family::new_with_constructor(iteration_tokens_histogram as fn() -> Histogram); + registry.register( + "vllm:iteration_tokens_total", + "Histogram of computed prompt and generated tokens per nonempty engine output batch.", + iteration_tokens_total.clone(), + ); + // Request-derived counters. let num_preemptions = Family::default(); registry.register( @@ -276,6 +290,7 @@ impl RequestMetrics { ); Self { + iteration_tokens_total, num_preemptions, prompt_tokens, prompt_tokens_by_source, diff --git a/rust/src/mock-engine/src/engine.rs b/rust/src/mock-engine/src/engine.rs index c216829159..38f9b21be1 100644 --- a/rust/src/mock-engine/src/engine.rs +++ b/rust/src/mock-engine/src/engine.rs @@ -19,6 +19,7 @@ use vllm_engine_core_client::protocol::output::{ UtilityCallOutput, }; use vllm_engine_core_client::protocol::request::EngineCoreRequest; +use vllm_engine_core_client::protocol::stats::PrefillStats; use vllm_engine_core_client::protocol::utility::{ EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope, }; @@ -187,6 +188,11 @@ impl ActiveRequest { /// Advance this request by one mock engine step. fn step(&mut self, opt: &Opt) -> EngineCoreOutput { + let prefill_stats = (self.generated == 0).then(|| PrefillStats { + num_prompt_tokens: self.prompt_len as u32, + num_computed_tokens: self.prompt_len as u32, + ..Default::default() + }); let remaining = self.max_tokens - self.generated; let chunk_len = remaining.min(opt.output_token_chunk_size); let mut new_token_ids = Vec::with_capacity(chunk_len); @@ -196,11 +202,14 @@ impl ActiveRequest { self.generated += chunk_len; let finished = self.generated >= self.max_tokens; - request_output( - self.request_id.clone(), - new_token_ids, - finished.then_some(EngineCoreFinishReason::Length), - ) + EngineCoreOutput { + prefill_stats, + ..request_output( + self.request_id.clone(), + new_token_ids, + finished.then_some(EngineCoreFinishReason::Length), + ) + } } } diff --git a/rust/src/mock-engine/src/tests.rs b/rust/src/mock-engine/src/tests.rs index 9bfee620ec..bee710fd0c 100644 --- a/rust/src/mock-engine/src/tests.rs +++ b/rust/src/mock-engine/src/tests.rs @@ -10,6 +10,7 @@ use tokio_util::sync::CancellationToken; use vllm_engine_core_client::protocol::output::EngineCoreFinishReason; use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; +use vllm_engine_core_client::protocol::stats::PrefillStats; use vllm_engine_core_client::test_utils::IpcNamespace; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode}; @@ -119,12 +120,22 @@ async fn chunk_size_one_outputs_one_token_per_update() { let first = stream.next().await.expect("first").expect("first ok"); assert_eq!(first.new_token_ids.len(), 1); assert_eq!(first.finish_reason, None); + assert_eq!( + first.prefill_stats, + Some(PrefillStats { + num_prompt_tokens: 3, + num_computed_tokens: 3, + ..Default::default() + }) + ); let second = stream.next().await.expect("second").expect("second ok"); assert_eq!(second.new_token_ids.len(), 1); assert_eq!(second.finish_reason, None); + assert!(second.prefill_stats.is_none()); let third = stream.next().await.expect("third").expect("third ok"); assert_eq!(third.new_token_ids.len(), 1); assert_eq!(third.finish_reason, Some(EngineCoreFinishReason::Length)); + assert!(third.prefill_stats.is_none()); assert!(stream.next().await.is_none()); shutdown_mock(client, shutdown, task).await;