[Rust Frontend] Add iteration token histogram (#56990)

Co-authored-by: Codex <noreply@openai.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-09-15 12:12:15 +00:00
committed by GitHub
co-authored by Codex
parent bb55077411
commit 78eaaad2c7
5 changed files with 90 additions and 14 deletions
+25 -1
View File
@@ -19,7 +19,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
use crate::client::stream::EngineCoreStreamOutput;
use crate::client::{AbortCause, AbortRequest};
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder};
use crate::metrics::{IterationMetricHandles, LoraInfoExporter, SchedulerStatsRecorder};
use crate::protocol::encode_msgpack;
use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
@@ -47,6 +47,7 @@ pub(crate) struct ClientInner {
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
msgpack_zero_copy_threshold: usize,
scheduler_stats_recorder: SchedulerStatsRecorder,
iteration_metrics: BTreeMap<u32, IterationMetricHandles>,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
health_error: ArcSwapOption<Error>,
@@ -64,12 +65,21 @@ impl ClientInner {
) -> Self {
let scheduler_stats_recorder =
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
let iteration_metrics = engines
.iter()
.filter_map(|engine| {
let engine = engine.engine_id.engine_index()?;
let handles = IterationMetricHandles::new(&METRICS.request, &model_name, engine);
Some((engine, handles))
})
.collect();
Self {
input_send,
handle,
model_name,
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
scheduler_stats_recorder,
iteration_metrics,
request_reg: Mutex::new(RequestRegistry::new(engines)),
utility_reg: Mutex::new(UtilityRegistry::default()),
health_error: ArcSwapOption::empty(),
@@ -415,6 +425,8 @@ pub(crate) async fn run_output_dispatcher_loop(
match outputs {
EngineCoreOutputs::RequestBatch(batch) => {
let has_outputs = !batch.outputs.is_empty();
let mut iteration_tokens = 0_u64;
let senders = inner.take_senders_for_outputs(&batch.outputs);
for (output, sender) in batch.outputs.into_iter().zip(senders) {
let request_id = output.request_id.clone();
@@ -423,6 +435,12 @@ pub(crate) async fn run_output_dispatcher_loop(
continue;
};
iteration_tokens += output.new_token_ids.len() as u64;
if let Some(prefill) = &output.prefill_stats {
// The engine emits prefill_stats once, on the first output.
iteration_tokens += u64::from(prefill.num_computed_tokens);
}
let wrapped_output = EngineCoreStreamOutput {
engine_index: batch.engine_index,
timestamp: batch.timestamp,
@@ -433,6 +451,12 @@ pub(crate) async fn run_output_dispatcher_loop(
}
}
if has_outputs
&& let Some(handles) = inner.iteration_metrics.get(&batch.engine_index)
{
handles.iteration_tokens_total.observe(iteration_tokens as f64);
}
// The sender for normally-finished requests should have already been removed
// from the registry when their final output was dispatched
// above. This serves as a safety net to capture any
+19 -2
View File
@@ -8,8 +8,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
use vllm_metrics::{
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily,
MooncakeOperationLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge,
WaitingReasonLabels,
MooncakeOperationLabels, RequestMetrics, SchedulerLogStatsAccumulator, SchedulerMetrics,
U64Counter, U64Gauge, WaitingReasonLabels,
};
use crate::protocol::stats::{
@@ -20,6 +20,23 @@ use crate::transport::ConnectedEngine;
const WAITING_REASON_CAPACITY: &str = "capacity";
const WAITING_REASON_DEFERRED: &str = "deferred";
/// Cached output-batch metric handles for one model and engine index.
pub(crate) struct IterationMetricHandles {
pub iteration_tokens_total: HistogramMetric,
}
impl IterationMetricHandles {
pub(crate) fn new(metrics: &RequestMetrics, model_name: &str, engine: u32) -> Self {
let labels = EngineLabels {
model_name: model_name.to_string(),
engine,
};
Self {
iteration_tokens_total: metrics.iteration_tokens_total.get_or_create_owned(&labels),
}
}
}
/// Cached scheduler-stats metric handles for all engines connected to one
/// frontend client.
pub(crate) struct SchedulerStatsRecorder {
+21 -6
View File
@@ -21,6 +21,9 @@ const REQUEST_LATENCY_BUCKETS: [f64; 21] = [
480.0, 960.0, 1920.0, 7680.0,
];
const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0];
const ITERATION_TOKENS_BUCKETS: [f64; 13] = [
1.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0, 16384.0,
];
fn build_1_2_5_buckets(max_value: u32) -> Vec<f64> {
let mut buckets = Vec::new();
@@ -66,6 +69,10 @@ fn request_params_n_histogram() -> Histogram {
Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied())
}
fn iteration_tokens_histogram() -> Histogram {
Histogram::new(ITERATION_TOKENS_BUCKETS.iter().copied())
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct FinishedReasonLabels {
pub model_name: String,
@@ -83,8 +90,12 @@ pub struct PromptTokenSourceLabels {
pub(crate) type FinishedReasonCounterFamily = Family<FinishedReasonLabels, U64Counter>;
pub(crate) type PromptTokenSourceCounterFamily = Family<PromptTokenSourceLabels, U64Counter>;
/// Request-lifecycle Prometheus families exported from the `llm` layer.
/// Request and output-batch Prometheus families.
pub struct RequestMetrics {
/// Computed prompt and generated tokens per nonempty engine output batch,
/// recorded by the engine-core client for requests tracked by this frontend.
pub iteration_tokens_total: HistogramFamily,
// Request-derived counters.
pub num_preemptions: Family<EngineLabels, U64Counter>,
pub prompt_tokens: Family<EngineLabels, U64Counter>,
@@ -92,11 +103,6 @@ pub struct RequestMetrics {
pub prompt_tokens_cached: Family<EngineLabels, U64Counter>,
pub generation_tokens: Family<EngineLabels, U64Counter>,
// We intentionally don't support iteration-level histograms for now, since it seems to make
// more sense if the engine maintains these metrics and frontend simply forwards.
//
// pub iteration_tokens_total: HistogramFamily,
// Request lifecycle counters and histograms.
pub request_success: FinishedReasonCounterFamily,
pub request_prompt_tokens: HistogramFamily,
@@ -118,6 +124,14 @@ pub struct RequestMetrics {
impl RequestMetrics {
/// Register the request-oriented metric families into the shared registry.
pub(crate) fn register(registry: &mut Registry) -> Self {
let iteration_tokens_total =
Family::new_with_constructor(iteration_tokens_histogram as fn() -> Histogram);
registry.register(
"vllm:iteration_tokens_total",
"Histogram of computed prompt and generated tokens per nonempty engine output batch.",
iteration_tokens_total.clone(),
);
// Request-derived counters.
let num_preemptions = Family::default();
registry.register(
@@ -276,6 +290,7 @@ impl RequestMetrics {
);
Self {
iteration_tokens_total,
num_preemptions,
prompt_tokens,
prompt_tokens_by_source,
+14 -5
View File
@@ -19,6 +19,7 @@ use vllm_engine_core_client::protocol::output::{
UtilityCallOutput,
};
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::protocol::utility::{
EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope,
};
@@ -187,6 +188,11 @@ impl ActiveRequest {
/// Advance this request by one mock engine step.
fn step(&mut self, opt: &Opt) -> EngineCoreOutput {
let prefill_stats = (self.generated == 0).then(|| PrefillStats {
num_prompt_tokens: self.prompt_len as u32,
num_computed_tokens: self.prompt_len as u32,
..Default::default()
});
let remaining = self.max_tokens - self.generated;
let chunk_len = remaining.min(opt.output_token_chunk_size);
let mut new_token_ids = Vec::with_capacity(chunk_len);
@@ -196,11 +202,14 @@ impl ActiveRequest {
self.generated += chunk_len;
let finished = self.generated >= self.max_tokens;
request_output(
self.request_id.clone(),
new_token_ids,
finished.then_some(EngineCoreFinishReason::Length),
)
EngineCoreOutput {
prefill_stats,
..request_output(
self.request_id.clone(),
new_token_ids,
finished.then_some(EngineCoreFinishReason::Length),
)
}
}
}
+11
View File
@@ -10,6 +10,7 @@ use tokio_util::sync::CancellationToken;
use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::test_utils::IpcNamespace;
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
@@ -119,12 +120,22 @@ async fn chunk_size_one_outputs_one_token_per_update() {
let first = stream.next().await.expect("first").expect("first ok");
assert_eq!(first.new_token_ids.len(), 1);
assert_eq!(first.finish_reason, None);
assert_eq!(
first.prefill_stats,
Some(PrefillStats {
num_prompt_tokens: 3,
num_computed_tokens: 3,
..Default::default()
})
);
let second = stream.next().await.expect("second").expect("second ok");
assert_eq!(second.new_token_ids.len(), 1);
assert_eq!(second.finish_reason, None);
assert!(second.prefill_stats.is_none());
let third = stream.next().await.expect("third").expect("third ok");
assert_eq!(third.new_token_ids.len(), 1);
assert_eq!(third.finish_reason, Some(EngineCoreFinishReason::Length));
assert!(third.prefill_stats.is_none());
assert!(stream.next().await.is_none());
shutdown_mock(client, shutdown, task).await;