[Rust Frontend] Cache metric handles for scheduler & request stats (#47444)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-06 13:02:59 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8b79971bb9
commit 8f4c69b222
9 changed files with 350 additions and 244 deletions
+11 -4
View File
@@ -88,14 +88,21 @@ impl Llm {
// Record internal engine-core request ID in the current tracing span.
Span::current().record("engine_request_id", &internal_request_id);
let arrival_time = prepared.engine_request.arrival_time;
let max_tokens_param =
(prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens);
let prompt_len = prepared.prompt_token_ids().len() as u32;
let stream = self.client.call(prepared.engine_request).await?;
let request_metrics = RequestMetricsTracker::new(
self.client.model_name().to_string(),
prepared.engine_request.arrival_time,
prepared.prompt_token_ids().len() as u32,
(prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens),
stream.engine_index(),
arrival_time,
prompt_len,
max_tokens_param,
1,
);
let stream = self.client.call(prepared.engine_request).await?;
let guard = self.inflight.track(external_request_id, internal_request_id);
Ok(GenerateOutputStream::new(
+1 -6
View File
@@ -248,12 +248,7 @@ impl Stream for GenerateOutputStream {
};
let received_at = current_unix_timestamp_secs();
self.request_metrics.observe_output(
raw.engine_index,
raw.timestamp,
received_at,
&raw.output,
);
self.request_metrics.observe_output(raw.timestamp, received_at, &raw.output);
let raw = raw.output;
+150 -142
View File
@@ -5,15 +5,12 @@ use vllm_engine_core_client::protocol::output::{
};
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_metrics::{
EngineLabels, FinishedReasonLabels, METRICS, PromptTokenSourceLabels, RequestMetrics,
EngineLabels, Family, FinishedReasonLabels, HistogramMetric, METRICS, PromptTokenSourceLabels,
U64Counter,
};
use crate::FinishReason;
fn metrics() -> &'static RequestMetrics {
&METRICS.request
}
const PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE: &str = "local_compute";
const PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT: &str = "local_cache_hit";
const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer";
@@ -29,9 +26,11 @@ const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer";
///
/// Original Python update flow:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/engine/output_processor.py#L600-L677>
#[derive(Debug, Clone)]
#[derive(Clone)]
pub(crate) struct RequestMetricsTracker {
model_name: String,
/// Cached request metric handles for this request's model and engine index.
handles: RequestMetricHandles,
arrival_time: f64,
prompt_len: u32,
max_tokens_param: Option<u32>,
@@ -44,7 +43,38 @@ pub(crate) struct RequestMetricsTracker {
first_token_latency: f64,
num_generation_tokens: u32,
latest_num_cached_tokens: u32,
last_seen_engine_index: u32,
}
/// Cached request metric handles for one model and engine index.
#[derive(Clone)]
struct RequestMetricHandles {
labels: EngineLabels,
// Request-derived counters.
num_preemptions: U64Counter,
prompt_tokens: U64Counter,
prompt_tokens_local_compute: U64Counter,
prompt_tokens_local_cache_hit: U64Counter,
prompt_tokens_external_kv_transfer: U64Counter,
prompt_tokens_cached: U64Counter,
generation_tokens: U64Counter,
// Request lifecycle counters and histograms.
request_success: Family<FinishedReasonLabels, U64Counter>,
request_prompt_tokens: HistogramMetric,
request_generation_tokens: HistogramMetric,
request_max_num_generation_tokens: HistogramMetric,
request_params_max_tokens: HistogramMetric,
request_params_n: HistogramMetric,
request_prefill_kv_computed_tokens: HistogramMetric,
time_to_first_token_seconds: HistogramMetric,
inter_token_latency_seconds: HistogramMetric,
e2e_request_latency_seconds: HistogramMetric,
request_queue_time_seconds: HistogramMetric,
request_prefill_time_seconds: HistogramMetric,
request_decode_time_seconds: HistogramMetric,
request_inference_time_seconds: HistogramMetric,
request_time_per_output_token_seconds: HistogramMetric,
}
impl RequestMetricsTracker {
@@ -52,13 +82,14 @@ impl RequestMetricsTracker {
/// context.
pub(crate) fn new(
model_name: String,
engine_index: u32,
arrival_time: f64,
prompt_len: u32,
max_tokens_param: Option<u32>,
n_param: u32,
) -> Self {
Self {
model_name,
handles: resolve_request_metric_handles(&model_name, engine_index),
arrival_time,
prompt_len,
max_tokens_param,
@@ -71,7 +102,6 @@ impl RequestMetricsTracker {
first_token_latency: 0.0,
num_generation_tokens: 0,
latest_num_cached_tokens: 0,
last_seen_engine_index: 0,
}
}
@@ -81,23 +111,18 @@ impl RequestMetricsTracker {
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L331-L384>
pub(crate) fn observe_output(
&mut self,
engine_index: u32,
batch_timestamp: f64,
received_at: f64,
output: &EngineCoreOutput,
) {
self.last_seen_engine_index = engine_index;
if let Some(prefill_stats) = &output.prefill_stats {
self.latest_num_cached_tokens = prefill_stats.num_cached_tokens;
}
self.num_generation_tokens += output.new_token_ids.len() as u32;
metrics()
.generation_tokens
.get_or_create(&engine_labels(&self.model_name, engine_index))
.inc_by(output.new_token_ids.len() as u64);
self.handles.generation_tokens.inc_by(output.new_token_ids.len() as u64);
if let Some(events) = &output.events {
self.observe_events(engine_index, events);
self.observe_events(events);
}
// Only outputs that actually carry tokens drive token-timing metrics.
@@ -107,22 +132,16 @@ impl RequestMetricsTracker {
if !output.new_token_ids.is_empty() {
if self.is_prefilling {
if let Some(prefill_stats) = &output.prefill_stats {
record_prompt_tokens(&self.model_name, engine_index, prefill_stats);
self.record_prompt_tokens(prefill_stats);
}
self.first_token_latency = received_at - self.arrival_time;
observe_time_to_first_token_seconds(
&self.model_name,
engine_index,
self.first_token_latency,
);
self.handles.time_to_first_token_seconds.observe(self.first_token_latency);
self.first_token_ts = batch_timestamp;
self.is_prefilling = false;
} else if self.last_token_ts > 0.0 {
observe_inter_token_latency_seconds(
&self.model_name,
engine_index,
batch_timestamp - self.last_token_ts,
);
self.handles
.inter_token_latency_seconds
.observe(batch_timestamp - self.last_token_ts);
}
self.last_token_ts = batch_timestamp;
@@ -135,7 +154,6 @@ impl RequestMetricsTracker {
/// Original Python finished-request stats:
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/v1/metrics/stats.py#L222-L237>
pub(crate) fn record_finished(&self, received_at: f64, finish_reason: FinishReason) {
let labels = engine_labels(&self.model_name, self.last_seen_engine_index);
let prefill_kv_computed_tokens =
self.prompt_len.saturating_sub(self.latest_num_cached_tokens);
let e2e_latency_seconds = received_at - self.arrival_time;
@@ -150,57 +168,47 @@ impl RequestMetricsTracker {
0.0
};
record_request_success(&self.model_name, self.last_seen_engine_index, finish_reason);
metrics()
.request_prompt_tokens
.get_or_create(&labels)
.observe(self.prompt_len as f64);
metrics()
self.record_request_success(finish_reason);
self.handles.request_prompt_tokens.observe(self.prompt_len as f64);
self.handles
.request_generation_tokens
.get_or_create(&labels)
.observe(self.num_generation_tokens as f64);
metrics()
self.handles
.request_max_num_generation_tokens
.get_or_create(&labels)
.observe(self.num_generation_tokens as f64);
if let Some(max_tokens_param) = self.max_tokens_param {
metrics()
.request_params_max_tokens
.get_or_create(&labels)
.observe(max_tokens_param as f64);
self.handles.request_params_max_tokens.observe(max_tokens_param as f64);
}
metrics().request_params_n.get_or_create(&labels).observe(self.n_param as f64);
metrics()
self.handles.request_params_n.observe(self.n_param as f64);
self.handles
.request_prefill_kv_computed_tokens
.get_or_create(&labels)
.observe(prefill_kv_computed_tokens as f64);
metrics()
.e2e_request_latency_seconds
.get_or_create(&labels)
.observe(e2e_latency_seconds);
metrics()
.request_queue_time_seconds
.get_or_create(&labels)
.observe(queue_time_seconds);
metrics()
.request_prefill_time_seconds
.get_or_create(&labels)
.observe(prefill_time_seconds);
metrics()
.request_decode_time_seconds
.get_or_create(&labels)
.observe(decode_time_seconds);
metrics()
.request_inference_time_seconds
.get_or_create(&labels)
.observe(inference_time_seconds);
metrics()
self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds);
self.handles.request_queue_time_seconds.observe(queue_time_seconds);
self.handles.request_prefill_time_seconds.observe(prefill_time_seconds);
self.handles.request_decode_time_seconds.observe(decode_time_seconds);
self.handles.request_inference_time_seconds.observe(inference_time_seconds);
self.handles
.request_time_per_output_token_seconds
.get_or_create(&labels)
.observe(time_per_output_token_seconds);
}
fn observe_events(&mut self, engine_index: u32, events: &[EngineCoreEvent]) {
/// Record prompt token counters through cached metric handles.
fn record_prompt_tokens(&self, prefill_stats: &PrefillStats) {
let computed = prefill_stats.num_computed_tokens as u64;
let local_cache_hit = prefill_stats.num_local_cached_tokens as u64;
let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64;
self.handles.prompt_tokens.inc_by(prefill_stats.num_prompt_tokens as u64);
self.handles.prompt_tokens_local_compute.inc_by(computed);
self.handles.prompt_tokens_local_cache_hit.inc_by(local_cache_hit);
self.handles.prompt_tokens_external_kv_transfer.inc_by(external_kv_transfer);
self.handles.prompt_tokens_cached.inc_by(prefill_stats.num_cached_tokens as u64);
}
/// Record request event counters through cached metric handles.
fn observe_events(&mut self, events: &[EngineCoreEvent]) {
for event in events {
match event.r#type {
EngineCoreEventType::Queued => {
@@ -212,46 +220,86 @@ impl RequestMetricsTracker {
}
}
EngineCoreEventType::Preempted => {
metrics()
.num_preemptions
.get_or_create(&engine_labels(&self.model_name, engine_index))
.inc();
self.handles.num_preemptions.inc();
}
}
}
}
}
fn engine_labels(model_name: &str, engine: u32) -> EngineLabels {
EngineLabels {
model_name: model_name.to_string(),
engine,
/// Increment the request-success counter for the terminal finish reason.
fn record_request_success(&self, finish_reason: FinishReason) {
self.handles
.request_success
.get_or_create(&FinishedReasonLabels {
model_name: self.handles.labels.model_name.clone(),
engine: self.handles.labels.engine,
finished_reason: finish_reason.as_str(),
})
.inc();
}
}
fn observe_time_to_first_token_seconds(model_name: &str, engine: u32, seconds: f64) {
metrics()
.time_to_first_token_seconds
.get_or_create(&engine_labels(model_name, engine))
.observe(seconds);
}
/// Resolve fixed request metric handles for one model and engine index.
fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles {
let metrics = &METRICS.request;
let labels = EngineLabels {
model_name: model_name.to_string(),
engine,
};
fn observe_inter_token_latency_seconds(model_name: &str, engine: u32, seconds: f64) {
metrics()
.inter_token_latency_seconds
.get_or_create(&engine_labels(model_name, engine))
.observe(seconds);
}
fn record_request_success(model_name: &str, engine: u32, finish_reason: FinishReason) {
metrics()
.request_success
.get_or_create(&FinishedReasonLabels {
model_name: model_name.to_string(),
engine,
finished_reason: finish_reason.as_str(),
})
.inc();
RequestMetricHandles {
num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels),
prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels),
prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned(
&prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE),
),
prompt_tokens_local_cache_hit: metrics.prompt_tokens_by_source.get_or_create_owned(
&prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT),
),
prompt_tokens_external_kv_transfer: metrics.prompt_tokens_by_source.get_or_create_owned(
&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER,
),
),
prompt_tokens_cached: metrics.prompt_tokens_cached.get_or_create_owned(&labels),
generation_tokens: metrics.generation_tokens.get_or_create_owned(&labels),
request_success: metrics.request_success.clone(),
request_prompt_tokens: metrics.request_prompt_tokens.get_or_create_owned(&labels),
request_generation_tokens: metrics.request_generation_tokens.get_or_create_owned(&labels),
request_max_num_generation_tokens: metrics
.request_max_num_generation_tokens
.get_or_create_owned(&labels),
request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels),
request_params_n: metrics.request_params_n.get_or_create_owned(&labels),
request_prefill_kv_computed_tokens: metrics
.request_prefill_kv_computed_tokens
.get_or_create_owned(&labels),
time_to_first_token_seconds: metrics
.time_to_first_token_seconds
.get_or_create_owned(&labels),
inter_token_latency_seconds: metrics
.inter_token_latency_seconds
.get_or_create_owned(&labels),
e2e_request_latency_seconds: metrics
.e2e_request_latency_seconds
.get_or_create_owned(&labels),
request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels),
request_prefill_time_seconds: metrics
.request_prefill_time_seconds
.get_or_create_owned(&labels),
request_decode_time_seconds: metrics
.request_decode_time_seconds
.get_or_create_owned(&labels),
request_inference_time_seconds: metrics
.request_inference_time_seconds
.get_or_create_owned(&labels),
request_time_per_output_token_seconds: metrics
.request_time_per_output_token_seconds
.get_or_create_owned(&labels),
labels,
}
}
fn prompt_token_source_labels(
@@ -266,45 +314,6 @@ fn prompt_token_source_labels(
}
}
fn record_prompt_tokens(model_name: &str, engine: u32, prefill_stats: &PrefillStats) {
let computed = prefill_stats.num_computed_tokens as u64;
let local_cache_hit = prefill_stats.num_local_cached_tokens as u64;
let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64;
metrics()
.prompt_tokens
.get_or_create(&engine_labels(model_name, engine))
.inc_by(prefill_stats.num_prompt_tokens as u64);
metrics()
.prompt_tokens_by_source
.get_or_create(&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE,
))
.inc_by(computed);
metrics()
.prompt_tokens_by_source
.get_or_create(&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT,
))
.inc_by(local_cache_hit);
metrics()
.prompt_tokens_by_source
.get_or_create(&prompt_token_source_labels(
model_name,
engine,
PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER,
))
.inc_by(external_kv_transfer);
metrics()
.prompt_tokens_cached
.get_or_create(&engine_labels(model_name, engine))
.inc_by(prefill_stats.num_cached_tokens as u64);
}
fn diff_or_zero(end: f64, start: f64) -> f64 {
if end > 0.0 && start > 0.0 && end >= start {
end - start
@@ -337,10 +346,10 @@ mod tests {
#[test]
fn tracker_updates_timing_state_across_prefill_decode_and_finish() {
let mut tracker = RequestMetricsTracker::new("model".to_string(), 100.0, 64, Some(128), 1);
let mut tracker =
RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1);
tracker.observe_output(
2,
10.0,
100.2,
&vllm_engine_core_client::protocol::output::EngineCoreOutput {
@@ -368,7 +377,6 @@ mod tests {
},
);
tracker.observe_output(
2,
11.5,
100.4,
&vllm_engine_core_client::protocol::output::EngineCoreOutput {
@@ -384,7 +392,7 @@ mod tests {
);
assert!(!tracker.is_prefilling);
assert_eq!(tracker.last_seen_engine_index, 2);
assert_eq!(tracker.handles.labels.engine, 2);
assert_eq!(tracker.num_generation_tokens, 3);
assert_eq!(tracker.queued_ts, 8.0);
assert_eq!(tracker.scheduled_ts, 9.0);
+3 -3
View File
@@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::{
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
};
@@ -699,7 +699,7 @@ async fn abort_by_external_id_aborts_all_internal_requests() {
async fn generate_records_request_metrics_in_prometheus_output() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-metrics".to_vec();
let engine_id = EngineId::from_engine_index(4);
let model_name = request_metrics_model_name("metrics-model");
let (shutdown_tx, engine_task) = spawn_mock_engine_task(
@@ -832,7 +832,7 @@ async fn generate_records_request_metrics_in_prometheus_output() {
async fn dropping_stream_records_abort_terminal_request_metrics() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = b"engine-metrics-drop".to_vec();
let engine_id = EngineId::from_engine_index(5);
let model_name = request_metrics_model_name("metrics-drop-model");
let (shutdown_tx, engine_task) = spawn_mock_engine_task(