diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index babc817932..eb899e3e78 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -512,6 +512,7 @@ impl EngineCoreClient { Ok(EngineCoreOutputStream::new( request_id, + engine_id.engine_index().unwrap_or(0), self.abort_tx.clone(), rx, )) diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index 3465817a45..c91a93d271 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -15,7 +15,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti use crate::client::stream::EngineCoreStreamOutput; use crate::client::{AbortCause, AbortRequest}; use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output}; -use crate::metrics::{LoraInfoExporter, record_scheduler_stats}; +use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder}; use crate::protocol::encode_msgpack; use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs}; use crate::protocol::request::EngineCoreRequestType; @@ -29,6 +29,7 @@ pub(crate) struct ClientInner { /// The runtime handle used for sending messages to the engine. handle: Handle, model_name: String, + scheduler_stats_recorder: SchedulerStatsRecorder, request_reg: Mutex, utility_reg: Mutex, health_error: ArcSwapOption, @@ -43,10 +44,13 @@ impl ClientInner { model_name: String, engines: &[ConnectedEngine], ) -> Self { + let scheduler_stats_recorder = + SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines); Self { input_send, handle, model_name, + scheduler_stats_recorder, request_reg: Mutex::new(RequestRegistry::new(engines)), utility_reg: Mutex::new(UtilityRegistry::default()), health_error: ArcSwapOption::empty(), @@ -389,12 +393,7 @@ pub(crate) async fn run_output_dispatcher_loop( "dropping scheduler stats for unknown engine" ); } - record_scheduler_stats( - &METRICS.scheduler, - inner.model_name(), - batch.engine_index, - scheduler_stats, - ); + inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats); } // The engine's scheduler stats never carry adapter names; diff --git a/rust/src/engine-core-client/src/client/stream.rs b/rust/src/engine-core-client/src/client/stream.rs index 56c6a7cb66..b0ea180795 100644 --- a/rust/src/engine-core-client/src/client/stream.rs +++ b/rust/src/engine-core-client/src/client/stream.rs @@ -45,6 +45,7 @@ impl Deref for EngineCoreStreamOutput { /// `finish_reason` is non-`None`. pub struct EngineCoreOutputStream { request_id: String, + engine_index: u32, abort_tx: mpsc::UnboundedSender, state: State, rx: OutputReceiver, @@ -53,11 +54,13 @@ pub struct EngineCoreOutputStream { impl EngineCoreOutputStream { pub(crate) fn new( request_id: String, + engine_index: u32, abort_tx: mpsc::UnboundedSender, rx: OutputReceiver, ) -> Self { Self { request_id, + engine_index, abort_tx, state: State::Running, rx, @@ -68,6 +71,11 @@ impl EngineCoreOutputStream { pub fn request_id(&self) -> &str { &self.request_id } + + /// Return the index of the engine that owns this request. + pub fn engine_index(&self) -> u32 { + self.engine_index + } } impl Stream for EngineCoreOutputStream { diff --git a/rust/src/engine-core-client/src/metrics.rs b/rust/src/engine-core-client/src/metrics.rs index e129fdd6a2..05744db42d 100644 --- a/rust/src/engine-core-client/src/metrics.rs +++ b/rust/src/engine-core-client/src/metrics.rs @@ -1,90 +1,190 @@ +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::time::{SystemTime, UNIX_EPOCH}; use vllm_metrics::{ - EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics, + EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames, + LoraInfoLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge, WaitingReasonLabels, }; use crate::protocol::stats::SchedulerStats; +use crate::transport::ConnectedEngine; const WAITING_REASON_CAPACITY: &str = "capacity"; const WAITING_REASON_DEFERRED: &str = "deferred"; -/// Record the scheduler-stats-backed metrics for one engine at one point in -/// time. -pub(crate) fn record_scheduler_stats( - metrics: &SchedulerMetrics, - model_name: impl Into, - engine: u32, - stats: &SchedulerStats, -) { - let model_name = model_name.into(); - let labels = EngineLabels { - model_name: model_name.clone(), - engine, - }; +/// Cached scheduler-stats metric handles for all engines connected to one +/// frontend client. +pub(crate) struct SchedulerStatsRecorder { + engines: BTreeMap, +} + +/// Per-engine cached metric handles used while recording `SchedulerStats`. +struct SchedulerStatsHandles { + // Base labels reused for dynamic child labels. + labels: EngineLabels, // Scheduler state gauges. - metrics.scheduler_running.get_or_create(&labels).set(stats.num_running_reqs); - metrics - .scheduler_waiting - .get_or_create(&labels) - .set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs); - metrics - .scheduler_waiting_by_reason - .get_or_create(&WaitingReasonLabels { - model_name: model_name.clone(), - engine, - reason: WAITING_REASON_CAPACITY, - }) - .set(stats.num_waiting_reqs); - metrics - .scheduler_waiting_by_reason - .get_or_create(&WaitingReasonLabels { - model_name: model_name.clone(), - engine, - reason: WAITING_REASON_DEFERRED, - }) - .set(stats.num_skipped_waiting_reqs); - metrics.kv_cache_usage.get_or_create(&labels).set(stats.kv_cache_usage); + scheduler_running: U64Gauge, + scheduler_waiting: U64Gauge, + scheduler_waiting_capacity: U64Gauge, + scheduler_waiting_deferred: U64Gauge, + kv_cache_usage: F64Gauge, // Prefix-cache counters, including the connector-backed external cache path. - metrics - .prefix_cache_queries - .get_or_create(&labels) - .inc_by(stats.prefix_cache_stats.base.queries); - metrics - .prefix_cache_hits - .get_or_create(&labels) - .inc_by(stats.prefix_cache_stats.base.hits); + prefix_cache_queries: U64Counter, + prefix_cache_hits: U64Counter, + external_prefix_cache_queries: U64Counter, + external_prefix_cache_hits: U64Counter, + + // Speculative decoding counters. + spec_decode_num_drafts: U64Counter, + spec_decode_num_draft_tokens: U64Counter, + spec_decode_num_accepted_tokens: U64Counter, + spec_decode_num_accepted_tokens_per_pos: Family, + + // Per-engine performance / MFU counters. + estimated_flops_per_gpu: U64Counter, + estimated_read_bytes_per_gpu: U64Counter, + estimated_write_bytes_per_gpu: U64Counter, + + // Sampled KV-cache residency histograms. + kv_block_lifetime_seconds: HistogramMetric, + kv_block_idle_before_evict_seconds: HistogramMetric, + kv_block_reuse_gap_seconds: HistogramMetric, + + // Non-Prometheus interval accumulator for periodic text-log helpers. + log_stats: SchedulerLogStatsAccumulator, +} + +impl SchedulerStatsRecorder { + /// Resolve the fixed-label metric handles for the connected engines. + pub(crate) fn new( + metrics: &SchedulerMetrics, + model_name: &str, + engines: &[ConnectedEngine], + ) -> Self { + let engines = engines + .iter() + .filter_map(|engine| { + let engine = engine.engine_id.engine_index()?; + Some(( + engine, + resolve_scheduler_stats_handles(metrics, model_name, engine), + )) + }) + .collect(); + + Self { engines } + } + + /// Record one scheduler-stats payload for the given engine index. + pub(crate) fn record(&self, engine_index: u32, stats: &SchedulerStats) { + if let Some(handles) = self.engines.get(&engine_index) { + record_scheduler_stats_with_handles(handles, stats); + } + } +} + +/// Resolve all fixed-label scheduler metrics for one engine. +fn resolve_scheduler_stats_handles( + metrics: &SchedulerMetrics, + model_name: &str, + engine: u32, +) -> SchedulerStatsHandles { + let labels = EngineLabels { + model_name: model_name.to_string(), + engine, + }; + let capacity = WaitingReasonLabels { + model_name: model_name.to_string(), + engine, + reason: WAITING_REASON_CAPACITY, + }; + let deferred = WaitingReasonLabels { + model_name: model_name.to_string(), + engine, + reason: WAITING_REASON_DEFERRED, + }; + + SchedulerStatsHandles { + scheduler_running: metrics.scheduler_running.get_or_create_owned(&labels), + scheduler_waiting: metrics.scheduler_waiting.get_or_create_owned(&labels), + scheduler_waiting_capacity: metrics + .scheduler_waiting_by_reason + .get_or_create_owned(&capacity), + scheduler_waiting_deferred: metrics + .scheduler_waiting_by_reason + .get_or_create_owned(&deferred), + kv_cache_usage: metrics.kv_cache_usage.get_or_create_owned(&labels), + prefix_cache_queries: metrics.prefix_cache_queries.get_or_create_owned(&labels), + prefix_cache_hits: metrics.prefix_cache_hits.get_or_create_owned(&labels), + external_prefix_cache_queries: metrics + .external_prefix_cache_queries + .get_or_create_owned(&labels), + external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels), + spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels), + spec_decode_num_draft_tokens: metrics + .spec_decode_num_draft_tokens + .get_or_create_owned(&labels), + spec_decode_num_accepted_tokens: metrics + .spec_decode_num_accepted_tokens + .get_or_create_owned(&labels), + spec_decode_num_accepted_tokens_per_pos: metrics + .spec_decode_num_accepted_tokens_per_pos + .clone(), + log_stats: metrics.log_stats.get_or_create_owned(&labels), + estimated_flops_per_gpu: metrics.estimated_flops_per_gpu.get_or_create_owned(&labels), + estimated_read_bytes_per_gpu: metrics + .estimated_read_bytes_per_gpu + .get_or_create_owned(&labels), + estimated_write_bytes_per_gpu: metrics + .estimated_write_bytes_per_gpu + .get_or_create_owned(&labels), + kv_block_lifetime_seconds: metrics.kv_block_lifetime_seconds.get_or_create_owned(&labels), + kv_block_idle_before_evict_seconds: metrics + .kv_block_idle_before_evict_seconds + .get_or_create_owned(&labels), + kv_block_reuse_gap_seconds: metrics.kv_block_reuse_gap_seconds.get_or_create_owned(&labels), + labels, + } +} + +/// Record scheduler-stats values through pre-resolved metric handles. +fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &SchedulerStats) { + // Scheduler state gauges. + handles.scheduler_running.set(stats.num_running_reqs); + handles + .scheduler_waiting + .set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs); + handles.scheduler_waiting_capacity.set(stats.num_waiting_reqs); + handles.scheduler_waiting_deferred.set(stats.num_skipped_waiting_reqs); + handles.kv_cache_usage.set(stats.kv_cache_usage); + + // Prefix-cache counters, including the connector-backed external cache path. + handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries); + handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits); if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats { - metrics + handles .external_prefix_cache_queries - .get_or_create(&labels) .inc_by(connector_prefix_cache_stats.base.queries); - metrics + handles .external_prefix_cache_hits - .get_or_create(&labels) .inc_by(connector_prefix_cache_stats.base.hits); } // Speculative decoding counters. if let Some(spec_decoding_stats) = &stats.spec_decoding_stats { - metrics - .spec_decode_num_drafts - .get_or_create(&labels) - .inc_by(spec_decoding_stats.num_drafts); - metrics + handles.spec_decode_num_drafts.inc_by(spec_decoding_stats.num_drafts); + handles .spec_decode_num_draft_tokens - .get_or_create(&labels) .inc_by(spec_decoding_stats.num_draft_tokens); - metrics + handles .spec_decode_num_accepted_tokens - .get_or_create(&labels) .inc_by(spec_decoding_stats.num_accepted_tokens); - metrics.log_stats.get_or_create(&labels).observe_spec_decode( + handles.log_stats.observe_spec_decode( spec_decoding_stats.num_drafts, &spec_decoding_stats.num_accepted_tokens_per_pos, ); @@ -92,11 +192,11 @@ pub(crate) fn record_scheduler_stats( for (position, accepted_tokens) in spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate() { - metrics + handles .spec_decode_num_accepted_tokens_per_pos .get_or_create(&EnginePositionLabels { - model_name: model_name.clone(), - engine, + model_name: handles.labels.model_name.clone(), + engine: handles.labels.engine, position: position as u32, }) .inc_by(accepted_tokens); @@ -109,22 +209,13 @@ pub(crate) fn record_scheduler_stats( || perf_stats.num_read_bytes_per_gpu != 0 || perf_stats.num_write_bytes_per_gpu != 0) { - metrics - .estimated_flops_per_gpu - .get_or_create(&labels) - .inc_by(perf_stats.num_flops_per_gpu); - metrics - .estimated_read_bytes_per_gpu - .get_or_create(&labels) - .inc_by(perf_stats.num_read_bytes_per_gpu); - metrics - .estimated_write_bytes_per_gpu - .get_or_create(&labels) - .inc_by(perf_stats.num_write_bytes_per_gpu); + handles.estimated_flops_per_gpu.inc_by(perf_stats.num_flops_per_gpu); + handles.estimated_read_bytes_per_gpu.inc_by(perf_stats.num_read_bytes_per_gpu); + handles.estimated_write_bytes_per_gpu.inc_by(perf_stats.num_write_bytes_per_gpu); } if let Some(cudagraph_stats) = &stats.cudagraph_stats { - metrics.log_stats.get_or_create(&labels).observe_cudagraph( + handles.log_stats.observe_cudagraph( cudagraph_stats.num_unpadded_tokens, cudagraph_stats.num_padded_tokens, cudagraph_stats.num_paddings, @@ -134,16 +225,11 @@ pub(crate) fn record_scheduler_stats( // Sampled KV-cache residency histograms. if !stats.kv_cache_eviction_events.is_empty() { - let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels); - let kv_block_idle_before_evict_seconds = - metrics.kv_block_idle_before_evict_seconds.get_or_create(&labels); - let kv_block_reuse_gap_seconds = metrics.kv_block_reuse_gap_seconds.get_or_create(&labels); - for event in &stats.kv_cache_eviction_events { - kv_block_lifetime_seconds.observe(event.lifetime_seconds); - kv_block_idle_before_evict_seconds.observe(event.idle_seconds); + handles.kv_block_lifetime_seconds.observe(event.lifetime_seconds); + handles.kv_block_idle_before_evict_seconds.observe(event.idle_seconds); for reuse_gap_seconds in &event.reuse_gaps_seconds { - kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds); + handles.kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds); } } } diff --git a/rust/src/llm/src/lib.rs b/rust/src/llm/src/lib.rs index ce15a970c6..b8e72632f1 100644 --- a/rust/src/llm/src/lib.rs +++ b/rust/src/llm/src/lib.rs @@ -88,14 +88,21 @@ impl Llm { // Record internal engine-core request ID in the current tracing span. Span::current().record("engine_request_id", &internal_request_id); + let arrival_time = prepared.engine_request.arrival_time; + let max_tokens_param = + (prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens); + let prompt_len = prepared.prompt_token_ids().len() as u32; + + let stream = self.client.call(prepared.engine_request).await?; + let request_metrics = RequestMetricsTracker::new( self.client.model_name().to_string(), - prepared.engine_request.arrival_time, - prepared.prompt_token_ids().len() as u32, - (prepared.engine_request.sampling_params.as_ref()).map(|p| p.max_tokens), + stream.engine_index(), + arrival_time, + prompt_len, + max_tokens_param, 1, ); - let stream = self.client.call(prepared.engine_request).await?; let guard = self.inflight.track(external_request_id, internal_request_id); Ok(GenerateOutputStream::new( diff --git a/rust/src/llm/src/output.rs b/rust/src/llm/src/output.rs index 609088ec6b..d1d7e5f46e 100644 --- a/rust/src/llm/src/output.rs +++ b/rust/src/llm/src/output.rs @@ -248,12 +248,7 @@ impl Stream for GenerateOutputStream { }; let received_at = current_unix_timestamp_secs(); - self.request_metrics.observe_output( - raw.engine_index, - raw.timestamp, - received_at, - &raw.output, - ); + self.request_metrics.observe_output(raw.timestamp, received_at, &raw.output); let raw = raw.output; diff --git a/rust/src/llm/src/request_metrics.rs b/rust/src/llm/src/request_metrics.rs index 38795a7092..f57c54df35 100644 --- a/rust/src/llm/src/request_metrics.rs +++ b/rust/src/llm/src/request_metrics.rs @@ -5,15 +5,12 @@ use vllm_engine_core_client::protocol::output::{ }; use vllm_engine_core_client::protocol::stats::PrefillStats; use vllm_metrics::{ - EngineLabels, FinishedReasonLabels, METRICS, PromptTokenSourceLabels, RequestMetrics, + EngineLabels, Family, FinishedReasonLabels, HistogramMetric, METRICS, PromptTokenSourceLabels, + U64Counter, }; use crate::FinishReason; -fn metrics() -> &'static RequestMetrics { - &METRICS.request -} - const PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE: &str = "local_compute"; const PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT: &str = "local_cache_hit"; const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer"; @@ -29,9 +26,11 @@ const PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER: &str = "external_kv_transfer"; /// /// Original Python update flow: /// -#[derive(Debug, Clone)] +#[derive(Clone)] pub(crate) struct RequestMetricsTracker { - model_name: String, + /// Cached request metric handles for this request's model and engine index. + handles: RequestMetricHandles, + arrival_time: f64, prompt_len: u32, max_tokens_param: Option, @@ -44,7 +43,38 @@ pub(crate) struct RequestMetricsTracker { first_token_latency: f64, num_generation_tokens: u32, latest_num_cached_tokens: u32, - last_seen_engine_index: u32, +} + +/// Cached request metric handles for one model and engine index. +#[derive(Clone)] +struct RequestMetricHandles { + labels: EngineLabels, + + // Request-derived counters. + num_preemptions: U64Counter, + prompt_tokens: U64Counter, + prompt_tokens_local_compute: U64Counter, + prompt_tokens_local_cache_hit: U64Counter, + prompt_tokens_external_kv_transfer: U64Counter, + prompt_tokens_cached: U64Counter, + generation_tokens: U64Counter, + + // Request lifecycle counters and histograms. + request_success: Family, + request_prompt_tokens: HistogramMetric, + request_generation_tokens: HistogramMetric, + request_max_num_generation_tokens: HistogramMetric, + request_params_max_tokens: HistogramMetric, + request_params_n: HistogramMetric, + request_prefill_kv_computed_tokens: HistogramMetric, + time_to_first_token_seconds: HistogramMetric, + inter_token_latency_seconds: HistogramMetric, + e2e_request_latency_seconds: HistogramMetric, + request_queue_time_seconds: HistogramMetric, + request_prefill_time_seconds: HistogramMetric, + request_decode_time_seconds: HistogramMetric, + request_inference_time_seconds: HistogramMetric, + request_time_per_output_token_seconds: HistogramMetric, } impl RequestMetricsTracker { @@ -52,13 +82,14 @@ impl RequestMetricsTracker { /// context. pub(crate) fn new( model_name: String, + engine_index: u32, arrival_time: f64, prompt_len: u32, max_tokens_param: Option, n_param: u32, ) -> Self { Self { - model_name, + handles: resolve_request_metric_handles(&model_name, engine_index), arrival_time, prompt_len, max_tokens_param, @@ -71,7 +102,6 @@ impl RequestMetricsTracker { first_token_latency: 0.0, num_generation_tokens: 0, latest_num_cached_tokens: 0, - last_seen_engine_index: 0, } } @@ -81,23 +111,18 @@ impl RequestMetricsTracker { /// pub(crate) fn observe_output( &mut self, - engine_index: u32, batch_timestamp: f64, received_at: f64, output: &EngineCoreOutput, ) { - self.last_seen_engine_index = engine_index; if let Some(prefill_stats) = &output.prefill_stats { self.latest_num_cached_tokens = prefill_stats.num_cached_tokens; } self.num_generation_tokens += output.new_token_ids.len() as u32; - metrics() - .generation_tokens - .get_or_create(&engine_labels(&self.model_name, engine_index)) - .inc_by(output.new_token_ids.len() as u64); + self.handles.generation_tokens.inc_by(output.new_token_ids.len() as u64); if let Some(events) = &output.events { - self.observe_events(engine_index, events); + self.observe_events(events); } // Only outputs that actually carry tokens drive token-timing metrics. @@ -107,22 +132,16 @@ impl RequestMetricsTracker { if !output.new_token_ids.is_empty() { if self.is_prefilling { if let Some(prefill_stats) = &output.prefill_stats { - record_prompt_tokens(&self.model_name, engine_index, prefill_stats); + self.record_prompt_tokens(prefill_stats); } self.first_token_latency = received_at - self.arrival_time; - observe_time_to_first_token_seconds( - &self.model_name, - engine_index, - self.first_token_latency, - ); + self.handles.time_to_first_token_seconds.observe(self.first_token_latency); self.first_token_ts = batch_timestamp; self.is_prefilling = false; } else if self.last_token_ts > 0.0 { - observe_inter_token_latency_seconds( - &self.model_name, - engine_index, - batch_timestamp - self.last_token_ts, - ); + self.handles + .inter_token_latency_seconds + .observe(batch_timestamp - self.last_token_ts); } self.last_token_ts = batch_timestamp; @@ -135,7 +154,6 @@ impl RequestMetricsTracker { /// Original Python finished-request stats: /// pub(crate) fn record_finished(&self, received_at: f64, finish_reason: FinishReason) { - let labels = engine_labels(&self.model_name, self.last_seen_engine_index); let prefill_kv_computed_tokens = self.prompt_len.saturating_sub(self.latest_num_cached_tokens); let e2e_latency_seconds = received_at - self.arrival_time; @@ -150,57 +168,47 @@ impl RequestMetricsTracker { 0.0 }; - record_request_success(&self.model_name, self.last_seen_engine_index, finish_reason); - metrics() - .request_prompt_tokens - .get_or_create(&labels) - .observe(self.prompt_len as f64); - metrics() + self.record_request_success(finish_reason); + + self.handles.request_prompt_tokens.observe(self.prompt_len as f64); + self.handles .request_generation_tokens - .get_or_create(&labels) .observe(self.num_generation_tokens as f64); - metrics() + self.handles .request_max_num_generation_tokens - .get_or_create(&labels) .observe(self.num_generation_tokens as f64); if let Some(max_tokens_param) = self.max_tokens_param { - metrics() - .request_params_max_tokens - .get_or_create(&labels) - .observe(max_tokens_param as f64); + self.handles.request_params_max_tokens.observe(max_tokens_param as f64); } - metrics().request_params_n.get_or_create(&labels).observe(self.n_param as f64); - metrics() + self.handles.request_params_n.observe(self.n_param as f64); + self.handles .request_prefill_kv_computed_tokens - .get_or_create(&labels) .observe(prefill_kv_computed_tokens as f64); - metrics() - .e2e_request_latency_seconds - .get_or_create(&labels) - .observe(e2e_latency_seconds); - metrics() - .request_queue_time_seconds - .get_or_create(&labels) - .observe(queue_time_seconds); - metrics() - .request_prefill_time_seconds - .get_or_create(&labels) - .observe(prefill_time_seconds); - metrics() - .request_decode_time_seconds - .get_or_create(&labels) - .observe(decode_time_seconds); - metrics() - .request_inference_time_seconds - .get_or_create(&labels) - .observe(inference_time_seconds); - metrics() + self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds); + self.handles.request_queue_time_seconds.observe(queue_time_seconds); + self.handles.request_prefill_time_seconds.observe(prefill_time_seconds); + self.handles.request_decode_time_seconds.observe(decode_time_seconds); + self.handles.request_inference_time_seconds.observe(inference_time_seconds); + self.handles .request_time_per_output_token_seconds - .get_or_create(&labels) .observe(time_per_output_token_seconds); } - fn observe_events(&mut self, engine_index: u32, events: &[EngineCoreEvent]) { + /// Record prompt token counters through cached metric handles. + fn record_prompt_tokens(&self, prefill_stats: &PrefillStats) { + let computed = prefill_stats.num_computed_tokens as u64; + let local_cache_hit = prefill_stats.num_local_cached_tokens as u64; + let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64; + + self.handles.prompt_tokens.inc_by(prefill_stats.num_prompt_tokens as u64); + self.handles.prompt_tokens_local_compute.inc_by(computed); + self.handles.prompt_tokens_local_cache_hit.inc_by(local_cache_hit); + self.handles.prompt_tokens_external_kv_transfer.inc_by(external_kv_transfer); + self.handles.prompt_tokens_cached.inc_by(prefill_stats.num_cached_tokens as u64); + } + + /// Record request event counters through cached metric handles. + fn observe_events(&mut self, events: &[EngineCoreEvent]) { for event in events { match event.r#type { EngineCoreEventType::Queued => { @@ -212,46 +220,86 @@ impl RequestMetricsTracker { } } EngineCoreEventType::Preempted => { - metrics() - .num_preemptions - .get_or_create(&engine_labels(&self.model_name, engine_index)) - .inc(); + self.handles.num_preemptions.inc(); } } } } -} -fn engine_labels(model_name: &str, engine: u32) -> EngineLabels { - EngineLabels { - model_name: model_name.to_string(), - engine, + /// Increment the request-success counter for the terminal finish reason. + fn record_request_success(&self, finish_reason: FinishReason) { + self.handles + .request_success + .get_or_create(&FinishedReasonLabels { + model_name: self.handles.labels.model_name.clone(), + engine: self.handles.labels.engine, + finished_reason: finish_reason.as_str(), + }) + .inc(); } } -fn observe_time_to_first_token_seconds(model_name: &str, engine: u32, seconds: f64) { - metrics() - .time_to_first_token_seconds - .get_or_create(&engine_labels(model_name, engine)) - .observe(seconds); -} +/// Resolve fixed request metric handles for one model and engine index. +fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles { + let metrics = &METRICS.request; + let labels = EngineLabels { + model_name: model_name.to_string(), + engine, + }; -fn observe_inter_token_latency_seconds(model_name: &str, engine: u32, seconds: f64) { - metrics() - .inter_token_latency_seconds - .get_or_create(&engine_labels(model_name, engine)) - .observe(seconds); -} - -fn record_request_success(model_name: &str, engine: u32, finish_reason: FinishReason) { - metrics() - .request_success - .get_or_create(&FinishedReasonLabels { - model_name: model_name.to_string(), - engine, - finished_reason: finish_reason.as_str(), - }) - .inc(); + RequestMetricHandles { + num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels), + prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels), + prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned( + &prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE), + ), + prompt_tokens_local_cache_hit: metrics.prompt_tokens_by_source.get_or_create_owned( + &prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT), + ), + prompt_tokens_external_kv_transfer: metrics.prompt_tokens_by_source.get_or_create_owned( + &prompt_token_source_labels( + model_name, + engine, + PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER, + ), + ), + prompt_tokens_cached: metrics.prompt_tokens_cached.get_or_create_owned(&labels), + generation_tokens: metrics.generation_tokens.get_or_create_owned(&labels), + request_success: metrics.request_success.clone(), + request_prompt_tokens: metrics.request_prompt_tokens.get_or_create_owned(&labels), + request_generation_tokens: metrics.request_generation_tokens.get_or_create_owned(&labels), + request_max_num_generation_tokens: metrics + .request_max_num_generation_tokens + .get_or_create_owned(&labels), + request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels), + request_params_n: metrics.request_params_n.get_or_create_owned(&labels), + request_prefill_kv_computed_tokens: metrics + .request_prefill_kv_computed_tokens + .get_or_create_owned(&labels), + time_to_first_token_seconds: metrics + .time_to_first_token_seconds + .get_or_create_owned(&labels), + inter_token_latency_seconds: metrics + .inter_token_latency_seconds + .get_or_create_owned(&labels), + e2e_request_latency_seconds: metrics + .e2e_request_latency_seconds + .get_or_create_owned(&labels), + request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels), + request_prefill_time_seconds: metrics + .request_prefill_time_seconds + .get_or_create_owned(&labels), + request_decode_time_seconds: metrics + .request_decode_time_seconds + .get_or_create_owned(&labels), + request_inference_time_seconds: metrics + .request_inference_time_seconds + .get_or_create_owned(&labels), + request_time_per_output_token_seconds: metrics + .request_time_per_output_token_seconds + .get_or_create_owned(&labels), + labels, + } } fn prompt_token_source_labels( @@ -266,45 +314,6 @@ fn prompt_token_source_labels( } } -fn record_prompt_tokens(model_name: &str, engine: u32, prefill_stats: &PrefillStats) { - let computed = prefill_stats.num_computed_tokens as u64; - let local_cache_hit = prefill_stats.num_local_cached_tokens as u64; - let external_kv_transfer = prefill_stats.num_external_cached_tokens as u64; - - metrics() - .prompt_tokens - .get_or_create(&engine_labels(model_name, engine)) - .inc_by(prefill_stats.num_prompt_tokens as u64); - metrics() - .prompt_tokens_by_source - .get_or_create(&prompt_token_source_labels( - model_name, - engine, - PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE, - )) - .inc_by(computed); - metrics() - .prompt_tokens_by_source - .get_or_create(&prompt_token_source_labels( - model_name, - engine, - PROMPT_TOKEN_SOURCE_LOCAL_CACHE_HIT, - )) - .inc_by(local_cache_hit); - metrics() - .prompt_tokens_by_source - .get_or_create(&prompt_token_source_labels( - model_name, - engine, - PROMPT_TOKEN_SOURCE_EXTERNAL_KV_TRANSFER, - )) - .inc_by(external_kv_transfer); - metrics() - .prompt_tokens_cached - .get_or_create(&engine_labels(model_name, engine)) - .inc_by(prefill_stats.num_cached_tokens as u64); -} - fn diff_or_zero(end: f64, start: f64) -> f64 { if end > 0.0 && start > 0.0 && end >= start { end - start @@ -337,10 +346,10 @@ mod tests { #[test] fn tracker_updates_timing_state_across_prefill_decode_and_finish() { - let mut tracker = RequestMetricsTracker::new("model".to_string(), 100.0, 64, Some(128), 1); + let mut tracker = + RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1); tracker.observe_output( - 2, 10.0, 100.2, &vllm_engine_core_client::protocol::output::EngineCoreOutput { @@ -368,7 +377,6 @@ mod tests { }, ); tracker.observe_output( - 2, 11.5, 100.4, &vllm_engine_core_client::protocol::output::EngineCoreOutput { @@ -384,7 +392,7 @@ mod tests { ); assert!(!tracker.is_prefilling); - assert_eq!(tracker.last_seen_engine_index, 2); + assert_eq!(tracker.handles.labels.engine, 2); assert_eq!(tracker.num_generation_tokens, 3); assert_eq!(tracker.queued_ts, 8.0); assert_eq!(tracker.scheduled_ts, 9.0); diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index 9911841ead..8581b1ac08 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; use vllm_engine_core_client::protocol::stats::PrefillStats; use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; -use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig}; +use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::{ Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm, }; @@ -699,7 +699,7 @@ async fn abort_by_external_id_aborts_all_internal_requests() { async fn generate_records_request_metrics_in_prometheus_output() { let ipc = IpcNamespace::new().unwrap(); let handshake_address = ipc.handshake_endpoint(); - let engine_id = b"engine-metrics".to_vec(); + let engine_id = EngineId::from_engine_index(4); let model_name = request_metrics_model_name("metrics-model"); let (shutdown_tx, engine_task) = spawn_mock_engine_task( @@ -832,7 +832,7 @@ async fn generate_records_request_metrics_in_prometheus_output() { async fn dropping_stream_records_abort_terminal_request_metrics() { let ipc = IpcNamespace::new().unwrap(); let handshake_address = ipc.handshake_endpoint(); - let engine_id = b"engine-metrics-drop".to_vec(); + let engine_id = EngineId::from_engine_index(5); let model_name = request_metrics_model_name("metrics-drop-model"); let (shutdown_tx, engine_task) = spawn_mock_engine_task( diff --git a/rust/src/metrics/src/lib.rs b/rust/src/metrics/src/lib.rs index 8f0db53d3f..ca650fbada 100644 --- a/rust/src/metrics/src/lib.rs +++ b/rust/src/metrics/src/lib.rs @@ -4,7 +4,7 @@ use std::sync::atomic::AtomicU64; use prometheus_client::encoding::text::encode; use prometheus_client::metrics::counter::Counter; -use prometheus_client::metrics::family::Family; +pub use prometheus_client::metrics::family::Family; use prometheus_client::metrics::gauge::Gauge; use prometheus_client::metrics::histogram::Histogram; use prometheus_client::registry::Registry; @@ -23,6 +23,8 @@ pub use scheduler::*; pub type U64Counter = Counter; pub type U64Gauge = Gauge; pub type F64Gauge = Gauge; +/// Histogram metric handle cloned out of a Prometheus family. +pub type HistogramMetric = Histogram; pub(crate) type HistogramFamily = Family Histogram>; /// Shared Prometheus registry for frontend metrics.