[Rust Frontend] Cache metric handles for scheduler & request stats (#47444)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-06 13:02:59 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8b79971bb9
commit 8f4c69b222
9 changed files with 350 additions and 244 deletions
@@ -512,6 +512,7 @@ impl EngineCoreClient {
Ok(EngineCoreOutputStream::new(
request_id,
engine_id.engine_index().unwrap_or(0),
self.abort_tx.clone(),
rx,
))
@@ -15,7 +15,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
use crate::client::stream::EngineCoreStreamOutput;
use crate::client::{AbortCause, AbortRequest};
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder};
use crate::protocol::encode_msgpack;
use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
use crate::protocol::request::EngineCoreRequestType;
@@ -29,6 +29,7 @@ pub(crate) struct ClientInner {
/// The runtime handle used for sending messages to the engine.
handle: Handle,
model_name: String,
scheduler_stats_recorder: SchedulerStatsRecorder,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
health_error: ArcSwapOption<Error>,
@@ -43,10 +44,13 @@ impl ClientInner {
model_name: String,
engines: &[ConnectedEngine],
) -> Self {
let scheduler_stats_recorder =
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
Self {
input_send,
handle,
model_name,
scheduler_stats_recorder,
request_reg: Mutex::new(RequestRegistry::new(engines)),
utility_reg: Mutex::new(UtilityRegistry::default()),
health_error: ArcSwapOption::empty(),
@@ -389,12 +393,7 @@ pub(crate) async fn run_output_dispatcher_loop(
"dropping scheduler stats for unknown engine"
);
}
record_scheduler_stats(
&METRICS.scheduler,
inner.model_name(),
batch.engine_index,
scheduler_stats,
);
inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats);
}
// The engine's scheduler stats never carry adapter names;
@@ -45,6 +45,7 @@ impl Deref for EngineCoreStreamOutput {
/// `finish_reason` is non-`None`.
pub struct EngineCoreOutputStream {
request_id: String,
engine_index: u32,
abort_tx: mpsc::UnboundedSender<AbortRequest>,
state: State,
rx: OutputReceiver,
@@ -53,11 +54,13 @@ pub struct EngineCoreOutputStream {
impl EngineCoreOutputStream {
pub(crate) fn new(
request_id: String,
engine_index: u32,
abort_tx: mpsc::UnboundedSender<AbortRequest>,
rx: OutputReceiver,
) -> Self {
Self {
request_id,
engine_index,
abort_tx,
state: State::Running,
rx,
@@ -68,6 +71,11 @@ impl EngineCoreOutputStream {
pub fn request_id(&self) -> &str {
&self.request_id
}
/// Return the index of the engine that owns this request.
pub fn engine_index(&self) -> u32 {
self.engine_index
}
}
impl Stream for EngineCoreOutputStream {
+167 -81
View File
@@ -1,90 +1,190 @@
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH};
use vllm_metrics::{
EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics,
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
LoraInfoLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge,
WaitingReasonLabels,
};
use crate::protocol::stats::SchedulerStats;
use crate::transport::ConnectedEngine;
const WAITING_REASON_CAPACITY: &str = "capacity";
const WAITING_REASON_DEFERRED: &str = "deferred";
/// Record the scheduler-stats-backed metrics for one engine at one point in
/// time.
pub(crate) fn record_scheduler_stats(
metrics: &SchedulerMetrics,
model_name: impl Into<String>,
engine: u32,
stats: &SchedulerStats,
) {
let model_name = model_name.into();
let labels = EngineLabels {
model_name: model_name.clone(),
engine,
};
/// Cached scheduler-stats metric handles for all engines connected to one
/// frontend client.
pub(crate) struct SchedulerStatsRecorder {
engines: BTreeMap<u32, SchedulerStatsHandles>,
}
/// Per-engine cached metric handles used while recording `SchedulerStats`.
struct SchedulerStatsHandles {
// Base labels reused for dynamic child labels.
labels: EngineLabels,
// Scheduler state gauges.
metrics.scheduler_running.get_or_create(&labels).set(stats.num_running_reqs);
metrics
.scheduler_waiting
.get_or_create(&labels)
.set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
metrics
.scheduler_waiting_by_reason
.get_or_create(&WaitingReasonLabels {
model_name: model_name.clone(),
engine,
reason: WAITING_REASON_CAPACITY,
})
.set(stats.num_waiting_reqs);
metrics
.scheduler_waiting_by_reason
.get_or_create(&WaitingReasonLabels {
model_name: model_name.clone(),
engine,
reason: WAITING_REASON_DEFERRED,
})
.set(stats.num_skipped_waiting_reqs);
metrics.kv_cache_usage.get_or_create(&labels).set(stats.kv_cache_usage);
scheduler_running: U64Gauge,
scheduler_waiting: U64Gauge,
scheduler_waiting_capacity: U64Gauge,
scheduler_waiting_deferred: U64Gauge,
kv_cache_usage: F64Gauge,
// Prefix-cache counters, including the connector-backed external cache path.
metrics
.prefix_cache_queries
.get_or_create(&labels)
.inc_by(stats.prefix_cache_stats.base.queries);
metrics
.prefix_cache_hits
.get_or_create(&labels)
.inc_by(stats.prefix_cache_stats.base.hits);
prefix_cache_queries: U64Counter,
prefix_cache_hits: U64Counter,
external_prefix_cache_queries: U64Counter,
external_prefix_cache_hits: U64Counter,
// Speculative decoding counters.
spec_decode_num_drafts: U64Counter,
spec_decode_num_draft_tokens: U64Counter,
spec_decode_num_accepted_tokens: U64Counter,
spec_decode_num_accepted_tokens_per_pos: Family<EnginePositionLabels, U64Counter>,
// Per-engine performance / MFU counters.
estimated_flops_per_gpu: U64Counter,
estimated_read_bytes_per_gpu: U64Counter,
estimated_write_bytes_per_gpu: U64Counter,
// Sampled KV-cache residency histograms.
kv_block_lifetime_seconds: HistogramMetric,
kv_block_idle_before_evict_seconds: HistogramMetric,
kv_block_reuse_gap_seconds: HistogramMetric,
// Non-Prometheus interval accumulator for periodic text-log helpers.
log_stats: SchedulerLogStatsAccumulator,
}
impl SchedulerStatsRecorder {
/// Resolve the fixed-label metric handles for the connected engines.
pub(crate) fn new(
metrics: &SchedulerMetrics,
model_name: &str,
engines: &[ConnectedEngine],
) -> Self {
let engines = engines
.iter()
.filter_map(|engine| {
let engine = engine.engine_id.engine_index()?;
Some((
engine,
resolve_scheduler_stats_handles(metrics, model_name, engine),
))
})
.collect();
Self { engines }
}
/// Record one scheduler-stats payload for the given engine index.
pub(crate) fn record(&self, engine_index: u32, stats: &SchedulerStats) {
if let Some(handles) = self.engines.get(&engine_index) {
record_scheduler_stats_with_handles(handles, stats);
}
}
}
/// Resolve all fixed-label scheduler metrics for one engine.
fn resolve_scheduler_stats_handles(
metrics: &SchedulerMetrics,
model_name: &str,
engine: u32,
) -> SchedulerStatsHandles {
let labels = EngineLabels {
model_name: model_name.to_string(),
engine,
};
let capacity = WaitingReasonLabels {
model_name: model_name.to_string(),
engine,
reason: WAITING_REASON_CAPACITY,
};
let deferred = WaitingReasonLabels {
model_name: model_name.to_string(),
engine,
reason: WAITING_REASON_DEFERRED,
};
SchedulerStatsHandles {
scheduler_running: metrics.scheduler_running.get_or_create_owned(&labels),
scheduler_waiting: metrics.scheduler_waiting.get_or_create_owned(&labels),
scheduler_waiting_capacity: metrics
.scheduler_waiting_by_reason
.get_or_create_owned(&capacity),
scheduler_waiting_deferred: metrics
.scheduler_waiting_by_reason
.get_or_create_owned(&deferred),
kv_cache_usage: metrics.kv_cache_usage.get_or_create_owned(&labels),
prefix_cache_queries: metrics.prefix_cache_queries.get_or_create_owned(&labels),
prefix_cache_hits: metrics.prefix_cache_hits.get_or_create_owned(&labels),
external_prefix_cache_queries: metrics
.external_prefix_cache_queries
.get_or_create_owned(&labels),
external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels),
spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels),
spec_decode_num_draft_tokens: metrics
.spec_decode_num_draft_tokens
.get_or_create_owned(&labels),
spec_decode_num_accepted_tokens: metrics
.spec_decode_num_accepted_tokens
.get_or_create_owned(&labels),
spec_decode_num_accepted_tokens_per_pos: metrics
.spec_decode_num_accepted_tokens_per_pos
.clone(),
log_stats: metrics.log_stats.get_or_create_owned(&labels),
estimated_flops_per_gpu: metrics.estimated_flops_per_gpu.get_or_create_owned(&labels),
estimated_read_bytes_per_gpu: metrics
.estimated_read_bytes_per_gpu
.get_or_create_owned(&labels),
estimated_write_bytes_per_gpu: metrics
.estimated_write_bytes_per_gpu
.get_or_create_owned(&labels),
kv_block_lifetime_seconds: metrics.kv_block_lifetime_seconds.get_or_create_owned(&labels),
kv_block_idle_before_evict_seconds: metrics
.kv_block_idle_before_evict_seconds
.get_or_create_owned(&labels),
kv_block_reuse_gap_seconds: metrics.kv_block_reuse_gap_seconds.get_or_create_owned(&labels),
labels,
}
}
/// Record scheduler-stats values through pre-resolved metric handles.
fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &SchedulerStats) {
// Scheduler state gauges.
handles.scheduler_running.set(stats.num_running_reqs);
handles
.scheduler_waiting
.set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
handles.scheduler_waiting_capacity.set(stats.num_waiting_reqs);
handles.scheduler_waiting_deferred.set(stats.num_skipped_waiting_reqs);
handles.kv_cache_usage.set(stats.kv_cache_usage);
// Prefix-cache counters, including the connector-backed external cache path.
handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries);
handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits);
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
metrics
handles
.external_prefix_cache_queries
.get_or_create(&labels)
.inc_by(connector_prefix_cache_stats.base.queries);
metrics
handles
.external_prefix_cache_hits
.get_or_create(&labels)
.inc_by(connector_prefix_cache_stats.base.hits);
}
// Speculative decoding counters.
if let Some(spec_decoding_stats) = &stats.spec_decoding_stats {
metrics
.spec_decode_num_drafts
.get_or_create(&labels)
.inc_by(spec_decoding_stats.num_drafts);
metrics
handles.spec_decode_num_drafts.inc_by(spec_decoding_stats.num_drafts);
handles
.spec_decode_num_draft_tokens
.get_or_create(&labels)
.inc_by(spec_decoding_stats.num_draft_tokens);
metrics
handles
.spec_decode_num_accepted_tokens
.get_or_create(&labels)
.inc_by(spec_decoding_stats.num_accepted_tokens);
metrics.log_stats.get_or_create(&labels).observe_spec_decode(
handles.log_stats.observe_spec_decode(
spec_decoding_stats.num_drafts,
&spec_decoding_stats.num_accepted_tokens_per_pos,
);
@@ -92,11 +192,11 @@ pub(crate) fn record_scheduler_stats(
for (position, accepted_tokens) in
spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate()
{
metrics
handles
.spec_decode_num_accepted_tokens_per_pos
.get_or_create(&EnginePositionLabels {
model_name: model_name.clone(),
engine,
model_name: handles.labels.model_name.clone(),
engine: handles.labels.engine,
position: position as u32,
})
.inc_by(accepted_tokens);
@@ -109,22 +209,13 @@ pub(crate) fn record_scheduler_stats(
|| perf_stats.num_read_bytes_per_gpu != 0
|| perf_stats.num_write_bytes_per_gpu != 0)
{
metrics
.estimated_flops_per_gpu
.get_or_create(&labels)
.inc_by(perf_stats.num_flops_per_gpu);
metrics
.estimated_read_bytes_per_gpu
.get_or_create(&labels)
.inc_by(perf_stats.num_read_bytes_per_gpu);
metrics
.estimated_write_bytes_per_gpu
.get_or_create(&labels)
.inc_by(perf_stats.num_write_bytes_per_gpu);
handles.estimated_flops_per_gpu.inc_by(perf_stats.num_flops_per_gpu);
handles.estimated_read_bytes_per_gpu.inc_by(perf_stats.num_read_bytes_per_gpu);
handles.estimated_write_bytes_per_gpu.inc_by(perf_stats.num_write_bytes_per_gpu);
}
if let Some(cudagraph_stats) = &stats.cudagraph_stats {
metrics.log_stats.get_or_create(&labels).observe_cudagraph(
handles.log_stats.observe_cudagraph(
cudagraph_stats.num_unpadded_tokens,
cudagraph_stats.num_padded_tokens,
cudagraph_stats.num_paddings,
@@ -134,16 +225,11 @@ pub(crate) fn record_scheduler_stats(
// Sampled KV-cache residency histograms.
if !stats.kv_cache_eviction_events.is_empty() {
let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels);
let kv_block_idle_before_evict_seconds =
metrics.kv_block_idle_before_evict_seconds.get_or_create(&labels);
let kv_block_reuse_gap_seconds = metrics.kv_block_reuse_gap_seconds.get_or_create(&labels);
for event in &stats.kv_cache_eviction_events {
kv_block_lifetime_seconds.observe(event.lifetime_seconds);
kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
handles.kv_block_lifetime_seconds.observe(event.lifetime_seconds);
handles.kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
for reuse_gap_seconds in &event.reuse_gaps_seconds {
kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
handles.kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
}
}
}