[Rust Frontend] Improve scheduler stats logging parity (#47435)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-07-02 15:55:25 +01:00
committed by GitHub
parent 178fd56094
commit ec0ffaacc8
6 changed files with 521 additions and 30 deletions
+119 -2
View File
@@ -1,4 +1,5 @@
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, Mutex};
use itertools::Itertools as _;
use prometheus_client::encoding::{EncodeLabelSet, EncodeLabelValue, LabelValueEncoder};
@@ -62,6 +63,90 @@ pub struct LoraInfoLabels {
pub waiting_lora_adapters: LoraAdapterNames,
}
/// CUDA graph sample key used for periodic text-log aggregation.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct CudagraphLogKey {
pub num_unpadded_tokens: u64,
pub num_padded_tokens: u64,
pub num_paddings: u64,
pub runtime_mode: String,
}
/// Raw scheduler stats accumulated for one periodic text-log interval.
#[derive(Default)]
pub struct SchedulerLogStatsInterval {
pub spec_num_drafts: u64,
pub spec_accepted_tokens_per_pos: Vec<u64>,
pub cudagraph_counts: BTreeMap<CudagraphLogKey, u64>,
}
impl SchedulerLogStatsInterval {
/// Merge another drained interval into this one.
pub fn merge(&mut self, other: Self) {
self.spec_num_drafts += other.spec_num_drafts;
if self.spec_accepted_tokens_per_pos.len() < other.spec_accepted_tokens_per_pos.len() {
self.spec_accepted_tokens_per_pos
.resize(other.spec_accepted_tokens_per_pos.len(), 0);
}
for (position, accepted_tokens) in
other.spec_accepted_tokens_per_pos.into_iter().enumerate()
{
self.spec_accepted_tokens_per_pos[position] += accepted_tokens;
}
for (key, count) in other.cudagraph_counts {
*self.cudagraph_counts.entry(key).or_default() += count;
}
}
}
/// Internal, non-Prometheus accumulator for periodic text logs that need raw
/// scheduler DTOs.
#[derive(Clone, Default)]
pub struct SchedulerLogStatsAccumulator {
inner: Arc<Mutex<SchedulerLogStatsInterval>>,
}
impl SchedulerLogStatsAccumulator {
/// Observe spec-decoding fields needed for per-position text-log rates.
pub fn observe_spec_decode(&self, num_drafts: u64, accepted_tokens_per_pos: &[u64]) {
let mut inner = self.inner.lock().expect("scheduler log stats accumulator poisoned");
inner.spec_num_drafts += num_drafts;
if inner.spec_accepted_tokens_per_pos.len() < accepted_tokens_per_pos.len() {
inner.spec_accepted_tokens_per_pos.resize(accepted_tokens_per_pos.len(), 0);
}
for (position, accepted_tokens) in accepted_tokens_per_pos.iter().copied().enumerate() {
inner.spec_accepted_tokens_per_pos[position] += accepted_tokens;
}
}
/// Observe one CUDA graph runtime sample for the interval table.
pub fn observe_cudagraph(
&self,
num_unpadded_tokens: u64,
num_padded_tokens: u64,
num_paddings: u64,
runtime_mode: &str,
) {
let mut inner = self.inner.lock().expect("scheduler log stats accumulator poisoned");
let key = CudagraphLogKey {
num_unpadded_tokens,
num_padded_tokens,
num_paddings,
runtime_mode: runtime_mode.to_string(),
};
*inner.cudagraph_counts.entry(key).or_default() += 1;
}
/// Drain and reset the current text-log interval.
pub fn drain(&self) -> SchedulerLogStatsInterval {
let mut inner = self.inner.lock().expect("scheduler log stats accumulator poisoned");
std::mem::take(&mut *inner)
}
}
/// Scheduler/batch-scoped Prometheus families exported from `SchedulerStats`.
pub struct SchedulerMetrics {
// Scheduler state gauges.
@@ -95,6 +180,9 @@ pub struct SchedulerMetrics {
pub kv_block_lifetime_seconds: HistogramFamily,
pub kv_block_idle_before_evict_seconds: HistogramFamily,
pub kv_block_reuse_gap_seconds: HistogramFamily,
/// Non-Prometheus interval accumulators for periodic text-log helpers.
pub log_stats: Family<EngineLabels, SchedulerLogStatsAccumulator>,
}
impl SchedulerMetrics {
@@ -265,13 +353,14 @@ impl SchedulerMetrics {
kv_block_lifetime_seconds,
kv_block_idle_before_evict_seconds,
kv_block_reuse_gap_seconds,
log_stats: Family::default(),
}
}
}
#[cfg(test)]
mod tests {
use crate::{EngineLabels, Metrics};
use crate::{CudagraphLogKey, EngineLabels, Metrics, SchedulerLogStatsAccumulator};
#[test]
fn perf_counters_render_with_a_single_total_suffix() {
@@ -301,4 +390,32 @@ mod tests {
assert!(!rendered.contains("vllm:estimated_read_bytes_per_gpu_total_total"));
assert!(!rendered.contains("vllm:estimated_write_bytes_per_gpu_total_total"));
}
#[test]
fn log_stats_accumulator_drains_interval_data() {
let accumulator = SchedulerLogStatsAccumulator::default();
accumulator.observe_spec_decode(2, &[1, 2]);
accumulator.observe_spec_decode(3, &[3, 4, 5]);
accumulator.observe_cudagraph(8, 16, 8, "FULL");
accumulator.observe_cudagraph(8, 16, 8, "FULL");
let interval = accumulator.drain();
assert_eq!(interval.spec_num_drafts, 5);
assert_eq!(interval.spec_accepted_tokens_per_pos, vec![4, 6, 5]);
assert_eq!(
interval
.cudagraph_counts
.get(&CudagraphLogKey {
num_unpadded_tokens: 8,
num_padded_tokens: 16,
num_paddings: 8,
runtime_mode: "FULL".to_string(),
})
.copied(),
Some(2)
);
assert_eq!(accumulator.drain().spec_num_drafts, 0);
}
}