[Metrics] Expose cached prompt tokens by cache tier (#56318)

Signed-off-by: Cam Quilici <cjquilici@gmail.com>
Signed-off-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Cam Quilici <cameron@semianalysis.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Cameron Quilici
2026-10-06 14:52:05 -04:00
committed by GitHub
co-authored by Cam Quilici Nick Hill Yifan Qiao
parent ad0f67a7ec
commit e37e51dd24
36 changed files with 1005 additions and 44 deletions
+23 -5
View File
@@ -6,14 +6,15 @@ use std::collections::BTreeSet;
use std::time::{SystemTime, UNIX_EPOCH};
use vllm_metrics::{
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily,
MooncakeOperationLabels, RequestMetrics, SchedulerLogStatsAccumulator, SchedulerMetrics,
U64Counter, U64Gauge, WaitingReasonLabels,
CacheHitSourceLabels, EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric,
LoraAdapterNames, LoraInfoLabels, MooncakeOperationCounterFamily,
MooncakeOperationHistogramFamily, MooncakeOperationLabels, RequestMetrics,
SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge, WaitingReasonLabels,
};
use crate::protocol::stats::{
KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats, SchedulerStats,
CachedTokensBySource, KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats,
SchedulerStats,
};
use crate::transport::ConnectedEngine;
@@ -60,6 +61,8 @@ struct SchedulerStatsHandles {
prefix_cache_hits: U64Counter,
external_prefix_cache_queries: U64Counter,
external_prefix_cache_hits: U64Counter,
/// One handle per `CachedTokensBySource::SOURCES` entry.
prompt_tokens_cached_by_source: [U64Counter; 5],
// Speculative decoding counters.
spec_decode_num_drafts: U64Counter,
@@ -165,6 +168,15 @@ fn resolve_scheduler_stats_handles(
.external_prefix_cache_queries
.get_or_create_owned(&labels),
external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels),
prompt_tokens_cached_by_source: CachedTokensBySource::SOURCES.map(|source| {
metrics
.prompt_tokens_cached_by_source
.get_or_create_owned(&CacheHitSourceLabels {
model_name: model_name.to_string(),
engine,
source,
})
}),
spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels),
spec_decode_num_draft_tokens: metrics
.spec_decode_num_draft_tokens
@@ -220,6 +232,8 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
// Prefix-cache counters, including the connector-backed external cache path.
handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries);
handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits);
// Local prefix-cache hits are the `device` tier.
handles.prompt_tokens_cached_by_source[0].inc_by(stats.prefix_cache_stats.base.hits);
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
handles
@@ -228,6 +242,10 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
handles
.external_prefix_cache_hits
.inc_by(connector_prefix_cache_stats.base.hits);
let counts = connector_prefix_cache_stats.hits_by_source.counts();
for (handle, count) in handles.prompt_tokens_cached_by_source.iter().zip(counts) {
handle.inc_by(count);
}
}
// Speculative decoding counters.
@@ -21,6 +21,34 @@ pub struct BaseCacheStats {
pub hits: u64,
}
/// Cache-hit tokens per source tier. Python sends a sparse
/// `dict[CacheHitSource, int]`, so absent tiers default to zero.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct CachedTokensBySource {
pub device: u64,
pub host: u64,
pub p2p: u64,
pub disk: u64,
pub external_unspecified: u64,
}
impl CachedTokensBySource {
/// Label values, in the same order as [`Self::counts`].
pub const SOURCES: [&'static str; 5] =
["device", "host", "p2p", "disk", "external_unspecified"];
pub fn counts(&self) -> [u64; 5] {
[
self.device,
self.host,
self.p2p,
self.disk,
self.external_unspecified,
]
}
}
/// Stores prefix cache hit statistics.
/// - `reset`: Whether `reset_prefix_cache` was invoked.
/// - `queries`: Refers to the number of tokens that were queried.
@@ -38,6 +66,9 @@ pub struct PrefixCacheStats {
pub preempted_queries: u64,
/// The `hits` number for preempted requests.
pub preempted_hits: u64,
/// `hits` split by the cache tier that supplied them (connector stats only).
#[serde(default)]
pub hits_by_source: CachedTokensBySource,
}
/// Single KV cache block eviction sample.
@@ -304,3 +335,31 @@ pub struct SchedulerStats {
/// Estimated MFU/performance stats, when enabled.
pub perf_stats: Option<PerfStats>,
}
#[cfg(test)]
mod tests {
use super::{CachedTokensBySource, PrefixCacheStats};
#[test]
fn prefix_cache_stats_hits_by_source_defaults_and_decodes() {
// Engines without per-source attribution omit the field entirely.
let legacy = serde_json::json!({
"reset": false, "requests": 1, "queries": 8, "hits": 4,
"preempted_requests": 0, "preempted_queries": 0, "preempted_hits": 0
});
let wire = rmp_serde::to_vec_named(&legacy).unwrap();
let stats: PrefixCacheStats = rmp_serde::from_slice(&wire).unwrap();
assert_eq!(stats.base.hits, 4);
assert_eq!(stats.hits_by_source, CachedTokensBySource::default());
let payload = serde_json::json!({
"reset": false, "requests": 1, "queries": 16, "hits": 12,
"preempted_requests": 0, "preempted_queries": 0, "preempted_hits": 0,
"hits_by_source": {"host": 8, "p2p": 4}
});
let wire = rmp_serde::to_vec_named(&payload).unwrap();
let stats: PrefixCacheStats = rmp_serde::from_slice(&wire).unwrap();
assert_eq!(stats.hits_by_source.counts(), [0, 8, 4, 0, 0]);
assert_eq!(CachedTokensBySource::SOURCES[1], "host");
}
}
+16
View File
@@ -86,6 +86,13 @@ pub struct EngineLabels {
pub engine: u32,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct CacheHitSourceLabels {
pub model_name: String,
pub engine: u32,
pub source: &'static str,
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
pub struct EnginePositionLabels {
pub model_name: String,
@@ -236,6 +243,7 @@ pub struct SchedulerMetrics {
pub prefix_cache_hits: Family<EngineLabels, U64Counter>,
pub external_prefix_cache_queries: Family<EngineLabels, U64Counter>,
pub external_prefix_cache_hits: Family<EngineLabels, U64Counter>,
pub prompt_tokens_cached_by_source: Family<CacheHitSourceLabels, U64Counter>,
// Speculative decoding counters.
pub spec_decode_num_drafts: Family<EngineLabels, U64Counter>,
@@ -339,6 +347,13 @@ impl SchedulerMetrics {
);
let external_prefix_cache_hits = Family::default();
let prompt_tokens_cached_by_source = Family::default();
registry.register(
"vllm:prompt_tokens_cached_by_source",
"Prefix-cache hit tokens by the cache tier that supplied them: device (local HBM), host (offloaded to DRAM), disk, p2p (transferred from another vLLM instance) or external_unspecified. Counted at admission; sums to vllm:prefix_cache_hits + vllm:external_prefix_cache_hits.",
prompt_tokens_cached_by_source.clone(),
);
registry.register(
"vllm:external_prefix_cache_hits",
"External prefix cache hits from KV connector cross-instance cache sharing, in terms of number of cached tokens.",
@@ -528,6 +543,7 @@ impl SchedulerMetrics {
prefix_cache_hits,
external_prefix_cache_queries,
external_prefix_cache_hits,
prompt_tokens_cached_by_source,
spec_decode_num_drafts,
spec_decode_num_draft_tokens,
spec_decode_num_accepted_tokens,