[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -6,14 +6,15 @@ use std::collections::BTreeSet;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use vllm_metrics::{
|
||||
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
|
||||
LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily,
|
||||
MooncakeOperationLabels, RequestMetrics, SchedulerLogStatsAccumulator, SchedulerMetrics,
|
||||
U64Counter, U64Gauge, WaitingReasonLabels,
|
||||
CacheHitSourceLabels, EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric,
|
||||
LoraAdapterNames, LoraInfoLabels, MooncakeOperationCounterFamily,
|
||||
MooncakeOperationHistogramFamily, MooncakeOperationLabels, RequestMetrics,
|
||||
SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge, WaitingReasonLabels,
|
||||
};
|
||||
|
||||
use crate::protocol::stats::{
|
||||
KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats, SchedulerStats,
|
||||
CachedTokensBySource, KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats,
|
||||
SchedulerStats,
|
||||
};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
@@ -60,6 +61,8 @@ struct SchedulerStatsHandles {
|
||||
prefix_cache_hits: U64Counter,
|
||||
external_prefix_cache_queries: U64Counter,
|
||||
external_prefix_cache_hits: U64Counter,
|
||||
/// One handle per `CachedTokensBySource::SOURCES` entry.
|
||||
prompt_tokens_cached_by_source: [U64Counter; 5],
|
||||
|
||||
// Speculative decoding counters.
|
||||
spec_decode_num_drafts: U64Counter,
|
||||
@@ -165,6 +168,15 @@ fn resolve_scheduler_stats_handles(
|
||||
.external_prefix_cache_queries
|
||||
.get_or_create_owned(&labels),
|
||||
external_prefix_cache_hits: metrics.external_prefix_cache_hits.get_or_create_owned(&labels),
|
||||
prompt_tokens_cached_by_source: CachedTokensBySource::SOURCES.map(|source| {
|
||||
metrics
|
||||
.prompt_tokens_cached_by_source
|
||||
.get_or_create_owned(&CacheHitSourceLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
source,
|
||||
})
|
||||
}),
|
||||
spec_decode_num_drafts: metrics.spec_decode_num_drafts.get_or_create_owned(&labels),
|
||||
spec_decode_num_draft_tokens: metrics
|
||||
.spec_decode_num_draft_tokens
|
||||
@@ -220,6 +232,8 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
|
||||
// Prefix-cache counters, including the connector-backed external cache path.
|
||||
handles.prefix_cache_queries.inc_by(stats.prefix_cache_stats.base.queries);
|
||||
handles.prefix_cache_hits.inc_by(stats.prefix_cache_stats.base.hits);
|
||||
// Local prefix-cache hits are the `device` tier.
|
||||
handles.prompt_tokens_cached_by_source[0].inc_by(stats.prefix_cache_stats.base.hits);
|
||||
|
||||
if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
|
||||
handles
|
||||
@@ -228,6 +242,10 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
|
||||
handles
|
||||
.external_prefix_cache_hits
|
||||
.inc_by(connector_prefix_cache_stats.base.hits);
|
||||
let counts = connector_prefix_cache_stats.hits_by_source.counts();
|
||||
for (handle, count) in handles.prompt_tokens_cached_by_source.iter().zip(counts) {
|
||||
handle.inc_by(count);
|
||||
}
|
||||
}
|
||||
|
||||
// Speculative decoding counters.
|
||||
|
||||
@@ -21,6 +21,34 @@ pub struct BaseCacheStats {
|
||||
pub hits: u64,
|
||||
}
|
||||
|
||||
/// Cache-hit tokens per source tier. Python sends a sparse
|
||||
/// `dict[CacheHitSource, int]`, so absent tiers default to zero.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct CachedTokensBySource {
|
||||
pub device: u64,
|
||||
pub host: u64,
|
||||
pub p2p: u64,
|
||||
pub disk: u64,
|
||||
pub external_unspecified: u64,
|
||||
}
|
||||
|
||||
impl CachedTokensBySource {
|
||||
/// Label values, in the same order as [`Self::counts`].
|
||||
pub const SOURCES: [&'static str; 5] =
|
||||
["device", "host", "p2p", "disk", "external_unspecified"];
|
||||
|
||||
pub fn counts(&self) -> [u64; 5] {
|
||||
[
|
||||
self.device,
|
||||
self.host,
|
||||
self.p2p,
|
||||
self.disk,
|
||||
self.external_unspecified,
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores prefix cache hit statistics.
|
||||
/// - `reset`: Whether `reset_prefix_cache` was invoked.
|
||||
/// - `queries`: Refers to the number of tokens that were queried.
|
||||
@@ -38,6 +66,9 @@ pub struct PrefixCacheStats {
|
||||
pub preempted_queries: u64,
|
||||
/// The `hits` number for preempted requests.
|
||||
pub preempted_hits: u64,
|
||||
/// `hits` split by the cache tier that supplied them (connector stats only).
|
||||
#[serde(default)]
|
||||
pub hits_by_source: CachedTokensBySource,
|
||||
}
|
||||
|
||||
/// Single KV cache block eviction sample.
|
||||
@@ -304,3 +335,31 @@ pub struct SchedulerStats {
|
||||
/// Estimated MFU/performance stats, when enabled.
|
||||
pub perf_stats: Option<PerfStats>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CachedTokensBySource, PrefixCacheStats};
|
||||
|
||||
#[test]
|
||||
fn prefix_cache_stats_hits_by_source_defaults_and_decodes() {
|
||||
// Engines without per-source attribution omit the field entirely.
|
||||
let legacy = serde_json::json!({
|
||||
"reset": false, "requests": 1, "queries": 8, "hits": 4,
|
||||
"preempted_requests": 0, "preempted_queries": 0, "preempted_hits": 0
|
||||
});
|
||||
let wire = rmp_serde::to_vec_named(&legacy).unwrap();
|
||||
let stats: PrefixCacheStats = rmp_serde::from_slice(&wire).unwrap();
|
||||
assert_eq!(stats.base.hits, 4);
|
||||
assert_eq!(stats.hits_by_source, CachedTokensBySource::default());
|
||||
|
||||
let payload = serde_json::json!({
|
||||
"reset": false, "requests": 1, "queries": 16, "hits": 12,
|
||||
"preempted_requests": 0, "preempted_queries": 0, "preempted_hits": 0,
|
||||
"hits_by_source": {"host": 8, "p2p": 4}
|
||||
});
|
||||
let wire = rmp_serde::to_vec_named(&payload).unwrap();
|
||||
let stats: PrefixCacheStats = rmp_serde::from_slice(&wire).unwrap();
|
||||
assert_eq!(stats.hits_by_source.counts(), [0, 8, 4, 0, 0]);
|
||||
assert_eq!(CachedTokensBySource::SOURCES[1], "host");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user