[Rust Frontend] Record Mooncake/NIXL KV-connector metrics (#52755)
Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Markov Ilya <markovilya197@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Ilya Markov
co-authored by
Bugen Zhao
Bugen Zhao
parent
8a728663c1
commit
ae2d1ca96f
@@ -7,11 +7,14 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use vllm_metrics::{
|
||||
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
|
||||
LoraInfoLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge,
|
||||
LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily,
|
||||
MooncakeOperationLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge,
|
||||
WaitingReasonLabels,
|
||||
};
|
||||
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::stats::{
|
||||
KvConnectorStats, MooncakeStats, MultiConnectorStats, NixlStats, SchedulerStats,
|
||||
};
|
||||
use crate::transport::ConnectedEngine;
|
||||
|
||||
const WAITING_REASON_CAPACITY: &str = "capacity";
|
||||
@@ -57,6 +60,24 @@ struct SchedulerStatsHandles {
|
||||
kv_block_idle_before_evict_seconds: HistogramMetric,
|
||||
kv_block_reuse_gap_seconds: HistogramMetric,
|
||||
|
||||
// Mooncake store connector telemetry, decoded from `kv_connector_stats`.
|
||||
// Kept as `Family` (not pre-resolved) because `operation`/`status` are
|
||||
// dynamic per-record labels.
|
||||
mooncake_operation_time_seconds: MooncakeOperationHistogramFamily,
|
||||
mooncake_operation_total: MooncakeOperationCounterFamily,
|
||||
mooncake_operation_keys_total: MooncakeOperationCounterFamily,
|
||||
mooncake_operation_bytes_total: MooncakeOperationCounterFamily,
|
||||
mooncake_operation_failed_keys_total: MooncakeOperationCounterFamily,
|
||||
|
||||
// NIXL connector telemetry, decoded from `kv_connector_stats`.
|
||||
nixl_xfer_time_seconds: HistogramMetric,
|
||||
nixl_post_time_seconds: HistogramMetric,
|
||||
nixl_bytes_transferred: HistogramMetric,
|
||||
nixl_num_descriptors: HistogramMetric,
|
||||
nixl_num_failed_transfers: U64Counter,
|
||||
nixl_num_failed_notifications: U64Counter,
|
||||
nixl_num_kv_expired_reqs: U64Counter,
|
||||
|
||||
// Non-Prometheus interval accumulator for periodic text-log helpers.
|
||||
log_stats: SchedulerLogStatsAccumulator,
|
||||
}
|
||||
@@ -150,6 +171,20 @@ fn resolve_scheduler_stats_handles(
|
||||
.kv_block_idle_before_evict_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
kv_block_reuse_gap_seconds: metrics.kv_block_reuse_gap_seconds.get_or_create_owned(&labels),
|
||||
mooncake_operation_time_seconds: metrics.mooncake_operation_time_seconds.clone(),
|
||||
mooncake_operation_total: metrics.mooncake_operation_total.clone(),
|
||||
mooncake_operation_keys_total: metrics.mooncake_operation_keys_total.clone(),
|
||||
mooncake_operation_bytes_total: metrics.mooncake_operation_bytes_total.clone(),
|
||||
mooncake_operation_failed_keys_total: metrics.mooncake_operation_failed_keys_total.clone(),
|
||||
nixl_xfer_time_seconds: metrics.nixl_xfer_time_seconds.get_or_create_owned(&labels),
|
||||
nixl_post_time_seconds: metrics.nixl_post_time_seconds.get_or_create_owned(&labels),
|
||||
nixl_bytes_transferred: metrics.nixl_bytes_transferred.get_or_create_owned(&labels),
|
||||
nixl_num_descriptors: metrics.nixl_num_descriptors.get_or_create_owned(&labels),
|
||||
nixl_num_failed_transfers: metrics.nixl_num_failed_transfers.get_or_create_owned(&labels),
|
||||
nixl_num_failed_notifications: metrics
|
||||
.nixl_num_failed_notifications
|
||||
.get_or_create_owned(&labels),
|
||||
nixl_num_kv_expired_reqs: metrics.nixl_num_kv_expired_reqs.get_or_create_owned(&labels),
|
||||
labels,
|
||||
}
|
||||
}
|
||||
@@ -236,6 +271,79 @@ fn record_scheduler_stats_with_handles(handles: &SchedulerStatsHandles, stats: &
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Connector-specific KV transfer stats. A bare connector reports its own
|
||||
// flat payload; MultiConnector reports connector class name -> flat child
|
||||
// payload.
|
||||
if let Some(kv_connector_stats) = &stats.kv_connector_stats {
|
||||
match kv_connector_stats {
|
||||
KvConnectorStats::Nixl(stats) => record_nixl_stats(handles, stats),
|
||||
KvConnectorStats::Mooncake(stats) => record_mooncake_stats(handles, stats),
|
||||
KvConnectorStats::Multi(stats) => record_multi_connector_stats(handles, stats),
|
||||
KvConnectorStats::Other(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_multi_connector_stats(handles: &SchedulerStatsHandles, stats: &MultiConnectorStats) {
|
||||
for nixl in [&stats.nixl, &stats.nixl_pull, &stats.nixl_push].into_iter().flatten() {
|
||||
record_nixl_stats(handles, nixl);
|
||||
}
|
||||
if let Some(mooncake) = &stats.mooncake {
|
||||
record_mooncake_stats(handles, mooncake);
|
||||
}
|
||||
}
|
||||
|
||||
fn record_mooncake_stats(handles: &SchedulerStatsHandles, stats: &MooncakeStats) {
|
||||
for (operation, records) in &stats.0 {
|
||||
for record in records {
|
||||
let labels = MooncakeOperationLabels {
|
||||
model_name: handles.labels.model_name.clone(),
|
||||
engine: handles.labels.engine,
|
||||
operation: operation.as_str().to_string(),
|
||||
status: record.status.as_str().to_string(),
|
||||
};
|
||||
handles
|
||||
.mooncake_operation_time_seconds
|
||||
.get_or_create(&labels)
|
||||
.observe(record.duration_seconds);
|
||||
handles.mooncake_operation_total.get_or_create(&labels).inc();
|
||||
handles
|
||||
.mooncake_operation_keys_total
|
||||
.get_or_create(&labels)
|
||||
.inc_by(record.num_keys);
|
||||
handles
|
||||
.mooncake_operation_bytes_total
|
||||
.get_or_create(&labels)
|
||||
.inc_by(record.num_bytes);
|
||||
handles
|
||||
.mooncake_operation_failed_keys_total
|
||||
.get_or_create(&labels)
|
||||
.inc_by(record.num_failed_keys);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_nixl_stats(handles: &SchedulerStatsHandles, stats: &NixlStats) {
|
||||
for value in &stats.transfer_duration {
|
||||
handles.nixl_xfer_time_seconds.observe(*value);
|
||||
}
|
||||
for value in &stats.post_duration {
|
||||
handles.nixl_post_time_seconds.observe(*value);
|
||||
}
|
||||
for value in &stats.bytes_transferred {
|
||||
handles.nixl_bytes_transferred.observe(*value as f64);
|
||||
}
|
||||
for value in &stats.num_descriptors {
|
||||
handles.nixl_num_descriptors.observe(*value as f64);
|
||||
}
|
||||
handles
|
||||
.nixl_num_failed_transfers
|
||||
.inc_by(stats.num_failed_transfers.iter().sum());
|
||||
handles
|
||||
.nixl_num_failed_notifications
|
||||
.inc_by(stats.num_failed_notifications.iter().sum());
|
||||
handles.nixl_num_kv_expired_reqs.inc_by(stats.num_kv_expired_reqs.iter().sum());
|
||||
}
|
||||
|
||||
/// Exports `vllm:lora_requests_info` as a single series covering all LoRA
|
||||
@@ -286,12 +394,16 @@ fn now_unix_secs() -> f64 {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use expect_test::expect;
|
||||
use vllm_metrics::Metrics;
|
||||
|
||||
use crate::metrics::LoraInfoExporter;
|
||||
use crate::protocol::stats::{
|
||||
KvConnectorStats, MooncakeOperation, MooncakeRecord, MooncakeStats, MooncakeStatus,
|
||||
MultiConnectorStats, NixlStats, SchedulerStats,
|
||||
};
|
||||
|
||||
fn names(values: &[&str]) -> BTreeSet<String> {
|
||||
values.iter().map(|name| (*name).to_string()).collect()
|
||||
@@ -345,4 +457,70 @@ mod tests {
|
||||
exporter.update(&metrics.scheduler, names(&[]), names(&[]));
|
||||
expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
|
||||
}
|
||||
|
||||
fn nixl_stats() -> NixlStats {
|
||||
NixlStats {
|
||||
transfer_duration: vec![0.01, 0.02],
|
||||
post_duration: vec![0.001, 0.002],
|
||||
bytes_transferred: vec![4096, 8192],
|
||||
num_descriptors: vec![2, 4],
|
||||
num_failed_transfers: vec![],
|
||||
num_failed_notifications: vec![],
|
||||
num_kv_expired_reqs: vec![1],
|
||||
}
|
||||
}
|
||||
|
||||
fn mooncake_stats() -> MooncakeStats {
|
||||
MooncakeStats(BTreeMap::from([(
|
||||
MooncakeOperation::LoadGet,
|
||||
vec![MooncakeRecord {
|
||||
duration_seconds: 0.05,
|
||||
num_keys: 3,
|
||||
num_bytes: 1024,
|
||||
status: MooncakeStatus::Ok,
|
||||
num_failed_keys: 0,
|
||||
}],
|
||||
)]))
|
||||
}
|
||||
|
||||
/// Records one MultiConnector payload into Mooncake and NIXL metrics.
|
||||
#[test]
|
||||
fn kv_connector_stats_are_recorded_into_mooncake_and_nixl_metrics() {
|
||||
let metrics = Metrics::new();
|
||||
let handles = super::resolve_scheduler_stats_handles(&metrics.scheduler, "model", 0);
|
||||
|
||||
let stats = SchedulerStats {
|
||||
kv_connector_stats: Some(KvConnectorStats::Multi(Box::new(MultiConnectorStats {
|
||||
nixl: Some(nixl_stats()),
|
||||
mooncake: Some(mooncake_stats()),
|
||||
..Default::default()
|
||||
}))),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
super::record_scheduler_stats_with_handles(&handles, &stats);
|
||||
|
||||
let rendered = metrics.render().unwrap();
|
||||
assert!(rendered.contains(
|
||||
"vllm:mooncake_store_operation_total{model_name=\"model\",engine=\"0\",\
|
||||
operation=\"load_get\",status=\"ok\"} 1"
|
||||
));
|
||||
assert!(rendered.contains(
|
||||
"vllm:mooncake_store_operation_keys_total{model_name=\"model\",engine=\"0\",\
|
||||
operation=\"load_get\",status=\"ok\"} 3"
|
||||
));
|
||||
assert!(rendered.contains(
|
||||
"vllm:mooncake_store_operation_bytes_total{model_name=\"model\",engine=\"0\",\
|
||||
operation=\"load_get\",status=\"ok\"} 1024"
|
||||
));
|
||||
assert!(
|
||||
rendered.contains(
|
||||
"vllm:nixl_num_kv_expired_reqs_total{model_name=\"model\",engine=\"0\"} 1"
|
||||
)
|
||||
);
|
||||
assert!(
|
||||
rendered
|
||||
.contains("vllm:nixl_xfer_time_seconds_count{model_name=\"model\",engine=\"0\"} 2")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,6 @@ use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::OpaqueValue;
|
||||
|
||||
/// Stores cache hit statistics.
|
||||
///
|
||||
/// Original Python definition:
|
||||
@@ -158,6 +156,120 @@ pub struct CudagraphStats {
|
||||
pub runtime_mode: String,
|
||||
}
|
||||
|
||||
/// KV connector telemetry DTOs carried by [`SchedulerStats`].
|
||||
pub mod kv_connector {
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::OpaqueValue;
|
||||
|
||||
/// NIXL connector transfer telemetry.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct NixlStats {
|
||||
/// Time spent transferring each successful request.
|
||||
pub transfer_duration: Vec<f64>,
|
||||
/// Time spent posting each successful transfer.
|
||||
pub post_duration: Vec<f64>,
|
||||
/// Bytes transferred by each successful request.
|
||||
pub bytes_transferred: Vec<u64>,
|
||||
/// Descriptor count for each successful request.
|
||||
pub num_descriptors: Vec<u64>,
|
||||
/// Failure counter increments collected since the previous update.
|
||||
pub num_failed_transfers: Vec<u64>,
|
||||
/// Notification failure counter increments collected since the previous update.
|
||||
pub num_failed_notifications: Vec<u64>,
|
||||
/// Expired-request counter increments collected since the previous update.
|
||||
pub num_kv_expired_reqs: Vec<u64>,
|
||||
}
|
||||
|
||||
/// Mooncake store operation name.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MooncakeOperation {
|
||||
SaveExists,
|
||||
SavePut,
|
||||
LoadGet,
|
||||
LookupExists,
|
||||
}
|
||||
|
||||
impl MooncakeOperation {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::SaveExists => "save_exists",
|
||||
Self::SavePut => "save_put",
|
||||
Self::LoadGet => "load_get",
|
||||
Self::LookupExists => "lookup_exists",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mooncake store operation status.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum MooncakeStatus {
|
||||
Ok,
|
||||
Error,
|
||||
PartialFailure,
|
||||
}
|
||||
|
||||
impl MooncakeStatus {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Ok => "ok",
|
||||
Self::Error => "error",
|
||||
Self::PartialFailure => "partial_failure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One Mooncake store operation sample.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct MooncakeRecord {
|
||||
pub duration_seconds: f64,
|
||||
pub num_keys: u64,
|
||||
pub num_bytes: u64,
|
||||
pub status: MooncakeStatus,
|
||||
pub num_failed_keys: u64,
|
||||
}
|
||||
|
||||
/// Mooncake store telemetry grouped by operation.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(transparent)]
|
||||
pub struct MooncakeStats(pub BTreeMap<MooncakeOperation, Vec<MooncakeRecord>>);
|
||||
|
||||
/// Telemetry emitted by a `MultiConnector` with a flat child payload map.
|
||||
#[serde_with::skip_serializing_none]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct MultiConnectorStats {
|
||||
#[serde(rename = "NixlConnector")]
|
||||
pub nixl: Option<NixlStats>,
|
||||
#[serde(rename = "NixlPullConnector")]
|
||||
pub nixl_pull: Option<NixlStats>,
|
||||
#[serde(rename = "NixlPushConnector")]
|
||||
pub nixl_push: Option<NixlStats>,
|
||||
#[serde(rename = "MooncakeStoreConnector")]
|
||||
pub mooncake: Option<MooncakeStats>,
|
||||
/// Child connector payloads without a Rust telemetry implementation.
|
||||
#[serde(flatten)]
|
||||
pub other: BTreeMap<String, OpaqueValue>,
|
||||
}
|
||||
|
||||
/// Connector-specific scheduler telemetry.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum KvConnectorStats {
|
||||
Nixl(NixlStats),
|
||||
Mooncake(MooncakeStats),
|
||||
Multi(Box<MultiConnectorStats>),
|
||||
Other(BTreeMap<String, OpaqueValue>),
|
||||
}
|
||||
}
|
||||
|
||||
pub use kv_connector::*;
|
||||
|
||||
/// Stats associated with the scheduler.
|
||||
///
|
||||
/// Original Python definition:
|
||||
@@ -185,8 +297,8 @@ pub struct SchedulerStats {
|
||||
pub kv_cache_eviction_events: Vec<KvCacheEvictionEvent>,
|
||||
/// Speculative decoding scheduler stats, when enabled.
|
||||
pub spec_decoding_stats: Option<SpecDecodingStats>,
|
||||
/// Connector-specific KV transfer stats, kept opaque for now.
|
||||
pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
|
||||
/// Connector-specific KV transfer stats.
|
||||
pub kv_connector_stats: Option<KvConnectorStats>,
|
||||
/// CUDA graph runtime stats when graph metrics are enabled.
|
||||
pub cudagraph_stats: Option<CudagraphStats>,
|
||||
/// Estimated MFU/performance stats, when enabled.
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::protocol::output::{
|
||||
};
|
||||
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
|
||||
use crate::protocol::sampling::EngineCoreSamplingParams;
|
||||
use crate::protocol::stats::SchedulerStats;
|
||||
use crate::protocol::stats::{KvConnectorStats, MooncakeOperation, SchedulerStats};
|
||||
use crate::protocol::tensor::{WireArrayData, WireTensor};
|
||||
use crate::protocol::utility::{UtilityOutput, UtilityResultEnvelope};
|
||||
use crate::test_utils::{
|
||||
@@ -2635,6 +2635,10 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
let inline_prompt_frames = lines.next().expect("missing inline prompt logprobs fixture line");
|
||||
let multipart_prompt_frames =
|
||||
lines.next().expect("missing multipart prompt logprobs fixture line");
|
||||
let nixl_stats_hex = lines.next().expect("missing NIXL stats fixture line");
|
||||
let mooncake_stats_hex = lines.next().expect("missing Mooncake stats fixture line");
|
||||
let multi_connector_stats_hex =
|
||||
lines.next().expect("missing MultiConnector stats fixture line");
|
||||
let ready_response_hex = lines.next().expect("missing ready response fixture line");
|
||||
|
||||
let request_bytes = hex::decode(request_hex).unwrap();
|
||||
@@ -2799,6 +2803,28 @@ fn python_msgpack_fixtures_match_rust_encoding() {
|
||||
.expect("multipart prompt logprobs decoded"),
|
||||
);
|
||||
|
||||
let nixl_stats: KvConnectorStats =
|
||||
rmp_serde::from_slice(&hex::decode(nixl_stats_hex).unwrap()).unwrap();
|
||||
assert!(matches!(nixl_stats, KvConnectorStats::Nixl(_)));
|
||||
|
||||
let mooncake_stats: KvConnectorStats =
|
||||
rmp_serde::from_slice(&hex::decode(mooncake_stats_hex).unwrap()).unwrap();
|
||||
assert!(matches!(
|
||||
mooncake_stats,
|
||||
KvConnectorStats::Mooncake(stats)
|
||||
if stats.0.contains_key(&MooncakeOperation::LoadGet)
|
||||
));
|
||||
|
||||
let multi_connector_stats: KvConnectorStats =
|
||||
rmp_serde::from_slice(&hex::decode(multi_connector_stats_hex).unwrap()).unwrap();
|
||||
assert!(matches!(
|
||||
multi_connector_stats,
|
||||
KvConnectorStats::Multi(stats)
|
||||
if stats.nixl.is_some()
|
||||
&& stats.mooncake.is_some()
|
||||
&& stats.other.contains_key("UnsupportedConnector")
|
||||
));
|
||||
|
||||
let map_keys = |bytes: &[u8]| -> BTreeSet<String> {
|
||||
match decode_value(bytes) {
|
||||
Value::Map(entries) => entries
|
||||
|
||||
@@ -456,6 +456,32 @@ ready_response = EngineCoreReadyResponse(
|
||||
),
|
||||
)
|
||||
|
||||
nixl_stats = {
|
||||
"transfer_duration": [0.01, 0.02],
|
||||
"post_duration": [0.001, 0.002],
|
||||
"bytes_transferred": [4096, 8192],
|
||||
"num_descriptors": [2, 4],
|
||||
"num_failed_transfers": [],
|
||||
"num_failed_notifications": [],
|
||||
"num_kv_expired_reqs": [1],
|
||||
}
|
||||
mooncake_stats = {
|
||||
"load_get": [
|
||||
{
|
||||
"duration_seconds": 0.05,
|
||||
"num_keys": 3,
|
||||
"num_bytes": 1024,
|
||||
"status": "ok",
|
||||
"num_failed_keys": 0,
|
||||
}
|
||||
]
|
||||
}
|
||||
multi_connector_stats = {
|
||||
"NixlConnector": nixl_stats,
|
||||
"MooncakeStoreConnector": mooncake_stats,
|
||||
"UnsupportedConnector": {"sample_count": 1},
|
||||
}
|
||||
|
||||
print(msgspec.msgpack.encode(request).hex())
|
||||
print(msgspec.msgpack.encode(defaults_request).hex())
|
||||
print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
|
||||
@@ -475,4 +501,7 @@ print(
|
||||
for frame in encode_output_frames(multipart_prompt_logprobs, size_threshold=1)
|
||||
)
|
||||
)
|
||||
print(msgspec.msgpack.encode(nixl_stats).hex())
|
||||
print(msgspec.msgpack.encode(mooncake_stats).hex())
|
||||
print(msgspec.msgpack.encode(multi_connector_stats).hex())
|
||||
print(msgspec.msgpack.encode(ready_response).hex())
|
||||
|
||||
Reference in New Issue
Block a user