[Rust Frontend] Improve scheduler stats logging parity (#47435)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -372,6 +372,14 @@ impl EngineCoreClient {
|
||||
self.engines.len()
|
||||
}
|
||||
|
||||
/// Return the engine-side indices connected to this client.
|
||||
pub fn engine_indices(&self) -> Vec<u32> {
|
||||
self.engines
|
||||
.iter()
|
||||
.map(|engine| engine.engine_id.engine_index().expect("engine id must encode as u16"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return the engine identities of all engines connected to this client.
|
||||
pub fn engine_identities(&self) -> Vec<&[u8]> {
|
||||
self.engines.iter().map(|engine| &*engine.engine_id).collect()
|
||||
|
||||
@@ -84,6 +84,10 @@ pub(crate) fn record_scheduler_stats(
|
||||
.spec_decode_num_accepted_tokens
|
||||
.get_or_create(&labels)
|
||||
.inc_by(spec_decoding_stats.num_accepted_tokens);
|
||||
metrics.log_stats.get_or_create(&labels).observe_spec_decode(
|
||||
spec_decoding_stats.num_drafts,
|
||||
&spec_decoding_stats.num_accepted_tokens_per_pos,
|
||||
);
|
||||
|
||||
for (position, accepted_tokens) in
|
||||
spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate()
|
||||
@@ -119,6 +123,15 @@ pub(crate) fn record_scheduler_stats(
|
||||
.inc_by(perf_stats.num_write_bytes_per_gpu);
|
||||
}
|
||||
|
||||
if let Some(cudagraph_stats) = &stats.cudagraph_stats {
|
||||
metrics.log_stats.get_or_create(&labels).observe_cudagraph(
|
||||
cudagraph_stats.num_unpadded_tokens,
|
||||
cudagraph_stats.num_padded_tokens,
|
||||
cudagraph_stats.num_paddings,
|
||||
&cudagraph_stats.runtime_mode,
|
||||
);
|
||||
}
|
||||
|
||||
// Sampled KV-cache residency histograms.
|
||||
if !stats.kv_cache_eviction_events.is_empty() {
|
||||
let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels);
|
||||
|
||||
@@ -141,7 +141,7 @@ pub struct PerfStats {
|
||||
/// Original Python definition:
|
||||
/// <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/compilation/cuda_graph.py#L28-L33>
|
||||
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
|
||||
pub struct CudagraphStat {
|
||||
pub struct CudagraphStats {
|
||||
/// Number of real tokens in the captured batch before padding.
|
||||
pub num_unpadded_tokens: u64,
|
||||
/// Number of padded tokens in the captured batch.
|
||||
@@ -182,7 +182,7 @@ pub struct SchedulerStats {
|
||||
/// Connector-specific KV transfer stats, kept opaque for now.
|
||||
pub kv_connector_stats: Option<BTreeMap<String, OpaqueValue>>,
|
||||
/// CUDA graph runtime stats when graph metrics are enabled.
|
||||
pub cudagraph_stats: Option<CudagraphStat>,
|
||||
pub cudagraph_stats: Option<CudagraphStats>,
|
||||
/// Estimated MFU/performance stats, when enabled.
|
||||
pub perf_stats: Option<PerfStats>,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user