[Bugfix][Rust Frontend] Skip engine-derived metrics under --disable-log-stats (#59205)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-09-29 11:01:54 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f5ed3d9c03
commit afac509a33
16 changed files with 248 additions and 34 deletions
+1
View File
@@ -230,6 +230,7 @@ async fn run_mm_processor_with_engine(
coordinator_mode: None,
model_name: args.model.clone(),
client_index: 0,
engine_stats_enabled: true,
})
.await
.context("failed to connect to engine core")?;
@@ -70,6 +70,7 @@ async fn main() -> Result<()> {
coordinator_mode: None,
model_name: args.model.clone(),
client_index: CLIENT_INDEX,
engine_stats_enabled: true,
})
.await
.context("failed to connect to external vLLM engine")?;
+3 -1
View File
@@ -380,7 +380,9 @@ pub struct SharedRuntimeArgs {
pub api_key: Vec<String>,
/// Disable periodic logging of engine statistics (throughput, queue depth,
/// cache usage).
/// cache usage). Engines also stop recording stats, so metrics derived from
/// engine-reported scheduler stats and request lifecycle events are not
/// exported.
#[arg(long)]
#[serde(default)]
pub disable_log_stats: bool,
@@ -118,6 +118,7 @@ async fn main() -> Result<()> {
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
engine_stats_enabled: true,
})
.await
.context("failed to connect to external vLLM engine")?;
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
engine_stats_enabled: true,
})
.await
.context("failed to connect to external vLLM engine")?;
+18
View File
@@ -155,6 +155,11 @@ pub struct EngineCoreClientConfig {
pub model_name: String,
/// Frontend client index stamped onto every request.
pub client_index: u32,
/// Whether the connected engines record stats, i.e. were not started with
/// `--disable-log-stats`. When `false`, engines emit neither
/// `SchedulerStats` nor request lifecycle events, so frontend metrics
/// derived from them are not exported.
pub engine_stats_enabled: bool,
}
impl EngineCoreClientConfig {
@@ -173,6 +178,7 @@ impl EngineCoreClientConfig {
coordinator_mode: None,
model_name: String::new(),
client_index: 0,
engine_stats_enabled: true,
}
}
@@ -193,6 +199,12 @@ impl EngineCoreClientConfig {
self
}
/// Set whether the connected engines record stats.
pub fn with_engine_stats_enabled(mut self, engine_stats_enabled: bool) -> Self {
self.engine_stats_enabled = engine_stats_enabled;
self
}
/// Override the optional coordinator mode for this client config.
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
self.coordinator_mode = coordinator_mode;
@@ -361,6 +373,7 @@ impl EngineCoreClient {
connected.input_send,
runtime.handle().clone(),
config.model_name.clone(),
config.engine_stats_enabled,
&engines,
));
let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
@@ -546,6 +559,11 @@ impl EngineCoreClient {
self.inner.model_name()
}
/// Return whether the connected engines record stats.
pub fn engine_stats_enabled(&self) -> bool {
self.config.engine_stats_enabled
}
/// Return whether the client still considers the engine healthy.
pub fn is_healthy(&self) -> bool {
self.inner.is_healthy()
+21 -7
View File
@@ -46,7 +46,12 @@ pub(crate) struct ClientInner {
/// Per-tensor byte threshold loaded from env variable
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
msgpack_zero_copy_threshold: usize,
scheduler_stats_recorder: SchedulerStatsRecorder,
/// Whether the connected engines record stats. See
/// [`crate::EngineCoreClientConfig::engine_stats_enabled`].
engine_stats_enabled: bool,
/// Scheduler-stats metric handles, present only when engines record stats
/// so that gauges are not exported with values no engine ever reports.
scheduler_stats_recorder: Option<SchedulerStatsRecorder>,
iteration_metrics: BTreeMap<u32, IterationMetricHandles>,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
@@ -61,10 +66,11 @@ impl ClientInner {
input_send: RouterSendHalf,
handle: Handle,
model_name: String,
engine_stats_enabled: bool,
engines: &[ConnectedEngine],
) -> Self {
let scheduler_stats_recorder =
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
let scheduler_stats_recorder = engine_stats_enabled
.then(|| SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines));
let iteration_metrics = engines
.iter()
.filter_map(|engine| {
@@ -78,6 +84,7 @@ impl ClientInner {
handle,
model_name,
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
engine_stats_enabled,
scheduler_stats_recorder,
iteration_metrics,
request_reg: Mutex::new(RequestRegistry::new(engines)),
@@ -412,7 +419,9 @@ pub(crate) async fn run_output_dispatcher_loop(
inner: Arc<ClientInner>,
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
) {
let mut lora_info = LoraInfoExporter::default();
// LoRA phases come from request lifecycle events, which engines only emit
// when they record stats.
let mut lora_info = inner.engine_stats_enabled.then(LoraInfoExporter::default);
let result: Result<()> = async {
loop {
@@ -475,14 +484,18 @@ pub(crate) async fn run_output_dispatcher_loop(
"dropping scheduler stats for unknown engine"
);
}
inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats);
if let Some(recorder) = &inner.scheduler_stats_recorder {
recorder.record(batch.engine_index, scheduler_stats);
}
}
// The engine's scheduler stats never carry adapter names;
// the gauge is derived from the registry's frontend-side
// request tracking instead.
let (running, waiting) = inner.lora_adapter_states();
lora_info.update(&METRICS.scheduler, running, waiting);
if let Some(lora_info) = &mut lora_info {
let (running, waiting) = inner.lora_adapter_states();
lora_info.update(&METRICS.scheduler, running, waiting);
}
}
EngineCoreOutputs::Utility(utility) => {
let call_id = utility.output.call_id;
@@ -530,6 +543,7 @@ mod tests {
send,
Handle::current(),
"test-model".to_string(),
true,
&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: default_ready_response(),
@@ -299,6 +299,7 @@ fn handshake_test_config(
coordinator_mode,
model_name: model_name.to_string(),
client_index,
engine_stats_enabled: true,
}
}
@@ -322,6 +323,7 @@ fn bootstrapped_test_config(
coordinator_mode,
model_name: "test-model".to_string(),
client_index,
engine_stats_enabled: true,
}
}
@@ -122,6 +122,7 @@ async fn main() -> Result<()> {
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
engine_stats_enabled: true,
})
.await
.context("failed to connect to external vLLM engine")?;
+1
View File
@@ -106,6 +106,7 @@ impl Llm {
prompt_len,
max_tokens_param,
1,
self.client.engine_stats_enabled(),
);
let guard = self.inflight.track(external_request_id, internal_request_id);
+53 -24
View File
@@ -55,7 +55,6 @@ struct RequestMetricHandles {
labels: EngineLabels,
// Request-derived counters.
num_preemptions: U64Counter,
prompt_tokens: U64Counter,
prompt_tokens_local_compute: U64Counter,
prompt_tokens_local_cache_hit: U64Counter,
@@ -70,16 +69,27 @@ struct RequestMetricHandles {
request_max_num_generation_tokens: HistogramMetric,
request_params_max_tokens: HistogramMetric,
request_params_n: HistogramMetric,
request_num_preemptions: HistogramMetric,
request_prefill_kv_computed_tokens: HistogramMetric,
time_to_first_token_seconds: HistogramMetric,
inter_token_latency_seconds: HistogramMetric,
e2e_request_latency_seconds: HistogramMetric,
request_decode_time_seconds: HistogramMetric,
request_time_per_output_token_seconds: HistogramMetric,
/// Present only when engines record stats.
events: Option<EventMetricHandles>,
}
/// Cached handles for request metrics derived from engine-core lifecycle
/// events (`QUEUED`, `SCHEDULED`, `PREEMPTED`), which engines only emit when
/// they record stats.
#[derive(Clone)]
struct EventMetricHandles {
num_preemptions: U64Counter,
request_num_preemptions: HistogramMetric,
request_queue_time_seconds: HistogramMetric,
request_prefill_time_seconds: HistogramMetric,
request_decode_time_seconds: HistogramMetric,
request_inference_time_seconds: HistogramMetric,
request_time_per_output_token_seconds: HistogramMetric,
}
impl RequestMetricsTracker {
@@ -92,9 +102,14 @@ impl RequestMetricsTracker {
prompt_len: u32,
max_tokens_param: Option<u32>,
n_param: u32,
engine_stats_enabled: bool,
) -> Self {
Self {
handles: resolve_request_metric_handles(&model_name, engine_index),
handles: resolve_request_metric_handles(
&model_name,
engine_index,
engine_stats_enabled,
),
arrival_time,
prompt_len,
max_tokens_param,
@@ -163,10 +178,7 @@ impl RequestMetricsTracker {
let prefill_kv_computed_tokens =
self.prompt_len.saturating_sub(self.latest_num_cached_tokens);
let e2e_latency_seconds = received_at - self.arrival_time;
let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts);
let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts);
let decode_time_seconds = diff_or_zero(self.last_token_ts, self.first_token_ts);
let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts);
let time_per_output_token_seconds = if self.num_generation_tokens > 1 {
diff_or_zero(self.last_token_ts, self.first_token_ts)
/ (self.num_generation_tokens - 1) as f64
@@ -187,18 +199,25 @@ impl RequestMetricsTracker {
self.handles.request_params_max_tokens.observe(max_tokens_param as f64);
}
self.handles.request_params_n.observe(self.n_param as f64);
self.handles.request_num_preemptions.observe(self.num_preemptions as f64);
self.handles
.request_prefill_kv_computed_tokens
.observe(prefill_kv_computed_tokens as f64);
self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds);
self.handles.request_queue_time_seconds.observe(queue_time_seconds);
self.handles.request_prefill_time_seconds.observe(prefill_time_seconds);
self.handles.request_decode_time_seconds.observe(decode_time_seconds);
self.handles.request_inference_time_seconds.observe(inference_time_seconds);
self.handles
.request_time_per_output_token_seconds
.observe(time_per_output_token_seconds);
if let Some(events) = &self.handles.events {
let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts);
let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts);
let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts);
events.request_num_preemptions.observe(self.num_preemptions as f64);
events.request_queue_time_seconds.observe(queue_time_seconds);
events.request_prefill_time_seconds.observe(prefill_time_seconds);
events.request_inference_time_seconds.observe(inference_time_seconds);
}
}
/// Record prompt token counters through cached metric handles.
@@ -228,7 +247,9 @@ impl RequestMetricsTracker {
}
EngineCoreEventType::Preempted => {
self.num_preemptions += 1;
self.handles.num_preemptions.inc();
if let Some(events) = &self.handles.events {
events.num_preemptions.inc();
}
}
}
}
@@ -248,7 +269,11 @@ impl RequestMetricsTracker {
}
/// Resolve fixed request metric handles for one model and engine index.
fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles {
fn resolve_request_metric_handles(
model_name: &str,
engine: u32,
engine_stats_enabled: bool,
) -> RequestMetricHandles {
let metrics = &METRICS.request;
let labels = EngineLabels {
model_name: model_name.to_string(),
@@ -256,7 +281,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
};
RequestMetricHandles {
num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels),
prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels),
prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned(
&prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE),
@@ -281,7 +305,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
.get_or_create_owned(&labels),
request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels),
request_params_n: metrics.request_params_n.get_or_create_owned(&labels),
request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels),
request_prefill_kv_computed_tokens: metrics
.request_prefill_kv_computed_tokens
.get_or_create_owned(&labels),
@@ -294,19 +317,25 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
e2e_request_latency_seconds: metrics
.e2e_request_latency_seconds
.get_or_create_owned(&labels),
request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels),
request_prefill_time_seconds: metrics
.request_prefill_time_seconds
.get_or_create_owned(&labels),
request_decode_time_seconds: metrics
.request_decode_time_seconds
.get_or_create_owned(&labels),
request_inference_time_seconds: metrics
.request_inference_time_seconds
.get_or_create_owned(&labels),
request_time_per_output_token_seconds: metrics
.request_time_per_output_token_seconds
.get_or_create_owned(&labels),
events: engine_stats_enabled.then(|| EventMetricHandles {
num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels),
request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels),
request_queue_time_seconds: metrics
.request_queue_time_seconds
.get_or_create_owned(&labels),
request_prefill_time_seconds: metrics
.request_prefill_time_seconds
.get_or_create_owned(&labels),
request_inference_time_seconds: metrics
.request_inference_time_seconds
.get_or_create_owned(&labels),
}),
labels,
}
}
@@ -356,7 +385,7 @@ mod tests {
#[test]
fn tracker_updates_timing_state_across_prefill_decode_and_finish() {
let mut tracker =
RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1);
RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1, true);
tracker.observe_output(
10.0,
+139 -1
View File
@@ -6,12 +6,16 @@ use std::sync::{Arc, Once};
use std::time::Duration;
use futures::StreamExt as _;
use tokio::sync::oneshot;
use tokio::time::timeout;
use tracing_subscriber::EnvFilter;
use uuid::Uuid;
use vllm_engine_core_client::mock_engine::default_ready_response;
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
use vllm_engine_core_client::protocol::logprobs::{
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
};
use vllm_engine_core_client::protocol::lora::LoraRequest;
use vllm_engine_core_client::protocol::output::{
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
EngineCoreOutputs, RequestBatchOutputs,
@@ -20,7 +24,9 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest;
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
use vllm_engine_core_client::protocol::sampling_mask::{MaybeWireSamplingMask, SamplingMask};
use vllm_engine_core_client::protocol::stats::PrefillStats;
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::test_utils::{
IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::{
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
@@ -868,6 +874,138 @@ async fn generate_records_request_metrics_in_prometheus_output() {
assert!(rendered.contains(&format!(
"vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
)));
assert!(rendered.contains(&format!(
"vllm:request_queue_time_seconds_sum{{model_name=\"{model_name}\",engine=\"4\"}} 1.0"
)));
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
llm.shutdown().await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn generate_without_engine_stats_exports_only_frontend_observed_metrics() {
let ipc = IpcNamespace::new().unwrap();
let handshake_address = ipc.handshake_endpoint();
let engine_id = EngineId::from_engine_index(6);
let model_name = request_metrics_model_name("metrics-no-log-stats-model");
let lora_name = request_metrics_model_name("metrics-no-log-stats-lora");
let (finish_tx, finish_rx) = oneshot::channel::<()>();
let (shutdown_tx, engine_task) = spawn_mock_engine_task_with_ready(
handshake_address.clone(),
engine_id,
EngineCoreReadyResponse {
supports_lora: true,
max_loras: 1,
..default_ready_response()
},
move |dealer, push| {
Box::pin(async move {
let add = recv_engine_message(dealer).await;
assert_eq!(add[0].as_ref(), &[0x00]);
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
// Engines started with `--disable-log-stats` attach neither
// lifecycle events nor scheduler stats to their outputs.
for token in [1, 2] {
send_outputs(
push,
RequestBatchOutputs {
engine_index: 6,
timestamp: 10.0 + f64::from(token),
outputs: vec![request_output(&request.request_id, vec![token], None)],
..Default::default()
}
.into(),
)
.await;
}
finish_rx.await.unwrap();
send_outputs(
push,
RequestBatchOutputs {
engine_index: 6,
timestamp: 13.0,
outputs: vec![request_output(
&request.request_id,
vec![3],
Some(EngineCoreFinishReason::Length),
)],
finished_requests: Some(BTreeSet::from([request.request_id])),
..Default::default()
}
.into(),
)
.await;
})
},
);
let client = EngineCoreClient::connect(
EngineCoreClientConfig::new_single(handshake_address)
.with_model_name(&model_name)
.with_engine_stats_enabled(false)
.with_local_input_output_addresses(
Some(ipc.input_endpoint()),
Some(ipc.output_endpoint()),
),
)
.await
.unwrap();
let llm = Llm::new(client);
let mut request = sample_generate_request("req-no-log-stats", 8);
request.arrival_time = None;
request.lora_request = Some(LoraRequest {
lora_name: lora_name.clone(),
lora_int_id: 1,
lora_path: "/unused".to_string(),
base_model_name: None,
tensorizer_config_dict: None,
load_inplace: false,
is_3d_lora_weight: false,
});
let mut stream = llm.generate(request).await.unwrap();
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]);
// The dispatcher handles the second batch only after finishing the first
// one, including its LoRA info update.
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![2]);
assert!(!METRICS.render().unwrap().contains(&lora_name));
finish_tx.send(()).unwrap();
let final_output = stream.next().await.unwrap().unwrap();
assert_eq!(final_output.finish_reason, Some(FinishReason::Length));
assert!(stream.next().await.is_none());
let rendered = METRICS.render().unwrap();
for metric in [
"vllm:time_to_first_token_seconds_count",
"vllm:e2e_request_latency_seconds_count",
"vllm:request_decode_time_seconds_count",
] {
assert!(
rendered.contains(&format!(
"{metric}{{model_name=\"{model_name}\",engine=\"6\"}} 1"
)),
"{metric} should be exported"
);
}
for metric in [
"vllm:num_requests_running",
"vllm:kv_cache_usage_perc",
"vllm:num_preemptions_total",
"vllm:request_num_preemptions_count",
"vllm:request_queue_time_seconds_count",
"vllm:request_prefill_time_seconds_count",
"vllm:request_inference_time_seconds_count",
] {
assert!(
!rendered.contains(&format!("{metric}{{model_name=\"{model_name}\"")),
"{metric} should not be exported"
);
}
let _ = shutdown_tx.send(());
engine_task.await.unwrap();
+1
View File
@@ -29,6 +29,7 @@ fn client_config(handshake_address: String, engine_count: usize) -> EngineCoreCl
coordinator_mode: None,
model_name: "mock-model".to_string(),
client_index: 0,
engine_stats_enabled: true,
}
}
+3 -1
View File
@@ -268,7 +268,9 @@ pub struct Config {
#[educe(Debug(method(fmt_redacted_api_keys)))]
pub api_keys: Vec<String>,
/// When `true`, suppress periodic stats logging (throughput, queue depth,
/// cache usage).
/// cache usage). Engines also stop recording stats, so metrics derived from
/// engine-reported scheduler stats and request lifecycle events are not
/// exported.
pub disable_log_stats: bool,
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
/// started.
+1
View File
@@ -2234,6 +2234,7 @@ async fn control_aggregates_multi_engine_capacity() {
coordinator_mode: None,
model_name: "test-model".to_string(),
client_index: 0,
engine_stats_enabled: true,
};
let client_task = tokio::spawn(EngineCoreClient::connect(client_config));
let mut engine_sockets = Vec::new();
+1
View File
@@ -130,6 +130,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
coordinator_mode,
model_name: metrics_model_name,
client_index: 0,
engine_stats_enabled: !config.disable_log_stats,
})
.await
.context("failed to connect to engine core")?;