diff --git a/rust/src/bench/src/mm_processor.rs b/rust/src/bench/src/mm_processor.rs index 5b26181f16..3454dc7993 100644 --- a/rust/src/bench/src/mm_processor.rs +++ b/rust/src/bench/src/mm_processor.rs @@ -230,6 +230,7 @@ async fn run_mm_processor_with_engine( coordinator_mode: None, model_name: args.model.clone(), client_index: 0, + engine_stats_enabled: true, }) .await .context("failed to connect to engine core")?; diff --git a/rust/src/chat/examples/external_engine_chat_qwen.rs b/rust/src/chat/examples/external_engine_chat_qwen.rs index 4f62982036..1ba4d8a71a 100644 --- a/rust/src/chat/examples/external_engine_chat_qwen.rs +++ b/rust/src/chat/examples/external_engine_chat_qwen.rs @@ -70,6 +70,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: CLIENT_INDEX, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index c5a7c75baf..f57d43a75f 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -380,7 +380,9 @@ pub struct SharedRuntimeArgs { pub api_key: Vec, /// Disable periodic logging of engine statistics (throughput, queue depth, - /// cache usage). + /// cache usage). Engines also stop recording stats, so metrics derived from + /// engine-reported scheduler stats and request lifecycle events are not + /// exported. #[arg(long)] #[serde(default)] pub disable_log_stats: bool, diff --git a/rust/src/engine-core-client/examples/external_engine_logprobs.rs b/rust/src/engine-core-client/examples/external_engine_logprobs.rs index 891598c9f7..d4ec116f17 100644 --- a/rust/src/engine-core-client/examples/external_engine_logprobs.rs +++ b/rust/src/engine-core-client/examples/external_engine_logprobs.rs @@ -118,6 +118,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: args.client_index, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/engine-core-client/examples/external_engine_utility_call.rs b/rust/src/engine-core-client/examples/external_engine_utility_call.rs index b1826bb76e..8b73a12ce2 100644 --- a/rust/src/engine-core-client/examples/external_engine_utility_call.rs +++ b/rust/src/engine-core-client/examples/external_engine_utility_call.rs @@ -68,6 +68,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: args.client_index, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index aa6aacf0e0..0514292b90 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -155,6 +155,11 @@ pub struct EngineCoreClientConfig { pub model_name: String, /// Frontend client index stamped onto every request. pub client_index: u32, + /// Whether the connected engines record stats, i.e. were not started with + /// `--disable-log-stats`. When `false`, engines emit neither + /// `SchedulerStats` nor request lifecycle events, so frontend metrics + /// derived from them are not exported. + pub engine_stats_enabled: bool, } impl EngineCoreClientConfig { @@ -173,6 +178,7 @@ impl EngineCoreClientConfig { coordinator_mode: None, model_name: String::new(), client_index: 0, + engine_stats_enabled: true, } } @@ -193,6 +199,12 @@ impl EngineCoreClientConfig { self } + /// Set whether the connected engines record stats. + pub fn with_engine_stats_enabled(mut self, engine_stats_enabled: bool) -> Self { + self.engine_stats_enabled = engine_stats_enabled; + self + } + /// Override the optional coordinator mode for this client config. pub fn with_coordinator_mode(mut self, coordinator_mode: Option) -> Self { self.coordinator_mode = coordinator_mode; @@ -361,6 +373,7 @@ impl EngineCoreClient { connected.input_send, runtime.handle().clone(), config.model_name.clone(), + config.engine_stats_enabled, &engines, )); let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop( @@ -546,6 +559,11 @@ impl EngineCoreClient { self.inner.model_name() } + /// Return whether the connected engines record stats. + pub fn engine_stats_enabled(&self) -> bool { + self.config.engine_stats_enabled + } + /// Return whether the client still considers the engine healthy. pub fn is_healthy(&self) -> bool { self.inner.is_healthy() diff --git a/rust/src/engine-core-client/src/client/imp.rs b/rust/src/engine-core-client/src/client/imp.rs index 7318fa01b8..c8abb95bbd 100644 --- a/rust/src/engine-core-client/src/client/imp.rs +++ b/rust/src/engine-core-client/src/client/imp.rs @@ -46,7 +46,12 @@ pub(crate) struct ClientInner { /// Per-tensor byte threshold loaded from env variable /// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created. msgpack_zero_copy_threshold: usize, - scheduler_stats_recorder: SchedulerStatsRecorder, + /// Whether the connected engines record stats. See + /// [`crate::EngineCoreClientConfig::engine_stats_enabled`]. + engine_stats_enabled: bool, + /// Scheduler-stats metric handles, present only when engines record stats + /// so that gauges are not exported with values no engine ever reports. + scheduler_stats_recorder: Option, iteration_metrics: BTreeMap, request_reg: Mutex, utility_reg: Mutex, @@ -61,10 +66,11 @@ impl ClientInner { input_send: RouterSendHalf, handle: Handle, model_name: String, + engine_stats_enabled: bool, engines: &[ConnectedEngine], ) -> Self { - let scheduler_stats_recorder = - SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines); + let scheduler_stats_recorder = engine_stats_enabled + .then(|| SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines)); let iteration_metrics = engines .iter() .filter_map(|engine| { @@ -78,6 +84,7 @@ impl ClientInner { handle, model_name, msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(), + engine_stats_enabled, scheduler_stats_recorder, iteration_metrics, request_reg: Mutex::new(RequestRegistry::new(engines)), @@ -412,7 +419,9 @@ pub(crate) async fn run_output_dispatcher_loop( inner: Arc, mut output_rx: mpsc::Receiver>, ) { - let mut lora_info = LoraInfoExporter::default(); + // LoRA phases come from request lifecycle events, which engines only emit + // when they record stats. + let mut lora_info = inner.engine_stats_enabled.then(LoraInfoExporter::default); let result: Result<()> = async { loop { @@ -475,14 +484,18 @@ pub(crate) async fn run_output_dispatcher_loop( "dropping scheduler stats for unknown engine" ); } - inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats); + if let Some(recorder) = &inner.scheduler_stats_recorder { + recorder.record(batch.engine_index, scheduler_stats); + } } // The engine's scheduler stats never carry adapter names; // the gauge is derived from the registry's frontend-side // request tracking instead. - let (running, waiting) = inner.lora_adapter_states(); - lora_info.update(&METRICS.scheduler, running, waiting); + if let Some(lora_info) = &mut lora_info { + let (running, waiting) = inner.lora_adapter_states(); + lora_info.update(&METRICS.scheduler, running, waiting); + } } EngineCoreOutputs::Utility(utility) => { let call_id = utility.output.call_id; @@ -530,6 +543,7 @@ mod tests { send, Handle::current(), "test-model".to_string(), + true, &[ConnectedEngine { engine_id: EngineId::from(b"engine-0"), ready_response: default_ready_response(), diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index cdc3f718bd..5f2dc9c81e 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -299,6 +299,7 @@ fn handshake_test_config( coordinator_mode, model_name: model_name.to_string(), client_index, + engine_stats_enabled: true, } } @@ -322,6 +323,7 @@ fn bootstrapped_test_config( coordinator_mode, model_name: "test-model".to_string(), client_index, + engine_stats_enabled: true, } } diff --git a/rust/src/llm/examples/external_engine_smoke.rs b/rust/src/llm/examples/external_engine_smoke.rs index a28639e912..c965ea3038 100644 --- a/rust/src/llm/examples/external_engine_smoke.rs +++ b/rust/src/llm/examples/external_engine_smoke.rs @@ -122,6 +122,7 @@ async fn main() -> Result<()> { coordinator_mode: None, model_name: args.model.clone(), client_index: args.client_index, + engine_stats_enabled: true, }) .await .context("failed to connect to external vLLM engine")?; diff --git a/rust/src/llm/src/lib.rs b/rust/src/llm/src/lib.rs index 5aa2479193..015624b409 100644 --- a/rust/src/llm/src/lib.rs +++ b/rust/src/llm/src/lib.rs @@ -106,6 +106,7 @@ impl Llm { prompt_len, max_tokens_param, 1, + self.client.engine_stats_enabled(), ); let guard = self.inflight.track(external_request_id, internal_request_id); diff --git a/rust/src/llm/src/request_metrics.rs b/rust/src/llm/src/request_metrics.rs index 43802eaf4a..98f9cadd1a 100644 --- a/rust/src/llm/src/request_metrics.rs +++ b/rust/src/llm/src/request_metrics.rs @@ -55,7 +55,6 @@ struct RequestMetricHandles { labels: EngineLabels, // Request-derived counters. - num_preemptions: U64Counter, prompt_tokens: U64Counter, prompt_tokens_local_compute: U64Counter, prompt_tokens_local_cache_hit: U64Counter, @@ -70,16 +69,27 @@ struct RequestMetricHandles { request_max_num_generation_tokens: HistogramMetric, request_params_max_tokens: HistogramMetric, request_params_n: HistogramMetric, - request_num_preemptions: HistogramMetric, request_prefill_kv_computed_tokens: HistogramMetric, time_to_first_token_seconds: HistogramMetric, inter_token_latency_seconds: HistogramMetric, e2e_request_latency_seconds: HistogramMetric, + request_decode_time_seconds: HistogramMetric, + request_time_per_output_token_seconds: HistogramMetric, + + /// Present only when engines record stats. + events: Option, +} + +/// Cached handles for request metrics derived from engine-core lifecycle +/// events (`QUEUED`, `SCHEDULED`, `PREEMPTED`), which engines only emit when +/// they record stats. +#[derive(Clone)] +struct EventMetricHandles { + num_preemptions: U64Counter, + request_num_preemptions: HistogramMetric, request_queue_time_seconds: HistogramMetric, request_prefill_time_seconds: HistogramMetric, - request_decode_time_seconds: HistogramMetric, request_inference_time_seconds: HistogramMetric, - request_time_per_output_token_seconds: HistogramMetric, } impl RequestMetricsTracker { @@ -92,9 +102,14 @@ impl RequestMetricsTracker { prompt_len: u32, max_tokens_param: Option, n_param: u32, + engine_stats_enabled: bool, ) -> Self { Self { - handles: resolve_request_metric_handles(&model_name, engine_index), + handles: resolve_request_metric_handles( + &model_name, + engine_index, + engine_stats_enabled, + ), arrival_time, prompt_len, max_tokens_param, @@ -163,10 +178,7 @@ impl RequestMetricsTracker { let prefill_kv_computed_tokens = self.prompt_len.saturating_sub(self.latest_num_cached_tokens); let e2e_latency_seconds = received_at - self.arrival_time; - let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts); - let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts); let decode_time_seconds = diff_or_zero(self.last_token_ts, self.first_token_ts); - let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts); let time_per_output_token_seconds = if self.num_generation_tokens > 1 { diff_or_zero(self.last_token_ts, self.first_token_ts) / (self.num_generation_tokens - 1) as f64 @@ -187,18 +199,25 @@ impl RequestMetricsTracker { self.handles.request_params_max_tokens.observe(max_tokens_param as f64); } self.handles.request_params_n.observe(self.n_param as f64); - self.handles.request_num_preemptions.observe(self.num_preemptions as f64); self.handles .request_prefill_kv_computed_tokens .observe(prefill_kv_computed_tokens as f64); self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds); - self.handles.request_queue_time_seconds.observe(queue_time_seconds); - self.handles.request_prefill_time_seconds.observe(prefill_time_seconds); self.handles.request_decode_time_seconds.observe(decode_time_seconds); - self.handles.request_inference_time_seconds.observe(inference_time_seconds); self.handles .request_time_per_output_token_seconds .observe(time_per_output_token_seconds); + + if let Some(events) = &self.handles.events { + let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts); + let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts); + let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts); + + events.request_num_preemptions.observe(self.num_preemptions as f64); + events.request_queue_time_seconds.observe(queue_time_seconds); + events.request_prefill_time_seconds.observe(prefill_time_seconds); + events.request_inference_time_seconds.observe(inference_time_seconds); + } } /// Record prompt token counters through cached metric handles. @@ -228,7 +247,9 @@ impl RequestMetricsTracker { } EngineCoreEventType::Preempted => { self.num_preemptions += 1; - self.handles.num_preemptions.inc(); + if let Some(events) = &self.handles.events { + events.num_preemptions.inc(); + } } } } @@ -248,7 +269,11 @@ impl RequestMetricsTracker { } /// Resolve fixed request metric handles for one model and engine index. -fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles { +fn resolve_request_metric_handles( + model_name: &str, + engine: u32, + engine_stats_enabled: bool, +) -> RequestMetricHandles { let metrics = &METRICS.request; let labels = EngineLabels { model_name: model_name.to_string(), @@ -256,7 +281,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri }; RequestMetricHandles { - num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels), prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels), prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned( &prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE), @@ -281,7 +305,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri .get_or_create_owned(&labels), request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels), request_params_n: metrics.request_params_n.get_or_create_owned(&labels), - request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels), request_prefill_kv_computed_tokens: metrics .request_prefill_kv_computed_tokens .get_or_create_owned(&labels), @@ -294,19 +317,25 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri e2e_request_latency_seconds: metrics .e2e_request_latency_seconds .get_or_create_owned(&labels), - request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels), - request_prefill_time_seconds: metrics - .request_prefill_time_seconds - .get_or_create_owned(&labels), request_decode_time_seconds: metrics .request_decode_time_seconds .get_or_create_owned(&labels), - request_inference_time_seconds: metrics - .request_inference_time_seconds - .get_or_create_owned(&labels), request_time_per_output_token_seconds: metrics .request_time_per_output_token_seconds .get_or_create_owned(&labels), + events: engine_stats_enabled.then(|| EventMetricHandles { + num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels), + request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels), + request_queue_time_seconds: metrics + .request_queue_time_seconds + .get_or_create_owned(&labels), + request_prefill_time_seconds: metrics + .request_prefill_time_seconds + .get_or_create_owned(&labels), + request_inference_time_seconds: metrics + .request_inference_time_seconds + .get_or_create_owned(&labels), + }), labels, } } @@ -356,7 +385,7 @@ mod tests { #[test] fn tracker_updates_timing_state_across_prefill_decode_and_finish() { let mut tracker = - RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1); + RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1, true); tracker.observe_output( 10.0, diff --git a/rust/src/llm/tests/generate.rs b/rust/src/llm/tests/generate.rs index efd48eba77..c596c89ba3 100644 --- a/rust/src/llm/tests/generate.rs +++ b/rust/src/llm/tests/generate.rs @@ -6,12 +6,16 @@ use std::sync::{Arc, Once}; use std::time::Duration; use futures::StreamExt as _; +use tokio::sync::oneshot; use tokio::time::timeout; use tracing_subscriber::EnvFilter; use uuid::Uuid; +use vllm_engine_core_client::mock_engine::default_ready_response; +use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse; use vllm_engine_core_client::protocol::logprobs::{ Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob, }; +use vllm_engine_core_client::protocol::lora::LoraRequest; use vllm_engine_core_client::protocol::output::{ EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, RequestBatchOutputs, @@ -20,7 +24,9 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest; use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams; use vllm_engine_core_client::protocol::sampling_mask::{MaybeWireSamplingMask, SamplingMask}; use vllm_engine_core_client::protocol::stats::PrefillStats; -use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; +use vllm_engine_core_client::test_utils::{ + IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready, +}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::{ Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm, @@ -868,6 +874,138 @@ async fn generate_records_request_metrics_in_prometheus_output() { assert!(rendered.contains(&format!( "vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1" ))); + assert!(rendered.contains(&format!( + "vllm:request_queue_time_seconds_sum{{model_name=\"{model_name}\",engine=\"4\"}} 1.0" + ))); + + let _ = shutdown_tx.send(()); + engine_task.await.unwrap(); + llm.shutdown().await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn generate_without_engine_stats_exports_only_frontend_observed_metrics() { + let ipc = IpcNamespace::new().unwrap(); + let handshake_address = ipc.handshake_endpoint(); + let engine_id = EngineId::from_engine_index(6); + let model_name = request_metrics_model_name("metrics-no-log-stats-model"); + let lora_name = request_metrics_model_name("metrics-no-log-stats-lora"); + let (finish_tx, finish_rx) = oneshot::channel::<()>(); + + let (shutdown_tx, engine_task) = spawn_mock_engine_task_with_ready( + handshake_address.clone(), + engine_id, + EngineCoreReadyResponse { + supports_lora: true, + max_loras: 1, + ..default_ready_response() + }, + move |dealer, push| { + Box::pin(async move { + let add = recv_engine_message(dealer).await; + assert_eq!(add[0].as_ref(), &[0x00]); + let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap(); + + // Engines started with `--disable-log-stats` attach neither + // lifecycle events nor scheduler stats to their outputs. + for token in [1, 2] { + send_outputs( + push, + RequestBatchOutputs { + engine_index: 6, + timestamp: 10.0 + f64::from(token), + outputs: vec![request_output(&request.request_id, vec![token], None)], + ..Default::default() + } + .into(), + ) + .await; + } + + finish_rx.await.unwrap(); + send_outputs( + push, + RequestBatchOutputs { + engine_index: 6, + timestamp: 13.0, + outputs: vec![request_output( + &request.request_id, + vec![3], + Some(EngineCoreFinishReason::Length), + )], + finished_requests: Some(BTreeSet::from([request.request_id])), + ..Default::default() + } + .into(), + ) + .await; + }) + }, + ); + + let client = EngineCoreClient::connect( + EngineCoreClientConfig::new_single(handshake_address) + .with_model_name(&model_name) + .with_engine_stats_enabled(false) + .with_local_input_output_addresses( + Some(ipc.input_endpoint()), + Some(ipc.output_endpoint()), + ), + ) + .await + .unwrap(); + let llm = Llm::new(client); + let mut request = sample_generate_request("req-no-log-stats", 8); + request.arrival_time = None; + request.lora_request = Some(LoraRequest { + lora_name: lora_name.clone(), + lora_int_id: 1, + lora_path: "/unused".to_string(), + base_model_name: None, + tensorizer_config_dict: None, + load_inplace: false, + is_3d_lora_weight: false, + }); + let mut stream = llm.generate(request).await.unwrap(); + + assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]); + // The dispatcher handles the second batch only after finishing the first + // one, including its LoRA info update. + assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![2]); + assert!(!METRICS.render().unwrap().contains(&lora_name)); + + finish_tx.send(()).unwrap(); + let final_output = stream.next().await.unwrap().unwrap(); + assert_eq!(final_output.finish_reason, Some(FinishReason::Length)); + assert!(stream.next().await.is_none()); + + let rendered = METRICS.render().unwrap(); + for metric in [ + "vllm:time_to_first_token_seconds_count", + "vllm:e2e_request_latency_seconds_count", + "vllm:request_decode_time_seconds_count", + ] { + assert!( + rendered.contains(&format!( + "{metric}{{model_name=\"{model_name}\",engine=\"6\"}} 1" + )), + "{metric} should be exported" + ); + } + for metric in [ + "vllm:num_requests_running", + "vllm:kv_cache_usage_perc", + "vllm:num_preemptions_total", + "vllm:request_num_preemptions_count", + "vllm:request_queue_time_seconds_count", + "vllm:request_prefill_time_seconds_count", + "vllm:request_inference_time_seconds_count", + ] { + assert!( + !rendered.contains(&format!("{metric}{{model_name=\"{model_name}\"")), + "{metric} should not be exported" + ); + } let _ = shutdown_tx.send(()); engine_task.await.unwrap(); diff --git a/rust/src/mock-engine/src/tests.rs b/rust/src/mock-engine/src/tests.rs index c1cfbb49cb..71b4286033 100644 --- a/rust/src/mock-engine/src/tests.rs +++ b/rust/src/mock-engine/src/tests.rs @@ -29,6 +29,7 @@ fn client_config(handshake_address: String, engine_count: usize) -> EngineCoreCl coordinator_mode: None, model_name: "mock-model".to_string(), client_index: 0, + engine_stats_enabled: true, } } diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index c1c580e75a..a9a77a9a9e 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -268,7 +268,9 @@ pub struct Config { #[educe(Debug(method(fmt_redacted_api_keys)))] pub api_keys: Vec, /// When `true`, suppress periodic stats logging (throughput, queue depth, - /// cache usage). + /// cache usage). Engines also stop recording stats, so metrics derived from + /// engine-reported scheduler stats and request lifecycle events are not + /// exported. pub disable_log_stats: bool, /// TCP port for the gRPC Inference service. When `None`, no gRPC server is /// started. diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 18b6e6eb29..21b240c4b1 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -2234,6 +2234,7 @@ async fn control_aggregates_multi_engine_capacity() { coordinator_mode: None, model_name: "test-model".to_string(), client_index: 0, + engine_stats_enabled: true, }; let client_task = tokio::spawn(EngineCoreClient::connect(client_config)); let mut engine_sockets = Vec::new(); diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 91564db73c..d5f9e1ea55 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -130,6 +130,7 @@ async fn build_state(config: &Config) -> Result> { coordinator_mode, model_name: metrics_model_name, client_index: 0, + engine_stats_enabled: !config.disable_log_stats, }) .await .context("failed to connect to engine core")?;