[Bugfix][Rust Frontend] Skip engine-derived metrics under --disable-log-stats (#59205)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-09-29 11:01:54 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f5ed3d9c03
commit afac509a33
16 changed files with 248 additions and 34 deletions
@@ -118,6 +118,7 @@ async fn main() -> Result<()> {
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
engine_stats_enabled: true,
})
.await
.context("failed to connect to external vLLM engine")?;
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
coordinator_mode: None,
model_name: args.model.clone(),
client_index: args.client_index,
engine_stats_enabled: true,
})
.await
.context("failed to connect to external vLLM engine")?;
+18
View File
@@ -155,6 +155,11 @@ pub struct EngineCoreClientConfig {
pub model_name: String,
/// Frontend client index stamped onto every request.
pub client_index: u32,
/// Whether the connected engines record stats, i.e. were not started with
/// `--disable-log-stats`. When `false`, engines emit neither
/// `SchedulerStats` nor request lifecycle events, so frontend metrics
/// derived from them are not exported.
pub engine_stats_enabled: bool,
}
impl EngineCoreClientConfig {
@@ -173,6 +178,7 @@ impl EngineCoreClientConfig {
coordinator_mode: None,
model_name: String::new(),
client_index: 0,
engine_stats_enabled: true,
}
}
@@ -193,6 +199,12 @@ impl EngineCoreClientConfig {
self
}
/// Set whether the connected engines record stats.
pub fn with_engine_stats_enabled(mut self, engine_stats_enabled: bool) -> Self {
self.engine_stats_enabled = engine_stats_enabled;
self
}
/// Override the optional coordinator mode for this client config.
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
self.coordinator_mode = coordinator_mode;
@@ -361,6 +373,7 @@ impl EngineCoreClient {
connected.input_send,
runtime.handle().clone(),
config.model_name.clone(),
config.engine_stats_enabled,
&engines,
));
let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
@@ -546,6 +559,11 @@ impl EngineCoreClient {
self.inner.model_name()
}
/// Return whether the connected engines record stats.
pub fn engine_stats_enabled(&self) -> bool {
self.config.engine_stats_enabled
}
/// Return whether the client still considers the engine healthy.
pub fn is_healthy(&self) -> bool {
self.inner.is_healthy()
+21 -7
View File
@@ -46,7 +46,12 @@ pub(crate) struct ClientInner {
/// Per-tensor byte threshold loaded from env variable
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
msgpack_zero_copy_threshold: usize,
scheduler_stats_recorder: SchedulerStatsRecorder,
/// Whether the connected engines record stats. See
/// [`crate::EngineCoreClientConfig::engine_stats_enabled`].
engine_stats_enabled: bool,
/// Scheduler-stats metric handles, present only when engines record stats
/// so that gauges are not exported with values no engine ever reports.
scheduler_stats_recorder: Option<SchedulerStatsRecorder>,
iteration_metrics: BTreeMap<u32, IterationMetricHandles>,
request_reg: Mutex<RequestRegistry>,
utility_reg: Mutex<UtilityRegistry>,
@@ -61,10 +66,11 @@ impl ClientInner {
input_send: RouterSendHalf,
handle: Handle,
model_name: String,
engine_stats_enabled: bool,
engines: &[ConnectedEngine],
) -> Self {
let scheduler_stats_recorder =
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
let scheduler_stats_recorder = engine_stats_enabled
.then(|| SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines));
let iteration_metrics = engines
.iter()
.filter_map(|engine| {
@@ -78,6 +84,7 @@ impl ClientInner {
handle,
model_name,
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
engine_stats_enabled,
scheduler_stats_recorder,
iteration_metrics,
request_reg: Mutex::new(RequestRegistry::new(engines)),
@@ -412,7 +419,9 @@ pub(crate) async fn run_output_dispatcher_loop(
inner: Arc<ClientInner>,
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
) {
let mut lora_info = LoraInfoExporter::default();
// LoRA phases come from request lifecycle events, which engines only emit
// when they record stats.
let mut lora_info = inner.engine_stats_enabled.then(LoraInfoExporter::default);
let result: Result<()> = async {
loop {
@@ -475,14 +484,18 @@ pub(crate) async fn run_output_dispatcher_loop(
"dropping scheduler stats for unknown engine"
);
}
inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats);
if let Some(recorder) = &inner.scheduler_stats_recorder {
recorder.record(batch.engine_index, scheduler_stats);
}
}
// The engine's scheduler stats never carry adapter names;
// the gauge is derived from the registry's frontend-side
// request tracking instead.
let (running, waiting) = inner.lora_adapter_states();
lora_info.update(&METRICS.scheduler, running, waiting);
if let Some(lora_info) = &mut lora_info {
let (running, waiting) = inner.lora_adapter_states();
lora_info.update(&METRICS.scheduler, running, waiting);
}
}
EngineCoreOutputs::Utility(utility) => {
let call_id = utility.output.call_id;
@@ -530,6 +543,7 @@ mod tests {
send,
Handle::current(),
"test-model".to_string(),
true,
&[ConnectedEngine {
engine_id: EngineId::from(b"engine-0"),
ready_response: default_ready_response(),
@@ -299,6 +299,7 @@ fn handshake_test_config(
coordinator_mode,
model_name: model_name.to_string(),
client_index,
engine_stats_enabled: true,
}
}
@@ -322,6 +323,7 @@ fn bootstrapped_test_config(
coordinator_mode,
model_name: "test-model".to_string(),
client_index,
engine_stats_enabled: true,
}
}