[Bugfix][Rust Frontend] Skip engine-derived metrics under --disable-log-stats (#59205)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f5ed3d9c03
commit
afac509a33
@@ -118,6 +118,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
@@ -155,6 +155,11 @@ pub struct EngineCoreClientConfig {
|
||||
pub model_name: String,
|
||||
/// Frontend client index stamped onto every request.
|
||||
pub client_index: u32,
|
||||
/// Whether the connected engines record stats, i.e. were not started with
|
||||
/// `--disable-log-stats`. When `false`, engines emit neither
|
||||
/// `SchedulerStats` nor request lifecycle events, so frontend metrics
|
||||
/// derived from them are not exported.
|
||||
pub engine_stats_enabled: bool,
|
||||
}
|
||||
|
||||
impl EngineCoreClientConfig {
|
||||
@@ -173,6 +178,7 @@ impl EngineCoreClientConfig {
|
||||
coordinator_mode: None,
|
||||
model_name: String::new(),
|
||||
client_index: 0,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,6 +199,12 @@ impl EngineCoreClientConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether the connected engines record stats.
|
||||
pub fn with_engine_stats_enabled(mut self, engine_stats_enabled: bool) -> Self {
|
||||
self.engine_stats_enabled = engine_stats_enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the optional coordinator mode for this client config.
|
||||
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
|
||||
self.coordinator_mode = coordinator_mode;
|
||||
@@ -361,6 +373,7 @@ impl EngineCoreClient {
|
||||
connected.input_send,
|
||||
runtime.handle().clone(),
|
||||
config.model_name.clone(),
|
||||
config.engine_stats_enabled,
|
||||
&engines,
|
||||
));
|
||||
let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
|
||||
@@ -546,6 +559,11 @@ impl EngineCoreClient {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
/// Return whether the connected engines record stats.
|
||||
pub fn engine_stats_enabled(&self) -> bool {
|
||||
self.config.engine_stats_enabled
|
||||
}
|
||||
|
||||
/// Return whether the client still considers the engine healthy.
|
||||
pub fn is_healthy(&self) -> bool {
|
||||
self.inner.is_healthy()
|
||||
|
||||
@@ -46,7 +46,12 @@ pub(crate) struct ClientInner {
|
||||
/// Per-tensor byte threshold loaded from env variable
|
||||
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
|
||||
msgpack_zero_copy_threshold: usize,
|
||||
scheduler_stats_recorder: SchedulerStatsRecorder,
|
||||
/// Whether the connected engines record stats. See
|
||||
/// [`crate::EngineCoreClientConfig::engine_stats_enabled`].
|
||||
engine_stats_enabled: bool,
|
||||
/// Scheduler-stats metric handles, present only when engines record stats
|
||||
/// so that gauges are not exported with values no engine ever reports.
|
||||
scheduler_stats_recorder: Option<SchedulerStatsRecorder>,
|
||||
iteration_metrics: BTreeMap<u32, IterationMetricHandles>,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
@@ -61,10 +66,11 @@ impl ClientInner {
|
||||
input_send: RouterSendHalf,
|
||||
handle: Handle,
|
||||
model_name: String,
|
||||
engine_stats_enabled: bool,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
let scheduler_stats_recorder =
|
||||
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
|
||||
let scheduler_stats_recorder = engine_stats_enabled
|
||||
.then(|| SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines));
|
||||
let iteration_metrics = engines
|
||||
.iter()
|
||||
.filter_map(|engine| {
|
||||
@@ -78,6 +84,7 @@ impl ClientInner {
|
||||
handle,
|
||||
model_name,
|
||||
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
|
||||
engine_stats_enabled,
|
||||
scheduler_stats_recorder,
|
||||
iteration_metrics,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
@@ -412,7 +419,9 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
|
||||
) {
|
||||
let mut lora_info = LoraInfoExporter::default();
|
||||
// LoRA phases come from request lifecycle events, which engines only emit
|
||||
// when they record stats.
|
||||
let mut lora_info = inner.engine_stats_enabled.then(LoraInfoExporter::default);
|
||||
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
@@ -475,14 +484,18 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
"dropping scheduler stats for unknown engine"
|
||||
);
|
||||
}
|
||||
inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats);
|
||||
if let Some(recorder) = &inner.scheduler_stats_recorder {
|
||||
recorder.record(batch.engine_index, scheduler_stats);
|
||||
}
|
||||
}
|
||||
|
||||
// The engine's scheduler stats never carry adapter names;
|
||||
// the gauge is derived from the registry's frontend-side
|
||||
// request tracking instead.
|
||||
let (running, waiting) = inner.lora_adapter_states();
|
||||
lora_info.update(&METRICS.scheduler, running, waiting);
|
||||
if let Some(lora_info) = &mut lora_info {
|
||||
let (running, waiting) = inner.lora_adapter_states();
|
||||
lora_info.update(&METRICS.scheduler, running, waiting);
|
||||
}
|
||||
}
|
||||
EngineCoreOutputs::Utility(utility) => {
|
||||
let call_id = utility.output.call_id;
|
||||
@@ -530,6 +543,7 @@ mod tests {
|
||||
send,
|
||||
Handle::current(),
|
||||
"test-model".to_string(),
|
||||
true,
|
||||
&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: default_ready_response(),
|
||||
|
||||
@@ -299,6 +299,7 @@ fn handshake_test_config(
|
||||
coordinator_mode,
|
||||
model_name: model_name.to_string(),
|
||||
client_index,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -322,6 +323,7 @@ fn bootstrapped_test_config(
|
||||
coordinator_mode,
|
||||
model_name: "test-model".to_string(),
|
||||
client_index,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user