[Bugfix][Rust Frontend] Skip engine-derived metrics under --disable-log-stats (#59205)
Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f5ed3d9c03
commit
afac509a33
@@ -230,6 +230,7 @@ async fn run_mm_processor_with_engine(
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: 0,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
@@ -70,6 +70,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: CLIENT_INDEX,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
@@ -380,7 +380,9 @@ pub struct SharedRuntimeArgs {
|
||||
pub api_key: Vec<String>,
|
||||
|
||||
/// Disable periodic logging of engine statistics (throughput, queue depth,
|
||||
/// cache usage).
|
||||
/// cache usage). Engines also stop recording stats, so metrics derived from
|
||||
/// engine-reported scheduler stats and request lifecycle events are not
|
||||
/// exported.
|
||||
#[arg(long)]
|
||||
#[serde(default)]
|
||||
pub disable_log_stats: bool,
|
||||
|
||||
@@ -118,6 +118,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
@@ -68,6 +68,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
@@ -155,6 +155,11 @@ pub struct EngineCoreClientConfig {
|
||||
pub model_name: String,
|
||||
/// Frontend client index stamped onto every request.
|
||||
pub client_index: u32,
|
||||
/// Whether the connected engines record stats, i.e. were not started with
|
||||
/// `--disable-log-stats`. When `false`, engines emit neither
|
||||
/// `SchedulerStats` nor request lifecycle events, so frontend metrics
|
||||
/// derived from them are not exported.
|
||||
pub engine_stats_enabled: bool,
|
||||
}
|
||||
|
||||
impl EngineCoreClientConfig {
|
||||
@@ -173,6 +178,7 @@ impl EngineCoreClientConfig {
|
||||
coordinator_mode: None,
|
||||
model_name: String::new(),
|
||||
client_index: 0,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -193,6 +199,12 @@ impl EngineCoreClientConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether the connected engines record stats.
|
||||
pub fn with_engine_stats_enabled(mut self, engine_stats_enabled: bool) -> Self {
|
||||
self.engine_stats_enabled = engine_stats_enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Override the optional coordinator mode for this client config.
|
||||
pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
|
||||
self.coordinator_mode = coordinator_mode;
|
||||
@@ -361,6 +373,7 @@ impl EngineCoreClient {
|
||||
connected.input_send,
|
||||
runtime.handle().clone(),
|
||||
config.model_name.clone(),
|
||||
config.engine_stats_enabled,
|
||||
&engines,
|
||||
));
|
||||
let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
|
||||
@@ -546,6 +559,11 @@ impl EngineCoreClient {
|
||||
self.inner.model_name()
|
||||
}
|
||||
|
||||
/// Return whether the connected engines record stats.
|
||||
pub fn engine_stats_enabled(&self) -> bool {
|
||||
self.config.engine_stats_enabled
|
||||
}
|
||||
|
||||
/// Return whether the client still considers the engine healthy.
|
||||
pub fn is_healthy(&self) -> bool {
|
||||
self.inner.is_healthy()
|
||||
|
||||
@@ -46,7 +46,12 @@ pub(crate) struct ClientInner {
|
||||
/// Per-tensor byte threshold loaded from env variable
|
||||
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
|
||||
msgpack_zero_copy_threshold: usize,
|
||||
scheduler_stats_recorder: SchedulerStatsRecorder,
|
||||
/// Whether the connected engines record stats. See
|
||||
/// [`crate::EngineCoreClientConfig::engine_stats_enabled`].
|
||||
engine_stats_enabled: bool,
|
||||
/// Scheduler-stats metric handles, present only when engines record stats
|
||||
/// so that gauges are not exported with values no engine ever reports.
|
||||
scheduler_stats_recorder: Option<SchedulerStatsRecorder>,
|
||||
iteration_metrics: BTreeMap<u32, IterationMetricHandles>,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
@@ -61,10 +66,11 @@ impl ClientInner {
|
||||
input_send: RouterSendHalf,
|
||||
handle: Handle,
|
||||
model_name: String,
|
||||
engine_stats_enabled: bool,
|
||||
engines: &[ConnectedEngine],
|
||||
) -> Self {
|
||||
let scheduler_stats_recorder =
|
||||
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
|
||||
let scheduler_stats_recorder = engine_stats_enabled
|
||||
.then(|| SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines));
|
||||
let iteration_metrics = engines
|
||||
.iter()
|
||||
.filter_map(|engine| {
|
||||
@@ -78,6 +84,7 @@ impl ClientInner {
|
||||
handle,
|
||||
model_name,
|
||||
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
|
||||
engine_stats_enabled,
|
||||
scheduler_stats_recorder,
|
||||
iteration_metrics,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
@@ -412,7 +419,9 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
inner: Arc<ClientInner>,
|
||||
mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
|
||||
) {
|
||||
let mut lora_info = LoraInfoExporter::default();
|
||||
// LoRA phases come from request lifecycle events, which engines only emit
|
||||
// when they record stats.
|
||||
let mut lora_info = inner.engine_stats_enabled.then(LoraInfoExporter::default);
|
||||
|
||||
let result: Result<()> = async {
|
||||
loop {
|
||||
@@ -475,14 +484,18 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
"dropping scheduler stats for unknown engine"
|
||||
);
|
||||
}
|
||||
inner.scheduler_stats_recorder.record(batch.engine_index, scheduler_stats);
|
||||
if let Some(recorder) = &inner.scheduler_stats_recorder {
|
||||
recorder.record(batch.engine_index, scheduler_stats);
|
||||
}
|
||||
}
|
||||
|
||||
// The engine's scheduler stats never carry adapter names;
|
||||
// the gauge is derived from the registry's frontend-side
|
||||
// request tracking instead.
|
||||
let (running, waiting) = inner.lora_adapter_states();
|
||||
lora_info.update(&METRICS.scheduler, running, waiting);
|
||||
if let Some(lora_info) = &mut lora_info {
|
||||
let (running, waiting) = inner.lora_adapter_states();
|
||||
lora_info.update(&METRICS.scheduler, running, waiting);
|
||||
}
|
||||
}
|
||||
EngineCoreOutputs::Utility(utility) => {
|
||||
let call_id = utility.output.call_id;
|
||||
@@ -530,6 +543,7 @@ mod tests {
|
||||
send,
|
||||
Handle::current(),
|
||||
"test-model".to_string(),
|
||||
true,
|
||||
&[ConnectedEngine {
|
||||
engine_id: EngineId::from(b"engine-0"),
|
||||
ready_response: default_ready_response(),
|
||||
|
||||
@@ -299,6 +299,7 @@ fn handshake_test_config(
|
||||
coordinator_mode,
|
||||
model_name: model_name.to_string(),
|
||||
client_index,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -322,6 +323,7 @@ fn bootstrapped_test_config(
|
||||
coordinator_mode,
|
||||
model_name: "test-model".to_string(),
|
||||
client_index,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ async fn main() -> Result<()> {
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: args.client_index,
|
||||
engine_stats_enabled: true,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to external vLLM engine")?;
|
||||
|
||||
@@ -106,6 +106,7 @@ impl Llm {
|
||||
prompt_len,
|
||||
max_tokens_param,
|
||||
1,
|
||||
self.client.engine_stats_enabled(),
|
||||
);
|
||||
let guard = self.inflight.track(external_request_id, internal_request_id);
|
||||
|
||||
|
||||
@@ -55,7 +55,6 @@ struct RequestMetricHandles {
|
||||
labels: EngineLabels,
|
||||
|
||||
// Request-derived counters.
|
||||
num_preemptions: U64Counter,
|
||||
prompt_tokens: U64Counter,
|
||||
prompt_tokens_local_compute: U64Counter,
|
||||
prompt_tokens_local_cache_hit: U64Counter,
|
||||
@@ -70,16 +69,27 @@ struct RequestMetricHandles {
|
||||
request_max_num_generation_tokens: HistogramMetric,
|
||||
request_params_max_tokens: HistogramMetric,
|
||||
request_params_n: HistogramMetric,
|
||||
request_num_preemptions: HistogramMetric,
|
||||
request_prefill_kv_computed_tokens: HistogramMetric,
|
||||
time_to_first_token_seconds: HistogramMetric,
|
||||
inter_token_latency_seconds: HistogramMetric,
|
||||
e2e_request_latency_seconds: HistogramMetric,
|
||||
request_decode_time_seconds: HistogramMetric,
|
||||
request_time_per_output_token_seconds: HistogramMetric,
|
||||
|
||||
/// Present only when engines record stats.
|
||||
events: Option<EventMetricHandles>,
|
||||
}
|
||||
|
||||
/// Cached handles for request metrics derived from engine-core lifecycle
|
||||
/// events (`QUEUED`, `SCHEDULED`, `PREEMPTED`), which engines only emit when
|
||||
/// they record stats.
|
||||
#[derive(Clone)]
|
||||
struct EventMetricHandles {
|
||||
num_preemptions: U64Counter,
|
||||
request_num_preemptions: HistogramMetric,
|
||||
request_queue_time_seconds: HistogramMetric,
|
||||
request_prefill_time_seconds: HistogramMetric,
|
||||
request_decode_time_seconds: HistogramMetric,
|
||||
request_inference_time_seconds: HistogramMetric,
|
||||
request_time_per_output_token_seconds: HistogramMetric,
|
||||
}
|
||||
|
||||
impl RequestMetricsTracker {
|
||||
@@ -92,9 +102,14 @@ impl RequestMetricsTracker {
|
||||
prompt_len: u32,
|
||||
max_tokens_param: Option<u32>,
|
||||
n_param: u32,
|
||||
engine_stats_enabled: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
handles: resolve_request_metric_handles(&model_name, engine_index),
|
||||
handles: resolve_request_metric_handles(
|
||||
&model_name,
|
||||
engine_index,
|
||||
engine_stats_enabled,
|
||||
),
|
||||
arrival_time,
|
||||
prompt_len,
|
||||
max_tokens_param,
|
||||
@@ -163,10 +178,7 @@ impl RequestMetricsTracker {
|
||||
let prefill_kv_computed_tokens =
|
||||
self.prompt_len.saturating_sub(self.latest_num_cached_tokens);
|
||||
let e2e_latency_seconds = received_at - self.arrival_time;
|
||||
let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts);
|
||||
let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts);
|
||||
let decode_time_seconds = diff_or_zero(self.last_token_ts, self.first_token_ts);
|
||||
let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts);
|
||||
let time_per_output_token_seconds = if self.num_generation_tokens > 1 {
|
||||
diff_or_zero(self.last_token_ts, self.first_token_ts)
|
||||
/ (self.num_generation_tokens - 1) as f64
|
||||
@@ -187,18 +199,25 @@ impl RequestMetricsTracker {
|
||||
self.handles.request_params_max_tokens.observe(max_tokens_param as f64);
|
||||
}
|
||||
self.handles.request_params_n.observe(self.n_param as f64);
|
||||
self.handles.request_num_preemptions.observe(self.num_preemptions as f64);
|
||||
self.handles
|
||||
.request_prefill_kv_computed_tokens
|
||||
.observe(prefill_kv_computed_tokens as f64);
|
||||
self.handles.e2e_request_latency_seconds.observe(e2e_latency_seconds);
|
||||
self.handles.request_queue_time_seconds.observe(queue_time_seconds);
|
||||
self.handles.request_prefill_time_seconds.observe(prefill_time_seconds);
|
||||
self.handles.request_decode_time_seconds.observe(decode_time_seconds);
|
||||
self.handles.request_inference_time_seconds.observe(inference_time_seconds);
|
||||
self.handles
|
||||
.request_time_per_output_token_seconds
|
||||
.observe(time_per_output_token_seconds);
|
||||
|
||||
if let Some(events) = &self.handles.events {
|
||||
let queue_time_seconds = diff_or_zero(self.scheduled_ts, self.queued_ts);
|
||||
let prefill_time_seconds = diff_or_zero(self.first_token_ts, self.scheduled_ts);
|
||||
let inference_time_seconds = diff_or_zero(self.last_token_ts, self.scheduled_ts);
|
||||
|
||||
events.request_num_preemptions.observe(self.num_preemptions as f64);
|
||||
events.request_queue_time_seconds.observe(queue_time_seconds);
|
||||
events.request_prefill_time_seconds.observe(prefill_time_seconds);
|
||||
events.request_inference_time_seconds.observe(inference_time_seconds);
|
||||
}
|
||||
}
|
||||
|
||||
/// Record prompt token counters through cached metric handles.
|
||||
@@ -228,7 +247,9 @@ impl RequestMetricsTracker {
|
||||
}
|
||||
EngineCoreEventType::Preempted => {
|
||||
self.num_preemptions += 1;
|
||||
self.handles.num_preemptions.inc();
|
||||
if let Some(events) = &self.handles.events {
|
||||
events.num_preemptions.inc();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -248,7 +269,11 @@ impl RequestMetricsTracker {
|
||||
}
|
||||
|
||||
/// Resolve fixed request metric handles for one model and engine index.
|
||||
fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetricHandles {
|
||||
fn resolve_request_metric_handles(
|
||||
model_name: &str,
|
||||
engine: u32,
|
||||
engine_stats_enabled: bool,
|
||||
) -> RequestMetricHandles {
|
||||
let metrics = &METRICS.request;
|
||||
let labels = EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
@@ -256,7 +281,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
|
||||
};
|
||||
|
||||
RequestMetricHandles {
|
||||
num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels),
|
||||
prompt_tokens: metrics.prompt_tokens.get_or_create_owned(&labels),
|
||||
prompt_tokens_local_compute: metrics.prompt_tokens_by_source.get_or_create_owned(
|
||||
&prompt_token_source_labels(model_name, engine, PROMPT_TOKEN_SOURCE_LOCAL_COMPUTE),
|
||||
@@ -281,7 +305,6 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
|
||||
.get_or_create_owned(&labels),
|
||||
request_params_max_tokens: metrics.request_params_max_tokens.get_or_create_owned(&labels),
|
||||
request_params_n: metrics.request_params_n.get_or_create_owned(&labels),
|
||||
request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels),
|
||||
request_prefill_kv_computed_tokens: metrics
|
||||
.request_prefill_kv_computed_tokens
|
||||
.get_or_create_owned(&labels),
|
||||
@@ -294,19 +317,25 @@ fn resolve_request_metric_handles(model_name: &str, engine: u32) -> RequestMetri
|
||||
e2e_request_latency_seconds: metrics
|
||||
.e2e_request_latency_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_queue_time_seconds: metrics.request_queue_time_seconds.get_or_create_owned(&labels),
|
||||
request_prefill_time_seconds: metrics
|
||||
.request_prefill_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_decode_time_seconds: metrics
|
||||
.request_decode_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_inference_time_seconds: metrics
|
||||
.request_inference_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_time_per_output_token_seconds: metrics
|
||||
.request_time_per_output_token_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
events: engine_stats_enabled.then(|| EventMetricHandles {
|
||||
num_preemptions: metrics.num_preemptions.get_or_create_owned(&labels),
|
||||
request_num_preemptions: metrics.request_num_preemptions.get_or_create_owned(&labels),
|
||||
request_queue_time_seconds: metrics
|
||||
.request_queue_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_prefill_time_seconds: metrics
|
||||
.request_prefill_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
request_inference_time_seconds: metrics
|
||||
.request_inference_time_seconds
|
||||
.get_or_create_owned(&labels),
|
||||
}),
|
||||
labels,
|
||||
}
|
||||
}
|
||||
@@ -356,7 +385,7 @@ mod tests {
|
||||
#[test]
|
||||
fn tracker_updates_timing_state_across_prefill_decode_and_finish() {
|
||||
let mut tracker =
|
||||
RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1);
|
||||
RequestMetricsTracker::new("model".to_string(), 2, 100.0, 64, Some(128), 1, true);
|
||||
|
||||
tracker.observe_output(
|
||||
10.0,
|
||||
|
||||
@@ -6,12 +6,16 @@ use std::sync::{Arc, Once};
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt as _;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::time::timeout;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use uuid::Uuid;
|
||||
use vllm_engine_core_client::mock_engine::default_ready_response;
|
||||
use vllm_engine_core_client::protocol::handshake::EngineCoreReadyResponse;
|
||||
use vllm_engine_core_client::protocol::logprobs::{
|
||||
Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::lora::LoraRequest;
|
||||
use vllm_engine_core_client::protocol::output::{
|
||||
EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
|
||||
EngineCoreOutputs, RequestBatchOutputs,
|
||||
@@ -20,7 +24,9 @@ use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::protocol::sampling_mask::{MaybeWireSamplingMask, SamplingMask};
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
|
||||
use vllm_engine_core_client::test_utils::{
|
||||
IpcNamespace, spawn_mock_engine_task, spawn_mock_engine_task_with_ready,
|
||||
};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
|
||||
use vllm_llm::{
|
||||
Error, FinishReason, GenerateOutputStreamExt as _, GeneratePromptInfo, GenerateRequest, Llm,
|
||||
@@ -868,6 +874,138 @@ async fn generate_records_request_metrics_in_prometheus_output() {
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_prefill_kv_computed_tokens_count{{model_name=\"{model_name}\",engine=\"4\"}} 1"
|
||||
)));
|
||||
assert!(rendered.contains(&format!(
|
||||
"vllm:request_queue_time_seconds_sum{{model_name=\"{model_name}\",engine=\"4\"}} 1.0"
|
||||
)));
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
llm.shutdown().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn generate_without_engine_stats_exports_only_frontend_observed_metrics() {
|
||||
let ipc = IpcNamespace::new().unwrap();
|
||||
let handshake_address = ipc.handshake_endpoint();
|
||||
let engine_id = EngineId::from_engine_index(6);
|
||||
let model_name = request_metrics_model_name("metrics-no-log-stats-model");
|
||||
let lora_name = request_metrics_model_name("metrics-no-log-stats-lora");
|
||||
let (finish_tx, finish_rx) = oneshot::channel::<()>();
|
||||
|
||||
let (shutdown_tx, engine_task) = spawn_mock_engine_task_with_ready(
|
||||
handshake_address.clone(),
|
||||
engine_id,
|
||||
EngineCoreReadyResponse {
|
||||
supports_lora: true,
|
||||
max_loras: 1,
|
||||
..default_ready_response()
|
||||
},
|
||||
move |dealer, push| {
|
||||
Box::pin(async move {
|
||||
let add = recv_engine_message(dealer).await;
|
||||
assert_eq!(add[0].as_ref(), &[0x00]);
|
||||
let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
|
||||
|
||||
// Engines started with `--disable-log-stats` attach neither
|
||||
// lifecycle events nor scheduler stats to their outputs.
|
||||
for token in [1, 2] {
|
||||
send_outputs(
|
||||
push,
|
||||
RequestBatchOutputs {
|
||||
engine_index: 6,
|
||||
timestamp: 10.0 + f64::from(token),
|
||||
outputs: vec![request_output(&request.request_id, vec![token], None)],
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
finish_rx.await.unwrap();
|
||||
send_outputs(
|
||||
push,
|
||||
RequestBatchOutputs {
|
||||
engine_index: 6,
|
||||
timestamp: 13.0,
|
||||
outputs: vec![request_output(
|
||||
&request.request_id,
|
||||
vec![3],
|
||||
Some(EngineCoreFinishReason::Length),
|
||||
)],
|
||||
finished_requests: Some(BTreeSet::from([request.request_id])),
|
||||
..Default::default()
|
||||
}
|
||||
.into(),
|
||||
)
|
||||
.await;
|
||||
})
|
||||
},
|
||||
);
|
||||
|
||||
let client = EngineCoreClient::connect(
|
||||
EngineCoreClientConfig::new_single(handshake_address)
|
||||
.with_model_name(&model_name)
|
||||
.with_engine_stats_enabled(false)
|
||||
.with_local_input_output_addresses(
|
||||
Some(ipc.input_endpoint()),
|
||||
Some(ipc.output_endpoint()),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let llm = Llm::new(client);
|
||||
let mut request = sample_generate_request("req-no-log-stats", 8);
|
||||
request.arrival_time = None;
|
||||
request.lora_request = Some(LoraRequest {
|
||||
lora_name: lora_name.clone(),
|
||||
lora_int_id: 1,
|
||||
lora_path: "/unused".to_string(),
|
||||
base_model_name: None,
|
||||
tensorizer_config_dict: None,
|
||||
load_inplace: false,
|
||||
is_3d_lora_weight: false,
|
||||
});
|
||||
let mut stream = llm.generate(request).await.unwrap();
|
||||
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![1]);
|
||||
// The dispatcher handles the second batch only after finishing the first
|
||||
// one, including its LoRA info update.
|
||||
assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![2]);
|
||||
assert!(!METRICS.render().unwrap().contains(&lora_name));
|
||||
|
||||
finish_tx.send(()).unwrap();
|
||||
let final_output = stream.next().await.unwrap().unwrap();
|
||||
assert_eq!(final_output.finish_reason, Some(FinishReason::Length));
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
let rendered = METRICS.render().unwrap();
|
||||
for metric in [
|
||||
"vllm:time_to_first_token_seconds_count",
|
||||
"vllm:e2e_request_latency_seconds_count",
|
||||
"vllm:request_decode_time_seconds_count",
|
||||
] {
|
||||
assert!(
|
||||
rendered.contains(&format!(
|
||||
"{metric}{{model_name=\"{model_name}\",engine=\"6\"}} 1"
|
||||
)),
|
||||
"{metric} should be exported"
|
||||
);
|
||||
}
|
||||
for metric in [
|
||||
"vllm:num_requests_running",
|
||||
"vllm:kv_cache_usage_perc",
|
||||
"vllm:num_preemptions_total",
|
||||
"vllm:request_num_preemptions_count",
|
||||
"vllm:request_queue_time_seconds_count",
|
||||
"vllm:request_prefill_time_seconds_count",
|
||||
"vllm:request_inference_time_seconds_count",
|
||||
] {
|
||||
assert!(
|
||||
!rendered.contains(&format!("{metric}{{model_name=\"{model_name}\"")),
|
||||
"{metric} should not be exported"
|
||||
);
|
||||
}
|
||||
|
||||
let _ = shutdown_tx.send(());
|
||||
engine_task.await.unwrap();
|
||||
|
||||
@@ -29,6 +29,7 @@ fn client_config(handshake_address: String, engine_count: usize) -> EngineCoreCl
|
||||
coordinator_mode: None,
|
||||
model_name: "mock-model".to_string(),
|
||||
client_index: 0,
|
||||
engine_stats_enabled: true,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -268,7 +268,9 @@ pub struct Config {
|
||||
#[educe(Debug(method(fmt_redacted_api_keys)))]
|
||||
pub api_keys: Vec<String>,
|
||||
/// When `true`, suppress periodic stats logging (throughput, queue depth,
|
||||
/// cache usage).
|
||||
/// cache usage). Engines also stop recording stats, so metrics derived from
|
||||
/// engine-reported scheduler stats and request lifecycle events are not
|
||||
/// exported.
|
||||
pub disable_log_stats: bool,
|
||||
/// TCP port for the gRPC Inference service. When `None`, no gRPC server is
|
||||
/// started.
|
||||
|
||||
@@ -2234,6 +2234,7 @@ async fn control_aggregates_multi_engine_capacity() {
|
||||
coordinator_mode: None,
|
||||
model_name: "test-model".to_string(),
|
||||
client_index: 0,
|
||||
engine_stats_enabled: true,
|
||||
};
|
||||
let client_task = tokio::spawn(EngineCoreClient::connect(client_config));
|
||||
let mut engine_sockets = Vec::new();
|
||||
|
||||
@@ -130,6 +130,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
|
||||
coordinator_mode,
|
||||
model_name: metrics_model_name,
|
||||
client_index: 0,
|
||||
engine_stats_enabled: !config.disable_log_stats,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
Reference in New Issue
Block a user