[Rust Frontend] Add iteration token histogram (#56990)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -19,7 +19,7 @@ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, Uti
|
||||
use crate::client::stream::EngineCoreStreamOutput;
|
||||
use crate::client::{AbortCause, AbortRequest};
|
||||
use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
|
||||
use crate::metrics::{LoraInfoExporter, SchedulerStatsRecorder};
|
||||
use crate::metrics::{IterationMetricHandles, LoraInfoExporter, SchedulerStatsRecorder};
|
||||
use crate::protocol::encode_msgpack;
|
||||
use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
|
||||
use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
|
||||
@@ -47,6 +47,7 @@ pub(crate) struct ClientInner {
|
||||
/// `VLLM_MSGPACK_ZERO_COPY_THRESHOLD` when this inner client is created.
|
||||
msgpack_zero_copy_threshold: usize,
|
||||
scheduler_stats_recorder: SchedulerStatsRecorder,
|
||||
iteration_metrics: BTreeMap<u32, IterationMetricHandles>,
|
||||
request_reg: Mutex<RequestRegistry>,
|
||||
utility_reg: Mutex<UtilityRegistry>,
|
||||
health_error: ArcSwapOption<Error>,
|
||||
@@ -64,12 +65,21 @@ impl ClientInner {
|
||||
) -> Self {
|
||||
let scheduler_stats_recorder =
|
||||
SchedulerStatsRecorder::new(&METRICS.scheduler, &model_name, engines);
|
||||
let iteration_metrics = engines
|
||||
.iter()
|
||||
.filter_map(|engine| {
|
||||
let engine = engine.engine_id.engine_index()?;
|
||||
let handles = IterationMetricHandles::new(&METRICS.request, &model_name, engine);
|
||||
Some((engine, handles))
|
||||
})
|
||||
.collect();
|
||||
Self {
|
||||
input_send,
|
||||
handle,
|
||||
model_name,
|
||||
msgpack_zero_copy_threshold: msgpack_zero_copy_threshold(),
|
||||
scheduler_stats_recorder,
|
||||
iteration_metrics,
|
||||
request_reg: Mutex::new(RequestRegistry::new(engines)),
|
||||
utility_reg: Mutex::new(UtilityRegistry::default()),
|
||||
health_error: ArcSwapOption::empty(),
|
||||
@@ -415,6 +425,8 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
|
||||
match outputs {
|
||||
EngineCoreOutputs::RequestBatch(batch) => {
|
||||
let has_outputs = !batch.outputs.is_empty();
|
||||
let mut iteration_tokens = 0_u64;
|
||||
let senders = inner.take_senders_for_outputs(&batch.outputs);
|
||||
for (output, sender) in batch.outputs.into_iter().zip(senders) {
|
||||
let request_id = output.request_id.clone();
|
||||
@@ -423,6 +435,12 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
continue;
|
||||
};
|
||||
|
||||
iteration_tokens += output.new_token_ids.len() as u64;
|
||||
if let Some(prefill) = &output.prefill_stats {
|
||||
// The engine emits prefill_stats once, on the first output.
|
||||
iteration_tokens += u64::from(prefill.num_computed_tokens);
|
||||
}
|
||||
|
||||
let wrapped_output = EngineCoreStreamOutput {
|
||||
engine_index: batch.engine_index,
|
||||
timestamp: batch.timestamp,
|
||||
@@ -433,6 +451,12 @@ pub(crate) async fn run_output_dispatcher_loop(
|
||||
}
|
||||
}
|
||||
|
||||
if has_outputs
|
||||
&& let Some(handles) = inner.iteration_metrics.get(&batch.engine_index)
|
||||
{
|
||||
handles.iteration_tokens_total.observe(iteration_tokens as f64);
|
||||
}
|
||||
|
||||
// The sender for normally-finished requests should have already been removed
|
||||
// from the registry when their final output was dispatched
|
||||
// above. This serves as a safety net to capture any
|
||||
|
||||
@@ -8,8 +8,8 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use vllm_metrics::{
|
||||
EngineLabels, EnginePositionLabels, F64Gauge, Family, HistogramMetric, LoraAdapterNames,
|
||||
LoraInfoLabels, MooncakeOperationCounterFamily, MooncakeOperationHistogramFamily,
|
||||
MooncakeOperationLabels, SchedulerLogStatsAccumulator, SchedulerMetrics, U64Counter, U64Gauge,
|
||||
WaitingReasonLabels,
|
||||
MooncakeOperationLabels, RequestMetrics, SchedulerLogStatsAccumulator, SchedulerMetrics,
|
||||
U64Counter, U64Gauge, WaitingReasonLabels,
|
||||
};
|
||||
|
||||
use crate::protocol::stats::{
|
||||
@@ -20,6 +20,23 @@ use crate::transport::ConnectedEngine;
|
||||
const WAITING_REASON_CAPACITY: &str = "capacity";
|
||||
const WAITING_REASON_DEFERRED: &str = "deferred";
|
||||
|
||||
/// Cached output-batch metric handles for one model and engine index.
|
||||
pub(crate) struct IterationMetricHandles {
|
||||
pub iteration_tokens_total: HistogramMetric,
|
||||
}
|
||||
|
||||
impl IterationMetricHandles {
|
||||
pub(crate) fn new(metrics: &RequestMetrics, model_name: &str, engine: u32) -> Self {
|
||||
let labels = EngineLabels {
|
||||
model_name: model_name.to_string(),
|
||||
engine,
|
||||
};
|
||||
Self {
|
||||
iteration_tokens_total: metrics.iteration_tokens_total.get_or_create_owned(&labels),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Cached scheduler-stats metric handles for all engines connected to one
|
||||
/// frontend client.
|
||||
pub(crate) struct SchedulerStatsRecorder {
|
||||
|
||||
@@ -21,6 +21,9 @@ const REQUEST_LATENCY_BUCKETS: [f64; 21] = [
|
||||
480.0, 960.0, 1920.0, 7680.0,
|
||||
];
|
||||
const REQUEST_PARAMS_N_BUCKETS: [f64; 5] = [1.0, 2.0, 5.0, 10.0, 20.0];
|
||||
const ITERATION_TOKENS_BUCKETS: [f64; 13] = [
|
||||
1.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0, 16384.0,
|
||||
];
|
||||
|
||||
fn build_1_2_5_buckets(max_value: u32) -> Vec<f64> {
|
||||
let mut buckets = Vec::new();
|
||||
@@ -66,6 +69,10 @@ fn request_params_n_histogram() -> Histogram {
|
||||
Histogram::new(REQUEST_PARAMS_N_BUCKETS.iter().copied())
|
||||
}
|
||||
|
||||
fn iteration_tokens_histogram() -> Histogram {
|
||||
Histogram::new(ITERATION_TOKENS_BUCKETS.iter().copied())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
|
||||
pub struct FinishedReasonLabels {
|
||||
pub model_name: String,
|
||||
@@ -83,8 +90,12 @@ pub struct PromptTokenSourceLabels {
|
||||
pub(crate) type FinishedReasonCounterFamily = Family<FinishedReasonLabels, U64Counter>;
|
||||
pub(crate) type PromptTokenSourceCounterFamily = Family<PromptTokenSourceLabels, U64Counter>;
|
||||
|
||||
/// Request-lifecycle Prometheus families exported from the `llm` layer.
|
||||
/// Request and output-batch Prometheus families.
|
||||
pub struct RequestMetrics {
|
||||
/// Computed prompt and generated tokens per nonempty engine output batch,
|
||||
/// recorded by the engine-core client for requests tracked by this frontend.
|
||||
pub iteration_tokens_total: HistogramFamily,
|
||||
|
||||
// Request-derived counters.
|
||||
pub num_preemptions: Family<EngineLabels, U64Counter>,
|
||||
pub prompt_tokens: Family<EngineLabels, U64Counter>,
|
||||
@@ -92,11 +103,6 @@ pub struct RequestMetrics {
|
||||
pub prompt_tokens_cached: Family<EngineLabels, U64Counter>,
|
||||
pub generation_tokens: Family<EngineLabels, U64Counter>,
|
||||
|
||||
// We intentionally don't support iteration-level histograms for now, since it seems to make
|
||||
// more sense if the engine maintains these metrics and frontend simply forwards.
|
||||
//
|
||||
// pub iteration_tokens_total: HistogramFamily,
|
||||
|
||||
// Request lifecycle counters and histograms.
|
||||
pub request_success: FinishedReasonCounterFamily,
|
||||
pub request_prompt_tokens: HistogramFamily,
|
||||
@@ -118,6 +124,14 @@ pub struct RequestMetrics {
|
||||
impl RequestMetrics {
|
||||
/// Register the request-oriented metric families into the shared registry.
|
||||
pub(crate) fn register(registry: &mut Registry) -> Self {
|
||||
let iteration_tokens_total =
|
||||
Family::new_with_constructor(iteration_tokens_histogram as fn() -> Histogram);
|
||||
registry.register(
|
||||
"vllm:iteration_tokens_total",
|
||||
"Histogram of computed prompt and generated tokens per nonempty engine output batch.",
|
||||
iteration_tokens_total.clone(),
|
||||
);
|
||||
|
||||
// Request-derived counters.
|
||||
let num_preemptions = Family::default();
|
||||
registry.register(
|
||||
@@ -276,6 +290,7 @@ impl RequestMetrics {
|
||||
);
|
||||
|
||||
Self {
|
||||
iteration_tokens_total,
|
||||
num_preemptions,
|
||||
prompt_tokens,
|
||||
prompt_tokens_by_source,
|
||||
|
||||
@@ -19,6 +19,7 @@ use vllm_engine_core_client::protocol::output::{
|
||||
UtilityCallOutput,
|
||||
};
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::protocol::utility::{
|
||||
EngineCoreUtilityRequest, UtilityOutput, UtilityResultEnvelope,
|
||||
};
|
||||
@@ -187,6 +188,11 @@ impl ActiveRequest {
|
||||
|
||||
/// Advance this request by one mock engine step.
|
||||
fn step(&mut self, opt: &Opt) -> EngineCoreOutput {
|
||||
let prefill_stats = (self.generated == 0).then(|| PrefillStats {
|
||||
num_prompt_tokens: self.prompt_len as u32,
|
||||
num_computed_tokens: self.prompt_len as u32,
|
||||
..Default::default()
|
||||
});
|
||||
let remaining = self.max_tokens - self.generated;
|
||||
let chunk_len = remaining.min(opt.output_token_chunk_size);
|
||||
let mut new_token_ids = Vec::with_capacity(chunk_len);
|
||||
@@ -196,11 +202,14 @@ impl ActiveRequest {
|
||||
self.generated += chunk_len;
|
||||
|
||||
let finished = self.generated >= self.max_tokens;
|
||||
request_output(
|
||||
self.request_id.clone(),
|
||||
new_token_ids,
|
||||
finished.then_some(EngineCoreFinishReason::Length),
|
||||
)
|
||||
EngineCoreOutput {
|
||||
prefill_stats,
|
||||
..request_output(
|
||||
self.request_id.clone(),
|
||||
new_token_ids,
|
||||
finished.then_some(EngineCoreFinishReason::Length),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
|
||||
use vllm_engine_core_client::protocol::request::EngineCoreRequest;
|
||||
use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
|
||||
use vllm_engine_core_client::protocol::stats::PrefillStats;
|
||||
use vllm_engine_core_client::test_utils::IpcNamespace;
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
|
||||
@@ -119,12 +120,22 @@ async fn chunk_size_one_outputs_one_token_per_update() {
|
||||
let first = stream.next().await.expect("first").expect("first ok");
|
||||
assert_eq!(first.new_token_ids.len(), 1);
|
||||
assert_eq!(first.finish_reason, None);
|
||||
assert_eq!(
|
||||
first.prefill_stats,
|
||||
Some(PrefillStats {
|
||||
num_prompt_tokens: 3,
|
||||
num_computed_tokens: 3,
|
||||
..Default::default()
|
||||
})
|
||||
);
|
||||
let second = stream.next().await.expect("second").expect("second ok");
|
||||
assert_eq!(second.new_token_ids.len(), 1);
|
||||
assert_eq!(second.finish_reason, None);
|
||||
assert!(second.prefill_stats.is_none());
|
||||
let third = stream.next().await.expect("third").expect("third ok");
|
||||
assert_eq!(third.new_token_ids.len(), 1);
|
||||
assert_eq!(third.finish_reason, Some(EngineCoreFinishReason::Length));
|
||||
assert!(third.prefill_stats.is_none());
|
||||
assert!(stream.next().await.is_none());
|
||||
|
||||
shutdown_mock(client, shutdown, task).await;
|
||||
|
||||
Reference in New Issue
Block a user