[Frontend][Rust] Add mm-processor benchmark for Rust frontend (#51922)
Signed-off-by: Karthik Gangula <gangula-karthik@users.noreply.github.com> Signed-off-by: gangula-karthik <gkarthik923@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Karthik Gangula <gangula-karthik@users.noreply.github.com> Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Karthik Gangula
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Isotr0py
parent
0961bbae28
commit
f84325c48c
@@ -3,13 +3,14 @@
|
||||
|
||||
//! Shared tracing subscriber and log formatting for vLLM Rust binaries.
|
||||
|
||||
pub mod timing;
|
||||
|
||||
use std::{env, fmt, process};
|
||||
|
||||
use time::UtcOffset;
|
||||
use time::macros::format_description;
|
||||
use tracing::level_filters::LevelFilter;
|
||||
use tracing::{Event, Level, Subscriber};
|
||||
use tracing_subscriber::Layer as _;
|
||||
use tracing_subscriber::filter::Targets;
|
||||
use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
|
||||
use tracing_subscriber::fmt::time::FormatTime;
|
||||
@@ -17,6 +18,7 @@ use tracing_subscriber::fmt::{FmtContext, FormattedFields};
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
use tracing_subscriber::util::SubscriberInitExt as _;
|
||||
use tracing_subscriber::{Layer, Registry};
|
||||
|
||||
const CYAN: &str = "\x1b[0;36m";
|
||||
const GREY: &str = "\x1b[90m";
|
||||
@@ -30,20 +32,36 @@ const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
|
||||
|
||||
/// Install the process-wide vLLM-style tracing subscriber.
|
||||
pub fn init_tracing(process_label: &str) {
|
||||
let _ = tracing_subscriber::registry().with(vllm_fmt_layer(process_label)).try_init();
|
||||
}
|
||||
|
||||
/// Install the process-wide vLLM-style tracing subscriber, with an
|
||||
/// additional layer (e.g. the multimodal preprocessing timing collector).
|
||||
pub fn init_tracing_with<L>(process_label: &str, layer: L)
|
||||
where
|
||||
L: Layer<Registry> + Send + Sync + 'static,
|
||||
{
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(layer)
|
||||
.with(vllm_fmt_layer(process_label))
|
||||
.try_init();
|
||||
}
|
||||
|
||||
/// The vLLM-style fmt layer filtered by `VLLM_LOGGING_LEVEL` / `RUST_LOG`.
|
||||
fn vllm_fmt_layer<S>(process_label: &str) -> impl Layer<S> + Send + Sync + 'static + use<S>
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
let filter = build_targets_filter(
|
||||
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
|
||||
env::var("RUST_LOG").ok().as_deref(),
|
||||
);
|
||||
let formatter = VllmEventFormatter::new(process_label);
|
||||
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.event_format(formatter)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter(filter),
|
||||
)
|
||||
.try_init();
|
||||
tracing_subscriber::fmt::layer()
|
||||
.event_format(formatter)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter(filter)
|
||||
}
|
||||
|
||||
/// Build the CLI log filter by merging the vLLM-style default level with
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Request-scoped span timing.
|
||||
//!
|
||||
//! Pipelines emit stage spans (target = the layer's configured target, a
|
||||
//! `stage` field naming the stage) inside a request span carrying a
|
||||
//! `request_id` field; [`RequestTimingLayer`] attributes each stage span's
|
||||
//! wall-clock time to the nearest ancestor `request_id` and accumulates per
|
||||
//! request. Callers that want timings install the layer on their subscriber;
|
||||
//! everyone else pays only the cost of untracked spans.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use tracing::field::{Field, Visit};
|
||||
use tracing::span::{Attributes, Span};
|
||||
use tracing::{Id, Subscriber, info_span};
|
||||
use tracing_subscriber::layer::{Context, Layer};
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
|
||||
/// `"{stage}_secs" => seconds` maps returned by [`RequestTimingStats::stat`].
|
||||
pub type StageStats = HashMap<String, f64>;
|
||||
|
||||
/// `request_id` extracted from a span, propagated to stage timings.
|
||||
struct RequestId(String);
|
||||
|
||||
/// Start timestamp of one in-flight stage span.
|
||||
struct StageStart {
|
||||
stage: String,
|
||||
start: Instant,
|
||||
}
|
||||
|
||||
/// Extracts the fields the layer cares about from span attributes.
|
||||
#[derive(Default)]
|
||||
struct SpanFields {
|
||||
request_id: Option<String>,
|
||||
stage: Option<String>,
|
||||
}
|
||||
|
||||
impl Visit for SpanFields {
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
match field.name() {
|
||||
"request_id" => self.request_id = Some(value.to_string()),
|
||||
"stage" => self.stage = Some(value.to_string()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
|
||||
match field.name() {
|
||||
"request_id" => self.request_id = Some(format!("{value:?}")),
|
||||
"stage" => self.stage = Some(format!("{value:?}")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `tracing` layer that aggregates stage-span timings per request, keyed by
|
||||
/// the nearest ancestor span with a `request_id` field.
|
||||
pub struct RequestTimingLayer {
|
||||
target: &'static str,
|
||||
stats: Arc<Mutex<HashMap<String, StageStats>>>,
|
||||
}
|
||||
|
||||
/// Drain handle for the timings collected by [`RequestTimingLayer`].
|
||||
pub struct RequestTimingStats {
|
||||
stats: Arc<Mutex<HashMap<String, StageStats>>>,
|
||||
}
|
||||
|
||||
impl RequestTimingLayer {
|
||||
/// Create the layer and its stats handle. Only spans whose target equals
|
||||
/// `target` and that carry a `stage` field are timed.
|
||||
pub fn new(target: &'static str) -> (Self, RequestTimingStats) {
|
||||
let stats = Arc::new(Mutex::new(HashMap::new()));
|
||||
(
|
||||
Self {
|
||||
target,
|
||||
stats: Arc::clone(&stats),
|
||||
},
|
||||
RequestTimingStats { stats },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestTimingStats {
|
||||
/// Drain and return `{request_id: {stage_secs}}` records.
|
||||
pub fn stat(&self) -> HashMap<String, StageStats> {
|
||||
std::mem::take(&mut *self.stats.lock().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for RequestTimingLayer
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
|
||||
let Some(span) = ctx.span(id) else {
|
||||
return;
|
||||
};
|
||||
let mut fields = SpanFields::default();
|
||||
attrs.record(&mut fields);
|
||||
let mut extensions = span.extensions_mut();
|
||||
if let Some(request_id) = fields.request_id {
|
||||
extensions.insert(RequestId(request_id));
|
||||
}
|
||||
if span.metadata().target() == self.target
|
||||
&& let Some(stage) = fields.stage
|
||||
{
|
||||
extensions.insert(StageStart {
|
||||
stage,
|
||||
start: Instant::now(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn on_close(&self, id: Id, ctx: Context<'_, S>) {
|
||||
let Some(span) = ctx.span(&id) else {
|
||||
return;
|
||||
};
|
||||
let (stage, start) = {
|
||||
let extensions = span.extensions();
|
||||
let Some(stage_start) = extensions.get::<StageStart>() else {
|
||||
return;
|
||||
};
|
||||
(stage_start.stage.clone(), stage_start.start)
|
||||
};
|
||||
let Some(request_id) = span
|
||||
.scope()
|
||||
.find_map(|ancestor| ancestor.extensions().get::<RequestId>().map(|id| id.0.clone()))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let mut stats = self.stats.lock().unwrap();
|
||||
*stats.entry(request_id).or_default().entry(format!("{stage}_secs")).or_default() +=
|
||||
start.elapsed().as_secs_f64();
|
||||
}
|
||||
}
|
||||
|
||||
/// Target of the multimodal preprocessing stage spans.
|
||||
const MM_STAGE_TARGET: &str = "mm_processor_timing";
|
||||
|
||||
/// Create the timing layer and its stats handle for multimodal preprocessing
|
||||
/// stage spans (`vllm-bench mm-processor`), mirroring the Python
|
||||
/// `TimingContext` / `MultiModalTimingRegistry`.
|
||||
pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) {
|
||||
RequestTimingLayer::new(MM_STAGE_TARGET)
|
||||
}
|
||||
|
||||
/// Span carrying the `request_id` used to attribute multimodal stage timings.
|
||||
pub fn mm_request_span(request_id: &str) -> Span {
|
||||
info_span!("mm_request", request_id)
|
||||
}
|
||||
|
||||
/// Span for one multimodal preprocessing stage; the elapsed time is recorded
|
||||
/// on close.
|
||||
pub fn mm_stage_span(stage: &'static str) -> Span {
|
||||
info_span!(target: MM_STAGE_TARGET, "mm_stage", stage)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tracing::info_span;
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
|
||||
const TARGET: &str = "test_request_timing";
|
||||
|
||||
fn stage_span(stage: &'static str) -> tracing::Span {
|
||||
info_span!(target: TARGET, "stage", stage)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_stage_timings_per_request() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
info_span!("request", request_id = "req-1").in_scope(|| {
|
||||
drop(stage_span("media_fetch"));
|
||||
stage_span("prompt_expansion").in_scope(|| {});
|
||||
});
|
||||
});
|
||||
let drained = stats.stat();
|
||||
assert!(drained["req-1"].contains_key("media_fetch_secs"));
|
||||
assert!(drained["req-1"].contains_key("prompt_expansion_secs"));
|
||||
// `stat` drains: a second read is empty.
|
||||
assert!(stats.stat().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attributes_to_nearest_request_id_ancestor() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
info_span!("request", request_id = "req-outer").in_scope(|| {
|
||||
info_span!("request", request_id = "req-inner").in_scope(|| {
|
||||
drop(stage_span("media_fetch"));
|
||||
});
|
||||
});
|
||||
});
|
||||
let drained = stats.stat();
|
||||
assert!(drained["req-inner"].contains_key("media_fetch_secs"));
|
||||
assert!(!drained.contains_key("req-outer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_without_request_id_is_dropped() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
drop(stage_span("media_fetch"));
|
||||
});
|
||||
assert!(stats.stat().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_targets_are_ignored() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
info_span!("request", request_id = "req-1").in_scope(|| {
|
||||
drop(info_span!("unrelated", stage = "media_fetch"));
|
||||
});
|
||||
});
|
||||
assert!(stats.stat().is_empty());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user