diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index b4917a303b..ad421ba41d 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -5735,6 +5735,11 @@ dependencies = [
"url",
"uuid",
"vllm-build-info",
+ "vllm-chat",
+ "vllm-engine-core-client",
+ "vllm-llm",
+ "vllm-managed-engine",
+ "vllm-text",
"vllm-tracing",
]
@@ -5784,6 +5789,7 @@ dependencies = [
"vllm-parser",
"vllm-text",
"vllm-tokenizer",
+ "vllm-tracing",
"xgrammar-structural-tag",
"zeromq",
]
diff --git a/rust/src/bench/AGENTS.md b/rust/src/bench/AGENTS.md
index 1f15165fa4..d9a17f63f2 100644
--- a/rust/src/bench/AGENTS.md
+++ b/rust/src/bench/AGENTS.md
@@ -23,7 +23,7 @@ cargo test -p vllm-bench -- --ignored
## Architecture
-- `src/main.rs` — Entry point, mimalloc, tokio runtime, mode dispatch (compare/sweep/multi-run/multi-turn/single)
+- `src/main.rs` — Entry point, mimalloc, tokio runtime, mode dispatch (compare/sweep/multi-run/multi-turn/single, `mm-processor` subcommand)
- `src/cli.rs` — clap derive CLI args (~50+ flags)
- `src/config.rs` — Validated config from CLI; `GoodputConfig`, `RampUpConfig`, sampling param merging
- `src/error.rs` — `BenchError` enum (Http, Json, Tokenizer, Config, EndpointTimeout, Backend, Io)
@@ -32,6 +32,7 @@ cargo test -p vllm-bench -- --ignored
- `src/sweep.rs` — Concurrency/rate parameter sweep (`--sweep-max-concurrency`, `--sweep-request-rate`)
- `src/multi_run.rs` — N-run aggregation with mean/std/min/max/CV (`--num-runs`)
- `src/compare.rs` — Side-by-side diff of two result JSON files (`--compare`)
+- `src/mm_processor.rs` — `vllm-bench mm-processor` subcommand: offline multimodal preprocessing latency benchmark against a managed headless engine (mirrors `vllm bench mm-processor`); per-stage timings come from `vllm_tracing::timing::RequestTimingLayer` (via `vllm_chat::mm_timing_layer`)
- `src/tokenizer.rs` — `TokenizerKind` enum: Local(HuggingFace), Tiktoken, OR Server-side `/tokenize`+`/detokenize` fallback
- `src/tiktoken.rs` — Tiktoken BPE loader (`.tiktoken`/`.model` files; built-in encodings o200k_base/cl100k_base; pat_str extraction from Python source)
- `src/hub.rs` — `HubRepo`: thin wrapper over hf-hub's async (reqwest/native-tls) API — file download and repo listing, optional revision; the sync ureq backend is unusable here because it pulls rustls, which `rust/deny.toml` bans
diff --git a/rust/src/bench/Cargo.toml b/rust/src/bench/Cargo.toml
index 5a3b14894a..60436d4251 100644
--- a/rust/src/bench/Cargo.toml
+++ b/rust/src/bench/Cargo.toml
@@ -36,6 +36,11 @@ tracing.workspace = true
url.workspace = true
uuid.workspace = true
vllm-build-info.workspace = true
+vllm-chat.workspace = true
+vllm-engine-core-client.workspace = true
+vllm-llm.workspace = true
+vllm-managed-engine.workspace = true
+vllm-text.workspace = true
vllm-tracing.workspace = true
[lints]
diff --git a/rust/src/bench/README.md b/rust/src/bench/README.md
index c8724d6cd6..fb1e5b529d 100644
--- a/rust/src/bench/README.md
+++ b/rust/src/bench/README.md
@@ -18,6 +18,7 @@ vllm-bench --backend vllm --base-url http://127.0.0.1:8000 \
- **Beyond a single run** — concurrency/rate **sweeps**, **multi-run** stats, **multi-turn** conversations, **LoRA** multi-adapter, and result **comparison**.
- **Steady-state metrics** — throughput/latency measured over the saturated plateau, excluding ramp-up and drain.
- **Parity** — JSON output schema and timing semantics match Python `vllm bench serve` exactly.
+- **Multimodal preprocessing** — `mm-processor` subcommand measures per-stage VLM preprocessing latency against a managed headless engine (mirrors `vllm bench mm-processor`).
### Performance vs. Python
@@ -404,6 +405,32 @@ vllm-bench \
+
+Multimodal preprocessing (offline)
+
+The `mm-processor` subcommand benchmarks the multimodal preprocessing pipeline
+itself (media fetch → per-modality preprocess → prompt expansion), mirroring
+`vllm bench mm-processor`. It spawns a managed headless Python engine (real
+weights, real encoder) — no serving endpoint needed — and reports per-stage
+timing (mean/median/std/percentiles) plus end-to-end latency.
+
+```bash
+vllm-bench mm-processor \
+ --model Qwen/Qwen2.5-VL-7B-Instruct \
+ --num-prompts 100 --num-warmups 5 \
+ --random-input-len 512 --random-output-len 128 \
+ --random-mm-base-items-per-request 1 \
+ --random-mm-limit-mm-per-prompt '{"image": 1, "video": 0}' \
+ --random-mm-bucket-config '{(1024, 800, 1): 1.0}' \
+ --metric-percentiles "50,90,99" \
+ --output-json mm_processor_stats.json
+```
+
+`--max-concurrency` defaults to `1`, matching Python's serial driver; raise it
+to measure a concurrent serving-style workload.
+
+
+
## Supported Backends
### Generation
@@ -705,6 +732,36 @@ With `--multi-turn`, `--num-prompts` controls the number of **conversations**, n
+
+mm-processor subcommand
+
+Offline multimodal preprocessing latency benchmark (mirrors `vllm bench mm-processor`). Spawns a managed headless Python engine and drives the full chat preprocessing pipeline.
+
+| Flag | Default | Description |
+| ------ | --------- | ------------- |
+| `--model` | (required) | Model to serve and benchmark (HF id or local path) |
+| `--num-prompts` | `10` | Prompts to process, excluding warmups |
+| `--num-warmups` | `1` | Warmup prompts processed and discarded before timing |
+| `--metric-percentiles` | `99` | Comma-separated percentiles to report |
+| `--output-json` | — | Path to write aggregate stats as JSON (same schema as Python) |
+| `--random-input-len` | `1024` | Text input length per prompt |
+| `--random-output-len` | `128` | Expected output length per prompt |
+| `--random-prefix-len` | `0` | Prefix token length per prompt |
+| `--random-range-ratio` | `0.0` | Input/output length range ratio (float or `{"input": i, "output": o}`) |
+| `--random-mm-base-items-per-request` | `1` | Base number of multimodal items per request |
+| `--random-mm-num-mm-items-range-ratio` | `0.0` | Range ratio (in [0, 1]) for the number of mm items per request |
+| `--random-mm-limit-mm-per-prompt` | `{"image": 255, "video": 1}` | Per-modality item caps |
+| `--random-mm-bucket-config` | `{(256,256,1): 0.5, (720,1280,1): 0.5}` | Bucket config |
+| `--seed` | `0` | Seed for dataset generation |
+| `--request-id-prefix` | `mm-proc-` | Request-id prefix for per-request timing keys |
+| `--max-concurrency` | `1` | Maximum concurrent requests (1 matches Python's serial driver) |
+| `--trust-remote-code` | `false` | Trust remote code for the tokenizer and the managed engine |
+| `--chat-template` | — | Chat-template override (inline text or path) |
+
+Managed headless-engine options (e.g. `--python`, `--data-parallel-address`, `--data-parallel-rpc-port`, `--data-parallel-size`, `--max-model-len`, `--python-args`) are shared with other managed-engine commands; see `vllm-bench mm-processor --help`.
+
+
+
## Tokenizer Support
Tokenizers are loaded with a three-tier fallback chain:
@@ -734,10 +791,11 @@ Use `--append-result` to append multiple runs to the same file in JSONL format.
```text
src/
-├── main.rs # Entry point, mimalloc, tokio runtime, mode dispatch
+├── main.rs # Entry point, mimalloc, tokio runtime, mode/subcommand dispatch
├── cli.rs # clap CLI argument definitions
├── config.rs # Validated config, goodput/ramp-up parsing
├── benchmark.rs # Core orchestrator (schedule, spawn, collect, verify, profile)
+├── mm_processor.rs # Offline mm preprocessing benchmark (mm-processor subcommand)
├── multi_turn.rs # Multi-turn conversation orchestrator (channel workers)
├── compare.rs # Result diff (--compare file_a.json file_b.json)
├── sweep.rs # Parameter sweep (--sweep-max-concurrency, --sweep-request-rate)
diff --git a/rust/src/bench/src/lib.rs b/rust/src/bench/src/lib.rs
index 0a4f19c0d6..8541c3b310 100644
--- a/rust/src/bench/src/lib.rs
+++ b/rust/src/bench/src/lib.rs
@@ -10,6 +10,7 @@ mod datasets;
mod error;
mod hub;
mod metrics;
+mod mm_processor;
mod multi_run;
mod multi_turn;
mod output;
@@ -25,6 +26,7 @@ pub use cli::{
BackendKind, BenchServeArgs, DatasetName, LoraAssignment, RampUpStrategy, SpeedBenchConfig,
};
use config::BenchConfig;
+pub use mm_processor::{MmProcessorArgs, run_mm_processor};
/// Prepare process-wide resources for a benchmark run.
pub fn prepare_process() {
diff --git a/rust/src/bench/src/main.rs b/rust/src/bench/src/main.rs
index 3485e22bb1..4a72b81892 100644
--- a/rust/src/bench/src/main.rs
+++ b/rust/src/bench/src/main.rs
@@ -6,22 +6,30 @@
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
use anyhow::Context;
-use clap::Parser;
+use clap::{Parser, Subcommand};
#[derive(Parser)]
#[command(
name = "vllm-bench",
- about = "Benchmark online serving throughput",
+ about = "Benchmark online serving throughput and offline multimodal preprocessing",
version = vllm_build_info::VERSION
)]
struct Cli {
#[command(flatten)]
args: vllm_bench::BenchServeArgs,
+
+ /// Optional subcommand; absent by default for the online serving benchmark.
+ #[command(subcommand)]
+ command: Option,
+}
+
+#[derive(Subcommand)]
+enum Command {
+ /// Offline multimodal preprocessing latency benchmark.
+ MmProcessor(vllm_bench::MmProcessorArgs),
}
fn main() -> anyhow::Result<()> {
- vllm_tracing::init_tracing("Bench");
-
let cli = Cli::parse();
vllm_bench::prepare_process();
@@ -30,5 +38,15 @@ fn main() -> anyhow::Result<()> {
.build()
.context("Failed to build tokio runtime")?;
- runtime.block_on(vllm_bench::run(cli.args))
+ match cli.command {
+ Some(Command::MmProcessor(args)) => {
+ let (timing_layer, timing_stats) = vllm_chat::mm_timing_layer();
+ vllm_tracing::init_tracing_with("Bench", timing_layer);
+ runtime.block_on(vllm_bench::run_mm_processor(args, timing_stats))
+ }
+ None => {
+ vllm_tracing::init_tracing("Bench");
+ runtime.block_on(vllm_bench::run(cli.args))
+ }
+ }
}
diff --git a/rust/src/bench/src/metrics/calculator.rs b/rust/src/bench/src/metrics/calculator.rs
index 11b68a3bfe..b292be42eb 100644
--- a/rust/src/bench/src/metrics/calculator.rs
+++ b/rust/src/bench/src/metrics/calculator.rs
@@ -216,20 +216,20 @@ pub fn calculate_metrics(
(metrics, actual_output_lens)
}
-fn mean(data: &[f64]) -> f64 {
+pub(crate) fn mean(data: &[f64]) -> f64 {
if data.is_empty() {
return 0.0;
}
data.iter().sum::() / data.len() as f64
}
-fn sort_clone(data: &[f64]) -> Vec {
+pub(crate) fn sort_clone(data: &[f64]) -> Vec {
let mut sorted = data.to_vec();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
sorted
}
-fn median_sorted(sorted: &[f64]) -> f64 {
+pub(crate) fn median_sorted(sorted: &[f64]) -> f64 {
if sorted.is_empty() {
return 0.0;
}
@@ -241,7 +241,7 @@ fn median_sorted(sorted: &[f64]) -> f64 {
}
}
-fn std_dev(data: &[f64]) -> f64 {
+pub(crate) fn std_dev(data: &[f64]) -> f64 {
if data.is_empty() {
return 0.0;
}
@@ -250,7 +250,7 @@ fn std_dev(data: &[f64]) -> f64 {
variance.sqrt()
}
-fn percentile_sorted(sorted_data: &[f64], p: f64) -> f64 {
+pub(crate) fn percentile_sorted(sorted_data: &[f64], p: f64) -> f64 {
if sorted_data.is_empty() {
return 0.0;
}
diff --git a/rust/src/bench/src/mm_processor.rs b/rust/src/bench/src/mm_processor.rs
new file mode 100644
index 0000000000..babcbf8080
--- /dev/null
+++ b/rust/src/bench/src/mm_processor.rs
@@ -0,0 +1,572 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+//! Multimodal preprocessing benchmark (`vllm-bench mm-processor`), mirroring
+//! `vllm bench mm-processor`.
+
+use std::collections::{BTreeMap, HashMap};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use anyhow::{Context, Result};
+use clap::Args;
+use futures::future::join_all;
+use serde::Serialize;
+use tokio::sync::Semaphore;
+
+use vllm_chat::{
+ ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRequest, LoadModelBackendsOptions,
+ MultiModalTimingStats, load_model_backends,
+};
+use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
+use vllm_llm::Llm;
+use vllm_managed_engine::cli::ManagedEngineArgs;
+use vllm_managed_engine::{ManagedEngineConfig, ManagedEngineHandle};
+use vllm_text::TextLlm;
+
+use crate::config::RangeRatio;
+use crate::datasets::SampleRequest;
+use crate::datasets::random_mm::{
+ MmBucketKey, MmLimitPerPrompt, generate_random_mm_dataset, parse_bucket_config,
+ parse_limit_mm_per_prompt,
+};
+use crate::error::BenchError;
+use crate::metrics::calculator::{mean, median_sorted, percentile_sorted, sort_clone, std_dev};
+use crate::tokenizer::{TokenizerKind, load_tokenizer};
+
+/// Seconds -> milliseconds.
+const SEC_TO_MS: f64 = 1000.0;
+
+/// Preferred column order; other stages appended alphabetically.
+const STAGE_ORDER: &[&str] = &[
+ "preprocessor_total_ms",
+ "media_fetch_ms",
+ "preprocess_image_ms",
+ "preprocess_video_ms",
+ "preprocess_audio_ms",
+ "prompt_expansion_ms",
+];
+
+/// Time to wait for managed engines to register.
+const ENGINE_READY_TIMEOUT: Duration = Duration::from_secs(600);
+
+/// Maximum time to wait for the managed engine to drain on shutdown.
+const ENGINE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(120);
+
+#[derive(Args, Debug, Clone)]
+pub struct MmProcessorArgs {
+ /// Model to serve and benchmark (Hugging Face id or local path).
+ #[arg(long)]
+ pub model: String,
+
+ /// Number of prompts to process, excluding warmups.
+ #[arg(long, default_value_t = 10)]
+ pub num_prompts: usize,
+
+ /// Number of warmup prompts processed and discarded before timing.
+ #[arg(long, default_value_t = 1)]
+ pub num_warmups: usize,
+
+ /// Comma-separated percentiles to report, e.g. "50,90,99".
+ #[arg(long, default_value = "99")]
+ pub metric_percentiles: String,
+
+ /// Path to write the aggregate stats as JSON.
+ #[arg(long)]
+ pub output_json: Option,
+
+ /// Text input length per prompt (before range scaling).
+ #[arg(long, default_value_t = 1024)]
+ pub random_input_len: usize,
+
+ /// Expected output length per prompt.
+ #[arg(long, default_value_t = 128)]
+ pub random_output_len: usize,
+
+ /// Prefix token length per prompt.
+ #[arg(long, default_value_t = 0)]
+ pub random_prefix_len: usize,
+
+ /// Input/output length range ratio: a float or `{"input": i, "output": o}`.
+ #[arg(long, default_value = "0.0")]
+ pub random_range_ratio: String,
+
+ /// Base number of multimodal items per request.
+ #[arg(long, default_value_t = 1)]
+ pub random_mm_base_items_per_request: usize,
+
+ /// Range ratio (in [0, 1]) for the number of mm items per request.
+ #[arg(long, default_value_t = 0.0)]
+ pub random_mm_num_mm_items_range_ratio: f64,
+
+ /// Per-modality item caps, e.g. `{"image": 255, "video": 1}`.
+ #[arg(long, default_value = "{\"image\": 255, \"video\": 1}")]
+ pub random_mm_limit_mm_per_prompt: String,
+
+ /// Bucket config, e.g. `{(256,256,1): 0.5, (720,1280,1): 0.5}`.
+ #[arg(long, default_value = "{(256,256,1): 0.5, (720,1280,1): 0.5}")]
+ pub random_mm_bucket_config: String,
+
+ /// Seed for dataset generation.
+ #[arg(long)]
+ pub seed: Option,
+
+ /// Request-id prefix for per-request timing keys.
+ #[arg(long, default_value = "mm-proc-")]
+ pub request_id_prefix: String,
+
+ /// Maximum concurrent requests (default 1, matching Python's serial driver).
+ #[arg(long, default_value_t = 1)]
+ pub max_concurrency: usize,
+
+ /// Trust remote code for the tokenizer and the managed engine.
+ #[arg(long, default_value_t = false)]
+ pub trust_remote_code: bool,
+
+ /// Optional chat-template override (inline text or path).
+ #[arg(long)]
+ pub chat_template: Option,
+
+ /// Managed headless-engine options.
+ #[command(flatten)]
+ pub engine: ManagedEngineArgs,
+}
+
+/// Entry point for `vllm-bench mm-processor`.
+pub async fn run_mm_processor(args: MmProcessorArgs, timing: MultiModalTimingStats) -> Result<()> {
+ if args.engine.data_parallel_size_local == Some(0) {
+ anyhow::bail!(
+ "`--data-parallel-size-local 0` is not supported; the benchmark requires \
+ a local managed engine"
+ );
+ }
+
+ // Spawn the managed headless Python engine (real weights, real encoder).
+ let handshake_port = args
+ .engine
+ .resolve_handshake_port()
+ .context("failed to allocate handshake port")?;
+ let mut python_args = args.engine.python_args.clone();
+ if args.trust_remote_code {
+ python_args.push("--trust-remote-code".to_string());
+ }
+ let engine_config = ManagedEngineConfig {
+ python: args.engine.python.clone(),
+ model: args.model.clone(),
+ handshake_host: args.engine.handshake_host.clone(),
+ handshake_port,
+ data_parallel_size: args.engine.data_parallel_size,
+ python_args,
+ };
+ eprintln!(
+ "Spawning managed headless engine: python={} model={} data_parallel_size={}",
+ engine_config.python, engine_config.model, engine_config.data_parallel_size
+ );
+ let engine = ManagedEngineHandle::spawn(engine_config)
+ .await
+ .context("failed to start managed Python headless engine")?;
+
+ let result = run_mm_processor_with_engine(args, handshake_port, &timing).await;
+ let shutdown_result = engine
+ .shutdown(ENGINE_SHUTDOWN_TIMEOUT)
+ .await
+ .context("failed to shut down managed engine");
+ result?;
+ shutdown_result?;
+ Ok(())
+}
+
+async fn run_mm_processor_with_engine(
+ args: MmProcessorArgs,
+ handshake_port: u16,
+ timing: &MultiModalTimingStats,
+) -> Result<()> {
+ let range_ratio =
+ RangeRatio::parse(&args.random_range_ratio).context("invalid --random-range-ratio")?;
+ let limit: MmLimitPerPrompt = parse_limit_mm_per_prompt(&args.random_mm_limit_mm_per_prompt)
+ .context("invalid --random-mm-limit-mm-per-prompt")?;
+ let buckets: Vec<(MmBucketKey, f64)> = parse_bucket_config(&args.random_mm_bucket_config)
+ .context("invalid --random-mm-bucket-config")?;
+ let percentiles = parse_percentiles(&args.metric_percentiles)?;
+ let seed = args.seed.unwrap_or(0);
+
+ // Offline tokenizer: local Hugging Face tokenizer or built-in tiktoken.
+ let tokenizer = load_tokenizer(&args.model, args.trust_remote_code, None)
+ .await
+ .context("failed to load tokenizer")?;
+
+ // Frontend backends: model configs + multimodal processor metadata.
+ let backends = load_model_backends(
+ &args.model,
+ LoadModelBackendsOptions {
+ chat_template: args.chat_template.clone(),
+ ..LoadModelBackendsOptions::default()
+ },
+ )
+ .await
+ .context("failed to load model backends")?;
+
+ if backends.chat_backend.multimodal_model_info().is_none() {
+ anyhow::bail!(
+ "model `{}` has no multimodal processor registered",
+ args.model
+ );
+ }
+
+ // Connect to the engine over the same transport the serving frontend uses.
+ let client = EngineCoreClient::connect(EngineCoreClientConfig {
+ transport_mode: TransportMode::HandshakeOwner {
+ handshake_address: format!("tcp://{}:{}", args.engine.handshake_host, handshake_port),
+ advertised_host: args.engine.handshake_host.clone(),
+ engine_count: args
+ .engine
+ .data_parallel_size_local
+ .unwrap_or(args.engine.data_parallel_size),
+ ready_timeout: ENGINE_READY_TIMEOUT,
+ local_input_address: None,
+ local_output_address: None,
+ },
+ coordinator_mode: None,
+ model_name: args.model.clone(),
+ client_index: 0,
+ })
+ .await
+ .context("failed to connect to engine core")?;
+
+ let chat = ChatLlm::new(
+ TextLlm::new(Llm::new(client), backends.text_backend),
+ backends.chat_backend,
+ );
+
+ let semaphore = Arc::new(Semaphore::new(args.max_concurrency));
+
+ // Warmup: generate a separate dataset, process it, then drain and discard
+ // the collected timings so cold caches do not skew the results.
+ if args.num_warmups > 0 {
+ let warmup = generate_dataset(
+ &tokenizer,
+ args.num_warmups,
+ seed.wrapping_add(1),
+ &args,
+ range_ratio,
+ &limit,
+ &buckets,
+ )?;
+ eprintln!("Processing {} warmup requests...", warmup.len());
+ run_batch(&chat, &warmup, &semaphore).await;
+ let _ = timing.stat();
+ }
+
+ let samples = generate_dataset(
+ &tokenizer,
+ args.num_prompts,
+ seed,
+ &args,
+ range_ratio,
+ &limit,
+ &buckets,
+ )?;
+ eprintln!("Processing {} requests...", samples.len());
+ let start_time = Instant::now();
+ let outcomes = run_batch(&chat, &samples, &semaphore).await;
+ let total_time = start_time.elapsed();
+
+ let completed = outcomes.iter().filter(|outcome| outcome.is_ok()).count();
+ let failed = outcomes.len() - completed;
+ let e2el_times: Vec = outcomes
+ .iter()
+ .filter_map(|outcome| outcome.as_ref().ok().map(|elapsed| elapsed.as_secs_f64()))
+ .collect();
+
+ let per_request = timing.stat();
+ let stats = aggregate_stats(per_request, &percentiles);
+
+ eprintln!(
+ "Processed {} requests (+{} warmup) for model `{}`",
+ samples.len(),
+ args.num_warmups,
+ args.model
+ );
+ print_report(&stats, &percentiles);
+ print_e2el_summary(completed, failed, total_time, &e2el_times, &percentiles);
+
+ if let Some(path) = &args.output_json {
+ write_result_json(path, completed, failed, &e2el_times, &percentiles, stats)?;
+ eprintln!("Wrote stats JSON to {path}");
+ }
+
+ chat.shutdown().await.context("failed to shut down chat facade")?;
+ Ok(())
+}
+
+/// Submit every sample through the chat pipeline and drive each stream to
+/// completion, returning per-request wall-clock latencies.
+async fn run_batch(
+ chat: &ChatLlm,
+ samples: &[SampleRequest],
+ semaphore: &Arc,
+) -> Vec> {
+ let submissions = samples.iter().map(|sample| {
+ let request = build_chat_request(sample);
+ let semaphore = semaphore.clone();
+ async move {
+ let _permit = semaphore.acquire().await.context("semaphore closed")?;
+ let request = request?;
+ let start = Instant::now();
+ chat.chat(request)
+ .await
+ .context("failed to submit request")?
+ .collect_message()
+ .await
+ .context("request failed")?;
+ Ok(start.elapsed())
+ }
+ });
+ join_all(submissions).await
+}
+
+fn generate_dataset(
+ tokenizer: &TokenizerKind,
+ num_requests: usize,
+ seed: u64,
+ args: &MmProcessorArgs,
+ range_ratio: RangeRatio,
+ limit: &MmLimitPerPrompt,
+ buckets: &[(MmBucketKey, f64)],
+) -> Result> {
+ generate_random_mm_dataset(
+ tokenizer,
+ num_requests,
+ args.random_input_len,
+ args.random_output_len,
+ args.random_prefix_len,
+ range_ratio,
+ seed,
+ &args.request_id_prefix,
+ args.random_mm_base_items_per_request,
+ args.random_mm_num_mm_items_range_ratio,
+ limit,
+ buckets,
+ false,
+ )
+ .map_err(|e| anyhow::anyhow!("{e}"))
+}
+
+/// Build a chat request from a sample (text prompt + `image_url` parts).
+fn build_chat_request(sample: &SampleRequest) -> Result {
+ let mut request = ChatRequest::for_test();
+ request.request_id = sample.request_id.clone().unwrap_or_else(|| "mm-proc".to_string());
+ request.sampling_params.temperature = Some(0.0);
+ request.sampling_params.max_tokens = Some(u32::try_from(sample.expected_output_len)?);
+
+ let mut parts = vec![ChatContentPart::text(sample.prompt.as_ref())];
+ if let Some(mm_items) = &sample.multi_modal_content {
+ for fragment in mm_items.iter() {
+ let url = extract_image_url(fragment)?;
+ parts.push(ChatContentPart::image_url(url));
+ }
+ }
+ request.messages = vec![ChatMessage::user(ChatContent::from(parts))];
+ Ok(request)
+}
+
+/// Extract the `image_url.url` string from a pre-serialized content fragment
+/// (`{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,..."}}`).
+fn extract_image_url(fragment: &str) -> Result {
+ let value: serde_json::Value = serde_json::from_str(fragment)
+ .with_context(|| format!("invalid multimodal fragment: {fragment}"))?;
+ let url = value
+ .get("image_url")
+ .and_then(|v| v.get("url"))
+ .and_then(|v| v.as_str())
+ .ok_or_else(|| BenchError::Config("fragment missing image_url.url".to_string()))?;
+ Ok(url.to_string())
+}
+
+fn parse_percentiles(raw: &str) -> Result> {
+ let mut values = Vec::new();
+ for part in raw.split(',') {
+ let part = part.trim();
+ if part.is_empty() {
+ continue;
+ }
+ let value: f64 = part.parse().with_context(|| format!("invalid percentile `{part}`"))?;
+ if !(0.0..=100.0).contains(&value) {
+ anyhow::bail!("percentile `{part}` out of range [0, 100]");
+ }
+ values.push(value);
+ }
+ Ok(values)
+}
+
+/// Aggregate per-request stage timings into `{stage_ms: {statistic: ms}}`,
+/// matching the Python `mm_processor_stats` schema orientation.
+fn aggregate_stats(
+ per_request: HashMap>,
+ percentiles: &[f64],
+) -> BTreeMap> {
+ let mut stage_values: BTreeMap> = BTreeMap::new();
+ for stage_map in per_request.into_values() {
+ for (stage, seconds) in stage_map {
+ stage_values.entry(stage).or_default().push(seconds);
+ }
+ }
+
+ stage_values
+ .iter()
+ .map(|(stage, values)| {
+ let sorted = sort_clone(values);
+ let mut stat_map = BTreeMap::new();
+ stat_map.insert("mean".to_string(), mean(values) * SEC_TO_MS);
+ stat_map.insert("median".to_string(), median_sorted(&sorted) * SEC_TO_MS);
+ stat_map.insert("std".to_string(), std_dev(values) * SEC_TO_MS);
+ for p in percentiles {
+ stat_map.insert(format!("p{p}"), percentile_sorted(&sorted, *p) * SEC_TO_MS);
+ }
+ (stage.replace("_secs", "_ms"), stat_map)
+ })
+ .collect()
+}
+
+/// Print a right-aligned plain-text table (pandas `to_string(index=False)` style).
+fn print_table(rows: &[Vec]) {
+ let ncols = rows.first().map_or(0, Vec::len);
+ let mut widths = vec![0usize; ncols];
+ for row in rows {
+ for (i, cell) in row.iter().enumerate() {
+ widths[i] = widths[i].max(cell.len());
+ }
+ }
+ for row in rows {
+ let line = row
+ .iter()
+ .enumerate()
+ .map(|(i, cell)| format!("{cell:>width$}", width = widths[i]))
+ .collect::>()
+ .join(" ");
+ println!("{line}");
+ }
+}
+
+/// Print a plain-text table with one row per stage and one column per
+/// statistic (mirrors the Python pandas report layout).
+fn print_report(stats: &BTreeMap>, percentiles: &[f64]) {
+ // Column order: mean/median/std then the requested percentiles, like Python.
+ let mut columns: Vec<(String, String)> =
+ [("mean", "Mean"), ("median", "Median"), ("std", "Std")]
+ .into_iter()
+ .map(|(key, label)| (key.to_string(), label.to_string()))
+ .collect();
+ columns.extend(percentiles.iter().map(|p| (format!("p{p}"), format!("P{p:?}"))));
+ for key in stats.values().flat_map(|map| map.keys()) {
+ if !columns.iter().any(|(k, _)| k == key) {
+ columns.push((key.clone(), key.clone()));
+ }
+ }
+
+ let mut stages: Vec<&String> = stats.keys().collect();
+ stages.sort_by_key(|stage| {
+ STAGE_ORDER
+ .iter()
+ .position(|s| s == stage)
+ .map(|i| (i as isize, (*stage).clone()))
+ .unwrap_or((isize::MAX, (*stage).clone()))
+ });
+
+ let mut rows: Vec> = vec![
+ std::iter::once("Stage".to_string())
+ .chain(columns.iter().map(|(_, label)| label.clone()))
+ .collect(),
+ ];
+ for stage in &stages {
+ let mut row = vec![(*stage).clone()];
+ for (key, _) in &columns {
+ row.push(
+ stats[*stage]
+ .get(key)
+ .map(|v| format!("{v:.2}"))
+ .unwrap_or_else(|| "-".to_string()),
+ );
+ }
+ rows.push(row);
+ }
+ println!("MM Processor Metrics:");
+ print_table(&rows);
+}
+
+/// Print the end-to-end latency summary (mirrors the Python report layout).
+fn print_e2el_summary(
+ completed: usize,
+ failed: usize,
+ total_time: Duration,
+ e2el_times_secs: &[f64],
+ percentiles: &[f64],
+) {
+ println!("Completed: {completed}, Failed: {failed}");
+ if e2el_times_secs.is_empty() {
+ println!("End-to-End Latency (ms): no completed requests");
+ return;
+ }
+
+ let e2el_ms: Vec = e2el_times_secs.iter().map(|secs| secs * SEC_TO_MS).collect();
+ let sorted = sort_clone(&e2el_ms);
+ let mut rows = vec![
+ vec!["Metric".to_string(), "Value (ms)".to_string()],
+ vec!["Mean".to_string(), format!("{:.2}", mean(&e2el_ms))],
+ vec![
+ "Median".to_string(),
+ format!("{:.2}", median_sorted(&sorted)),
+ ],
+ vec!["Std".to_string(), format!("{:.2}", std_dev(&e2el_ms))],
+ ];
+ for p in percentiles {
+ rows.push(vec![
+ format!("P{p:?}"),
+ format!("{:.2}", percentile_sorted(&sorted, *p)),
+ ]);
+ }
+ println!("End-to-End Latency (ms):");
+ print_table(&rows);
+ let throughput = completed as f64 / total_time.as_secs_f64();
+ println!("Request throughput: {throughput:.3} req/s");
+}
+
+/// JSON result schema mirroring `vllm bench mm-processor`.
+#[derive(Serialize)]
+struct BenchmarkResultJson {
+ completed: usize,
+ failed: usize,
+ mean_e2el_ms: f64,
+ median_e2el_ms: f64,
+ std_e2el_ms: f64,
+ percentiles_e2el_ms: Vec<(f64, f64)>,
+ /// `{stage_ms: {statistic: ms}}`, keyed by stage like the Python schema.
+ mm_processor_stats: BTreeMap>,
+}
+
+fn write_result_json(
+ path: &str,
+ completed: usize,
+ failed: usize,
+ e2el_times_secs: &[f64],
+ percentiles: &[f64],
+ mm_processor_stats: BTreeMap>,
+) -> Result<()> {
+ let e2el_ms: Vec = e2el_times_secs.iter().map(|secs| secs * SEC_TO_MS).collect();
+ let sorted = sort_clone(&e2el_ms);
+ let result = BenchmarkResultJson {
+ completed,
+ failed,
+ mean_e2el_ms: mean(&e2el_ms),
+ median_e2el_ms: median_sorted(&sorted),
+ std_e2el_ms: std_dev(&e2el_ms),
+ percentiles_e2el_ms: percentiles
+ .iter()
+ .map(|p| (*p, percentile_sorted(&sorted, *p)))
+ .collect(),
+ mm_processor_stats,
+ };
+ let json = serde_json::to_string_pretty(&result).context("failed to serialize result")?;
+ std::fs::write(path, format!("{json}\n")).with_context(|| format!("failed to write {path}"))?;
+ Ok(())
+}
diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml
index e92dedae71..1f1f1974ea 100644
--- a/rust/src/chat/Cargo.toml
+++ b/rust/src/chat/Cargo.toml
@@ -35,6 +35,7 @@ vllm-llm.workspace = true
vllm-parser.workspace = true
vllm-text.workspace = true
vllm-tokenizer.workspace = true
+vllm-tracing.workspace = true
xgrammar-structural-tag.workspace = true
[dev-dependencies]
diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs
index 806554b5a5..3295abf45f 100644
--- a/rust/src/chat/src/lib.rs
+++ b/rust/src/chat/src/lib.rs
@@ -46,6 +46,9 @@ pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessag
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
pub use vllm_llm::FinishReason;
pub use vllm_text::GenerationConfigMode;
+pub use vllm_tracing::timing::{
+ RequestTimingStats as MultiModalTimingStats, mm_request_span, mm_timing_layer,
+};
mod backend;
mod error;
diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs
index a5dd3ba8ea..362db4b7b7 100644
--- a/rust/src/chat/src/multimodal.rs
+++ b/rust/src/chat/src/multimodal.rs
@@ -26,7 +26,7 @@ use llm_multimodal::{
};
use serde::{Deserialize, Serialize};
use thiserror_ext::AsReport as _;
-use tracing::warn;
+use tracing::{Instrument as _, warn};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, MmFeatures, MmKwargsItem};
use vllm_text::Prompt;
@@ -47,6 +47,7 @@ mod video;
use self::expand::expand_prompt_token_ids;
pub use self::input::MultimodalInput;
+use vllm_tracing::timing::{mm_request_span, mm_stage_span};
/// Resolved multimodal support for one loaded model.
#[derive(Clone)]
@@ -562,7 +563,10 @@ pub(crate) async fn finalize_rendered_prompt(
.map_err(|error| multimodal!("{error}"))?,
Prompt::TokenIds(token_ids) => token_ids,
};
- let prepared = info.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype).await?;
+ let prepared = info
+ .prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype)
+ .instrument(mm_request_span(&request.request_id))
+ .await?;
Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared)))
}
@@ -753,27 +757,52 @@ impl MultimodalModelInfo {
prompt_token_ids: &mut Vec,
model_dtype: ModelDtype,
) -> Result {
- let media_parts_len = media_parts.len();
- if media_parts_len == 0 {
+ if media_parts.is_empty() {
return Ok(Vec::new());
}
+ self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype)
+ .instrument(mm_stage_span("preprocessor_total"))
+ .await
+ }
+
+ /// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing`
+ /// span aggregated per request by the `vllm-tracing` timing layer.
+ async fn prepare_multimodal_timed(
+ &self,
+ media_parts: Vec,
+ prompt_token_ids: &mut Vec,
+ model_dtype: ModelDtype,
+ ) -> Result {
+ let media_parts_len = media_parts.len();
self.validate_mm_limits(&media_parts)?;
- let fetched = self.fetch_media(media_parts).await?;
+ let fetched =
+ self.fetch_media(media_parts).instrument(mm_stage_span("media_fetch")).await?;
let mut prepared = Vec::new();
if !fetched.images.is_empty() {
- prepared
- .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
+ prepared.push(
+ self.prepare_images(fetched.images, fetched.image_uuids, model_dtype)
+ .instrument(mm_stage_span("preprocess_image"))
+ .await?,
+ );
}
if !fetched.videos.is_empty() {
- prepared
- .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
+ prepared.push(
+ self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype)
+ .instrument(mm_stage_span("preprocess_video"))
+ .await?,
+ );
}
if !fetched.audios.is_empty() {
- prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
+ prepared.push(
+ self.prepare_audios(fetched.audios, fetched.audio_uuids)
+ .instrument(mm_stage_span("preprocess_audio"))
+ .await?,
+ );
}
- let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
+ let mut ranges = mm_stage_span("prompt_expansion")
+ .in_scope(|| expand_prompt_token_ids(prompt_token_ids, &prepared))?;
let mut features = Vec::with_capacity(media_parts_len);
for media in prepared {
diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs
index eb3338a67f..3523f173e3 100644
--- a/rust/src/server/src/routes/inference/generate.rs
+++ b/rust/src/server/src/routes/inference/generate.rs
@@ -53,7 +53,12 @@ pub async fn generate(
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
let mm_features = if let Some(parts) = body.content_parts.take() {
- match state.chat.prepare_media(parts, &mut body.token_ids).await {
+ match state
+ .chat
+ .prepare_media(parts, &mut body.token_ids)
+ .instrument(vllm_chat::mm_request_span(&request_context.request_id))
+ .await
+ {
Ok(features) => features,
Err(e) => {
return ApiError::invalid_request(
diff --git a/rust/src/tracing/src/lib.rs b/rust/src/tracing/src/lib.rs
index eb6da9c0b2..f407b51f5e 100644
--- a/rust/src/tracing/src/lib.rs
+++ b/rust/src/tracing/src/lib.rs
@@ -3,13 +3,14 @@
//! Shared tracing subscriber and log formatting for vLLM Rust binaries.
+pub mod timing;
+
use std::{env, fmt, process};
use time::UtcOffset;
use time::macros::format_description;
use tracing::level_filters::LevelFilter;
use tracing::{Event, Level, Subscriber};
-use tracing_subscriber::Layer as _;
use tracing_subscriber::filter::Targets;
use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
use tracing_subscriber::fmt::time::FormatTime;
@@ -17,6 +18,7 @@ use tracing_subscriber::fmt::{FmtContext, FormattedFields};
use tracing_subscriber::layer::SubscriberExt as _;
use tracing_subscriber::registry::LookupSpan;
use tracing_subscriber::util::SubscriberInitExt as _;
+use tracing_subscriber::{Layer, Registry};
const CYAN: &str = "\x1b[0;36m";
const GREY: &str = "\x1b[90m";
@@ -30,20 +32,36 @@ const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
/// Install the process-wide vLLM-style tracing subscriber.
pub fn init_tracing(process_label: &str) {
+ let _ = tracing_subscriber::registry().with(vllm_fmt_layer(process_label)).try_init();
+}
+
+/// Install the process-wide vLLM-style tracing subscriber, with an
+/// additional layer (e.g. the multimodal preprocessing timing collector).
+pub fn init_tracing_with(process_label: &str, layer: L)
+where
+ L: Layer + Send + Sync + 'static,
+{
+ let _ = tracing_subscriber::registry()
+ .with(layer)
+ .with(vllm_fmt_layer(process_label))
+ .try_init();
+}
+
+/// The vLLM-style fmt layer filtered by `VLLM_LOGGING_LEVEL` / `RUST_LOG`.
+fn vllm_fmt_layer(process_label: &str) -> impl Layer + Send + Sync + 'static + use
+where
+ S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
let filter = build_targets_filter(
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
env::var("RUST_LOG").ok().as_deref(),
);
let formatter = VllmEventFormatter::new(process_label);
- let _ = tracing_subscriber::registry()
- .with(
- tracing_subscriber::fmt::layer()
- .event_format(formatter)
- .with_writer(std::io::stderr)
- .with_filter(filter),
- )
- .try_init();
+ tracing_subscriber::fmt::layer()
+ .event_format(formatter)
+ .with_writer(std::io::stderr)
+ .with_filter(filter)
}
/// Build the CLI log filter by merging the vLLM-style default level with
diff --git a/rust/src/tracing/src/timing.rs b/rust/src/tracing/src/timing.rs
new file mode 100644
index 0000000000..b0b68e533f
--- /dev/null
+++ b/rust/src/tracing/src/timing.rs
@@ -0,0 +1,229 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+//! Request-scoped span timing.
+//!
+//! Pipelines emit stage spans (target = the layer's configured target, a
+//! `stage` field naming the stage) inside a request span carrying a
+//! `request_id` field; [`RequestTimingLayer`] attributes each stage span's
+//! wall-clock time to the nearest ancestor `request_id` and accumulates per
+//! request. Callers that want timings install the layer on their subscriber;
+//! everyone else pays only the cost of untracked spans.
+
+use std::collections::HashMap;
+use std::fmt;
+use std::sync::{Arc, Mutex};
+use std::time::Instant;
+
+use tracing::field::{Field, Visit};
+use tracing::span::{Attributes, Span};
+use tracing::{Id, Subscriber, info_span};
+use tracing_subscriber::layer::{Context, Layer};
+use tracing_subscriber::registry::LookupSpan;
+
+/// `"{stage}_secs" => seconds` maps returned by [`RequestTimingStats::stat`].
+pub type StageStats = HashMap;
+
+/// `request_id` extracted from a span, propagated to stage timings.
+struct RequestId(String);
+
+/// Start timestamp of one in-flight stage span.
+struct StageStart {
+ stage: String,
+ start: Instant,
+}
+
+/// Extracts the fields the layer cares about from span attributes.
+#[derive(Default)]
+struct SpanFields {
+ request_id: Option,
+ stage: Option,
+}
+
+impl Visit for SpanFields {
+ fn record_str(&mut self, field: &Field, value: &str) {
+ match field.name() {
+ "request_id" => self.request_id = Some(value.to_string()),
+ "stage" => self.stage = Some(value.to_string()),
+ _ => {}
+ }
+ }
+
+ fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
+ match field.name() {
+ "request_id" => self.request_id = Some(format!("{value:?}")),
+ "stage" => self.stage = Some(format!("{value:?}")),
+ _ => {}
+ }
+ }
+}
+
+/// `tracing` layer that aggregates stage-span timings per request, keyed by
+/// the nearest ancestor span with a `request_id` field.
+pub struct RequestTimingLayer {
+ target: &'static str,
+ stats: Arc>>,
+}
+
+/// Drain handle for the timings collected by [`RequestTimingLayer`].
+pub struct RequestTimingStats {
+ stats: Arc>>,
+}
+
+impl RequestTimingLayer {
+ /// Create the layer and its stats handle. Only spans whose target equals
+ /// `target` and that carry a `stage` field are timed.
+ pub fn new(target: &'static str) -> (Self, RequestTimingStats) {
+ let stats = Arc::new(Mutex::new(HashMap::new()));
+ (
+ Self {
+ target,
+ stats: Arc::clone(&stats),
+ },
+ RequestTimingStats { stats },
+ )
+ }
+}
+
+impl RequestTimingStats {
+ /// Drain and return `{request_id: {stage_secs}}` records.
+ pub fn stat(&self) -> HashMap {
+ std::mem::take(&mut *self.stats.lock().unwrap())
+ }
+}
+
+impl Layer for RequestTimingLayer
+where
+ S: Subscriber + for<'lookup> LookupSpan<'lookup>,
+{
+ fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
+ let Some(span) = ctx.span(id) else {
+ return;
+ };
+ let mut fields = SpanFields::default();
+ attrs.record(&mut fields);
+ let mut extensions = span.extensions_mut();
+ if let Some(request_id) = fields.request_id {
+ extensions.insert(RequestId(request_id));
+ }
+ if span.metadata().target() == self.target
+ && let Some(stage) = fields.stage
+ {
+ extensions.insert(StageStart {
+ stage,
+ start: Instant::now(),
+ });
+ }
+ }
+
+ fn on_close(&self, id: Id, ctx: Context<'_, S>) {
+ let Some(span) = ctx.span(&id) else {
+ return;
+ };
+ let (stage, start) = {
+ let extensions = span.extensions();
+ let Some(stage_start) = extensions.get::() else {
+ return;
+ };
+ (stage_start.stage.clone(), stage_start.start)
+ };
+ let Some(request_id) = span
+ .scope()
+ .find_map(|ancestor| ancestor.extensions().get::().map(|id| id.0.clone()))
+ else {
+ return;
+ };
+ let mut stats = self.stats.lock().unwrap();
+ *stats.entry(request_id).or_default().entry(format!("{stage}_secs")).or_default() +=
+ start.elapsed().as_secs_f64();
+ }
+}
+
+/// Target of the multimodal preprocessing stage spans.
+const MM_STAGE_TARGET: &str = "mm_processor_timing";
+
+/// Create the timing layer and its stats handle for multimodal preprocessing
+/// stage spans (`vllm-bench mm-processor`), mirroring the Python
+/// `TimingContext` / `MultiModalTimingRegistry`.
+pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) {
+ RequestTimingLayer::new(MM_STAGE_TARGET)
+}
+
+/// Span carrying the `request_id` used to attribute multimodal stage timings.
+pub fn mm_request_span(request_id: &str) -> Span {
+ info_span!("mm_request", request_id)
+}
+
+/// Span for one multimodal preprocessing stage; the elapsed time is recorded
+/// on close.
+pub fn mm_stage_span(stage: &'static str) -> Span {
+ info_span!(target: MM_STAGE_TARGET, "mm_stage", stage)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use tracing::info_span;
+ use tracing_subscriber::layer::SubscriberExt as _;
+
+ const TARGET: &str = "test_request_timing";
+
+ fn stage_span(stage: &'static str) -> tracing::Span {
+ info_span!(target: TARGET, "stage", stage)
+ }
+
+ #[test]
+ fn records_stage_timings_per_request() {
+ let (layer, stats) = RequestTimingLayer::new(TARGET);
+ let subscriber = tracing_subscriber::registry().with(layer);
+ tracing::subscriber::with_default(subscriber, || {
+ info_span!("request", request_id = "req-1").in_scope(|| {
+ drop(stage_span("media_fetch"));
+ stage_span("prompt_expansion").in_scope(|| {});
+ });
+ });
+ let drained = stats.stat();
+ assert!(drained["req-1"].contains_key("media_fetch_secs"));
+ assert!(drained["req-1"].contains_key("prompt_expansion_secs"));
+ // `stat` drains: a second read is empty.
+ assert!(stats.stat().is_empty());
+ }
+
+ #[test]
+ fn attributes_to_nearest_request_id_ancestor() {
+ let (layer, stats) = RequestTimingLayer::new(TARGET);
+ let subscriber = tracing_subscriber::registry().with(layer);
+ tracing::subscriber::with_default(subscriber, || {
+ info_span!("request", request_id = "req-outer").in_scope(|| {
+ info_span!("request", request_id = "req-inner").in_scope(|| {
+ drop(stage_span("media_fetch"));
+ });
+ });
+ });
+ let drained = stats.stat();
+ assert!(drained["req-inner"].contains_key("media_fetch_secs"));
+ assert!(!drained.contains_key("req-outer"));
+ }
+
+ #[test]
+ fn stage_without_request_id_is_dropped() {
+ let (layer, stats) = RequestTimingLayer::new(TARGET);
+ let subscriber = tracing_subscriber::registry().with(layer);
+ tracing::subscriber::with_default(subscriber, || {
+ drop(stage_span("media_fetch"));
+ });
+ assert!(stats.stat().is_empty());
+ }
+
+ #[test]
+ fn other_targets_are_ignored() {
+ let (layer, stats) = RequestTimingLayer::new(TARGET);
+ let subscriber = tracing_subscriber::registry().with(layer);
+ tracing::subscriber::with_default(subscriber, || {
+ info_span!("request", request_id = "req-1").in_scope(|| {
+ drop(info_span!("unrelated", stage = "media_fetch"));
+ });
+ });
+ assert!(stats.stat().is_empty());
+ }
+}