From f84325c48c0acc1e3703103788c5f2976e719762 Mon Sep 17 00:00:00 2001 From: karthik <56480632+gangula-karthik@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:08:35 +0800 Subject: [PATCH] [Frontend][Rust] Add mm-processor benchmark for Rust frontend (#51922) Signed-off-by: Karthik Gangula Signed-off-by: gangula-karthik Signed-off-by: Isotr0py Co-authored-by: Karthik Gangula Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Isotr0py --- rust/Cargo.lock | 6 + rust/src/bench/AGENTS.md | 3 +- rust/src/bench/Cargo.toml | 5 + rust/src/bench/README.md | 60 +- rust/src/bench/src/lib.rs | 2 + rust/src/bench/src/main.rs | 28 +- rust/src/bench/src/metrics/calculator.rs | 10 +- rust/src/bench/src/mm_processor.rs | 572 ++++++++++++++++++ rust/src/chat/Cargo.toml | 1 + rust/src/chat/src/lib.rs | 3 + rust/src/chat/src/multimodal.rs | 51 +- .../server/src/routes/inference/generate.rs | 7 +- rust/src/tracing/src/lib.rs | 36 +- rust/src/tracing/src/timing.rs | 229 +++++++ 14 files changed, 980 insertions(+), 33 deletions(-) create mode 100644 rust/src/bench/src/mm_processor.rs create mode 100644 rust/src/tracing/src/timing.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index b4917a303b..ad421ba41d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5735,6 +5735,11 @@ dependencies = [ "url", "uuid", "vllm-build-info", + "vllm-chat", + "vllm-engine-core-client", + "vllm-llm", + "vllm-managed-engine", + "vllm-text", "vllm-tracing", ] @@ -5784,6 +5789,7 @@ dependencies = [ "vllm-parser", "vllm-text", "vllm-tokenizer", + "vllm-tracing", "xgrammar-structural-tag", "zeromq", ] diff --git a/rust/src/bench/AGENTS.md b/rust/src/bench/AGENTS.md index 1f15165fa4..d9a17f63f2 100644 --- a/rust/src/bench/AGENTS.md +++ b/rust/src/bench/AGENTS.md @@ -23,7 +23,7 @@ cargo test -p vllm-bench -- --ignored ## Architecture -- `src/main.rs` — Entry point, mimalloc, tokio runtime, mode dispatch (compare/sweep/multi-run/multi-turn/single) +- `src/main.rs` — Entry point, mimalloc, tokio runtime, mode dispatch (compare/sweep/multi-run/multi-turn/single, `mm-processor` subcommand) - `src/cli.rs` — clap derive CLI args (~50+ flags) - `src/config.rs` — Validated config from CLI; `GoodputConfig`, `RampUpConfig`, sampling param merging - `src/error.rs` — `BenchError` enum (Http, Json, Tokenizer, Config, EndpointTimeout, Backend, Io) @@ -32,6 +32,7 @@ cargo test -p vllm-bench -- --ignored - `src/sweep.rs` — Concurrency/rate parameter sweep (`--sweep-max-concurrency`, `--sweep-request-rate`) - `src/multi_run.rs` — N-run aggregation with mean/std/min/max/CV (`--num-runs`) - `src/compare.rs` — Side-by-side diff of two result JSON files (`--compare`) +- `src/mm_processor.rs` — `vllm-bench mm-processor` subcommand: offline multimodal preprocessing latency benchmark against a managed headless engine (mirrors `vllm bench mm-processor`); per-stage timings come from `vllm_tracing::timing::RequestTimingLayer` (via `vllm_chat::mm_timing_layer`) - `src/tokenizer.rs` — `TokenizerKind` enum: Local(HuggingFace), Tiktoken, OR Server-side `/tokenize`+`/detokenize` fallback - `src/tiktoken.rs` — Tiktoken BPE loader (`.tiktoken`/`.model` files; built-in encodings o200k_base/cl100k_base; pat_str extraction from Python source) - `src/hub.rs` — `HubRepo`: thin wrapper over hf-hub's async (reqwest/native-tls) API — file download and repo listing, optional revision; the sync ureq backend is unusable here because it pulls rustls, which `rust/deny.toml` bans diff --git a/rust/src/bench/Cargo.toml b/rust/src/bench/Cargo.toml index 5a3b14894a..60436d4251 100644 --- a/rust/src/bench/Cargo.toml +++ b/rust/src/bench/Cargo.toml @@ -36,6 +36,11 @@ tracing.workspace = true url.workspace = true uuid.workspace = true vllm-build-info.workspace = true +vllm-chat.workspace = true +vllm-engine-core-client.workspace = true +vllm-llm.workspace = true +vllm-managed-engine.workspace = true +vllm-text.workspace = true vllm-tracing.workspace = true [lints] diff --git a/rust/src/bench/README.md b/rust/src/bench/README.md index c8724d6cd6..fb1e5b529d 100644 --- a/rust/src/bench/README.md +++ b/rust/src/bench/README.md @@ -18,6 +18,7 @@ vllm-bench --backend vllm --base-url http://127.0.0.1:8000 \ - **Beyond a single run** — concurrency/rate **sweeps**, **multi-run** stats, **multi-turn** conversations, **LoRA** multi-adapter, and result **comparison**. - **Steady-state metrics** — throughput/latency measured over the saturated plateau, excluding ramp-up and drain. - **Parity** — JSON output schema and timing semantics match Python `vllm bench serve` exactly. +- **Multimodal preprocessing** — `mm-processor` subcommand measures per-stage VLM preprocessing latency against a managed headless engine (mirrors `vllm bench mm-processor`). ### Performance vs. Python @@ -404,6 +405,32 @@ vllm-bench \ +
+Multimodal preprocessing (offline) + +The `mm-processor` subcommand benchmarks the multimodal preprocessing pipeline +itself (media fetch → per-modality preprocess → prompt expansion), mirroring +`vllm bench mm-processor`. It spawns a managed headless Python engine (real +weights, real encoder) — no serving endpoint needed — and reports per-stage +timing (mean/median/std/percentiles) plus end-to-end latency. + +```bash +vllm-bench mm-processor \ + --model Qwen/Qwen2.5-VL-7B-Instruct \ + --num-prompts 100 --num-warmups 5 \ + --random-input-len 512 --random-output-len 128 \ + --random-mm-base-items-per-request 1 \ + --random-mm-limit-mm-per-prompt '{"image": 1, "video": 0}' \ + --random-mm-bucket-config '{(1024, 800, 1): 1.0}' \ + --metric-percentiles "50,90,99" \ + --output-json mm_processor_stats.json +``` + +`--max-concurrency` defaults to `1`, matching Python's serial driver; raise it +to measure a concurrent serving-style workload. + +
+ ## Supported Backends ### Generation @@ -705,6 +732,36 @@ With `--multi-turn`, `--num-prompts` controls the number of **conversations**, n +
+mm-processor subcommand + +Offline multimodal preprocessing latency benchmark (mirrors `vllm bench mm-processor`). Spawns a managed headless Python engine and drives the full chat preprocessing pipeline. + +| Flag | Default | Description | +| ------ | --------- | ------------- | +| `--model` | (required) | Model to serve and benchmark (HF id or local path) | +| `--num-prompts` | `10` | Prompts to process, excluding warmups | +| `--num-warmups` | `1` | Warmup prompts processed and discarded before timing | +| `--metric-percentiles` | `99` | Comma-separated percentiles to report | +| `--output-json` | — | Path to write aggregate stats as JSON (same schema as Python) | +| `--random-input-len` | `1024` | Text input length per prompt | +| `--random-output-len` | `128` | Expected output length per prompt | +| `--random-prefix-len` | `0` | Prefix token length per prompt | +| `--random-range-ratio` | `0.0` | Input/output length range ratio (float or `{"input": i, "output": o}`) | +| `--random-mm-base-items-per-request` | `1` | Base number of multimodal items per request | +| `--random-mm-num-mm-items-range-ratio` | `0.0` | Range ratio (in [0, 1]) for the number of mm items per request | +| `--random-mm-limit-mm-per-prompt` | `{"image": 255, "video": 1}` | Per-modality item caps | +| `--random-mm-bucket-config` | `{(256,256,1): 0.5, (720,1280,1): 0.5}` | Bucket config | +| `--seed` | `0` | Seed for dataset generation | +| `--request-id-prefix` | `mm-proc-` | Request-id prefix for per-request timing keys | +| `--max-concurrency` | `1` | Maximum concurrent requests (1 matches Python's serial driver) | +| `--trust-remote-code` | `false` | Trust remote code for the tokenizer and the managed engine | +| `--chat-template` | — | Chat-template override (inline text or path) | + +Managed headless-engine options (e.g. `--python`, `--data-parallel-address`, `--data-parallel-rpc-port`, `--data-parallel-size`, `--max-model-len`, `--python-args`) are shared with other managed-engine commands; see `vllm-bench mm-processor --help`. + +
+ ## Tokenizer Support Tokenizers are loaded with a three-tier fallback chain: @@ -734,10 +791,11 @@ Use `--append-result` to append multiple runs to the same file in JSONL format. ```text src/ -├── main.rs # Entry point, mimalloc, tokio runtime, mode dispatch +├── main.rs # Entry point, mimalloc, tokio runtime, mode/subcommand dispatch ├── cli.rs # clap CLI argument definitions ├── config.rs # Validated config, goodput/ramp-up parsing ├── benchmark.rs # Core orchestrator (schedule, spawn, collect, verify, profile) +├── mm_processor.rs # Offline mm preprocessing benchmark (mm-processor subcommand) ├── multi_turn.rs # Multi-turn conversation orchestrator (channel workers) ├── compare.rs # Result diff (--compare file_a.json file_b.json) ├── sweep.rs # Parameter sweep (--sweep-max-concurrency, --sweep-request-rate) diff --git a/rust/src/bench/src/lib.rs b/rust/src/bench/src/lib.rs index 0a4f19c0d6..8541c3b310 100644 --- a/rust/src/bench/src/lib.rs +++ b/rust/src/bench/src/lib.rs @@ -10,6 +10,7 @@ mod datasets; mod error; mod hub; mod metrics; +mod mm_processor; mod multi_run; mod multi_turn; mod output; @@ -25,6 +26,7 @@ pub use cli::{ BackendKind, BenchServeArgs, DatasetName, LoraAssignment, RampUpStrategy, SpeedBenchConfig, }; use config::BenchConfig; +pub use mm_processor::{MmProcessorArgs, run_mm_processor}; /// Prepare process-wide resources for a benchmark run. pub fn prepare_process() { diff --git a/rust/src/bench/src/main.rs b/rust/src/bench/src/main.rs index 3485e22bb1..4a72b81892 100644 --- a/rust/src/bench/src/main.rs +++ b/rust/src/bench/src/main.rs @@ -6,22 +6,30 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; use anyhow::Context; -use clap::Parser; +use clap::{Parser, Subcommand}; #[derive(Parser)] #[command( name = "vllm-bench", - about = "Benchmark online serving throughput", + about = "Benchmark online serving throughput and offline multimodal preprocessing", version = vllm_build_info::VERSION )] struct Cli { #[command(flatten)] args: vllm_bench::BenchServeArgs, + + /// Optional subcommand; absent by default for the online serving benchmark. + #[command(subcommand)] + command: Option, +} + +#[derive(Subcommand)] +enum Command { + /// Offline multimodal preprocessing latency benchmark. + MmProcessor(vllm_bench::MmProcessorArgs), } fn main() -> anyhow::Result<()> { - vllm_tracing::init_tracing("Bench"); - let cli = Cli::parse(); vllm_bench::prepare_process(); @@ -30,5 +38,15 @@ fn main() -> anyhow::Result<()> { .build() .context("Failed to build tokio runtime")?; - runtime.block_on(vllm_bench::run(cli.args)) + match cli.command { + Some(Command::MmProcessor(args)) => { + let (timing_layer, timing_stats) = vllm_chat::mm_timing_layer(); + vllm_tracing::init_tracing_with("Bench", timing_layer); + runtime.block_on(vllm_bench::run_mm_processor(args, timing_stats)) + } + None => { + vllm_tracing::init_tracing("Bench"); + runtime.block_on(vllm_bench::run(cli.args)) + } + } } diff --git a/rust/src/bench/src/metrics/calculator.rs b/rust/src/bench/src/metrics/calculator.rs index 11b68a3bfe..b292be42eb 100644 --- a/rust/src/bench/src/metrics/calculator.rs +++ b/rust/src/bench/src/metrics/calculator.rs @@ -216,20 +216,20 @@ pub fn calculate_metrics( (metrics, actual_output_lens) } -fn mean(data: &[f64]) -> f64 { +pub(crate) fn mean(data: &[f64]) -> f64 { if data.is_empty() { return 0.0; } data.iter().sum::() / data.len() as f64 } -fn sort_clone(data: &[f64]) -> Vec { +pub(crate) fn sort_clone(data: &[f64]) -> Vec { let mut sorted = data.to_vec(); sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); sorted } -fn median_sorted(sorted: &[f64]) -> f64 { +pub(crate) fn median_sorted(sorted: &[f64]) -> f64 { if sorted.is_empty() { return 0.0; } @@ -241,7 +241,7 @@ fn median_sorted(sorted: &[f64]) -> f64 { } } -fn std_dev(data: &[f64]) -> f64 { +pub(crate) fn std_dev(data: &[f64]) -> f64 { if data.is_empty() { return 0.0; } @@ -250,7 +250,7 @@ fn std_dev(data: &[f64]) -> f64 { variance.sqrt() } -fn percentile_sorted(sorted_data: &[f64], p: f64) -> f64 { +pub(crate) fn percentile_sorted(sorted_data: &[f64], p: f64) -> f64 { if sorted_data.is_empty() { return 0.0; } diff --git a/rust/src/bench/src/mm_processor.rs b/rust/src/bench/src/mm_processor.rs new file mode 100644 index 0000000000..babcbf8080 --- /dev/null +++ b/rust/src/bench/src/mm_processor.rs @@ -0,0 +1,572 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Multimodal preprocessing benchmark (`vllm-bench mm-processor`), mirroring +//! `vllm bench mm-processor`. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use clap::Args; +use futures::future::join_all; +use serde::Serialize; +use tokio::sync::Semaphore; + +use vllm_chat::{ + ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRequest, LoadModelBackendsOptions, + MultiModalTimingStats, load_model_backends, +}; +use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode}; +use vllm_llm::Llm; +use vllm_managed_engine::cli::ManagedEngineArgs; +use vllm_managed_engine::{ManagedEngineConfig, ManagedEngineHandle}; +use vllm_text::TextLlm; + +use crate::config::RangeRatio; +use crate::datasets::SampleRequest; +use crate::datasets::random_mm::{ + MmBucketKey, MmLimitPerPrompt, generate_random_mm_dataset, parse_bucket_config, + parse_limit_mm_per_prompt, +}; +use crate::error::BenchError; +use crate::metrics::calculator::{mean, median_sorted, percentile_sorted, sort_clone, std_dev}; +use crate::tokenizer::{TokenizerKind, load_tokenizer}; + +/// Seconds -> milliseconds. +const SEC_TO_MS: f64 = 1000.0; + +/// Preferred column order; other stages appended alphabetically. +const STAGE_ORDER: &[&str] = &[ + "preprocessor_total_ms", + "media_fetch_ms", + "preprocess_image_ms", + "preprocess_video_ms", + "preprocess_audio_ms", + "prompt_expansion_ms", +]; + +/// Time to wait for managed engines to register. +const ENGINE_READY_TIMEOUT: Duration = Duration::from_secs(600); + +/// Maximum time to wait for the managed engine to drain on shutdown. +const ENGINE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(120); + +#[derive(Args, Debug, Clone)] +pub struct MmProcessorArgs { + /// Model to serve and benchmark (Hugging Face id or local path). + #[arg(long)] + pub model: String, + + /// Number of prompts to process, excluding warmups. + #[arg(long, default_value_t = 10)] + pub num_prompts: usize, + + /// Number of warmup prompts processed and discarded before timing. + #[arg(long, default_value_t = 1)] + pub num_warmups: usize, + + /// Comma-separated percentiles to report, e.g. "50,90,99". + #[arg(long, default_value = "99")] + pub metric_percentiles: String, + + /// Path to write the aggregate stats as JSON. + #[arg(long)] + pub output_json: Option, + + /// Text input length per prompt (before range scaling). + #[arg(long, default_value_t = 1024)] + pub random_input_len: usize, + + /// Expected output length per prompt. + #[arg(long, default_value_t = 128)] + pub random_output_len: usize, + + /// Prefix token length per prompt. + #[arg(long, default_value_t = 0)] + pub random_prefix_len: usize, + + /// Input/output length range ratio: a float or `{"input": i, "output": o}`. + #[arg(long, default_value = "0.0")] + pub random_range_ratio: String, + + /// Base number of multimodal items per request. + #[arg(long, default_value_t = 1)] + pub random_mm_base_items_per_request: usize, + + /// Range ratio (in [0, 1]) for the number of mm items per request. + #[arg(long, default_value_t = 0.0)] + pub random_mm_num_mm_items_range_ratio: f64, + + /// Per-modality item caps, e.g. `{"image": 255, "video": 1}`. + #[arg(long, default_value = "{\"image\": 255, \"video\": 1}")] + pub random_mm_limit_mm_per_prompt: String, + + /// Bucket config, e.g. `{(256,256,1): 0.5, (720,1280,1): 0.5}`. + #[arg(long, default_value = "{(256,256,1): 0.5, (720,1280,1): 0.5}")] + pub random_mm_bucket_config: String, + + /// Seed for dataset generation. + #[arg(long)] + pub seed: Option, + + /// Request-id prefix for per-request timing keys. + #[arg(long, default_value = "mm-proc-")] + pub request_id_prefix: String, + + /// Maximum concurrent requests (default 1, matching Python's serial driver). + #[arg(long, default_value_t = 1)] + pub max_concurrency: usize, + + /// Trust remote code for the tokenizer and the managed engine. + #[arg(long, default_value_t = false)] + pub trust_remote_code: bool, + + /// Optional chat-template override (inline text or path). + #[arg(long)] + pub chat_template: Option, + + /// Managed headless-engine options. + #[command(flatten)] + pub engine: ManagedEngineArgs, +} + +/// Entry point for `vllm-bench mm-processor`. +pub async fn run_mm_processor(args: MmProcessorArgs, timing: MultiModalTimingStats) -> Result<()> { + if args.engine.data_parallel_size_local == Some(0) { + anyhow::bail!( + "`--data-parallel-size-local 0` is not supported; the benchmark requires \ + a local managed engine" + ); + } + + // Spawn the managed headless Python engine (real weights, real encoder). + let handshake_port = args + .engine + .resolve_handshake_port() + .context("failed to allocate handshake port")?; + let mut python_args = args.engine.python_args.clone(); + if args.trust_remote_code { + python_args.push("--trust-remote-code".to_string()); + } + let engine_config = ManagedEngineConfig { + python: args.engine.python.clone(), + model: args.model.clone(), + handshake_host: args.engine.handshake_host.clone(), + handshake_port, + data_parallel_size: args.engine.data_parallel_size, + python_args, + }; + eprintln!( + "Spawning managed headless engine: python={} model={} data_parallel_size={}", + engine_config.python, engine_config.model, engine_config.data_parallel_size + ); + let engine = ManagedEngineHandle::spawn(engine_config) + .await + .context("failed to start managed Python headless engine")?; + + let result = run_mm_processor_with_engine(args, handshake_port, &timing).await; + let shutdown_result = engine + .shutdown(ENGINE_SHUTDOWN_TIMEOUT) + .await + .context("failed to shut down managed engine"); + result?; + shutdown_result?; + Ok(()) +} + +async fn run_mm_processor_with_engine( + args: MmProcessorArgs, + handshake_port: u16, + timing: &MultiModalTimingStats, +) -> Result<()> { + let range_ratio = + RangeRatio::parse(&args.random_range_ratio).context("invalid --random-range-ratio")?; + let limit: MmLimitPerPrompt = parse_limit_mm_per_prompt(&args.random_mm_limit_mm_per_prompt) + .context("invalid --random-mm-limit-mm-per-prompt")?; + let buckets: Vec<(MmBucketKey, f64)> = parse_bucket_config(&args.random_mm_bucket_config) + .context("invalid --random-mm-bucket-config")?; + let percentiles = parse_percentiles(&args.metric_percentiles)?; + let seed = args.seed.unwrap_or(0); + + // Offline tokenizer: local Hugging Face tokenizer or built-in tiktoken. + let tokenizer = load_tokenizer(&args.model, args.trust_remote_code, None) + .await + .context("failed to load tokenizer")?; + + // Frontend backends: model configs + multimodal processor metadata. + let backends = load_model_backends( + &args.model, + LoadModelBackendsOptions { + chat_template: args.chat_template.clone(), + ..LoadModelBackendsOptions::default() + }, + ) + .await + .context("failed to load model backends")?; + + if backends.chat_backend.multimodal_model_info().is_none() { + anyhow::bail!( + "model `{}` has no multimodal processor registered", + args.model + ); + } + + // Connect to the engine over the same transport the serving frontend uses. + let client = EngineCoreClient::connect(EngineCoreClientConfig { + transport_mode: TransportMode::HandshakeOwner { + handshake_address: format!("tcp://{}:{}", args.engine.handshake_host, handshake_port), + advertised_host: args.engine.handshake_host.clone(), + engine_count: args + .engine + .data_parallel_size_local + .unwrap_or(args.engine.data_parallel_size), + ready_timeout: ENGINE_READY_TIMEOUT, + local_input_address: None, + local_output_address: None, + }, + coordinator_mode: None, + model_name: args.model.clone(), + client_index: 0, + }) + .await + .context("failed to connect to engine core")?; + + let chat = ChatLlm::new( + TextLlm::new(Llm::new(client), backends.text_backend), + backends.chat_backend, + ); + + let semaphore = Arc::new(Semaphore::new(args.max_concurrency)); + + // Warmup: generate a separate dataset, process it, then drain and discard + // the collected timings so cold caches do not skew the results. + if args.num_warmups > 0 { + let warmup = generate_dataset( + &tokenizer, + args.num_warmups, + seed.wrapping_add(1), + &args, + range_ratio, + &limit, + &buckets, + )?; + eprintln!("Processing {} warmup requests...", warmup.len()); + run_batch(&chat, &warmup, &semaphore).await; + let _ = timing.stat(); + } + + let samples = generate_dataset( + &tokenizer, + args.num_prompts, + seed, + &args, + range_ratio, + &limit, + &buckets, + )?; + eprintln!("Processing {} requests...", samples.len()); + let start_time = Instant::now(); + let outcomes = run_batch(&chat, &samples, &semaphore).await; + let total_time = start_time.elapsed(); + + let completed = outcomes.iter().filter(|outcome| outcome.is_ok()).count(); + let failed = outcomes.len() - completed; + let e2el_times: Vec = outcomes + .iter() + .filter_map(|outcome| outcome.as_ref().ok().map(|elapsed| elapsed.as_secs_f64())) + .collect(); + + let per_request = timing.stat(); + let stats = aggregate_stats(per_request, &percentiles); + + eprintln!( + "Processed {} requests (+{} warmup) for model `{}`", + samples.len(), + args.num_warmups, + args.model + ); + print_report(&stats, &percentiles); + print_e2el_summary(completed, failed, total_time, &e2el_times, &percentiles); + + if let Some(path) = &args.output_json { + write_result_json(path, completed, failed, &e2el_times, &percentiles, stats)?; + eprintln!("Wrote stats JSON to {path}"); + } + + chat.shutdown().await.context("failed to shut down chat facade")?; + Ok(()) +} + +/// Submit every sample through the chat pipeline and drive each stream to +/// completion, returning per-request wall-clock latencies. +async fn run_batch( + chat: &ChatLlm, + samples: &[SampleRequest], + semaphore: &Arc, +) -> Vec> { + let submissions = samples.iter().map(|sample| { + let request = build_chat_request(sample); + let semaphore = semaphore.clone(); + async move { + let _permit = semaphore.acquire().await.context("semaphore closed")?; + let request = request?; + let start = Instant::now(); + chat.chat(request) + .await + .context("failed to submit request")? + .collect_message() + .await + .context("request failed")?; + Ok(start.elapsed()) + } + }); + join_all(submissions).await +} + +fn generate_dataset( + tokenizer: &TokenizerKind, + num_requests: usize, + seed: u64, + args: &MmProcessorArgs, + range_ratio: RangeRatio, + limit: &MmLimitPerPrompt, + buckets: &[(MmBucketKey, f64)], +) -> Result> { + generate_random_mm_dataset( + tokenizer, + num_requests, + args.random_input_len, + args.random_output_len, + args.random_prefix_len, + range_ratio, + seed, + &args.request_id_prefix, + args.random_mm_base_items_per_request, + args.random_mm_num_mm_items_range_ratio, + limit, + buckets, + false, + ) + .map_err(|e| anyhow::anyhow!("{e}")) +} + +/// Build a chat request from a sample (text prompt + `image_url` parts). +fn build_chat_request(sample: &SampleRequest) -> Result { + let mut request = ChatRequest::for_test(); + request.request_id = sample.request_id.clone().unwrap_or_else(|| "mm-proc".to_string()); + request.sampling_params.temperature = Some(0.0); + request.sampling_params.max_tokens = Some(u32::try_from(sample.expected_output_len)?); + + let mut parts = vec![ChatContentPart::text(sample.prompt.as_ref())]; + if let Some(mm_items) = &sample.multi_modal_content { + for fragment in mm_items.iter() { + let url = extract_image_url(fragment)?; + parts.push(ChatContentPart::image_url(url)); + } + } + request.messages = vec![ChatMessage::user(ChatContent::from(parts))]; + Ok(request) +} + +/// Extract the `image_url.url` string from a pre-serialized content fragment +/// (`{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,..."}}`). +fn extract_image_url(fragment: &str) -> Result { + let value: serde_json::Value = serde_json::from_str(fragment) + .with_context(|| format!("invalid multimodal fragment: {fragment}"))?; + let url = value + .get("image_url") + .and_then(|v| v.get("url")) + .and_then(|v| v.as_str()) + .ok_or_else(|| BenchError::Config("fragment missing image_url.url".to_string()))?; + Ok(url.to_string()) +} + +fn parse_percentiles(raw: &str) -> Result> { + let mut values = Vec::new(); + for part in raw.split(',') { + let part = part.trim(); + if part.is_empty() { + continue; + } + let value: f64 = part.parse().with_context(|| format!("invalid percentile `{part}`"))?; + if !(0.0..=100.0).contains(&value) { + anyhow::bail!("percentile `{part}` out of range [0, 100]"); + } + values.push(value); + } + Ok(values) +} + +/// Aggregate per-request stage timings into `{stage_ms: {statistic: ms}}`, +/// matching the Python `mm_processor_stats` schema orientation. +fn aggregate_stats( + per_request: HashMap>, + percentiles: &[f64], +) -> BTreeMap> { + let mut stage_values: BTreeMap> = BTreeMap::new(); + for stage_map in per_request.into_values() { + for (stage, seconds) in stage_map { + stage_values.entry(stage).or_default().push(seconds); + } + } + + stage_values + .iter() + .map(|(stage, values)| { + let sorted = sort_clone(values); + let mut stat_map = BTreeMap::new(); + stat_map.insert("mean".to_string(), mean(values) * SEC_TO_MS); + stat_map.insert("median".to_string(), median_sorted(&sorted) * SEC_TO_MS); + stat_map.insert("std".to_string(), std_dev(values) * SEC_TO_MS); + for p in percentiles { + stat_map.insert(format!("p{p}"), percentile_sorted(&sorted, *p) * SEC_TO_MS); + } + (stage.replace("_secs", "_ms"), stat_map) + }) + .collect() +} + +/// Print a right-aligned plain-text table (pandas `to_string(index=False)` style). +fn print_table(rows: &[Vec]) { + let ncols = rows.first().map_or(0, Vec::len); + let mut widths = vec![0usize; ncols]; + for row in rows { + for (i, cell) in row.iter().enumerate() { + widths[i] = widths[i].max(cell.len()); + } + } + for row in rows { + let line = row + .iter() + .enumerate() + .map(|(i, cell)| format!("{cell:>width$}", width = widths[i])) + .collect::>() + .join(" "); + println!("{line}"); + } +} + +/// Print a plain-text table with one row per stage and one column per +/// statistic (mirrors the Python pandas report layout). +fn print_report(stats: &BTreeMap>, percentiles: &[f64]) { + // Column order: mean/median/std then the requested percentiles, like Python. + let mut columns: Vec<(String, String)> = + [("mean", "Mean"), ("median", "Median"), ("std", "Std")] + .into_iter() + .map(|(key, label)| (key.to_string(), label.to_string())) + .collect(); + columns.extend(percentiles.iter().map(|p| (format!("p{p}"), format!("P{p:?}")))); + for key in stats.values().flat_map(|map| map.keys()) { + if !columns.iter().any(|(k, _)| k == key) { + columns.push((key.clone(), key.clone())); + } + } + + let mut stages: Vec<&String> = stats.keys().collect(); + stages.sort_by_key(|stage| { + STAGE_ORDER + .iter() + .position(|s| s == stage) + .map(|i| (i as isize, (*stage).clone())) + .unwrap_or((isize::MAX, (*stage).clone())) + }); + + let mut rows: Vec> = vec![ + std::iter::once("Stage".to_string()) + .chain(columns.iter().map(|(_, label)| label.clone())) + .collect(), + ]; + for stage in &stages { + let mut row = vec![(*stage).clone()]; + for (key, _) in &columns { + row.push( + stats[*stage] + .get(key) + .map(|v| format!("{v:.2}")) + .unwrap_or_else(|| "-".to_string()), + ); + } + rows.push(row); + } + println!("MM Processor Metrics:"); + print_table(&rows); +} + +/// Print the end-to-end latency summary (mirrors the Python report layout). +fn print_e2el_summary( + completed: usize, + failed: usize, + total_time: Duration, + e2el_times_secs: &[f64], + percentiles: &[f64], +) { + println!("Completed: {completed}, Failed: {failed}"); + if e2el_times_secs.is_empty() { + println!("End-to-End Latency (ms): no completed requests"); + return; + } + + let e2el_ms: Vec = e2el_times_secs.iter().map(|secs| secs * SEC_TO_MS).collect(); + let sorted = sort_clone(&e2el_ms); + let mut rows = vec![ + vec!["Metric".to_string(), "Value (ms)".to_string()], + vec!["Mean".to_string(), format!("{:.2}", mean(&e2el_ms))], + vec![ + "Median".to_string(), + format!("{:.2}", median_sorted(&sorted)), + ], + vec!["Std".to_string(), format!("{:.2}", std_dev(&e2el_ms))], + ]; + for p in percentiles { + rows.push(vec![ + format!("P{p:?}"), + format!("{:.2}", percentile_sorted(&sorted, *p)), + ]); + } + println!("End-to-End Latency (ms):"); + print_table(&rows); + let throughput = completed as f64 / total_time.as_secs_f64(); + println!("Request throughput: {throughput:.3} req/s"); +} + +/// JSON result schema mirroring `vllm bench mm-processor`. +#[derive(Serialize)] +struct BenchmarkResultJson { + completed: usize, + failed: usize, + mean_e2el_ms: f64, + median_e2el_ms: f64, + std_e2el_ms: f64, + percentiles_e2el_ms: Vec<(f64, f64)>, + /// `{stage_ms: {statistic: ms}}`, keyed by stage like the Python schema. + mm_processor_stats: BTreeMap>, +} + +fn write_result_json( + path: &str, + completed: usize, + failed: usize, + e2el_times_secs: &[f64], + percentiles: &[f64], + mm_processor_stats: BTreeMap>, +) -> Result<()> { + let e2el_ms: Vec = e2el_times_secs.iter().map(|secs| secs * SEC_TO_MS).collect(); + let sorted = sort_clone(&e2el_ms); + let result = BenchmarkResultJson { + completed, + failed, + mean_e2el_ms: mean(&e2el_ms), + median_e2el_ms: median_sorted(&sorted), + std_e2el_ms: std_dev(&e2el_ms), + percentiles_e2el_ms: percentiles + .iter() + .map(|p| (*p, percentile_sorted(&sorted, *p))) + .collect(), + mm_processor_stats, + }; + let json = serde_json::to_string_pretty(&result).context("failed to serialize result")?; + std::fs::write(path, format!("{json}\n")).with_context(|| format!("failed to write {path}"))?; + Ok(()) +} diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index e92dedae71..1f1f1974ea 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -35,6 +35,7 @@ vllm-llm.workspace = true vllm-parser.workspace = true vllm-text.workspace = true vllm-tokenizer.workspace = true +vllm-tracing.workspace = true xgrammar-structural-tag.workspace = true [dev-dependencies] diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 806554b5a5..3295abf45f 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -46,6 +46,9 @@ pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessag pub use vllm_engine_core_client::protocol::multimodal::MmFeatures; pub use vllm_llm::FinishReason; pub use vllm_text::GenerationConfigMode; +pub use vllm_tracing::timing::{ + RequestTimingStats as MultiModalTimingStats, mm_request_span, mm_timing_layer, +}; mod backend; mod error; diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index a5dd3ba8ea..362db4b7b7 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -26,7 +26,7 @@ use llm_multimodal::{ }; use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport as _; -use tracing::warn; +use tracing::{Instrument as _, warn}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, MmFeatures, MmKwargsItem}; use vllm_text::Prompt; @@ -47,6 +47,7 @@ mod video; use self::expand::expand_prompt_token_ids; pub use self::input::MultimodalInput; +use vllm_tracing::timing::{mm_request_span, mm_stage_span}; /// Resolved multimodal support for one loaded model. #[derive(Clone)] @@ -562,7 +563,10 @@ pub(crate) async fn finalize_rendered_prompt( .map_err(|error| multimodal!("{error}"))?, Prompt::TokenIds(token_ids) => token_ids, }; - let prepared = info.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype).await?; + let prepared = info + .prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype) + .instrument(mm_request_span(&request.request_id)) + .await?; Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared))) } @@ -753,27 +757,52 @@ impl MultimodalModelInfo { prompt_token_ids: &mut Vec, model_dtype: ModelDtype, ) -> Result { - let media_parts_len = media_parts.len(); - if media_parts_len == 0 { + if media_parts.is_empty() { return Ok(Vec::new()); } + self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype) + .instrument(mm_stage_span("preprocessor_total")) + .await + } + + /// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing` + /// span aggregated per request by the `vllm-tracing` timing layer. + async fn prepare_multimodal_timed( + &self, + media_parts: Vec, + prompt_token_ids: &mut Vec, + model_dtype: ModelDtype, + ) -> Result { + let media_parts_len = media_parts.len(); self.validate_mm_limits(&media_parts)?; - let fetched = self.fetch_media(media_parts).await?; + let fetched = + self.fetch_media(media_parts).instrument(mm_stage_span("media_fetch")).await?; let mut prepared = Vec::new(); if !fetched.images.is_empty() { - prepared - .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?); + prepared.push( + self.prepare_images(fetched.images, fetched.image_uuids, model_dtype) + .instrument(mm_stage_span("preprocess_image")) + .await?, + ); } if !fetched.videos.is_empty() { - prepared - .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?); + prepared.push( + self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype) + .instrument(mm_stage_span("preprocess_video")) + .await?, + ); } if !fetched.audios.is_empty() { - prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?); + prepared.push( + self.prepare_audios(fetched.audios, fetched.audio_uuids) + .instrument(mm_stage_span("preprocess_audio")) + .await?, + ); } - let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?; + let mut ranges = mm_stage_span("prompt_expansion") + .in_scope(|| expand_prompt_token_ids(prompt_token_ids, &prepared))?; let mut features = Vec::with_capacity(media_parts_len); for media in prepared { diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index eb3338a67f..3523f173e3 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -53,7 +53,12 @@ pub async fn generate( let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await; let mm_features = if let Some(parts) = body.content_parts.take() { - match state.chat.prepare_media(parts, &mut body.token_ids).await { + match state + .chat + .prepare_media(parts, &mut body.token_ids) + .instrument(vllm_chat::mm_request_span(&request_context.request_id)) + .await + { Ok(features) => features, Err(e) => { return ApiError::invalid_request( diff --git a/rust/src/tracing/src/lib.rs b/rust/src/tracing/src/lib.rs index eb6da9c0b2..f407b51f5e 100644 --- a/rust/src/tracing/src/lib.rs +++ b/rust/src/tracing/src/lib.rs @@ -3,13 +3,14 @@ //! Shared tracing subscriber and log formatting for vLLM Rust binaries. +pub mod timing; + use std::{env, fmt, process}; use time::UtcOffset; use time::macros::format_description; use tracing::level_filters::LevelFilter; use tracing::{Event, Level, Subscriber}; -use tracing_subscriber::Layer as _; use tracing_subscriber::filter::Targets; use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer}; use tracing_subscriber::fmt::time::FormatTime; @@ -17,6 +18,7 @@ use tracing_subscriber::fmt::{FmtContext, FormattedFields}; use tracing_subscriber::layer::SubscriberExt as _; use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::util::SubscriberInitExt as _; +use tracing_subscriber::{Layer, Registry}; const CYAN: &str = "\x1b[0;36m"; const GREY: &str = "\x1b[90m"; @@ -30,20 +32,36 @@ const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] = /// Install the process-wide vLLM-style tracing subscriber. pub fn init_tracing(process_label: &str) { + let _ = tracing_subscriber::registry().with(vllm_fmt_layer(process_label)).try_init(); +} + +/// Install the process-wide vLLM-style tracing subscriber, with an +/// additional layer (e.g. the multimodal preprocessing timing collector). +pub fn init_tracing_with(process_label: &str, layer: L) +where + L: Layer + Send + Sync + 'static, +{ + let _ = tracing_subscriber::registry() + .with(layer) + .with(vllm_fmt_layer(process_label)) + .try_init(); +} + +/// The vLLM-style fmt layer filtered by `VLLM_LOGGING_LEVEL` / `RUST_LOG`. +fn vllm_fmt_layer(process_label: &str) -> impl Layer + Send + Sync + 'static + use +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, +{ let filter = build_targets_filter( env::var("VLLM_LOGGING_LEVEL").ok().as_deref(), env::var("RUST_LOG").ok().as_deref(), ); let formatter = VllmEventFormatter::new(process_label); - let _ = tracing_subscriber::registry() - .with( - tracing_subscriber::fmt::layer() - .event_format(formatter) - .with_writer(std::io::stderr) - .with_filter(filter), - ) - .try_init(); + tracing_subscriber::fmt::layer() + .event_format(formatter) + .with_writer(std::io::stderr) + .with_filter(filter) } /// Build the CLI log filter by merging the vLLM-style default level with diff --git a/rust/src/tracing/src/timing.rs b/rust/src/tracing/src/timing.rs new file mode 100644 index 0000000000..b0b68e533f --- /dev/null +++ b/rust/src/tracing/src/timing.rs @@ -0,0 +1,229 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Request-scoped span timing. +//! +//! Pipelines emit stage spans (target = the layer's configured target, a +//! `stage` field naming the stage) inside a request span carrying a +//! `request_id` field; [`RequestTimingLayer`] attributes each stage span's +//! wall-clock time to the nearest ancestor `request_id` and accumulates per +//! request. Callers that want timings install the layer on their subscriber; +//! everyone else pays only the cost of untracked spans. + +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +use tracing::field::{Field, Visit}; +use tracing::span::{Attributes, Span}; +use tracing::{Id, Subscriber, info_span}; +use tracing_subscriber::layer::{Context, Layer}; +use tracing_subscriber::registry::LookupSpan; + +/// `"{stage}_secs" => seconds` maps returned by [`RequestTimingStats::stat`]. +pub type StageStats = HashMap; + +/// `request_id` extracted from a span, propagated to stage timings. +struct RequestId(String); + +/// Start timestamp of one in-flight stage span. +struct StageStart { + stage: String, + start: Instant, +} + +/// Extracts the fields the layer cares about from span attributes. +#[derive(Default)] +struct SpanFields { + request_id: Option, + stage: Option, +} + +impl Visit for SpanFields { + fn record_str(&mut self, field: &Field, value: &str) { + match field.name() { + "request_id" => self.request_id = Some(value.to_string()), + "stage" => self.stage = Some(value.to_string()), + _ => {} + } + } + + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + match field.name() { + "request_id" => self.request_id = Some(format!("{value:?}")), + "stage" => self.stage = Some(format!("{value:?}")), + _ => {} + } + } +} + +/// `tracing` layer that aggregates stage-span timings per request, keyed by +/// the nearest ancestor span with a `request_id` field. +pub struct RequestTimingLayer { + target: &'static str, + stats: Arc>>, +} + +/// Drain handle for the timings collected by [`RequestTimingLayer`]. +pub struct RequestTimingStats { + stats: Arc>>, +} + +impl RequestTimingLayer { + /// Create the layer and its stats handle. Only spans whose target equals + /// `target` and that carry a `stage` field are timed. + pub fn new(target: &'static str) -> (Self, RequestTimingStats) { + let stats = Arc::new(Mutex::new(HashMap::new())); + ( + Self { + target, + stats: Arc::clone(&stats), + }, + RequestTimingStats { stats }, + ) + } +} + +impl RequestTimingStats { + /// Drain and return `{request_id: {stage_secs}}` records. + pub fn stat(&self) -> HashMap { + std::mem::take(&mut *self.stats.lock().unwrap()) + } +} + +impl Layer for RequestTimingLayer +where + S: Subscriber + for<'lookup> LookupSpan<'lookup>, +{ + fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) { + let Some(span) = ctx.span(id) else { + return; + }; + let mut fields = SpanFields::default(); + attrs.record(&mut fields); + let mut extensions = span.extensions_mut(); + if let Some(request_id) = fields.request_id { + extensions.insert(RequestId(request_id)); + } + if span.metadata().target() == self.target + && let Some(stage) = fields.stage + { + extensions.insert(StageStart { + stage, + start: Instant::now(), + }); + } + } + + fn on_close(&self, id: Id, ctx: Context<'_, S>) { + let Some(span) = ctx.span(&id) else { + return; + }; + let (stage, start) = { + let extensions = span.extensions(); + let Some(stage_start) = extensions.get::() else { + return; + }; + (stage_start.stage.clone(), stage_start.start) + }; + let Some(request_id) = span + .scope() + .find_map(|ancestor| ancestor.extensions().get::().map(|id| id.0.clone())) + else { + return; + }; + let mut stats = self.stats.lock().unwrap(); + *stats.entry(request_id).or_default().entry(format!("{stage}_secs")).or_default() += + start.elapsed().as_secs_f64(); + } +} + +/// Target of the multimodal preprocessing stage spans. +const MM_STAGE_TARGET: &str = "mm_processor_timing"; + +/// Create the timing layer and its stats handle for multimodal preprocessing +/// stage spans (`vllm-bench mm-processor`), mirroring the Python +/// `TimingContext` / `MultiModalTimingRegistry`. +pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) { + RequestTimingLayer::new(MM_STAGE_TARGET) +} + +/// Span carrying the `request_id` used to attribute multimodal stage timings. +pub fn mm_request_span(request_id: &str) -> Span { + info_span!("mm_request", request_id) +} + +/// Span for one multimodal preprocessing stage; the elapsed time is recorded +/// on close. +pub fn mm_stage_span(stage: &'static str) -> Span { + info_span!(target: MM_STAGE_TARGET, "mm_stage", stage) +} + +#[cfg(test)] +mod tests { + use super::*; + use tracing::info_span; + use tracing_subscriber::layer::SubscriberExt as _; + + const TARGET: &str = "test_request_timing"; + + fn stage_span(stage: &'static str) -> tracing::Span { + info_span!(target: TARGET, "stage", stage) + } + + #[test] + fn records_stage_timings_per_request() { + let (layer, stats) = RequestTimingLayer::new(TARGET); + let subscriber = tracing_subscriber::registry().with(layer); + tracing::subscriber::with_default(subscriber, || { + info_span!("request", request_id = "req-1").in_scope(|| { + drop(stage_span("media_fetch")); + stage_span("prompt_expansion").in_scope(|| {}); + }); + }); + let drained = stats.stat(); + assert!(drained["req-1"].contains_key("media_fetch_secs")); + assert!(drained["req-1"].contains_key("prompt_expansion_secs")); + // `stat` drains: a second read is empty. + assert!(stats.stat().is_empty()); + } + + #[test] + fn attributes_to_nearest_request_id_ancestor() { + let (layer, stats) = RequestTimingLayer::new(TARGET); + let subscriber = tracing_subscriber::registry().with(layer); + tracing::subscriber::with_default(subscriber, || { + info_span!("request", request_id = "req-outer").in_scope(|| { + info_span!("request", request_id = "req-inner").in_scope(|| { + drop(stage_span("media_fetch")); + }); + }); + }); + let drained = stats.stat(); + assert!(drained["req-inner"].contains_key("media_fetch_secs")); + assert!(!drained.contains_key("req-outer")); + } + + #[test] + fn stage_without_request_id_is_dropped() { + let (layer, stats) = RequestTimingLayer::new(TARGET); + let subscriber = tracing_subscriber::registry().with(layer); + tracing::subscriber::with_default(subscriber, || { + drop(stage_span("media_fetch")); + }); + assert!(stats.stat().is_empty()); + } + + #[test] + fn other_targets_are_ignored() { + let (layer, stats) = RequestTimingLayer::new(TARGET); + let subscriber = tracing_subscriber::registry().with(layer); + tracing::subscriber::with_default(subscriber, || { + info_span!("request", request_id = "req-1").in_scope(|| { + drop(info_span!("unrelated", stage = "media_fetch")); + }); + }); + assert!(stats.stat().is_empty()); + } +}