[Frontend][Rust] Add mm-processor benchmark for Rust frontend (#51922)
Signed-off-by: Karthik Gangula <gangula-karthik@users.noreply.github.com> Signed-off-by: gangula-karthik <gkarthik923@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Karthik Gangula <gangula-karthik@users.noreply.github.com> Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Karthik Gangula
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Isotr0py
parent
0961bbae28
commit
f84325c48c
Generated
+6
@@ -5735,6 +5735,11 @@ dependencies = [
|
||||
"url",
|
||||
"uuid",
|
||||
"vllm-build-info",
|
||||
"vllm-chat",
|
||||
"vllm-engine-core-client",
|
||||
"vllm-llm",
|
||||
"vllm-managed-engine",
|
||||
"vllm-text",
|
||||
"vllm-tracing",
|
||||
]
|
||||
|
||||
@@ -5784,6 +5789,7 @@ dependencies = [
|
||||
"vllm-parser",
|
||||
"vllm-text",
|
||||
"vllm-tokenizer",
|
||||
"vllm-tracing",
|
||||
"xgrammar-structural-tag",
|
||||
"zeromq",
|
||||
]
|
||||
|
||||
@@ -23,7 +23,7 @@ cargo test -p vllm-bench -- --ignored
|
||||
|
||||
## Architecture
|
||||
|
||||
- `src/main.rs` — Entry point, mimalloc, tokio runtime, mode dispatch (compare/sweep/multi-run/multi-turn/single)
|
||||
- `src/main.rs` — Entry point, mimalloc, tokio runtime, mode dispatch (compare/sweep/multi-run/multi-turn/single, `mm-processor` subcommand)
|
||||
- `src/cli.rs` — clap derive CLI args (~50+ flags)
|
||||
- `src/config.rs` — Validated config from CLI; `GoodputConfig`, `RampUpConfig`, sampling param merging
|
||||
- `src/error.rs` — `BenchError` enum (Http, Json, Tokenizer, Config, EndpointTimeout, Backend, Io)
|
||||
@@ -32,6 +32,7 @@ cargo test -p vllm-bench -- --ignored
|
||||
- `src/sweep.rs` — Concurrency/rate parameter sweep (`--sweep-max-concurrency`, `--sweep-request-rate`)
|
||||
- `src/multi_run.rs` — N-run aggregation with mean/std/min/max/CV (`--num-runs`)
|
||||
- `src/compare.rs` — Side-by-side diff of two result JSON files (`--compare`)
|
||||
- `src/mm_processor.rs` — `vllm-bench mm-processor` subcommand: offline multimodal preprocessing latency benchmark against a managed headless engine (mirrors `vllm bench mm-processor`); per-stage timings come from `vllm_tracing::timing::RequestTimingLayer` (via `vllm_chat::mm_timing_layer`)
|
||||
- `src/tokenizer.rs` — `TokenizerKind` enum: Local(HuggingFace), Tiktoken, OR Server-side `/tokenize`+`/detokenize` fallback
|
||||
- `src/tiktoken.rs` — Tiktoken BPE loader (`.tiktoken`/`.model` files; built-in encodings o200k_base/cl100k_base; pat_str extraction from Python source)
|
||||
- `src/hub.rs` — `HubRepo`: thin wrapper over hf-hub's async (reqwest/native-tls) API — file download and repo listing, optional revision; the sync ureq backend is unusable here because it pulls rustls, which `rust/deny.toml` bans
|
||||
|
||||
@@ -36,6 +36,11 @@ tracing.workspace = true
|
||||
url.workspace = true
|
||||
uuid.workspace = true
|
||||
vllm-build-info.workspace = true
|
||||
vllm-chat.workspace = true
|
||||
vllm-engine-core-client.workspace = true
|
||||
vllm-llm.workspace = true
|
||||
vllm-managed-engine.workspace = true
|
||||
vllm-text.workspace = true
|
||||
vllm-tracing.workspace = true
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -18,6 +18,7 @@ vllm-bench --backend vllm --base-url http://127.0.0.1:8000 \
|
||||
- **Beyond a single run** — concurrency/rate **sweeps**, **multi-run** stats, **multi-turn** conversations, **LoRA** multi-adapter, and result **comparison**.
|
||||
- **Steady-state metrics** — throughput/latency measured over the saturated plateau, excluding ramp-up and drain.
|
||||
- **Parity** — JSON output schema and timing semantics match Python `vllm bench serve` exactly.
|
||||
- **Multimodal preprocessing** — `mm-processor` subcommand measures per-stage VLM preprocessing latency against a managed headless engine (mirrors `vllm bench mm-processor`).
|
||||
|
||||
### Performance vs. Python
|
||||
|
||||
@@ -404,6 +405,32 @@ vllm-bench \
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b>Multimodal preprocessing (offline)</b></summary>
|
||||
|
||||
The `mm-processor` subcommand benchmarks the multimodal preprocessing pipeline
|
||||
itself (media fetch → per-modality preprocess → prompt expansion), mirroring
|
||||
`vllm bench mm-processor`. It spawns a managed headless Python engine (real
|
||||
weights, real encoder) — no serving endpoint needed — and reports per-stage
|
||||
timing (mean/median/std/percentiles) plus end-to-end latency.
|
||||
|
||||
```bash
|
||||
vllm-bench mm-processor \
|
||||
--model Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--num-prompts 100 --num-warmups 5 \
|
||||
--random-input-len 512 --random-output-len 128 \
|
||||
--random-mm-base-items-per-request 1 \
|
||||
--random-mm-limit-mm-per-prompt '{"image": 1, "video": 0}' \
|
||||
--random-mm-bucket-config '{(1024, 800, 1): 1.0}' \
|
||||
--metric-percentiles "50,90,99" \
|
||||
--output-json mm_processor_stats.json
|
||||
```
|
||||
|
||||
`--max-concurrency` defaults to `1`, matching Python's serial driver; raise it
|
||||
to measure a concurrent serving-style workload.
|
||||
|
||||
</details>
|
||||
|
||||
## Supported Backends
|
||||
|
||||
### Generation
|
||||
@@ -705,6 +732,36 @@ With `--multi-turn`, `--num-prompts` controls the number of **conversations**, n
|
||||
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary><b><code>mm-processor</code> subcommand</b></summary>
|
||||
|
||||
Offline multimodal preprocessing latency benchmark (mirrors `vllm bench mm-processor`). Spawns a managed headless Python engine and drives the full chat preprocessing pipeline.
|
||||
|
||||
| Flag | Default | Description |
|
||||
| ------ | --------- | ------------- |
|
||||
| `--model` | (required) | Model to serve and benchmark (HF id or local path) |
|
||||
| `--num-prompts` | `10` | Prompts to process, excluding warmups |
|
||||
| `--num-warmups` | `1` | Warmup prompts processed and discarded before timing |
|
||||
| `--metric-percentiles` | `99` | Comma-separated percentiles to report |
|
||||
| `--output-json` | — | Path to write aggregate stats as JSON (same schema as Python) |
|
||||
| `--random-input-len` | `1024` | Text input length per prompt |
|
||||
| `--random-output-len` | `128` | Expected output length per prompt |
|
||||
| `--random-prefix-len` | `0` | Prefix token length per prompt |
|
||||
| `--random-range-ratio` | `0.0` | Input/output length range ratio (float or `{"input": i, "output": o}`) |
|
||||
| `--random-mm-base-items-per-request` | `1` | Base number of multimodal items per request |
|
||||
| `--random-mm-num-mm-items-range-ratio` | `0.0` | Range ratio (in [0, 1]) for the number of mm items per request |
|
||||
| `--random-mm-limit-mm-per-prompt` | `{"image": 255, "video": 1}` | Per-modality item caps |
|
||||
| `--random-mm-bucket-config` | `{(256,256,1): 0.5, (720,1280,1): 0.5}` | Bucket config |
|
||||
| `--seed` | `0` | Seed for dataset generation |
|
||||
| `--request-id-prefix` | `mm-proc-` | Request-id prefix for per-request timing keys |
|
||||
| `--max-concurrency` | `1` | Maximum concurrent requests (1 matches Python's serial driver) |
|
||||
| `--trust-remote-code` | `false` | Trust remote code for the tokenizer and the managed engine |
|
||||
| `--chat-template` | — | Chat-template override (inline text or path) |
|
||||
|
||||
Managed headless-engine options (e.g. `--python`, `--data-parallel-address`, `--data-parallel-rpc-port`, `--data-parallel-size`, `--max-model-len`, `--python-args`) are shared with other managed-engine commands; see `vllm-bench mm-processor --help`.
|
||||
|
||||
</details>
|
||||
|
||||
## Tokenizer Support
|
||||
|
||||
Tokenizers are loaded with a three-tier fallback chain:
|
||||
@@ -734,10 +791,11 @@ Use `--append-result` to append multiple runs to the same file in JSONL format.
|
||||
|
||||
```text
|
||||
src/
|
||||
├── main.rs # Entry point, mimalloc, tokio runtime, mode dispatch
|
||||
├── main.rs # Entry point, mimalloc, tokio runtime, mode/subcommand dispatch
|
||||
├── cli.rs # clap CLI argument definitions
|
||||
├── config.rs # Validated config, goodput/ramp-up parsing
|
||||
├── benchmark.rs # Core orchestrator (schedule, spawn, collect, verify, profile)
|
||||
├── mm_processor.rs # Offline mm preprocessing benchmark (mm-processor subcommand)
|
||||
├── multi_turn.rs # Multi-turn conversation orchestrator (channel workers)
|
||||
├── compare.rs # Result diff (--compare file_a.json file_b.json)
|
||||
├── sweep.rs # Parameter sweep (--sweep-max-concurrency, --sweep-request-rate)
|
||||
|
||||
@@ -10,6 +10,7 @@ mod datasets;
|
||||
mod error;
|
||||
mod hub;
|
||||
mod metrics;
|
||||
mod mm_processor;
|
||||
mod multi_run;
|
||||
mod multi_turn;
|
||||
mod output;
|
||||
@@ -25,6 +26,7 @@ pub use cli::{
|
||||
BackendKind, BenchServeArgs, DatasetName, LoraAssignment, RampUpStrategy, SpeedBenchConfig,
|
||||
};
|
||||
use config::BenchConfig;
|
||||
pub use mm_processor::{MmProcessorArgs, run_mm_processor};
|
||||
|
||||
/// Prepare process-wide resources for a benchmark run.
|
||||
pub fn prepare_process() {
|
||||
|
||||
@@ -6,22 +6,30 @@
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
use anyhow::Context;
|
||||
use clap::Parser;
|
||||
use clap::{Parser, Subcommand};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "vllm-bench",
|
||||
about = "Benchmark online serving throughput",
|
||||
about = "Benchmark online serving throughput and offline multimodal preprocessing",
|
||||
version = vllm_build_info::VERSION
|
||||
)]
|
||||
struct Cli {
|
||||
#[command(flatten)]
|
||||
args: vllm_bench::BenchServeArgs,
|
||||
|
||||
/// Optional subcommand; absent by default for the online serving benchmark.
|
||||
#[command(subcommand)]
|
||||
command: Option<Command>,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Command {
|
||||
/// Offline multimodal preprocessing latency benchmark.
|
||||
MmProcessor(vllm_bench::MmProcessorArgs),
|
||||
}
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
vllm_tracing::init_tracing("Bench");
|
||||
|
||||
let cli = Cli::parse();
|
||||
vllm_bench::prepare_process();
|
||||
|
||||
@@ -30,5 +38,15 @@ fn main() -> anyhow::Result<()> {
|
||||
.build()
|
||||
.context("Failed to build tokio runtime")?;
|
||||
|
||||
runtime.block_on(vllm_bench::run(cli.args))
|
||||
match cli.command {
|
||||
Some(Command::MmProcessor(args)) => {
|
||||
let (timing_layer, timing_stats) = vllm_chat::mm_timing_layer();
|
||||
vllm_tracing::init_tracing_with("Bench", timing_layer);
|
||||
runtime.block_on(vllm_bench::run_mm_processor(args, timing_stats))
|
||||
}
|
||||
None => {
|
||||
vllm_tracing::init_tracing("Bench");
|
||||
runtime.block_on(vllm_bench::run(cli.args))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -216,20 +216,20 @@ pub fn calculate_metrics(
|
||||
(metrics, actual_output_lens)
|
||||
}
|
||||
|
||||
fn mean(data: &[f64]) -> f64 {
|
||||
pub(crate) fn mean(data: &[f64]) -> f64 {
|
||||
if data.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
data.iter().sum::<f64>() / data.len() as f64
|
||||
}
|
||||
|
||||
fn sort_clone(data: &[f64]) -> Vec<f64> {
|
||||
pub(crate) fn sort_clone(data: &[f64]) -> Vec<f64> {
|
||||
let mut sorted = data.to_vec();
|
||||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
|
||||
sorted
|
||||
}
|
||||
|
||||
fn median_sorted(sorted: &[f64]) -> f64 {
|
||||
pub(crate) fn median_sorted(sorted: &[f64]) -> f64 {
|
||||
if sorted.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
@@ -241,7 +241,7 @@ fn median_sorted(sorted: &[f64]) -> f64 {
|
||||
}
|
||||
}
|
||||
|
||||
fn std_dev(data: &[f64]) -> f64 {
|
||||
pub(crate) fn std_dev(data: &[f64]) -> f64 {
|
||||
if data.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
@@ -250,7 +250,7 @@ fn std_dev(data: &[f64]) -> f64 {
|
||||
variance.sqrt()
|
||||
}
|
||||
|
||||
fn percentile_sorted(sorted_data: &[f64], p: f64) -> f64 {
|
||||
pub(crate) fn percentile_sorted(sorted_data: &[f64], p: f64) -> f64 {
|
||||
if sorted_data.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,572 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Multimodal preprocessing benchmark (`vllm-bench mm-processor`), mirroring
|
||||
//! `vllm bench mm-processor`.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Args;
|
||||
use futures::future::join_all;
|
||||
use serde::Serialize;
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use vllm_chat::{
|
||||
ChatContent, ChatContentPart, ChatLlm, ChatMessage, ChatRequest, LoadModelBackendsOptions,
|
||||
MultiModalTimingStats, load_model_backends,
|
||||
};
|
||||
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
|
||||
use vllm_llm::Llm;
|
||||
use vllm_managed_engine::cli::ManagedEngineArgs;
|
||||
use vllm_managed_engine::{ManagedEngineConfig, ManagedEngineHandle};
|
||||
use vllm_text::TextLlm;
|
||||
|
||||
use crate::config::RangeRatio;
|
||||
use crate::datasets::SampleRequest;
|
||||
use crate::datasets::random_mm::{
|
||||
MmBucketKey, MmLimitPerPrompt, generate_random_mm_dataset, parse_bucket_config,
|
||||
parse_limit_mm_per_prompt,
|
||||
};
|
||||
use crate::error::BenchError;
|
||||
use crate::metrics::calculator::{mean, median_sorted, percentile_sorted, sort_clone, std_dev};
|
||||
use crate::tokenizer::{TokenizerKind, load_tokenizer};
|
||||
|
||||
/// Seconds -> milliseconds.
|
||||
const SEC_TO_MS: f64 = 1000.0;
|
||||
|
||||
/// Preferred column order; other stages appended alphabetically.
|
||||
const STAGE_ORDER: &[&str] = &[
|
||||
"preprocessor_total_ms",
|
||||
"media_fetch_ms",
|
||||
"preprocess_image_ms",
|
||||
"preprocess_video_ms",
|
||||
"preprocess_audio_ms",
|
||||
"prompt_expansion_ms",
|
||||
];
|
||||
|
||||
/// Time to wait for managed engines to register.
|
||||
const ENGINE_READY_TIMEOUT: Duration = Duration::from_secs(600);
|
||||
|
||||
/// Maximum time to wait for the managed engine to drain on shutdown.
|
||||
const ENGINE_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
|
||||
#[derive(Args, Debug, Clone)]
|
||||
pub struct MmProcessorArgs {
|
||||
/// Model to serve and benchmark (Hugging Face id or local path).
|
||||
#[arg(long)]
|
||||
pub model: String,
|
||||
|
||||
/// Number of prompts to process, excluding warmups.
|
||||
#[arg(long, default_value_t = 10)]
|
||||
pub num_prompts: usize,
|
||||
|
||||
/// Number of warmup prompts processed and discarded before timing.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub num_warmups: usize,
|
||||
|
||||
/// Comma-separated percentiles to report, e.g. "50,90,99".
|
||||
#[arg(long, default_value = "99")]
|
||||
pub metric_percentiles: String,
|
||||
|
||||
/// Path to write the aggregate stats as JSON.
|
||||
#[arg(long)]
|
||||
pub output_json: Option<String>,
|
||||
|
||||
/// Text input length per prompt (before range scaling).
|
||||
#[arg(long, default_value_t = 1024)]
|
||||
pub random_input_len: usize,
|
||||
|
||||
/// Expected output length per prompt.
|
||||
#[arg(long, default_value_t = 128)]
|
||||
pub random_output_len: usize,
|
||||
|
||||
/// Prefix token length per prompt.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
pub random_prefix_len: usize,
|
||||
|
||||
/// Input/output length range ratio: a float or `{"input": i, "output": o}`.
|
||||
#[arg(long, default_value = "0.0")]
|
||||
pub random_range_ratio: String,
|
||||
|
||||
/// Base number of multimodal items per request.
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub random_mm_base_items_per_request: usize,
|
||||
|
||||
/// Range ratio (in [0, 1]) for the number of mm items per request.
|
||||
#[arg(long, default_value_t = 0.0)]
|
||||
pub random_mm_num_mm_items_range_ratio: f64,
|
||||
|
||||
/// Per-modality item caps, e.g. `{"image": 255, "video": 1}`.
|
||||
#[arg(long, default_value = "{\"image\": 255, \"video\": 1}")]
|
||||
pub random_mm_limit_mm_per_prompt: String,
|
||||
|
||||
/// Bucket config, e.g. `{(256,256,1): 0.5, (720,1280,1): 0.5}`.
|
||||
#[arg(long, default_value = "{(256,256,1): 0.5, (720,1280,1): 0.5}")]
|
||||
pub random_mm_bucket_config: String,
|
||||
|
||||
/// Seed for dataset generation.
|
||||
#[arg(long)]
|
||||
pub seed: Option<u64>,
|
||||
|
||||
/// Request-id prefix for per-request timing keys.
|
||||
#[arg(long, default_value = "mm-proc-")]
|
||||
pub request_id_prefix: String,
|
||||
|
||||
/// Maximum concurrent requests (default 1, matching Python's serial driver).
|
||||
#[arg(long, default_value_t = 1)]
|
||||
pub max_concurrency: usize,
|
||||
|
||||
/// Trust remote code for the tokenizer and the managed engine.
|
||||
#[arg(long, default_value_t = false)]
|
||||
pub trust_remote_code: bool,
|
||||
|
||||
/// Optional chat-template override (inline text or path).
|
||||
#[arg(long)]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Managed headless-engine options.
|
||||
#[command(flatten)]
|
||||
pub engine: ManagedEngineArgs,
|
||||
}
|
||||
|
||||
/// Entry point for `vllm-bench mm-processor`.
|
||||
pub async fn run_mm_processor(args: MmProcessorArgs, timing: MultiModalTimingStats) -> Result<()> {
|
||||
if args.engine.data_parallel_size_local == Some(0) {
|
||||
anyhow::bail!(
|
||||
"`--data-parallel-size-local 0` is not supported; the benchmark requires \
|
||||
a local managed engine"
|
||||
);
|
||||
}
|
||||
|
||||
// Spawn the managed headless Python engine (real weights, real encoder).
|
||||
let handshake_port = args
|
||||
.engine
|
||||
.resolve_handshake_port()
|
||||
.context("failed to allocate handshake port")?;
|
||||
let mut python_args = args.engine.python_args.clone();
|
||||
if args.trust_remote_code {
|
||||
python_args.push("--trust-remote-code".to_string());
|
||||
}
|
||||
let engine_config = ManagedEngineConfig {
|
||||
python: args.engine.python.clone(),
|
||||
model: args.model.clone(),
|
||||
handshake_host: args.engine.handshake_host.clone(),
|
||||
handshake_port,
|
||||
data_parallel_size: args.engine.data_parallel_size,
|
||||
python_args,
|
||||
};
|
||||
eprintln!(
|
||||
"Spawning managed headless engine: python={} model={} data_parallel_size={}",
|
||||
engine_config.python, engine_config.model, engine_config.data_parallel_size
|
||||
);
|
||||
let engine = ManagedEngineHandle::spawn(engine_config)
|
||||
.await
|
||||
.context("failed to start managed Python headless engine")?;
|
||||
|
||||
let result = run_mm_processor_with_engine(args, handshake_port, &timing).await;
|
||||
let shutdown_result = engine
|
||||
.shutdown(ENGINE_SHUTDOWN_TIMEOUT)
|
||||
.await
|
||||
.context("failed to shut down managed engine");
|
||||
result?;
|
||||
shutdown_result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run_mm_processor_with_engine(
|
||||
args: MmProcessorArgs,
|
||||
handshake_port: u16,
|
||||
timing: &MultiModalTimingStats,
|
||||
) -> Result<()> {
|
||||
let range_ratio =
|
||||
RangeRatio::parse(&args.random_range_ratio).context("invalid --random-range-ratio")?;
|
||||
let limit: MmLimitPerPrompt = parse_limit_mm_per_prompt(&args.random_mm_limit_mm_per_prompt)
|
||||
.context("invalid --random-mm-limit-mm-per-prompt")?;
|
||||
let buckets: Vec<(MmBucketKey, f64)> = parse_bucket_config(&args.random_mm_bucket_config)
|
||||
.context("invalid --random-mm-bucket-config")?;
|
||||
let percentiles = parse_percentiles(&args.metric_percentiles)?;
|
||||
let seed = args.seed.unwrap_or(0);
|
||||
|
||||
// Offline tokenizer: local Hugging Face tokenizer or built-in tiktoken.
|
||||
let tokenizer = load_tokenizer(&args.model, args.trust_remote_code, None)
|
||||
.await
|
||||
.context("failed to load tokenizer")?;
|
||||
|
||||
// Frontend backends: model configs + multimodal processor metadata.
|
||||
let backends = load_model_backends(
|
||||
&args.model,
|
||||
LoadModelBackendsOptions {
|
||||
chat_template: args.chat_template.clone(),
|
||||
..LoadModelBackendsOptions::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.context("failed to load model backends")?;
|
||||
|
||||
if backends.chat_backend.multimodal_model_info().is_none() {
|
||||
anyhow::bail!(
|
||||
"model `{}` has no multimodal processor registered",
|
||||
args.model
|
||||
);
|
||||
}
|
||||
|
||||
// Connect to the engine over the same transport the serving frontend uses.
|
||||
let client = EngineCoreClient::connect(EngineCoreClientConfig {
|
||||
transport_mode: TransportMode::HandshakeOwner {
|
||||
handshake_address: format!("tcp://{}:{}", args.engine.handshake_host, handshake_port),
|
||||
advertised_host: args.engine.handshake_host.clone(),
|
||||
engine_count: args
|
||||
.engine
|
||||
.data_parallel_size_local
|
||||
.unwrap_or(args.engine.data_parallel_size),
|
||||
ready_timeout: ENGINE_READY_TIMEOUT,
|
||||
local_input_address: None,
|
||||
local_output_address: None,
|
||||
},
|
||||
coordinator_mode: None,
|
||||
model_name: args.model.clone(),
|
||||
client_index: 0,
|
||||
})
|
||||
.await
|
||||
.context("failed to connect to engine core")?;
|
||||
|
||||
let chat = ChatLlm::new(
|
||||
TextLlm::new(Llm::new(client), backends.text_backend),
|
||||
backends.chat_backend,
|
||||
);
|
||||
|
||||
let semaphore = Arc::new(Semaphore::new(args.max_concurrency));
|
||||
|
||||
// Warmup: generate a separate dataset, process it, then drain and discard
|
||||
// the collected timings so cold caches do not skew the results.
|
||||
if args.num_warmups > 0 {
|
||||
let warmup = generate_dataset(
|
||||
&tokenizer,
|
||||
args.num_warmups,
|
||||
seed.wrapping_add(1),
|
||||
&args,
|
||||
range_ratio,
|
||||
&limit,
|
||||
&buckets,
|
||||
)?;
|
||||
eprintln!("Processing {} warmup requests...", warmup.len());
|
||||
run_batch(&chat, &warmup, &semaphore).await;
|
||||
let _ = timing.stat();
|
||||
}
|
||||
|
||||
let samples = generate_dataset(
|
||||
&tokenizer,
|
||||
args.num_prompts,
|
||||
seed,
|
||||
&args,
|
||||
range_ratio,
|
||||
&limit,
|
||||
&buckets,
|
||||
)?;
|
||||
eprintln!("Processing {} requests...", samples.len());
|
||||
let start_time = Instant::now();
|
||||
let outcomes = run_batch(&chat, &samples, &semaphore).await;
|
||||
let total_time = start_time.elapsed();
|
||||
|
||||
let completed = outcomes.iter().filter(|outcome| outcome.is_ok()).count();
|
||||
let failed = outcomes.len() - completed;
|
||||
let e2el_times: Vec<f64> = outcomes
|
||||
.iter()
|
||||
.filter_map(|outcome| outcome.as_ref().ok().map(|elapsed| elapsed.as_secs_f64()))
|
||||
.collect();
|
||||
|
||||
let per_request = timing.stat();
|
||||
let stats = aggregate_stats(per_request, &percentiles);
|
||||
|
||||
eprintln!(
|
||||
"Processed {} requests (+{} warmup) for model `{}`",
|
||||
samples.len(),
|
||||
args.num_warmups,
|
||||
args.model
|
||||
);
|
||||
print_report(&stats, &percentiles);
|
||||
print_e2el_summary(completed, failed, total_time, &e2el_times, &percentiles);
|
||||
|
||||
if let Some(path) = &args.output_json {
|
||||
write_result_json(path, completed, failed, &e2el_times, &percentiles, stats)?;
|
||||
eprintln!("Wrote stats JSON to {path}");
|
||||
}
|
||||
|
||||
chat.shutdown().await.context("failed to shut down chat facade")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Submit every sample through the chat pipeline and drive each stream to
|
||||
/// completion, returning per-request wall-clock latencies.
|
||||
async fn run_batch(
|
||||
chat: &ChatLlm,
|
||||
samples: &[SampleRequest],
|
||||
semaphore: &Arc<Semaphore>,
|
||||
) -> Vec<Result<Duration>> {
|
||||
let submissions = samples.iter().map(|sample| {
|
||||
let request = build_chat_request(sample);
|
||||
let semaphore = semaphore.clone();
|
||||
async move {
|
||||
let _permit = semaphore.acquire().await.context("semaphore closed")?;
|
||||
let request = request?;
|
||||
let start = Instant::now();
|
||||
chat.chat(request)
|
||||
.await
|
||||
.context("failed to submit request")?
|
||||
.collect_message()
|
||||
.await
|
||||
.context("request failed")?;
|
||||
Ok(start.elapsed())
|
||||
}
|
||||
});
|
||||
join_all(submissions).await
|
||||
}
|
||||
|
||||
fn generate_dataset(
|
||||
tokenizer: &TokenizerKind,
|
||||
num_requests: usize,
|
||||
seed: u64,
|
||||
args: &MmProcessorArgs,
|
||||
range_ratio: RangeRatio,
|
||||
limit: &MmLimitPerPrompt,
|
||||
buckets: &[(MmBucketKey, f64)],
|
||||
) -> Result<Vec<SampleRequest>> {
|
||||
generate_random_mm_dataset(
|
||||
tokenizer,
|
||||
num_requests,
|
||||
args.random_input_len,
|
||||
args.random_output_len,
|
||||
args.random_prefix_len,
|
||||
range_ratio,
|
||||
seed,
|
||||
&args.request_id_prefix,
|
||||
args.random_mm_base_items_per_request,
|
||||
args.random_mm_num_mm_items_range_ratio,
|
||||
limit,
|
||||
buckets,
|
||||
false,
|
||||
)
|
||||
.map_err(|e| anyhow::anyhow!("{e}"))
|
||||
}
|
||||
|
||||
/// Build a chat request from a sample (text prompt + `image_url` parts).
|
||||
fn build_chat_request(sample: &SampleRequest) -> Result<ChatRequest> {
|
||||
let mut request = ChatRequest::for_test();
|
||||
request.request_id = sample.request_id.clone().unwrap_or_else(|| "mm-proc".to_string());
|
||||
request.sampling_params.temperature = Some(0.0);
|
||||
request.sampling_params.max_tokens = Some(u32::try_from(sample.expected_output_len)?);
|
||||
|
||||
let mut parts = vec![ChatContentPart::text(sample.prompt.as_ref())];
|
||||
if let Some(mm_items) = &sample.multi_modal_content {
|
||||
for fragment in mm_items.iter() {
|
||||
let url = extract_image_url(fragment)?;
|
||||
parts.push(ChatContentPart::image_url(url));
|
||||
}
|
||||
}
|
||||
request.messages = vec![ChatMessage::user(ChatContent::from(parts))];
|
||||
Ok(request)
|
||||
}
|
||||
|
||||
/// Extract the `image_url.url` string from a pre-serialized content fragment
|
||||
/// (`{"type":"image_url","image_url":{"url":"data:image/jpeg;base64,..."}}`).
|
||||
fn extract_image_url(fragment: &str) -> Result<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(fragment)
|
||||
.with_context(|| format!("invalid multimodal fragment: {fragment}"))?;
|
||||
let url = value
|
||||
.get("image_url")
|
||||
.and_then(|v| v.get("url"))
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| BenchError::Config("fragment missing image_url.url".to_string()))?;
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
fn parse_percentiles(raw: &str) -> Result<Vec<f64>> {
|
||||
let mut values = Vec::new();
|
||||
for part in raw.split(',') {
|
||||
let part = part.trim();
|
||||
if part.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let value: f64 = part.parse().with_context(|| format!("invalid percentile `{part}`"))?;
|
||||
if !(0.0..=100.0).contains(&value) {
|
||||
anyhow::bail!("percentile `{part}` out of range [0, 100]");
|
||||
}
|
||||
values.push(value);
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
/// Aggregate per-request stage timings into `{stage_ms: {statistic: ms}}`,
|
||||
/// matching the Python `mm_processor_stats` schema orientation.
|
||||
fn aggregate_stats(
|
||||
per_request: HashMap<String, HashMap<String, f64>>,
|
||||
percentiles: &[f64],
|
||||
) -> BTreeMap<String, BTreeMap<String, f64>> {
|
||||
let mut stage_values: BTreeMap<String, Vec<f64>> = BTreeMap::new();
|
||||
for stage_map in per_request.into_values() {
|
||||
for (stage, seconds) in stage_map {
|
||||
stage_values.entry(stage).or_default().push(seconds);
|
||||
}
|
||||
}
|
||||
|
||||
stage_values
|
||||
.iter()
|
||||
.map(|(stage, values)| {
|
||||
let sorted = sort_clone(values);
|
||||
let mut stat_map = BTreeMap::new();
|
||||
stat_map.insert("mean".to_string(), mean(values) * SEC_TO_MS);
|
||||
stat_map.insert("median".to_string(), median_sorted(&sorted) * SEC_TO_MS);
|
||||
stat_map.insert("std".to_string(), std_dev(values) * SEC_TO_MS);
|
||||
for p in percentiles {
|
||||
stat_map.insert(format!("p{p}"), percentile_sorted(&sorted, *p) * SEC_TO_MS);
|
||||
}
|
||||
(stage.replace("_secs", "_ms"), stat_map)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Print a right-aligned plain-text table (pandas `to_string(index=False)` style).
|
||||
fn print_table(rows: &[Vec<String>]) {
|
||||
let ncols = rows.first().map_or(0, Vec::len);
|
||||
let mut widths = vec![0usize; ncols];
|
||||
for row in rows {
|
||||
for (i, cell) in row.iter().enumerate() {
|
||||
widths[i] = widths[i].max(cell.len());
|
||||
}
|
||||
}
|
||||
for row in rows {
|
||||
let line = row
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, cell)| format!("{cell:>width$}", width = widths[i]))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
println!("{line}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Print a plain-text table with one row per stage and one column per
|
||||
/// statistic (mirrors the Python pandas report layout).
|
||||
fn print_report(stats: &BTreeMap<String, BTreeMap<String, f64>>, percentiles: &[f64]) {
|
||||
// Column order: mean/median/std then the requested percentiles, like Python.
|
||||
let mut columns: Vec<(String, String)> =
|
||||
[("mean", "Mean"), ("median", "Median"), ("std", "Std")]
|
||||
.into_iter()
|
||||
.map(|(key, label)| (key.to_string(), label.to_string()))
|
||||
.collect();
|
||||
columns.extend(percentiles.iter().map(|p| (format!("p{p}"), format!("P{p:?}"))));
|
||||
for key in stats.values().flat_map(|map| map.keys()) {
|
||||
if !columns.iter().any(|(k, _)| k == key) {
|
||||
columns.push((key.clone(), key.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
let mut stages: Vec<&String> = stats.keys().collect();
|
||||
stages.sort_by_key(|stage| {
|
||||
STAGE_ORDER
|
||||
.iter()
|
||||
.position(|s| s == stage)
|
||||
.map(|i| (i as isize, (*stage).clone()))
|
||||
.unwrap_or((isize::MAX, (*stage).clone()))
|
||||
});
|
||||
|
||||
let mut rows: Vec<Vec<String>> = vec![
|
||||
std::iter::once("Stage".to_string())
|
||||
.chain(columns.iter().map(|(_, label)| label.clone()))
|
||||
.collect(),
|
||||
];
|
||||
for stage in &stages {
|
||||
let mut row = vec![(*stage).clone()];
|
||||
for (key, _) in &columns {
|
||||
row.push(
|
||||
stats[*stage]
|
||||
.get(key)
|
||||
.map(|v| format!("{v:.2}"))
|
||||
.unwrap_or_else(|| "-".to_string()),
|
||||
);
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
println!("MM Processor Metrics:");
|
||||
print_table(&rows);
|
||||
}
|
||||
|
||||
/// Print the end-to-end latency summary (mirrors the Python report layout).
|
||||
fn print_e2el_summary(
|
||||
completed: usize,
|
||||
failed: usize,
|
||||
total_time: Duration,
|
||||
e2el_times_secs: &[f64],
|
||||
percentiles: &[f64],
|
||||
) {
|
||||
println!("Completed: {completed}, Failed: {failed}");
|
||||
if e2el_times_secs.is_empty() {
|
||||
println!("End-to-End Latency (ms): no completed requests");
|
||||
return;
|
||||
}
|
||||
|
||||
let e2el_ms: Vec<f64> = e2el_times_secs.iter().map(|secs| secs * SEC_TO_MS).collect();
|
||||
let sorted = sort_clone(&e2el_ms);
|
||||
let mut rows = vec![
|
||||
vec!["Metric".to_string(), "Value (ms)".to_string()],
|
||||
vec!["Mean".to_string(), format!("{:.2}", mean(&e2el_ms))],
|
||||
vec![
|
||||
"Median".to_string(),
|
||||
format!("{:.2}", median_sorted(&sorted)),
|
||||
],
|
||||
vec!["Std".to_string(), format!("{:.2}", std_dev(&e2el_ms))],
|
||||
];
|
||||
for p in percentiles {
|
||||
rows.push(vec![
|
||||
format!("P{p:?}"),
|
||||
format!("{:.2}", percentile_sorted(&sorted, *p)),
|
||||
]);
|
||||
}
|
||||
println!("End-to-End Latency (ms):");
|
||||
print_table(&rows);
|
||||
let throughput = completed as f64 / total_time.as_secs_f64();
|
||||
println!("Request throughput: {throughput:.3} req/s");
|
||||
}
|
||||
|
||||
/// JSON result schema mirroring `vllm bench mm-processor`.
|
||||
#[derive(Serialize)]
|
||||
struct BenchmarkResultJson {
|
||||
completed: usize,
|
||||
failed: usize,
|
||||
mean_e2el_ms: f64,
|
||||
median_e2el_ms: f64,
|
||||
std_e2el_ms: f64,
|
||||
percentiles_e2el_ms: Vec<(f64, f64)>,
|
||||
/// `{stage_ms: {statistic: ms}}`, keyed by stage like the Python schema.
|
||||
mm_processor_stats: BTreeMap<String, BTreeMap<String, f64>>,
|
||||
}
|
||||
|
||||
fn write_result_json(
|
||||
path: &str,
|
||||
completed: usize,
|
||||
failed: usize,
|
||||
e2el_times_secs: &[f64],
|
||||
percentiles: &[f64],
|
||||
mm_processor_stats: BTreeMap<String, BTreeMap<String, f64>>,
|
||||
) -> Result<()> {
|
||||
let e2el_ms: Vec<f64> = e2el_times_secs.iter().map(|secs| secs * SEC_TO_MS).collect();
|
||||
let sorted = sort_clone(&e2el_ms);
|
||||
let result = BenchmarkResultJson {
|
||||
completed,
|
||||
failed,
|
||||
mean_e2el_ms: mean(&e2el_ms),
|
||||
median_e2el_ms: median_sorted(&sorted),
|
||||
std_e2el_ms: std_dev(&e2el_ms),
|
||||
percentiles_e2el_ms: percentiles
|
||||
.iter()
|
||||
.map(|p| (*p, percentile_sorted(&sorted, *p)))
|
||||
.collect(),
|
||||
mm_processor_stats,
|
||||
};
|
||||
let json = serde_json::to_string_pretty(&result).context("failed to serialize result")?;
|
||||
std::fs::write(path, format!("{json}\n")).with_context(|| format!("failed to write {path}"))?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -35,6 +35,7 @@ vllm-llm.workspace = true
|
||||
vllm-parser.workspace = true
|
||||
vllm-text.workspace = true
|
||||
vllm-tokenizer.workspace = true
|
||||
vllm-tracing.workspace = true
|
||||
xgrammar-structural-tag.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -46,6 +46,9 @@ pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessag
|
||||
pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
|
||||
pub use vllm_llm::FinishReason;
|
||||
pub use vllm_text::GenerationConfigMode;
|
||||
pub use vllm_tracing::timing::{
|
||||
RequestTimingStats as MultiModalTimingStats, mm_request_span, mm_timing_layer,
|
||||
};
|
||||
|
||||
mod backend;
|
||||
mod error;
|
||||
|
||||
@@ -26,7 +26,7 @@ use llm_multimodal::{
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror_ext::AsReport as _;
|
||||
use tracing::warn;
|
||||
use tracing::{Instrument as _, warn};
|
||||
use vllm_engine_core_client::protocol::dtype::ModelDtype;
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, MmFeatures, MmKwargsItem};
|
||||
use vllm_text::Prompt;
|
||||
@@ -47,6 +47,7 @@ mod video;
|
||||
|
||||
use self::expand::expand_prompt_token_ids;
|
||||
pub use self::input::MultimodalInput;
|
||||
use vllm_tracing::timing::{mm_request_span, mm_stage_span};
|
||||
|
||||
/// Resolved multimodal support for one loaded model.
|
||||
#[derive(Clone)]
|
||||
@@ -562,7 +563,10 @@ pub(crate) async fn finalize_rendered_prompt(
|
||||
.map_err(|error| multimodal!("{error}"))?,
|
||||
Prompt::TokenIds(token_ids) => token_ids,
|
||||
};
|
||||
let prepared = info.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype).await?;
|
||||
let prepared = info
|
||||
.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype)
|
||||
.instrument(mm_request_span(&request.request_id))
|
||||
.await?;
|
||||
|
||||
Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared)))
|
||||
}
|
||||
@@ -753,27 +757,52 @@ impl MultimodalModelInfo {
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<MmFeatures> {
|
||||
let media_parts_len = media_parts.len();
|
||||
if media_parts_len == 0 {
|
||||
if media_parts.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
self.prepare_multimodal_timed(media_parts, prompt_token_ids, model_dtype)
|
||||
.instrument(mm_stage_span("preprocessor_total"))
|
||||
.await
|
||||
}
|
||||
|
||||
/// Timed body of [`Self::prepare_multimodal`]; each stage is a `tracing`
|
||||
/// span aggregated per request by the `vllm-tracing` timing layer.
|
||||
async fn prepare_multimodal_timed(
|
||||
&self,
|
||||
media_parts: Vec<MediaContentPart>,
|
||||
prompt_token_ids: &mut Vec<u32>,
|
||||
model_dtype: ModelDtype,
|
||||
) -> Result<MmFeatures> {
|
||||
let media_parts_len = media_parts.len();
|
||||
self.validate_mm_limits(&media_parts)?;
|
||||
let fetched = self.fetch_media(media_parts).await?;
|
||||
let fetched =
|
||||
self.fetch_media(media_parts).instrument(mm_stage_span("media_fetch")).await?;
|
||||
|
||||
let mut prepared = Vec::new();
|
||||
if !fetched.images.is_empty() {
|
||||
prepared
|
||||
.push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
|
||||
prepared.push(
|
||||
self.prepare_images(fetched.images, fetched.image_uuids, model_dtype)
|
||||
.instrument(mm_stage_span("preprocess_image"))
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
if !fetched.videos.is_empty() {
|
||||
prepared
|
||||
.push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
|
||||
prepared.push(
|
||||
self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype)
|
||||
.instrument(mm_stage_span("preprocess_video"))
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
if !fetched.audios.is_empty() {
|
||||
prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
|
||||
prepared.push(
|
||||
self.prepare_audios(fetched.audios, fetched.audio_uuids)
|
||||
.instrument(mm_stage_span("preprocess_audio"))
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
|
||||
let mut ranges = mm_stage_span("prompt_expansion")
|
||||
.in_scope(|| expand_prompt_token_ids(prompt_token_ids, &prepared))?;
|
||||
|
||||
let mut features = Vec::with_capacity(media_parts_len);
|
||||
for media in prepared {
|
||||
|
||||
@@ -53,7 +53,12 @@ pub async fn generate(
|
||||
let lora_resolution = state.resolve_model_with_loras(body.model.as_deref()).await;
|
||||
|
||||
let mm_features = if let Some(parts) = body.content_parts.take() {
|
||||
match state.chat.prepare_media(parts, &mut body.token_ids).await {
|
||||
match state
|
||||
.chat
|
||||
.prepare_media(parts, &mut body.token_ids)
|
||||
.instrument(vllm_chat::mm_request_span(&request_context.request_id))
|
||||
.await
|
||||
{
|
||||
Ok(features) => features,
|
||||
Err(e) => {
|
||||
return ApiError::invalid_request(
|
||||
|
||||
@@ -3,13 +3,14 @@
|
||||
|
||||
//! Shared tracing subscriber and log formatting for vLLM Rust binaries.
|
||||
|
||||
pub mod timing;
|
||||
|
||||
use std::{env, fmt, process};
|
||||
|
||||
use time::UtcOffset;
|
||||
use time::macros::format_description;
|
||||
use tracing::level_filters::LevelFilter;
|
||||
use tracing::{Event, Level, Subscriber};
|
||||
use tracing_subscriber::Layer as _;
|
||||
use tracing_subscriber::filter::Targets;
|
||||
use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
|
||||
use tracing_subscriber::fmt::time::FormatTime;
|
||||
@@ -17,6 +18,7 @@ use tracing_subscriber::fmt::{FmtContext, FormattedFields};
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
use tracing_subscriber::util::SubscriberInitExt as _;
|
||||
use tracing_subscriber::{Layer, Registry};
|
||||
|
||||
const CYAN: &str = "\x1b[0;36m";
|
||||
const GREY: &str = "\x1b[90m";
|
||||
@@ -30,20 +32,36 @@ const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
|
||||
|
||||
/// Install the process-wide vLLM-style tracing subscriber.
|
||||
pub fn init_tracing(process_label: &str) {
|
||||
let _ = tracing_subscriber::registry().with(vllm_fmt_layer(process_label)).try_init();
|
||||
}
|
||||
|
||||
/// Install the process-wide vLLM-style tracing subscriber, with an
|
||||
/// additional layer (e.g. the multimodal preprocessing timing collector).
|
||||
pub fn init_tracing_with<L>(process_label: &str, layer: L)
|
||||
where
|
||||
L: Layer<Registry> + Send + Sync + 'static,
|
||||
{
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(layer)
|
||||
.with(vllm_fmt_layer(process_label))
|
||||
.try_init();
|
||||
}
|
||||
|
||||
/// The vLLM-style fmt layer filtered by `VLLM_LOGGING_LEVEL` / `RUST_LOG`.
|
||||
fn vllm_fmt_layer<S>(process_label: &str) -> impl Layer<S> + Send + Sync + 'static + use<S>
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
let filter = build_targets_filter(
|
||||
env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
|
||||
env::var("RUST_LOG").ok().as_deref(),
|
||||
);
|
||||
let formatter = VllmEventFormatter::new(process_label);
|
||||
|
||||
let _ = tracing_subscriber::registry()
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.event_format(formatter)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter(filter),
|
||||
)
|
||||
.try_init();
|
||||
tracing_subscriber::fmt::layer()
|
||||
.event_format(formatter)
|
||||
.with_writer(std::io::stderr)
|
||||
.with_filter(filter)
|
||||
}
|
||||
|
||||
/// Build the CLI log filter by merging the vLLM-style default level with
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
//! Request-scoped span timing.
|
||||
//!
|
||||
//! Pipelines emit stage spans (target = the layer's configured target, a
|
||||
//! `stage` field naming the stage) inside a request span carrying a
|
||||
//! `request_id` field; [`RequestTimingLayer`] attributes each stage span's
|
||||
//! wall-clock time to the nearest ancestor `request_id` and accumulates per
|
||||
//! request. Callers that want timings install the layer on their subscriber;
|
||||
//! everyone else pays only the cost of untracked spans.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Instant;
|
||||
|
||||
use tracing::field::{Field, Visit};
|
||||
use tracing::span::{Attributes, Span};
|
||||
use tracing::{Id, Subscriber, info_span};
|
||||
use tracing_subscriber::layer::{Context, Layer};
|
||||
use tracing_subscriber::registry::LookupSpan;
|
||||
|
||||
/// `"{stage}_secs" => seconds` maps returned by [`RequestTimingStats::stat`].
|
||||
pub type StageStats = HashMap<String, f64>;
|
||||
|
||||
/// `request_id` extracted from a span, propagated to stage timings.
|
||||
struct RequestId(String);
|
||||
|
||||
/// Start timestamp of one in-flight stage span.
|
||||
struct StageStart {
|
||||
stage: String,
|
||||
start: Instant,
|
||||
}
|
||||
|
||||
/// Extracts the fields the layer cares about from span attributes.
|
||||
#[derive(Default)]
|
||||
struct SpanFields {
|
||||
request_id: Option<String>,
|
||||
stage: Option<String>,
|
||||
}
|
||||
|
||||
impl Visit for SpanFields {
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
match field.name() {
|
||||
"request_id" => self.request_id = Some(value.to_string()),
|
||||
"stage" => self.stage = Some(value.to_string()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
|
||||
match field.name() {
|
||||
"request_id" => self.request_id = Some(format!("{value:?}")),
|
||||
"stage" => self.stage = Some(format!("{value:?}")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `tracing` layer that aggregates stage-span timings per request, keyed by
|
||||
/// the nearest ancestor span with a `request_id` field.
|
||||
pub struct RequestTimingLayer {
|
||||
target: &'static str,
|
||||
stats: Arc<Mutex<HashMap<String, StageStats>>>,
|
||||
}
|
||||
|
||||
/// Drain handle for the timings collected by [`RequestTimingLayer`].
|
||||
pub struct RequestTimingStats {
|
||||
stats: Arc<Mutex<HashMap<String, StageStats>>>,
|
||||
}
|
||||
|
||||
impl RequestTimingLayer {
|
||||
/// Create the layer and its stats handle. Only spans whose target equals
|
||||
/// `target` and that carry a `stage` field are timed.
|
||||
pub fn new(target: &'static str) -> (Self, RequestTimingStats) {
|
||||
let stats = Arc::new(Mutex::new(HashMap::new()));
|
||||
(
|
||||
Self {
|
||||
target,
|
||||
stats: Arc::clone(&stats),
|
||||
},
|
||||
RequestTimingStats { stats },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestTimingStats {
|
||||
/// Drain and return `{request_id: {stage_secs}}` records.
|
||||
pub fn stat(&self) -> HashMap<String, StageStats> {
|
||||
std::mem::take(&mut *self.stats.lock().unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for RequestTimingLayer
|
||||
where
|
||||
S: Subscriber + for<'lookup> LookupSpan<'lookup>,
|
||||
{
|
||||
fn on_new_span(&self, attrs: &Attributes<'_>, id: &Id, ctx: Context<'_, S>) {
|
||||
let Some(span) = ctx.span(id) else {
|
||||
return;
|
||||
};
|
||||
let mut fields = SpanFields::default();
|
||||
attrs.record(&mut fields);
|
||||
let mut extensions = span.extensions_mut();
|
||||
if let Some(request_id) = fields.request_id {
|
||||
extensions.insert(RequestId(request_id));
|
||||
}
|
||||
if span.metadata().target() == self.target
|
||||
&& let Some(stage) = fields.stage
|
||||
{
|
||||
extensions.insert(StageStart {
|
||||
stage,
|
||||
start: Instant::now(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn on_close(&self, id: Id, ctx: Context<'_, S>) {
|
||||
let Some(span) = ctx.span(&id) else {
|
||||
return;
|
||||
};
|
||||
let (stage, start) = {
|
||||
let extensions = span.extensions();
|
||||
let Some(stage_start) = extensions.get::<StageStart>() else {
|
||||
return;
|
||||
};
|
||||
(stage_start.stage.clone(), stage_start.start)
|
||||
};
|
||||
let Some(request_id) = span
|
||||
.scope()
|
||||
.find_map(|ancestor| ancestor.extensions().get::<RequestId>().map(|id| id.0.clone()))
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let mut stats = self.stats.lock().unwrap();
|
||||
*stats.entry(request_id).or_default().entry(format!("{stage}_secs")).or_default() +=
|
||||
start.elapsed().as_secs_f64();
|
||||
}
|
||||
}
|
||||
|
||||
/// Target of the multimodal preprocessing stage spans.
|
||||
const MM_STAGE_TARGET: &str = "mm_processor_timing";
|
||||
|
||||
/// Create the timing layer and its stats handle for multimodal preprocessing
|
||||
/// stage spans (`vllm-bench mm-processor`), mirroring the Python
|
||||
/// `TimingContext` / `MultiModalTimingRegistry`.
|
||||
pub fn mm_timing_layer() -> (RequestTimingLayer, RequestTimingStats) {
|
||||
RequestTimingLayer::new(MM_STAGE_TARGET)
|
||||
}
|
||||
|
||||
/// Span carrying the `request_id` used to attribute multimodal stage timings.
|
||||
pub fn mm_request_span(request_id: &str) -> Span {
|
||||
info_span!("mm_request", request_id)
|
||||
}
|
||||
|
||||
/// Span for one multimodal preprocessing stage; the elapsed time is recorded
|
||||
/// on close.
|
||||
pub fn mm_stage_span(stage: &'static str) -> Span {
|
||||
info_span!(target: MM_STAGE_TARGET, "mm_stage", stage)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tracing::info_span;
|
||||
use tracing_subscriber::layer::SubscriberExt as _;
|
||||
|
||||
const TARGET: &str = "test_request_timing";
|
||||
|
||||
fn stage_span(stage: &'static str) -> tracing::Span {
|
||||
info_span!(target: TARGET, "stage", stage)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_stage_timings_per_request() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
info_span!("request", request_id = "req-1").in_scope(|| {
|
||||
drop(stage_span("media_fetch"));
|
||||
stage_span("prompt_expansion").in_scope(|| {});
|
||||
});
|
||||
});
|
||||
let drained = stats.stat();
|
||||
assert!(drained["req-1"].contains_key("media_fetch_secs"));
|
||||
assert!(drained["req-1"].contains_key("prompt_expansion_secs"));
|
||||
// `stat` drains: a second read is empty.
|
||||
assert!(stats.stat().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attributes_to_nearest_request_id_ancestor() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
info_span!("request", request_id = "req-outer").in_scope(|| {
|
||||
info_span!("request", request_id = "req-inner").in_scope(|| {
|
||||
drop(stage_span("media_fetch"));
|
||||
});
|
||||
});
|
||||
});
|
||||
let drained = stats.stat();
|
||||
assert!(drained["req-inner"].contains_key("media_fetch_secs"));
|
||||
assert!(!drained.contains_key("req-outer"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_without_request_id_is_dropped() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
drop(stage_span("media_fetch"));
|
||||
});
|
||||
assert!(stats.stat().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_targets_are_ignored() {
|
||||
let (layer, stats) = RequestTimingLayer::new(TARGET);
|
||||
let subscriber = tracing_subscriber::registry().with(layer);
|
||||
tracing::subscriber::with_default(subscriber, || {
|
||||
info_span!("request", request_id = "req-1").in_scope(|| {
|
||||
drop(info_span!("unrelated", stage = "media_fetch"));
|
||||
});
|
||||
});
|
||||
assert!(stats.stat().is_empty());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user