From 704f12aa1ed66db6307659f2996fd905a73570d4 Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Sat, 22 Aug 2026 08:01:40 -0500 Subject: [PATCH] [Pooling] Report input throughput for batched requests (#53213) Signed-off-by: Taneem Ibrahim Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- rust/src/bench/src/backends/mod.rs | 2 ++ rust/src/bench/src/backends/pooling.rs | 15 +++++++++++++-- rust/src/bench/src/compare.rs | 5 +++++ rust/src/bench/src/metrics/calculator.rs | 6 ++++++ rust/src/bench/src/metrics/mod.rs | 2 ++ rust/src/bench/src/multi_run.rs | 20 +++++++++++++++++--- rust/src/bench/src/output/console.rs | 6 ++++++ rust/src/bench/src/output/json.rs | 10 ++++++++++ rust/src/bench/src/sweep.rs | 3 +++ vllm/benchmarks/datasets/datasets.py | 2 +- vllm/benchmarks/lib/endpoint_request_func.py | 16 ++++++++++++++-- vllm/benchmarks/serve.py | 14 ++++++++++++++ 12 files changed, 93 insertions(+), 8 deletions(-) diff --git a/rust/src/bench/src/backends/mod.rs b/rust/src/bench/src/backends/mod.rs index 698a20e49a..c01018529f 100644 --- a/rust/src/bench/src/backends/mod.rs +++ b/rust/src/bench/src/backends/mod.rs @@ -101,6 +101,7 @@ pub struct RequestFuncOutput { pub prompt_len: usize, pub error: String, pub start_time: f64, + pub num_input_sequences: usize, } impl Default for RequestFuncOutput { @@ -116,6 +117,7 @@ impl Default for RequestFuncOutput { prompt_len: 0, error: String::new(), start_time: 0.0, + num_input_sequences: 1, } } } diff --git a/rust/src/bench/src/backends/pooling.rs b/rust/src/bench/src/backends/pooling.rs index 8f9d4deab9..ad43f480e2 100644 --- a/rust/src/bench/src/backends/pooling.rs +++ b/rust/src/bench/src/backends/pooling.rs @@ -39,9 +39,22 @@ impl PoolingBackend { input: &RequestFuncInput, client: &reqwest::Client, ) -> Result { + let payload = self.build_payload(input); + // Preserve client-side prompt_len as fallback if server doesn't report usage. let mut output = RequestFuncOutput { prompt_len: input.prompt_len, + num_input_sequences: match self.kind { + BackendKind::OpenaiEmbeddings | BackendKind::VllmPooling => { + input.prompt_list.as_ref().map_or(1, |list| list.len()) + } + BackendKind::OpenaiEmbeddingsChat => 1, + BackendKind::VllmRerank => payload + .get("documents") + .and_then(|documents| documents.as_array()) + .map_or(0, |documents| documents.len()), + _ => unreachable!("PoolingBackend with non-pooling kind"), + }, ..Default::default() }; @@ -51,8 +64,6 @@ impl PoolingBackend { &input.request_id, ); - let payload = self.build_payload(input); - let mut request = client.post(&input.api_url); for (k, v) in &headers { request = request.header(k, v); diff --git a/rust/src/bench/src/compare.rs b/rust/src/bench/src/compare.rs index 6d41b72b85..0b0dece1c3 100644 --- a/rust/src/bench/src/compare.rs +++ b/rust/src/bench/src/compare.rs @@ -16,6 +16,11 @@ const METRICS: &[MetricDef] = &[ key: "request_throughput", lower_is_better: false, }, + MetricDef { + label: "Input throughput (inputs/s)", + key: "input_sequence_throughput", + lower_is_better: false, + }, MetricDef { label: "Output throughput (tok/s)", key: "output_throughput", diff --git a/rust/src/bench/src/metrics/calculator.rs b/rust/src/bench/src/metrics/calculator.rs index 3bce8dd1a7..11b68a3bfe 100644 --- a/rust/src/bench/src/metrics/calculator.rs +++ b/rust/src/bench/src/metrics/calculator.rs @@ -184,8 +184,10 @@ pub fn calculate_metrics( completed, failed, total_input, + total_input_sequences: completed, total_output, request_throughput: completed as f64 / dur_s, + input_sequence_throughput: completed as f64 / dur_s, request_goodput, input_throughput: total_input as f64 / dur_s, output_throughput: total_output as f64 / dur_s, @@ -291,6 +293,7 @@ pub fn calculate_embedding_metrics( selected_percentiles: &[f64], ) -> BenchmarkMetrics { let mut total_input: usize = 0; + let mut total_input_sequences: usize = 0; let mut completed: usize = 0; let mut e2els: Vec = Vec::new(); @@ -299,6 +302,7 @@ pub fn calculate_embedding_metrics( e2els.push(output.latency); completed += 1; total_input += output.prompt_len; + total_input_sequences += output.num_input_sequences; } } @@ -344,8 +348,10 @@ pub fn calculate_embedding_metrics( completed, failed, total_input, + total_input_sequences, total_output: 0, request_throughput: completed as f64 / dur_s, + input_sequence_throughput: total_input_sequences as f64 / dur_s, request_goodput: 0.0, input_throughput: total_input as f64 / dur_s, output_throughput: 0.0, diff --git a/rust/src/bench/src/metrics/mod.rs b/rust/src/bench/src/metrics/mod.rs index 829291acc8..2acce568a8 100644 --- a/rust/src/bench/src/metrics/mod.rs +++ b/rust/src/bench/src/metrics/mod.rs @@ -25,8 +25,10 @@ pub struct BenchmarkMetrics { pub completed: usize, pub failed: usize, pub total_input: usize, + pub total_input_sequences: usize, pub total_output: usize, pub request_throughput: f64, + pub input_sequence_throughput: f64, pub request_goodput: f64, pub input_throughput: f64, pub output_throughput: f64, diff --git a/rust/src/bench/src/multi_run.rs b/rust/src/bench/src/multi_run.rs index 18ff7753b9..3d4518d7ce 100644 --- a/rust/src/bench/src/multi_run.rs +++ b/rust/src/bench/src/multi_run.rs @@ -7,6 +7,7 @@ use crate::error::Result; /// Key metrics extracted from a single run's JSON result. struct RunMetrics { request_throughput: f64, + input_sequence_throughput: Option, output_throughput: f64, total_token_throughput: f64, mean_ttft_ms: f64, @@ -38,6 +39,7 @@ impl RunMetrics { fn from_json(json: &serde_json::Value) -> Self { Self { request_throughput: get(json, "request_throughput"), + input_sequence_throughput: get_opt(json, "input_sequence_throughput"), output_throughput: get(json, "output_throughput"), total_token_throughput: get(json, "total_token_throughput"), mean_ttft_ms: get(json, "mean_ttft_ms"), @@ -71,6 +73,10 @@ fn get(json: &serde_json::Value, key: &str) -> f64 { json.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0) } +fn get_opt(json: &serde_json::Value, key: &str) -> Option { + json.get(key).and_then(|v| v.as_f64()) +} + fn get_ss_opt(json: &serde_json::Value, key: &str) -> Option { json.get("steady_state") .and_then(|ss| if ss.is_null() { None } else { Some(ss) }) @@ -119,8 +125,15 @@ fn print_multi_run_summary(runs: &[RunMetrics]) { let n = runs.len(); // Collect each metric into a series, compute stats - let stats = vec![ - compute_stats("Request throughput (req/s)", runs, |r| r.request_throughput), + let mut stats = vec![compute_stats("Request throughput (req/s)", runs, |r| { + r.request_throughput + })]; + if runs.iter().all(|r| r.input_sequence_throughput.is_some()) { + stats.push(compute_stats("Input throughput (inputs/s)", runs, |r| { + r.input_sequence_throughput.unwrap_or_default() + })); + } + stats.extend([ compute_stats("Output throughput (tok/s)", runs, |r| r.output_throughput), compute_stats("Total token throughput (tok/s)", runs, |r| { r.total_token_throughput @@ -138,7 +151,7 @@ fn print_multi_run_summary(runs: &[RunMetrics]) { compute_stats("Completed requests", runs, |r| r.completed), compute_stats("Failed requests", runs, |r| r.failed), compute_stats("Duration (s)", runs, |r| r.duration), - ]; + ]); println!("{:=^80}", format!(" Multi-Run Summary ({n} runs) ")); println!( @@ -307,6 +320,7 @@ mod tests { fn mk_run(ss: Option) -> RunMetrics { RunMetrics { request_throughput: 0.0, + input_sequence_throughput: None, output_throughput: 0.0, total_token_throughput: 0.0, mean_ttft_ms: 0.0, diff --git a/rust/src/bench/src/output/console.rs b/rust/src/bench/src/output/console.rs index ae0184e615..62d7455d7c 100644 --- a/rust/src/bench/src/output/console.rs +++ b/rust/src/bench/src/output/console.rs @@ -51,6 +51,12 @@ pub fn print_results( "{:<40} {:<10.2}", "Request throughput (req/s):", metrics.request_throughput ); + if is_pooling { + println!( + "{:<40} {:<10.2}", + "Input throughput (inputs/s):", metrics.input_sequence_throughput + ); + } if metrics.request_goodput > 0.0 { println!( "{:<40} {:<10.2}", diff --git a/rust/src/bench/src/output/json.rs b/rust/src/bench/src/output/json.rs index 6e92cf8a20..d2420529e8 100644 --- a/rust/src/bench/src/output/json.rs +++ b/rust/src/bench/src/output/json.rs @@ -98,6 +98,16 @@ pub fn build_result_json( "request_throughput".into(), serde_json::json!(metrics.request_throughput), ); + if is_pooling { + result.insert( + "total_input_sequences".into(), + serde_json::json!(metrics.total_input_sequences), + ); + result.insert( + "input_sequence_throughput".into(), + serde_json::json!(metrics.input_sequence_throughput), + ); + } if config.goodput.is_empty() { result.insert("request_goodput".into(), Value::Null); } else { diff --git a/rust/src/bench/src/sweep.rs b/rust/src/bench/src/sweep.rs index 2d695356dc..367e7adfc3 100644 --- a/rust/src/bench/src/sweep.rs +++ b/rust/src/bench/src/sweep.rs @@ -295,6 +295,7 @@ fn print_sweep_summary(param_name: &str, points: &[SweepPoint], summary_percenti fn build_summary_columns(summary_percentiles: &[f64]) -> Vec { let mut columns = vec![ SummaryColumn::new("Req/s", "request_throughput", 10), + SummaryColumn::new("Inputs/s", "input_sequence_throughput", 10), SummaryColumn::new("Tok/s", "output_throughput", 10), SummaryColumn::new("Total tok/s", "total_token_throughput", 12), SummaryColumn::new("SS req/s", SS_REQUEST_THROUGHPUT_KEY, 10), @@ -452,6 +453,7 @@ mod tests { headers, vec![ "Req/s", + "Inputs/s", "Tok/s", "Total tok/s", "SS req/s", @@ -495,6 +497,7 @@ mod tests { headers, vec![ "Req/s", + "Inputs/s", "Tok/s", "Total tok/s", "SS req/s", diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index d928123dde..3ca9fd6708 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -79,7 +79,7 @@ class SampleRequest: Represents a single inference request for benchmarking. """ - prompt: str | list[str] | list[int] | list[dict] + prompt: str | list[str] | list[int] | list[list[int]] | list[dict] prompt_len: int expected_output_len: int = 0 multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index 1023700da4..0680d666da 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -66,7 +66,7 @@ class StreamedResponseHandler: class RequestFuncInput: """The input for the request function.""" - prompt: str | list[str] | list[int] | list[dict[str, Any]] + prompt: str | list[str] | list[int] | list[list[int]] | list[dict[str, Any]] api_url: str prompt_len: int output_len: int @@ -100,6 +100,7 @@ class RequestFuncOutput: error: str = "" start_time: float = 0.0 input_audio_duration: float = 0.0 # in seconds + num_input_sequences: int = 1 class RequestFunc(Protocol): @@ -586,8 +587,9 @@ async def _run_pooling_request( payload: dict[str, Any], headers: dict[str, Any], pbar: tqdm | None = None, + num_input_sequences: int = 1, ) -> RequestFuncOutput: - output = RequestFuncOutput() + output = RequestFuncOutput(num_input_sequences=num_input_sequences) st = time.perf_counter() output.start_time = st try: @@ -617,6 +619,12 @@ async def _run_pooling_request( return output +def _get_num_input_sequences(prompt: Any) -> int: + if prompt and isinstance(prompt, list) and isinstance(prompt[0], (str, list)): + return len(prompt) + return 1 + + async def async_request_openai_embeddings( request_func_input: RequestFuncInput, session: aiohttp.ClientSession, @@ -645,6 +653,7 @@ async def async_request_openai_embeddings( payload=payload, headers=headers, pbar=pbar, + num_input_sequences=_get_num_input_sequences(request_func_input.prompt), ) @@ -681,6 +690,7 @@ async def async_request_vllm_rerank( payload=payload, headers=headers, pbar=pbar, + num_input_sequences=len(request_func_input.prompt) - 1, ) @@ -818,6 +828,7 @@ async def async_request_infinity_embeddings( payload=payload, headers=headers, pbar=pbar, + num_input_sequences=_get_num_input_sequences(request_func_input.prompt), ) @@ -866,6 +877,7 @@ async def async_request_vllm_pooling( payload=payload, headers=headers, pbar=pbar, + num_input_sequences=_get_num_input_sequences(request_func_input.prompt), ) diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 9b99368923..960668b93a 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -357,7 +357,9 @@ class EmbedBenchmarkMetrics: completed: int failed: int total_input: int + total_input_sequences: int request_throughput: float + input_sequence_throughput: float total_token_throughput: float mean_e2el_ms: float std_e2el_ms: float @@ -520,6 +522,7 @@ def calculate_metrics_for_embeddings( The calculated benchmark metrics. """ total_input = 0 + total_input_sequences = 0 completed = 0 failed = 0 e2els: list[float] = [] @@ -528,6 +531,7 @@ def calculate_metrics_for_embeddings( e2els.append(outputs[i].latency) completed += 1 total_input += outputs[i].prompt_len + total_input_sequences += outputs[i].num_input_sequences else: failed += 1 @@ -541,7 +545,9 @@ def calculate_metrics_for_embeddings( completed=completed, failed=failed, total_input=total_input, + total_input_sequences=total_input_sequences, request_throughput=completed / dur_s, + input_sequence_throughput=total_input_sequences / dur_s, total_token_throughput=total_input / dur_s, mean_e2el_ms=np.mean(e2els or 0) * 1000, std_e2el_ms=np.std(e2els or 0) * 1000, @@ -1181,6 +1187,12 @@ async def benchmark( "Request throughput (req/s):", metrics.request_throughput ) ) + if isinstance(metrics, EmbedBenchmarkMetrics): + print( + "{:<40} {:<10.2f}".format( + "Input throughput (inputs/s):", metrics.input_sequence_throughput + ) + ) if goodput_config_dict and isinstance(metrics, BenchmarkMetrics): print( "{:<40} {:<10.2f}".format( @@ -1284,7 +1296,9 @@ async def benchmark( "duration": benchmark_duration, "completed": metrics.completed, "total_input_tokens": metrics.total_input, + "total_input_sequences": metrics.total_input_sequences, "request_throughput": metrics.request_throughput, + "input_sequence_throughput": metrics.input_sequence_throughput, "total_token_throughput": metrics.total_token_throughput, "input_lens": [output.prompt_len for output in outputs], "errors": [output.error for output in outputs],