[Pooling] Report input throughput for batched requests (#53213)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-08-22 09:01:40 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 10704541aa
commit 704f12aa1e
12 changed files with 93 additions and 8 deletions
+2
View File
@@ -101,6 +101,7 @@ pub struct RequestFuncOutput {
pub prompt_len: usize,
pub error: String,
pub start_time: f64,
pub num_input_sequences: usize,
}
impl Default for RequestFuncOutput {
@@ -116,6 +117,7 @@ impl Default for RequestFuncOutput {
prompt_len: 0,
error: String::new(),
start_time: 0.0,
num_input_sequences: 1,
}
}
}
+13 -2
View File
@@ -39,9 +39,22 @@ impl PoolingBackend {
input: &RequestFuncInput,
client: &reqwest::Client,
) -> Result<RequestFuncOutput> {
let payload = self.build_payload(input);
// Preserve client-side prompt_len as fallback if server doesn't report usage.
let mut output = RequestFuncOutput {
prompt_len: input.prompt_len,
num_input_sequences: match self.kind {
BackendKind::OpenaiEmbeddings | BackendKind::VllmPooling => {
input.prompt_list.as_ref().map_or(1, |list| list.len())
}
BackendKind::OpenaiEmbeddingsChat => 1,
BackendKind::VllmRerank => payload
.get("documents")
.and_then(|documents| documents.as_array())
.map_or(0, |documents| documents.len()),
_ => unreachable!("PoolingBackend with non-pooling kind"),
},
..Default::default()
};
@@ -51,8 +64,6 @@ impl PoolingBackend {
&input.request_id,
);
let payload = self.build_payload(input);
let mut request = client.post(&input.api_url);
for (k, v) in &headers {
request = request.header(k, v);
+5
View File
@@ -16,6 +16,11 @@ const METRICS: &[MetricDef] = &[
key: "request_throughput",
lower_is_better: false,
},
MetricDef {
label: "Input throughput (inputs/s)",
key: "input_sequence_throughput",
lower_is_better: false,
},
MetricDef {
label: "Output throughput (tok/s)",
key: "output_throughput",
+6
View File
@@ -184,8 +184,10 @@ pub fn calculate_metrics(
completed,
failed,
total_input,
total_input_sequences: completed,
total_output,
request_throughput: completed as f64 / dur_s,
input_sequence_throughput: completed as f64 / dur_s,
request_goodput,
input_throughput: total_input as f64 / dur_s,
output_throughput: total_output as f64 / dur_s,
@@ -291,6 +293,7 @@ pub fn calculate_embedding_metrics(
selected_percentiles: &[f64],
) -> BenchmarkMetrics {
let mut total_input: usize = 0;
let mut total_input_sequences: usize = 0;
let mut completed: usize = 0;
let mut e2els: Vec<f64> = Vec::new();
@@ -299,6 +302,7 @@ pub fn calculate_embedding_metrics(
e2els.push(output.latency);
completed += 1;
total_input += output.prompt_len;
total_input_sequences += output.num_input_sequences;
}
}
@@ -344,8 +348,10 @@ pub fn calculate_embedding_metrics(
completed,
failed,
total_input,
total_input_sequences,
total_output: 0,
request_throughput: completed as f64 / dur_s,
input_sequence_throughput: total_input_sequences as f64 / dur_s,
request_goodput: 0.0,
input_throughput: total_input as f64 / dur_s,
output_throughput: 0.0,
+2
View File
@@ -25,8 +25,10 @@ pub struct BenchmarkMetrics {
pub completed: usize,
pub failed: usize,
pub total_input: usize,
pub total_input_sequences: usize,
pub total_output: usize,
pub request_throughput: f64,
pub input_sequence_throughput: f64,
pub request_goodput: f64,
pub input_throughput: f64,
pub output_throughput: f64,
+17 -3
View File
@@ -7,6 +7,7 @@ use crate::error::Result;
/// Key metrics extracted from a single run's JSON result.
struct RunMetrics {
request_throughput: f64,
input_sequence_throughput: Option<f64>,
output_throughput: f64,
total_token_throughput: f64,
mean_ttft_ms: f64,
@@ -38,6 +39,7 @@ impl RunMetrics {
fn from_json(json: &serde_json::Value) -> Self {
Self {
request_throughput: get(json, "request_throughput"),
input_sequence_throughput: get_opt(json, "input_sequence_throughput"),
output_throughput: get(json, "output_throughput"),
total_token_throughput: get(json, "total_token_throughput"),
mean_ttft_ms: get(json, "mean_ttft_ms"),
@@ -71,6 +73,10 @@ fn get(json: &serde_json::Value, key: &str) -> f64 {
json.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0)
}
fn get_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
json.get(key).and_then(|v| v.as_f64())
}
fn get_ss_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
json.get("steady_state")
.and_then(|ss| if ss.is_null() { None } else { Some(ss) })
@@ -119,8 +125,15 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
let n = runs.len();
// Collect each metric into a series, compute stats
let stats = vec![
compute_stats("Request throughput (req/s)", runs, |r| r.request_throughput),
let mut stats = vec![compute_stats("Request throughput (req/s)", runs, |r| {
r.request_throughput
})];
if runs.iter().all(|r| r.input_sequence_throughput.is_some()) {
stats.push(compute_stats("Input throughput (inputs/s)", runs, |r| {
r.input_sequence_throughput.unwrap_or_default()
}));
}
stats.extend([
compute_stats("Output throughput (tok/s)", runs, |r| r.output_throughput),
compute_stats("Total token throughput (tok/s)", runs, |r| {
r.total_token_throughput
@@ -138,7 +151,7 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
compute_stats("Completed requests", runs, |r| r.completed),
compute_stats("Failed requests", runs, |r| r.failed),
compute_stats("Duration (s)", runs, |r| r.duration),
];
]);
println!("{:=^80}", format!(" Multi-Run Summary ({n} runs) "));
println!(
@@ -307,6 +320,7 @@ mod tests {
fn mk_run(ss: Option<f64>) -> RunMetrics {
RunMetrics {
request_throughput: 0.0,
input_sequence_throughput: None,
output_throughput: 0.0,
total_token_throughput: 0.0,
mean_ttft_ms: 0.0,
+6
View File
@@ -51,6 +51,12 @@ pub fn print_results(
"{:<40} {:<10.2}",
"Request throughput (req/s):", metrics.request_throughput
);
if is_pooling {
println!(
"{:<40} {:<10.2}",
"Input throughput (inputs/s):", metrics.input_sequence_throughput
);
}
if metrics.request_goodput > 0.0 {
println!(
"{:<40} {:<10.2}",
+10
View File
@@ -98,6 +98,16 @@ pub fn build_result_json(
"request_throughput".into(),
serde_json::json!(metrics.request_throughput),
);
if is_pooling {
result.insert(
"total_input_sequences".into(),
serde_json::json!(metrics.total_input_sequences),
);
result.insert(
"input_sequence_throughput".into(),
serde_json::json!(metrics.input_sequence_throughput),
);
}
if config.goodput.is_empty() {
result.insert("request_goodput".into(), Value::Null);
} else {
+3
View File
@@ -295,6 +295,7 @@ fn print_sweep_summary(param_name: &str, points: &[SweepPoint], summary_percenti
fn build_summary_columns(summary_percentiles: &[f64]) -> Vec<SummaryColumn> {
let mut columns = vec![
SummaryColumn::new("Req/s", "request_throughput", 10),
SummaryColumn::new("Inputs/s", "input_sequence_throughput", 10),
SummaryColumn::new("Tok/s", "output_throughput", 10),
SummaryColumn::new("Total tok/s", "total_token_throughput", 12),
SummaryColumn::new("SS req/s", SS_REQUEST_THROUGHPUT_KEY, 10),
@@ -452,6 +453,7 @@ mod tests {
headers,
vec![
"Req/s",
"Inputs/s",
"Tok/s",
"Total tok/s",
"SS req/s",
@@ -495,6 +497,7 @@ mod tests {
headers,
vec![
"Req/s",
"Inputs/s",
"Tok/s",
"Total tok/s",
"SS req/s",