[Pooling] Report input throughput for batched requests (#53213)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
10704541aa
commit
704f12aa1e
@@ -101,6 +101,7 @@ pub struct RequestFuncOutput {
|
||||
pub prompt_len: usize,
|
||||
pub error: String,
|
||||
pub start_time: f64,
|
||||
pub num_input_sequences: usize,
|
||||
}
|
||||
|
||||
impl Default for RequestFuncOutput {
|
||||
@@ -116,6 +117,7 @@ impl Default for RequestFuncOutput {
|
||||
prompt_len: 0,
|
||||
error: String::new(),
|
||||
start_time: 0.0,
|
||||
num_input_sequences: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,9 +39,22 @@ impl PoolingBackend {
|
||||
input: &RequestFuncInput,
|
||||
client: &reqwest::Client,
|
||||
) -> Result<RequestFuncOutput> {
|
||||
let payload = self.build_payload(input);
|
||||
|
||||
// Preserve client-side prompt_len as fallback if server doesn't report usage.
|
||||
let mut output = RequestFuncOutput {
|
||||
prompt_len: input.prompt_len,
|
||||
num_input_sequences: match self.kind {
|
||||
BackendKind::OpenaiEmbeddings | BackendKind::VllmPooling => {
|
||||
input.prompt_list.as_ref().map_or(1, |list| list.len())
|
||||
}
|
||||
BackendKind::OpenaiEmbeddingsChat => 1,
|
||||
BackendKind::VllmRerank => payload
|
||||
.get("documents")
|
||||
.and_then(|documents| documents.as_array())
|
||||
.map_or(0, |documents| documents.len()),
|
||||
_ => unreachable!("PoolingBackend with non-pooling kind"),
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -51,8 +64,6 @@ impl PoolingBackend {
|
||||
&input.request_id,
|
||||
);
|
||||
|
||||
let payload = self.build_payload(input);
|
||||
|
||||
let mut request = client.post(&input.api_url);
|
||||
for (k, v) in &headers {
|
||||
request = request.header(k, v);
|
||||
|
||||
@@ -16,6 +16,11 @@ const METRICS: &[MetricDef] = &[
|
||||
key: "request_throughput",
|
||||
lower_is_better: false,
|
||||
},
|
||||
MetricDef {
|
||||
label: "Input throughput (inputs/s)",
|
||||
key: "input_sequence_throughput",
|
||||
lower_is_better: false,
|
||||
},
|
||||
MetricDef {
|
||||
label: "Output throughput (tok/s)",
|
||||
key: "output_throughput",
|
||||
|
||||
@@ -184,8 +184,10 @@ pub fn calculate_metrics(
|
||||
completed,
|
||||
failed,
|
||||
total_input,
|
||||
total_input_sequences: completed,
|
||||
total_output,
|
||||
request_throughput: completed as f64 / dur_s,
|
||||
input_sequence_throughput: completed as f64 / dur_s,
|
||||
request_goodput,
|
||||
input_throughput: total_input as f64 / dur_s,
|
||||
output_throughput: total_output as f64 / dur_s,
|
||||
@@ -291,6 +293,7 @@ pub fn calculate_embedding_metrics(
|
||||
selected_percentiles: &[f64],
|
||||
) -> BenchmarkMetrics {
|
||||
let mut total_input: usize = 0;
|
||||
let mut total_input_sequences: usize = 0;
|
||||
let mut completed: usize = 0;
|
||||
let mut e2els: Vec<f64> = Vec::new();
|
||||
|
||||
@@ -299,6 +302,7 @@ pub fn calculate_embedding_metrics(
|
||||
e2els.push(output.latency);
|
||||
completed += 1;
|
||||
total_input += output.prompt_len;
|
||||
total_input_sequences += output.num_input_sequences;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,8 +348,10 @@ pub fn calculate_embedding_metrics(
|
||||
completed,
|
||||
failed,
|
||||
total_input,
|
||||
total_input_sequences,
|
||||
total_output: 0,
|
||||
request_throughput: completed as f64 / dur_s,
|
||||
input_sequence_throughput: total_input_sequences as f64 / dur_s,
|
||||
request_goodput: 0.0,
|
||||
input_throughput: total_input as f64 / dur_s,
|
||||
output_throughput: 0.0,
|
||||
|
||||
@@ -25,8 +25,10 @@ pub struct BenchmarkMetrics {
|
||||
pub completed: usize,
|
||||
pub failed: usize,
|
||||
pub total_input: usize,
|
||||
pub total_input_sequences: usize,
|
||||
pub total_output: usize,
|
||||
pub request_throughput: f64,
|
||||
pub input_sequence_throughput: f64,
|
||||
pub request_goodput: f64,
|
||||
pub input_throughput: f64,
|
||||
pub output_throughput: f64,
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::error::Result;
|
||||
/// Key metrics extracted from a single run's JSON result.
|
||||
struct RunMetrics {
|
||||
request_throughput: f64,
|
||||
input_sequence_throughput: Option<f64>,
|
||||
output_throughput: f64,
|
||||
total_token_throughput: f64,
|
||||
mean_ttft_ms: f64,
|
||||
@@ -38,6 +39,7 @@ impl RunMetrics {
|
||||
fn from_json(json: &serde_json::Value) -> Self {
|
||||
Self {
|
||||
request_throughput: get(json, "request_throughput"),
|
||||
input_sequence_throughput: get_opt(json, "input_sequence_throughput"),
|
||||
output_throughput: get(json, "output_throughput"),
|
||||
total_token_throughput: get(json, "total_token_throughput"),
|
||||
mean_ttft_ms: get(json, "mean_ttft_ms"),
|
||||
@@ -71,6 +73,10 @@ fn get(json: &serde_json::Value, key: &str) -> f64 {
|
||||
json.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0)
|
||||
}
|
||||
|
||||
fn get_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
|
||||
json.get(key).and_then(|v| v.as_f64())
|
||||
}
|
||||
|
||||
fn get_ss_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
|
||||
json.get("steady_state")
|
||||
.and_then(|ss| if ss.is_null() { None } else { Some(ss) })
|
||||
@@ -119,8 +125,15 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
|
||||
let n = runs.len();
|
||||
|
||||
// Collect each metric into a series, compute stats
|
||||
let stats = vec![
|
||||
compute_stats("Request throughput (req/s)", runs, |r| r.request_throughput),
|
||||
let mut stats = vec![compute_stats("Request throughput (req/s)", runs, |r| {
|
||||
r.request_throughput
|
||||
})];
|
||||
if runs.iter().all(|r| r.input_sequence_throughput.is_some()) {
|
||||
stats.push(compute_stats("Input throughput (inputs/s)", runs, |r| {
|
||||
r.input_sequence_throughput.unwrap_or_default()
|
||||
}));
|
||||
}
|
||||
stats.extend([
|
||||
compute_stats("Output throughput (tok/s)", runs, |r| r.output_throughput),
|
||||
compute_stats("Total token throughput (tok/s)", runs, |r| {
|
||||
r.total_token_throughput
|
||||
@@ -138,7 +151,7 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
|
||||
compute_stats("Completed requests", runs, |r| r.completed),
|
||||
compute_stats("Failed requests", runs, |r| r.failed),
|
||||
compute_stats("Duration (s)", runs, |r| r.duration),
|
||||
];
|
||||
]);
|
||||
|
||||
println!("{:=^80}", format!(" Multi-Run Summary ({n} runs) "));
|
||||
println!(
|
||||
@@ -307,6 +320,7 @@ mod tests {
|
||||
fn mk_run(ss: Option<f64>) -> RunMetrics {
|
||||
RunMetrics {
|
||||
request_throughput: 0.0,
|
||||
input_sequence_throughput: None,
|
||||
output_throughput: 0.0,
|
||||
total_token_throughput: 0.0,
|
||||
mean_ttft_ms: 0.0,
|
||||
|
||||
@@ -51,6 +51,12 @@ pub fn print_results(
|
||||
"{:<40} {:<10.2}",
|
||||
"Request throughput (req/s):", metrics.request_throughput
|
||||
);
|
||||
if is_pooling {
|
||||
println!(
|
||||
"{:<40} {:<10.2}",
|
||||
"Input throughput (inputs/s):", metrics.input_sequence_throughput
|
||||
);
|
||||
}
|
||||
if metrics.request_goodput > 0.0 {
|
||||
println!(
|
||||
"{:<40} {:<10.2}",
|
||||
|
||||
@@ -98,6 +98,16 @@ pub fn build_result_json(
|
||||
"request_throughput".into(),
|
||||
serde_json::json!(metrics.request_throughput),
|
||||
);
|
||||
if is_pooling {
|
||||
result.insert(
|
||||
"total_input_sequences".into(),
|
||||
serde_json::json!(metrics.total_input_sequences),
|
||||
);
|
||||
result.insert(
|
||||
"input_sequence_throughput".into(),
|
||||
serde_json::json!(metrics.input_sequence_throughput),
|
||||
);
|
||||
}
|
||||
if config.goodput.is_empty() {
|
||||
result.insert("request_goodput".into(), Value::Null);
|
||||
} else {
|
||||
|
||||
@@ -295,6 +295,7 @@ fn print_sweep_summary(param_name: &str, points: &[SweepPoint], summary_percenti
|
||||
fn build_summary_columns(summary_percentiles: &[f64]) -> Vec<SummaryColumn> {
|
||||
let mut columns = vec![
|
||||
SummaryColumn::new("Req/s", "request_throughput", 10),
|
||||
SummaryColumn::new("Inputs/s", "input_sequence_throughput", 10),
|
||||
SummaryColumn::new("Tok/s", "output_throughput", 10),
|
||||
SummaryColumn::new("Total tok/s", "total_token_throughput", 12),
|
||||
SummaryColumn::new("SS req/s", SS_REQUEST_THROUGHPUT_KEY, 10),
|
||||
@@ -452,6 +453,7 @@ mod tests {
|
||||
headers,
|
||||
vec![
|
||||
"Req/s",
|
||||
"Inputs/s",
|
||||
"Tok/s",
|
||||
"Total tok/s",
|
||||
"SS req/s",
|
||||
@@ -495,6 +497,7 @@ mod tests {
|
||||
headers,
|
||||
vec![
|
||||
"Req/s",
|
||||
"Inputs/s",
|
||||
"Tok/s",
|
||||
"Total tok/s",
|
||||
"SS req/s",
|
||||
|
||||
Reference in New Issue
Block a user