[Pooling] Report input throughput for batched requests (#53213)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
10704541aa
commit
704f12aa1e
@@ -101,6 +101,7 @@ pub struct RequestFuncOutput {
|
||||
pub prompt_len: usize,
|
||||
pub error: String,
|
||||
pub start_time: f64,
|
||||
pub num_input_sequences: usize,
|
||||
}
|
||||
|
||||
impl Default for RequestFuncOutput {
|
||||
@@ -116,6 +117,7 @@ impl Default for RequestFuncOutput {
|
||||
prompt_len: 0,
|
||||
error: String::new(),
|
||||
start_time: 0.0,
|
||||
num_input_sequences: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,9 +39,22 @@ impl PoolingBackend {
|
||||
input: &RequestFuncInput,
|
||||
client: &reqwest::Client,
|
||||
) -> Result<RequestFuncOutput> {
|
||||
let payload = self.build_payload(input);
|
||||
|
||||
// Preserve client-side prompt_len as fallback if server doesn't report usage.
|
||||
let mut output = RequestFuncOutput {
|
||||
prompt_len: input.prompt_len,
|
||||
num_input_sequences: match self.kind {
|
||||
BackendKind::OpenaiEmbeddings | BackendKind::VllmPooling => {
|
||||
input.prompt_list.as_ref().map_or(1, |list| list.len())
|
||||
}
|
||||
BackendKind::OpenaiEmbeddingsChat => 1,
|
||||
BackendKind::VllmRerank => payload
|
||||
.get("documents")
|
||||
.and_then(|documents| documents.as_array())
|
||||
.map_or(0, |documents| documents.len()),
|
||||
_ => unreachable!("PoolingBackend with non-pooling kind"),
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -51,8 +64,6 @@ impl PoolingBackend {
|
||||
&input.request_id,
|
||||
);
|
||||
|
||||
let payload = self.build_payload(input);
|
||||
|
||||
let mut request = client.post(&input.api_url);
|
||||
for (k, v) in &headers {
|
||||
request = request.header(k, v);
|
||||
|
||||
@@ -16,6 +16,11 @@ const METRICS: &[MetricDef] = &[
|
||||
key: "request_throughput",
|
||||
lower_is_better: false,
|
||||
},
|
||||
MetricDef {
|
||||
label: "Input throughput (inputs/s)",
|
||||
key: "input_sequence_throughput",
|
||||
lower_is_better: false,
|
||||
},
|
||||
MetricDef {
|
||||
label: "Output throughput (tok/s)",
|
||||
key: "output_throughput",
|
||||
|
||||
@@ -184,8 +184,10 @@ pub fn calculate_metrics(
|
||||
completed,
|
||||
failed,
|
||||
total_input,
|
||||
total_input_sequences: completed,
|
||||
total_output,
|
||||
request_throughput: completed as f64 / dur_s,
|
||||
input_sequence_throughput: completed as f64 / dur_s,
|
||||
request_goodput,
|
||||
input_throughput: total_input as f64 / dur_s,
|
||||
output_throughput: total_output as f64 / dur_s,
|
||||
@@ -291,6 +293,7 @@ pub fn calculate_embedding_metrics(
|
||||
selected_percentiles: &[f64],
|
||||
) -> BenchmarkMetrics {
|
||||
let mut total_input: usize = 0;
|
||||
let mut total_input_sequences: usize = 0;
|
||||
let mut completed: usize = 0;
|
||||
let mut e2els: Vec<f64> = Vec::new();
|
||||
|
||||
@@ -299,6 +302,7 @@ pub fn calculate_embedding_metrics(
|
||||
e2els.push(output.latency);
|
||||
completed += 1;
|
||||
total_input += output.prompt_len;
|
||||
total_input_sequences += output.num_input_sequences;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,8 +348,10 @@ pub fn calculate_embedding_metrics(
|
||||
completed,
|
||||
failed,
|
||||
total_input,
|
||||
total_input_sequences,
|
||||
total_output: 0,
|
||||
request_throughput: completed as f64 / dur_s,
|
||||
input_sequence_throughput: total_input_sequences as f64 / dur_s,
|
||||
request_goodput: 0.0,
|
||||
input_throughput: total_input as f64 / dur_s,
|
||||
output_throughput: 0.0,
|
||||
|
||||
@@ -25,8 +25,10 @@ pub struct BenchmarkMetrics {
|
||||
pub completed: usize,
|
||||
pub failed: usize,
|
||||
pub total_input: usize,
|
||||
pub total_input_sequences: usize,
|
||||
pub total_output: usize,
|
||||
pub request_throughput: f64,
|
||||
pub input_sequence_throughput: f64,
|
||||
pub request_goodput: f64,
|
||||
pub input_throughput: f64,
|
||||
pub output_throughput: f64,
|
||||
|
||||
@@ -7,6 +7,7 @@ use crate::error::Result;
|
||||
/// Key metrics extracted from a single run's JSON result.
|
||||
struct RunMetrics {
|
||||
request_throughput: f64,
|
||||
input_sequence_throughput: Option<f64>,
|
||||
output_throughput: f64,
|
||||
total_token_throughput: f64,
|
||||
mean_ttft_ms: f64,
|
||||
@@ -38,6 +39,7 @@ impl RunMetrics {
|
||||
fn from_json(json: &serde_json::Value) -> Self {
|
||||
Self {
|
||||
request_throughput: get(json, "request_throughput"),
|
||||
input_sequence_throughput: get_opt(json, "input_sequence_throughput"),
|
||||
output_throughput: get(json, "output_throughput"),
|
||||
total_token_throughput: get(json, "total_token_throughput"),
|
||||
mean_ttft_ms: get(json, "mean_ttft_ms"),
|
||||
@@ -71,6 +73,10 @@ fn get(json: &serde_json::Value, key: &str) -> f64 {
|
||||
json.get(key).and_then(|v| v.as_f64()).unwrap_or(0.0)
|
||||
}
|
||||
|
||||
fn get_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
|
||||
json.get(key).and_then(|v| v.as_f64())
|
||||
}
|
||||
|
||||
fn get_ss_opt(json: &serde_json::Value, key: &str) -> Option<f64> {
|
||||
json.get("steady_state")
|
||||
.and_then(|ss| if ss.is_null() { None } else { Some(ss) })
|
||||
@@ -119,8 +125,15 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
|
||||
let n = runs.len();
|
||||
|
||||
// Collect each metric into a series, compute stats
|
||||
let stats = vec![
|
||||
compute_stats("Request throughput (req/s)", runs, |r| r.request_throughput),
|
||||
let mut stats = vec![compute_stats("Request throughput (req/s)", runs, |r| {
|
||||
r.request_throughput
|
||||
})];
|
||||
if runs.iter().all(|r| r.input_sequence_throughput.is_some()) {
|
||||
stats.push(compute_stats("Input throughput (inputs/s)", runs, |r| {
|
||||
r.input_sequence_throughput.unwrap_or_default()
|
||||
}));
|
||||
}
|
||||
stats.extend([
|
||||
compute_stats("Output throughput (tok/s)", runs, |r| r.output_throughput),
|
||||
compute_stats("Total token throughput (tok/s)", runs, |r| {
|
||||
r.total_token_throughput
|
||||
@@ -138,7 +151,7 @@ fn print_multi_run_summary(runs: &[RunMetrics]) {
|
||||
compute_stats("Completed requests", runs, |r| r.completed),
|
||||
compute_stats("Failed requests", runs, |r| r.failed),
|
||||
compute_stats("Duration (s)", runs, |r| r.duration),
|
||||
];
|
||||
]);
|
||||
|
||||
println!("{:=^80}", format!(" Multi-Run Summary ({n} runs) "));
|
||||
println!(
|
||||
@@ -307,6 +320,7 @@ mod tests {
|
||||
fn mk_run(ss: Option<f64>) -> RunMetrics {
|
||||
RunMetrics {
|
||||
request_throughput: 0.0,
|
||||
input_sequence_throughput: None,
|
||||
output_throughput: 0.0,
|
||||
total_token_throughput: 0.0,
|
||||
mean_ttft_ms: 0.0,
|
||||
|
||||
@@ -51,6 +51,12 @@ pub fn print_results(
|
||||
"{:<40} {:<10.2}",
|
||||
"Request throughput (req/s):", metrics.request_throughput
|
||||
);
|
||||
if is_pooling {
|
||||
println!(
|
||||
"{:<40} {:<10.2}",
|
||||
"Input throughput (inputs/s):", metrics.input_sequence_throughput
|
||||
);
|
||||
}
|
||||
if metrics.request_goodput > 0.0 {
|
||||
println!(
|
||||
"{:<40} {:<10.2}",
|
||||
|
||||
@@ -98,6 +98,16 @@ pub fn build_result_json(
|
||||
"request_throughput".into(),
|
||||
serde_json::json!(metrics.request_throughput),
|
||||
);
|
||||
if is_pooling {
|
||||
result.insert(
|
||||
"total_input_sequences".into(),
|
||||
serde_json::json!(metrics.total_input_sequences),
|
||||
);
|
||||
result.insert(
|
||||
"input_sequence_throughput".into(),
|
||||
serde_json::json!(metrics.input_sequence_throughput),
|
||||
);
|
||||
}
|
||||
if config.goodput.is_empty() {
|
||||
result.insert("request_goodput".into(), Value::Null);
|
||||
} else {
|
||||
|
||||
@@ -295,6 +295,7 @@ fn print_sweep_summary(param_name: &str, points: &[SweepPoint], summary_percenti
|
||||
fn build_summary_columns(summary_percentiles: &[f64]) -> Vec<SummaryColumn> {
|
||||
let mut columns = vec![
|
||||
SummaryColumn::new("Req/s", "request_throughput", 10),
|
||||
SummaryColumn::new("Inputs/s", "input_sequence_throughput", 10),
|
||||
SummaryColumn::new("Tok/s", "output_throughput", 10),
|
||||
SummaryColumn::new("Total tok/s", "total_token_throughput", 12),
|
||||
SummaryColumn::new("SS req/s", SS_REQUEST_THROUGHPUT_KEY, 10),
|
||||
@@ -452,6 +453,7 @@ mod tests {
|
||||
headers,
|
||||
vec![
|
||||
"Req/s",
|
||||
"Inputs/s",
|
||||
"Tok/s",
|
||||
"Total tok/s",
|
||||
"SS req/s",
|
||||
@@ -495,6 +497,7 @@ mod tests {
|
||||
headers,
|
||||
vec![
|
||||
"Req/s",
|
||||
"Inputs/s",
|
||||
"Tok/s",
|
||||
"Total tok/s",
|
||||
"SS req/s",
|
||||
|
||||
@@ -79,7 +79,7 @@ class SampleRequest:
|
||||
Represents a single inference request for benchmarking.
|
||||
"""
|
||||
|
||||
prompt: str | list[str] | list[int] | list[dict]
|
||||
prompt: str | list[str] | list[int] | list[list[int]] | list[dict]
|
||||
prompt_len: int
|
||||
expected_output_len: int = 0
|
||||
multi_modal_data: MultiModalDataDict | dict | list[dict] | None = None
|
||||
|
||||
@@ -66,7 +66,7 @@ class StreamedResponseHandler:
|
||||
class RequestFuncInput:
|
||||
"""The input for the request function."""
|
||||
|
||||
prompt: str | list[str] | list[int] | list[dict[str, Any]]
|
||||
prompt: str | list[str] | list[int] | list[list[int]] | list[dict[str, Any]]
|
||||
api_url: str
|
||||
prompt_len: int
|
||||
output_len: int
|
||||
@@ -100,6 +100,7 @@ class RequestFuncOutput:
|
||||
error: str = ""
|
||||
start_time: float = 0.0
|
||||
input_audio_duration: float = 0.0 # in seconds
|
||||
num_input_sequences: int = 1
|
||||
|
||||
|
||||
class RequestFunc(Protocol):
|
||||
@@ -586,8 +587,9 @@ async def _run_pooling_request(
|
||||
payload: dict[str, Any],
|
||||
headers: dict[str, Any],
|
||||
pbar: tqdm | None = None,
|
||||
num_input_sequences: int = 1,
|
||||
) -> RequestFuncOutput:
|
||||
output = RequestFuncOutput()
|
||||
output = RequestFuncOutput(num_input_sequences=num_input_sequences)
|
||||
st = time.perf_counter()
|
||||
output.start_time = st
|
||||
try:
|
||||
@@ -617,6 +619,12 @@ async def _run_pooling_request(
|
||||
return output
|
||||
|
||||
|
||||
def _get_num_input_sequences(prompt: Any) -> int:
|
||||
if prompt and isinstance(prompt, list) and isinstance(prompt[0], (str, list)):
|
||||
return len(prompt)
|
||||
return 1
|
||||
|
||||
|
||||
async def async_request_openai_embeddings(
|
||||
request_func_input: RequestFuncInput,
|
||||
session: aiohttp.ClientSession,
|
||||
@@ -645,6 +653,7 @@ async def async_request_openai_embeddings(
|
||||
payload=payload,
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
|
||||
)
|
||||
|
||||
|
||||
@@ -681,6 +690,7 @@ async def async_request_vllm_rerank(
|
||||
payload=payload,
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=len(request_func_input.prompt) - 1,
|
||||
)
|
||||
|
||||
|
||||
@@ -818,6 +828,7 @@ async def async_request_infinity_embeddings(
|
||||
payload=payload,
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
|
||||
)
|
||||
|
||||
|
||||
@@ -866,6 +877,7 @@ async def async_request_vllm_pooling(
|
||||
payload=payload,
|
||||
headers=headers,
|
||||
pbar=pbar,
|
||||
num_input_sequences=_get_num_input_sequences(request_func_input.prompt),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -357,7 +357,9 @@ class EmbedBenchmarkMetrics:
|
||||
completed: int
|
||||
failed: int
|
||||
total_input: int
|
||||
total_input_sequences: int
|
||||
request_throughput: float
|
||||
input_sequence_throughput: float
|
||||
total_token_throughput: float
|
||||
mean_e2el_ms: float
|
||||
std_e2el_ms: float
|
||||
@@ -520,6 +522,7 @@ def calculate_metrics_for_embeddings(
|
||||
The calculated benchmark metrics.
|
||||
"""
|
||||
total_input = 0
|
||||
total_input_sequences = 0
|
||||
completed = 0
|
||||
failed = 0
|
||||
e2els: list[float] = []
|
||||
@@ -528,6 +531,7 @@ def calculate_metrics_for_embeddings(
|
||||
e2els.append(outputs[i].latency)
|
||||
completed += 1
|
||||
total_input += outputs[i].prompt_len
|
||||
total_input_sequences += outputs[i].num_input_sequences
|
||||
else:
|
||||
failed += 1
|
||||
|
||||
@@ -541,7 +545,9 @@ def calculate_metrics_for_embeddings(
|
||||
completed=completed,
|
||||
failed=failed,
|
||||
total_input=total_input,
|
||||
total_input_sequences=total_input_sequences,
|
||||
request_throughput=completed / dur_s,
|
||||
input_sequence_throughput=total_input_sequences / dur_s,
|
||||
total_token_throughput=total_input / dur_s,
|
||||
mean_e2el_ms=np.mean(e2els or 0) * 1000,
|
||||
std_e2el_ms=np.std(e2els or 0) * 1000,
|
||||
@@ -1181,6 +1187,12 @@ async def benchmark(
|
||||
"Request throughput (req/s):", metrics.request_throughput
|
||||
)
|
||||
)
|
||||
if isinstance(metrics, EmbedBenchmarkMetrics):
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
"Input throughput (inputs/s):", metrics.input_sequence_throughput
|
||||
)
|
||||
)
|
||||
if goodput_config_dict and isinstance(metrics, BenchmarkMetrics):
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
@@ -1284,7 +1296,9 @@ async def benchmark(
|
||||
"duration": benchmark_duration,
|
||||
"completed": metrics.completed,
|
||||
"total_input_tokens": metrics.total_input,
|
||||
"total_input_sequences": metrics.total_input_sequences,
|
||||
"request_throughput": metrics.request_throughput,
|
||||
"input_sequence_throughput": metrics.input_sequence_throughput,
|
||||
"total_token_throughput": metrics.total_token_throughput,
|
||||
"input_lens": [output.prompt_len for output in outputs],
|
||||
"errors": [output.error for output in outputs],
|
||||
|
||||
Reference in New Issue
Block a user