[Bugfix] Account for client queue time in serve benchmarks (#54136)
Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com> Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
maithilijoshi20
co-authored by
maithilijoshi20
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
maithilijoshi20
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
92ccd2c306
commit
9e905f7450
@@ -664,7 +664,7 @@ vLLM's benchmark serving script provides sophisticated load pattern simulation c
|
||||
|
||||
- `--request-rate`: Controls the target request generation rate (requests per second). Set to `inf` for maximum throughput testing or finite values for controlled load simulation.
|
||||
- `--burstiness`: Controls traffic variability using a Gamma distribution (range: > 0). Lower values create bursty traffic, higher values create uniform traffic.
|
||||
- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure.
|
||||
- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure. When set, include `client_queue_time` in `--percentile-metrics` to report time spent waiting for the benchmark client's concurrency limit. With a finite `--request-rate`, `e2el_including_client_queue` reports schedule-relative end-to-end latency; it is omitted for `--request-rate=inf`, where all requests arrive at benchmark start.
|
||||
|
||||
These parameters work together to create realistic load patterns with carefully chosen defaults. The `--request-rate` parameter defaults to `inf` (infinite), which sends all requests immediately for maximum throughput testing. When set to finite values, it uses either a Poisson process (default `--burstiness=1.0`) or Gamma distribution for realistic request timing. The `--burstiness` parameter only takes effect when `--request-rate` is not infinite - a value of 1.0 creates natural Poisson traffic, while lower values (0.1-0.5) create bursty patterns and higher values (2.0-5.0) create uniform spacing. The `--max-concurrency` parameter defaults to `None` (unlimited) but can be set to simulate real-world constraints where a load balancer or API gateway limits concurrent connections. When combined, these parameters allow you to simulate everything from unrestricted stress testing (`--request-rate=inf`) to production-like scenarios with realistic arrival patterns and resource constraints.
|
||||
|
||||
|
||||
@@ -99,6 +99,8 @@ class RequestFuncOutput:
|
||||
prompt_len: int = 0
|
||||
error: str = ""
|
||||
start_time: float = 0.0
|
||||
# Time spent awaiting the benchmark client concurrency semaphore.
|
||||
client_queue_time: float = 0.0
|
||||
input_audio_duration: float = 0.0 # in seconds
|
||||
num_input_sequences: int = 1
|
||||
|
||||
|
||||
+61
-26
@@ -23,6 +23,7 @@ import asyncio
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
@@ -972,11 +973,17 @@ async def benchmark(
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
|
||||
async def limited_request_func(request_func_input, session, pbar):
|
||||
async def limited_request_func(
|
||||
request_func_input, session, pbar, request_arrival_time
|
||||
):
|
||||
async with semaphore:
|
||||
return await request_func(
|
||||
output = await request_func(
|
||||
request_func_input=request_func_input, session=session, pbar=pbar
|
||||
)
|
||||
# Preserve start_time as the time the request was sent. It is used by
|
||||
# throughput calculations; record client-side semaphore delay separately.
|
||||
output.client_queue_time = output.start_time - request_arrival_time
|
||||
return output
|
||||
|
||||
probe_outputs: list[RequestFuncOutput] = []
|
||||
probe_stop = asyncio.Event()
|
||||
@@ -1064,10 +1071,11 @@ async def benchmark(
|
||||
request_id=request_id,
|
||||
chat_messages=request.chat_messages,
|
||||
)
|
||||
request_arrival_time = time.perf_counter()
|
||||
tasks.append(
|
||||
asyncio.create_task(
|
||||
limited_request_func(
|
||||
request_func_input=request_func_input, session=session, pbar=pbar
|
||||
request_func_input, session, pbar, request_arrival_time
|
||||
)
|
||||
)
|
||||
)
|
||||
@@ -1284,7 +1292,9 @@ async def benchmark(
|
||||
"output_lens": actual_output_lens,
|
||||
"ttfts": [output.ttft for output in outputs],
|
||||
"itls": [output.itl for output in outputs],
|
||||
"latencies": [output.latency for output in outputs],
|
||||
"start_times": [output.start_time for output in outputs],
|
||||
"queue_times": [output.client_queue_time for output in outputs],
|
||||
"generated_texts": [output.generated_text for output in outputs],
|
||||
"errors": [output.error for output in outputs],
|
||||
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
|
||||
@@ -1301,9 +1311,22 @@ async def benchmark(
|
||||
"input_sequence_throughput": metrics.input_sequence_throughput,
|
||||
"total_token_throughput": metrics.total_token_throughput,
|
||||
"input_lens": [output.prompt_len for output in outputs],
|
||||
"latencies": [output.latency for output in outputs],
|
||||
"queue_times": [output.client_queue_time for output in outputs],
|
||||
"errors": [output.error for output in outputs],
|
||||
}
|
||||
|
||||
queue_times: list[float] | None = None
|
||||
e2els_including_queue: list[float] | None = None
|
||||
if max_concurrency is not None:
|
||||
queue_times = [output.client_queue_time for output in outputs if output.success]
|
||||
if not math.isinf(request_rate):
|
||||
e2els_including_queue = [
|
||||
output.latency + output.client_queue_time
|
||||
for output in outputs
|
||||
if output.success
|
||||
]
|
||||
|
||||
if probe_stats is not None:
|
||||
result.update(probe_stats)
|
||||
|
||||
@@ -1339,34 +1362,31 @@ async def benchmark(
|
||||
metric_name: str,
|
||||
# E.g., "Time to First Token"
|
||||
metric_header: str,
|
||||
values: list[float] | None = None,
|
||||
):
|
||||
# This function prints and adds statistics of the specified
|
||||
# metric.
|
||||
if metric_attribute_name not in selected_percentile_metrics:
|
||||
return
|
||||
if values is None:
|
||||
mean = getattr(metrics, f"mean_{metric_attribute_name}_ms")
|
||||
median = getattr(metrics, f"median_{metric_attribute_name}_ms")
|
||||
std = getattr(metrics, f"std_{metric_attribute_name}_ms")
|
||||
percentiles = getattr(metrics, f"percentiles_{metric_attribute_name}_ms")
|
||||
else:
|
||||
mean = np.mean(values or 0) * 1000
|
||||
median = np.median(values or 0) * 1000
|
||||
std = np.std(values or 0) * 1000
|
||||
percentiles = [
|
||||
(p, np.percentile(values or 0, p) * 1000) for p in selected_percentiles
|
||||
]
|
||||
print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-"))
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
f"Mean {metric_name} (ms):",
|
||||
getattr(metrics, f"mean_{metric_attribute_name}_ms"),
|
||||
)
|
||||
)
|
||||
print(
|
||||
"{:<40} {:<10.2f}".format(
|
||||
f"Median {metric_name} (ms):",
|
||||
getattr(metrics, f"median_{metric_attribute_name}_ms"),
|
||||
)
|
||||
)
|
||||
result[f"mean_{metric_attribute_name}_ms"] = getattr(
|
||||
metrics, f"mean_{metric_attribute_name}_ms"
|
||||
)
|
||||
result[f"median_{metric_attribute_name}_ms"] = getattr(
|
||||
metrics, f"median_{metric_attribute_name}_ms"
|
||||
)
|
||||
result[f"std_{metric_attribute_name}_ms"] = getattr(
|
||||
metrics, f"std_{metric_attribute_name}_ms"
|
||||
)
|
||||
for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"):
|
||||
print("{:<40} {:<10.2f}".format(f"Mean {metric_name} (ms):", mean))
|
||||
print("{:<40} {:<10.2f}".format(f"Median {metric_name} (ms):", median))
|
||||
result[f"mean_{metric_attribute_name}_ms"] = mean
|
||||
result[f"median_{metric_attribute_name}_ms"] = median
|
||||
result[f"std_{metric_attribute_name}_ms"] = std
|
||||
for p, value in percentiles:
|
||||
p_word = str(int(p)) if int(p) == p else str(p)
|
||||
print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value))
|
||||
result[f"p{p_word}_{metric_attribute_name}_ms"] = value
|
||||
@@ -1376,6 +1396,20 @@ async def benchmark(
|
||||
process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)")
|
||||
process_one_metric("itl", "ITL", "Inter-token Latency")
|
||||
process_one_metric("e2el", "E2EL", "End-to-end Latency")
|
||||
if queue_times is not None:
|
||||
process_one_metric(
|
||||
"client_queue_time",
|
||||
"Client Queue Time",
|
||||
"Client-side Queueing",
|
||||
queue_times,
|
||||
)
|
||||
if e2els_including_queue is not None:
|
||||
process_one_metric(
|
||||
"e2el_including_client_queue",
|
||||
"E2EL incl. Client Queue",
|
||||
"Queue-inclusive End-to-end Latency",
|
||||
e2els_including_queue,
|
||||
)
|
||||
|
||||
if diffusion_stats is not None:
|
||||
print("{s:{c}^{n}}".format(s="Diffusion Decoding", n=50, c="-"))
|
||||
@@ -1776,7 +1810,8 @@ def add_cli_args(parser: FlexibleArgumentParser):
|
||||
default=None,
|
||||
help="Comma-separated list of selected metrics to report percentiles. "
|
||||
"This argument specifies the metrics to report percentiles. "
|
||||
'Allowed metric names are "ttft", "tpot", "itl", "e2el". '
|
||||
'Allowed metric names are "ttft", "tpot", "itl", "e2el", '
|
||||
'"client_queue_time", "e2el_including_client_queue". '
|
||||
'If not specified, defaults to "ttft,tpot,itl" for generative models '
|
||||
'and "e2el" for pooling models.',
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user