[Bugfix] Account for client queue time in serve benchmarks (#54136)
Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com> Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
maithilijoshi20
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
92ccd2c306
commit
9e905f7450
@@ -664,7 +664,7 @@ vLLM's benchmark serving script provides sophisticated load pattern simulation c
|
|||||||
|
|
||||||
- `--request-rate`: Controls the target request generation rate (requests per second). Set to `inf` for maximum throughput testing or finite values for controlled load simulation.
|
- `--request-rate`: Controls the target request generation rate (requests per second). Set to `inf` for maximum throughput testing or finite values for controlled load simulation.
|
||||||
- `--burstiness`: Controls traffic variability using a Gamma distribution (range: > 0). Lower values create bursty traffic, higher values create uniform traffic.
|
- `--burstiness`: Controls traffic variability using a Gamma distribution (range: > 0). Lower values create bursty traffic, higher values create uniform traffic.
|
||||||
- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure.
|
- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure. When set, include `client_queue_time` in `--percentile-metrics` to report time spent waiting for the benchmark client's concurrency limit. With a finite `--request-rate`, `e2el_including_client_queue` reports schedule-relative end-to-end latency; it is omitted for `--request-rate=inf`, where all requests arrive at benchmark start.
|
||||||
|
|
||||||
These parameters work together to create realistic load patterns with carefully chosen defaults. The `--request-rate` parameter defaults to `inf` (infinite), which sends all requests immediately for maximum throughput testing. When set to finite values, it uses either a Poisson process (default `--burstiness=1.0`) or Gamma distribution for realistic request timing. The `--burstiness` parameter only takes effect when `--request-rate` is not infinite - a value of 1.0 creates natural Poisson traffic, while lower values (0.1-0.5) create bursty patterns and higher values (2.0-5.0) create uniform spacing. The `--max-concurrency` parameter defaults to `None` (unlimited) but can be set to simulate real-world constraints where a load balancer or API gateway limits concurrent connections. When combined, these parameters allow you to simulate everything from unrestricted stress testing (`--request-rate=inf`) to production-like scenarios with realistic arrival patterns and resource constraints.
|
These parameters work together to create realistic load patterns with carefully chosen defaults. The `--request-rate` parameter defaults to `inf` (infinite), which sends all requests immediately for maximum throughput testing. When set to finite values, it uses either a Poisson process (default `--burstiness=1.0`) or Gamma distribution for realistic request timing. The `--burstiness` parameter only takes effect when `--request-rate` is not infinite - a value of 1.0 creates natural Poisson traffic, while lower values (0.1-0.5) create bursty patterns and higher values (2.0-5.0) create uniform spacing. The `--max-concurrency` parameter defaults to `None` (unlimited) but can be set to simulate real-world constraints where a load balancer or API gateway limits concurrent connections. When combined, these parameters allow you to simulate everything from unrestricted stress testing (`--request-rate=inf`) to production-like scenarios with realistic arrival patterns and resource constraints.
|
||||||
|
|
||||||
|
|||||||
@@ -99,6 +99,8 @@ class RequestFuncOutput:
|
|||||||
prompt_len: int = 0
|
prompt_len: int = 0
|
||||||
error: str = ""
|
error: str = ""
|
||||||
start_time: float = 0.0
|
start_time: float = 0.0
|
||||||
|
# Time spent awaiting the benchmark client concurrency semaphore.
|
||||||
|
client_queue_time: float = 0.0
|
||||||
input_audio_duration: float = 0.0 # in seconds
|
input_audio_duration: float = 0.0 # in seconds
|
||||||
num_input_sequences: int = 1
|
num_input_sequences: int = 1
|
||||||
|
|
||||||
|
|||||||
+61
-26
@@ -23,6 +23,7 @@ import asyncio
|
|||||||
import contextlib
|
import contextlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
import shutil
|
import shutil
|
||||||
@@ -972,11 +973,17 @@ async def benchmark(
|
|||||||
else contextlib.nullcontext()
|
else contextlib.nullcontext()
|
||||||
)
|
)
|
||||||
|
|
||||||
async def limited_request_func(request_func_input, session, pbar):
|
async def limited_request_func(
|
||||||
|
request_func_input, session, pbar, request_arrival_time
|
||||||
|
):
|
||||||
async with semaphore:
|
async with semaphore:
|
||||||
return await request_func(
|
output = await request_func(
|
||||||
request_func_input=request_func_input, session=session, pbar=pbar
|
request_func_input=request_func_input, session=session, pbar=pbar
|
||||||
)
|
)
|
||||||
|
# Preserve start_time as the time the request was sent. It is used by
|
||||||
|
# throughput calculations; record client-side semaphore delay separately.
|
||||||
|
output.client_queue_time = output.start_time - request_arrival_time
|
||||||
|
return output
|
||||||
|
|
||||||
probe_outputs: list[RequestFuncOutput] = []
|
probe_outputs: list[RequestFuncOutput] = []
|
||||||
probe_stop = asyncio.Event()
|
probe_stop = asyncio.Event()
|
||||||
@@ -1064,10 +1071,11 @@ async def benchmark(
|
|||||||
request_id=request_id,
|
request_id=request_id,
|
||||||
chat_messages=request.chat_messages,
|
chat_messages=request.chat_messages,
|
||||||
)
|
)
|
||||||
|
request_arrival_time = time.perf_counter()
|
||||||
tasks.append(
|
tasks.append(
|
||||||
asyncio.create_task(
|
asyncio.create_task(
|
||||||
limited_request_func(
|
limited_request_func(
|
||||||
request_func_input=request_func_input, session=session, pbar=pbar
|
request_func_input, session, pbar, request_arrival_time
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -1284,7 +1292,9 @@ async def benchmark(
|
|||||||
"output_lens": actual_output_lens,
|
"output_lens": actual_output_lens,
|
||||||
"ttfts": [output.ttft for output in outputs],
|
"ttfts": [output.ttft for output in outputs],
|
||||||
"itls": [output.itl for output in outputs],
|
"itls": [output.itl for output in outputs],
|
||||||
|
"latencies": [output.latency for output in outputs],
|
||||||
"start_times": [output.start_time for output in outputs],
|
"start_times": [output.start_time for output in outputs],
|
||||||
|
"queue_times": [output.client_queue_time for output in outputs],
|
||||||
"generated_texts": [output.generated_text for output in outputs],
|
"generated_texts": [output.generated_text for output in outputs],
|
||||||
"errors": [output.error for output in outputs],
|
"errors": [output.error for output in outputs],
|
||||||
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
|
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
|
||||||
@@ -1301,9 +1311,22 @@ async def benchmark(
|
|||||||
"input_sequence_throughput": metrics.input_sequence_throughput,
|
"input_sequence_throughput": metrics.input_sequence_throughput,
|
||||||
"total_token_throughput": metrics.total_token_throughput,
|
"total_token_throughput": metrics.total_token_throughput,
|
||||||
"input_lens": [output.prompt_len for output in outputs],
|
"input_lens": [output.prompt_len for output in outputs],
|
||||||
|
"latencies": [output.latency for output in outputs],
|
||||||
|
"queue_times": [output.client_queue_time for output in outputs],
|
||||||
"errors": [output.error for output in outputs],
|
"errors": [output.error for output in outputs],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
queue_times: list[float] | None = None
|
||||||
|
e2els_including_queue: list[float] | None = None
|
||||||
|
if max_concurrency is not None:
|
||||||
|
queue_times = [output.client_queue_time for output in outputs if output.success]
|
||||||
|
if not math.isinf(request_rate):
|
||||||
|
e2els_including_queue = [
|
||||||
|
output.latency + output.client_queue_time
|
||||||
|
for output in outputs
|
||||||
|
if output.success
|
||||||
|
]
|
||||||
|
|
||||||
if probe_stats is not None:
|
if probe_stats is not None:
|
||||||
result.update(probe_stats)
|
result.update(probe_stats)
|
||||||
|
|
||||||
@@ -1339,34 +1362,31 @@ async def benchmark(
|
|||||||
metric_name: str,
|
metric_name: str,
|
||||||
# E.g., "Time to First Token"
|
# E.g., "Time to First Token"
|
||||||
metric_header: str,
|
metric_header: str,
|
||||||
|
values: list[float] | None = None,
|
||||||
):
|
):
|
||||||
# This function prints and adds statistics of the specified
|
# This function prints and adds statistics of the specified
|
||||||
# metric.
|
# metric.
|
||||||
if metric_attribute_name not in selected_percentile_metrics:
|
if metric_attribute_name not in selected_percentile_metrics:
|
||||||
return
|
return
|
||||||
|
if values is None:
|
||||||
|
mean = getattr(metrics, f"mean_{metric_attribute_name}_ms")
|
||||||
|
median = getattr(metrics, f"median_{metric_attribute_name}_ms")
|
||||||
|
std = getattr(metrics, f"std_{metric_attribute_name}_ms")
|
||||||
|
percentiles = getattr(metrics, f"percentiles_{metric_attribute_name}_ms")
|
||||||
|
else:
|
||||||
|
mean = np.mean(values or 0) * 1000
|
||||||
|
median = np.median(values or 0) * 1000
|
||||||
|
std = np.std(values or 0) * 1000
|
||||||
|
percentiles = [
|
||||||
|
(p, np.percentile(values or 0, p) * 1000) for p in selected_percentiles
|
||||||
|
]
|
||||||
print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-"))
|
print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-"))
|
||||||
print(
|
print("{:<40} {:<10.2f}".format(f"Mean {metric_name} (ms):", mean))
|
||||||
"{:<40} {:<10.2f}".format(
|
print("{:<40} {:<10.2f}".format(f"Median {metric_name} (ms):", median))
|
||||||
f"Mean {metric_name} (ms):",
|
result[f"mean_{metric_attribute_name}_ms"] = mean
|
||||||
getattr(metrics, f"mean_{metric_attribute_name}_ms"),
|
result[f"median_{metric_attribute_name}_ms"] = median
|
||||||
)
|
result[f"std_{metric_attribute_name}_ms"] = std
|
||||||
)
|
for p, value in percentiles:
|
||||||
print(
|
|
||||||
"{:<40} {:<10.2f}".format(
|
|
||||||
f"Median {metric_name} (ms):",
|
|
||||||
getattr(metrics, f"median_{metric_attribute_name}_ms"),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
result[f"mean_{metric_attribute_name}_ms"] = getattr(
|
|
||||||
metrics, f"mean_{metric_attribute_name}_ms"
|
|
||||||
)
|
|
||||||
result[f"median_{metric_attribute_name}_ms"] = getattr(
|
|
||||||
metrics, f"median_{metric_attribute_name}_ms"
|
|
||||||
)
|
|
||||||
result[f"std_{metric_attribute_name}_ms"] = getattr(
|
|
||||||
metrics, f"std_{metric_attribute_name}_ms"
|
|
||||||
)
|
|
||||||
for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"):
|
|
||||||
p_word = str(int(p)) if int(p) == p else str(p)
|
p_word = str(int(p)) if int(p) == p else str(p)
|
||||||
print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value))
|
print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value))
|
||||||
result[f"p{p_word}_{metric_attribute_name}_ms"] = value
|
result[f"p{p_word}_{metric_attribute_name}_ms"] = value
|
||||||
@@ -1376,6 +1396,20 @@ async def benchmark(
|
|||||||
process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)")
|
process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)")
|
||||||
process_one_metric("itl", "ITL", "Inter-token Latency")
|
process_one_metric("itl", "ITL", "Inter-token Latency")
|
||||||
process_one_metric("e2el", "E2EL", "End-to-end Latency")
|
process_one_metric("e2el", "E2EL", "End-to-end Latency")
|
||||||
|
if queue_times is not None:
|
||||||
|
process_one_metric(
|
||||||
|
"client_queue_time",
|
||||||
|
"Client Queue Time",
|
||||||
|
"Client-side Queueing",
|
||||||
|
queue_times,
|
||||||
|
)
|
||||||
|
if e2els_including_queue is not None:
|
||||||
|
process_one_metric(
|
||||||
|
"e2el_including_client_queue",
|
||||||
|
"E2EL incl. Client Queue",
|
||||||
|
"Queue-inclusive End-to-end Latency",
|
||||||
|
e2els_including_queue,
|
||||||
|
)
|
||||||
|
|
||||||
if diffusion_stats is not None:
|
if diffusion_stats is not None:
|
||||||
print("{s:{c}^{n}}".format(s="Diffusion Decoding", n=50, c="-"))
|
print("{s:{c}^{n}}".format(s="Diffusion Decoding", n=50, c="-"))
|
||||||
@@ -1776,7 +1810,8 @@ def add_cli_args(parser: FlexibleArgumentParser):
|
|||||||
default=None,
|
default=None,
|
||||||
help="Comma-separated list of selected metrics to report percentiles. "
|
help="Comma-separated list of selected metrics to report percentiles. "
|
||||||
"This argument specifies the metrics to report percentiles. "
|
"This argument specifies the metrics to report percentiles. "
|
||||||
'Allowed metric names are "ttft", "tpot", "itl", "e2el". '
|
'Allowed metric names are "ttft", "tpot", "itl", "e2el", '
|
||||||
|
'"client_queue_time", "e2el_including_client_queue". '
|
||||||
'If not specified, defaults to "ttft,tpot,itl" for generative models '
|
'If not specified, defaults to "ttft,tpot,itl" for generative models '
|
||||||
'and "e2el" for pooling models.',
|
'and "e2el" for pooling models.',
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user