diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 5ac9a6126d..d46ab04b0e 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -664,7 +664,7 @@ vLLM's benchmark serving script provides sophisticated load pattern simulation c - `--request-rate`: Controls the target request generation rate (requests per second). Set to `inf` for maximum throughput testing or finite values for controlled load simulation. - `--burstiness`: Controls traffic variability using a Gamma distribution (range: > 0). Lower values create bursty traffic, higher values create uniform traffic. -- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure. +- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure. When set, include `client_queue_time` in `--percentile-metrics` to report time spent waiting for the benchmark client's concurrency limit. With a finite `--request-rate`, `e2el_including_client_queue` reports schedule-relative end-to-end latency; it is omitted for `--request-rate=inf`, where all requests arrive at benchmark start. These parameters work together to create realistic load patterns with carefully chosen defaults. The `--request-rate` parameter defaults to `inf` (infinite), which sends all requests immediately for maximum throughput testing. When set to finite values, it uses either a Poisson process (default `--burstiness=1.0`) or Gamma distribution for realistic request timing. The `--burstiness` parameter only takes effect when `--request-rate` is not infinite - a value of 1.0 creates natural Poisson traffic, while lower values (0.1-0.5) create bursty patterns and higher values (2.0-5.0) create uniform spacing. The `--max-concurrency` parameter defaults to `None` (unlimited) but can be set to simulate real-world constraints where a load balancer or API gateway limits concurrent connections. When combined, these parameters allow you to simulate everything from unrestricted stress testing (`--request-rate=inf`) to production-like scenarios with realistic arrival patterns and resource constraints. diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index 0680d666da..ee6ea65633 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -99,6 +99,8 @@ class RequestFuncOutput: prompt_len: int = 0 error: str = "" start_time: float = 0.0 + # Time spent awaiting the benchmark client concurrency semaphore. + client_queue_time: float = 0.0 input_audio_duration: float = 0.0 # in seconds num_input_sequences: int = 1 diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 960668b93a..70d13770d5 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -23,6 +23,7 @@ import asyncio import contextlib import importlib.util import json +import math import os import random import shutil @@ -972,11 +973,17 @@ async def benchmark( else contextlib.nullcontext() ) - async def limited_request_func(request_func_input, session, pbar): + async def limited_request_func( + request_func_input, session, pbar, request_arrival_time + ): async with semaphore: - return await request_func( + output = await request_func( request_func_input=request_func_input, session=session, pbar=pbar ) + # Preserve start_time as the time the request was sent. It is used by + # throughput calculations; record client-side semaphore delay separately. + output.client_queue_time = output.start_time - request_arrival_time + return output probe_outputs: list[RequestFuncOutput] = [] probe_stop = asyncio.Event() @@ -1064,10 +1071,11 @@ async def benchmark( request_id=request_id, chat_messages=request.chat_messages, ) + request_arrival_time = time.perf_counter() tasks.append( asyncio.create_task( limited_request_func( - request_func_input=request_func_input, session=session, pbar=pbar + request_func_input, session, pbar, request_arrival_time ) ) ) @@ -1284,7 +1292,9 @@ async def benchmark( "output_lens": actual_output_lens, "ttfts": [output.ttft for output in outputs], "itls": [output.itl for output in outputs], + "latencies": [output.latency for output in outputs], "start_times": [output.start_time for output in outputs], + "queue_times": [output.client_queue_time for output in outputs], "generated_texts": [output.generated_text for output in outputs], "errors": [output.error for output in outputs], "max_output_tokens_per_s": metrics.max_output_tokens_per_s, @@ -1301,9 +1311,22 @@ async def benchmark( "input_sequence_throughput": metrics.input_sequence_throughput, "total_token_throughput": metrics.total_token_throughput, "input_lens": [output.prompt_len for output in outputs], + "latencies": [output.latency for output in outputs], + "queue_times": [output.client_queue_time for output in outputs], "errors": [output.error for output in outputs], } + queue_times: list[float] | None = None + e2els_including_queue: list[float] | None = None + if max_concurrency is not None: + queue_times = [output.client_queue_time for output in outputs if output.success] + if not math.isinf(request_rate): + e2els_including_queue = [ + output.latency + output.client_queue_time + for output in outputs + if output.success + ] + if probe_stats is not None: result.update(probe_stats) @@ -1339,34 +1362,31 @@ async def benchmark( metric_name: str, # E.g., "Time to First Token" metric_header: str, + values: list[float] | None = None, ): # This function prints and adds statistics of the specified # metric. if metric_attribute_name not in selected_percentile_metrics: return + if values is None: + mean = getattr(metrics, f"mean_{metric_attribute_name}_ms") + median = getattr(metrics, f"median_{metric_attribute_name}_ms") + std = getattr(metrics, f"std_{metric_attribute_name}_ms") + percentiles = getattr(metrics, f"percentiles_{metric_attribute_name}_ms") + else: + mean = np.mean(values or 0) * 1000 + median = np.median(values or 0) * 1000 + std = np.std(values or 0) * 1000 + percentiles = [ + (p, np.percentile(values or 0, p) * 1000) for p in selected_percentiles + ] print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-")) - print( - "{:<40} {:<10.2f}".format( - f"Mean {metric_name} (ms):", - getattr(metrics, f"mean_{metric_attribute_name}_ms"), - ) - ) - print( - "{:<40} {:<10.2f}".format( - f"Median {metric_name} (ms):", - getattr(metrics, f"median_{metric_attribute_name}_ms"), - ) - ) - result[f"mean_{metric_attribute_name}_ms"] = getattr( - metrics, f"mean_{metric_attribute_name}_ms" - ) - result[f"median_{metric_attribute_name}_ms"] = getattr( - metrics, f"median_{metric_attribute_name}_ms" - ) - result[f"std_{metric_attribute_name}_ms"] = getattr( - metrics, f"std_{metric_attribute_name}_ms" - ) - for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"): + print("{:<40} {:<10.2f}".format(f"Mean {metric_name} (ms):", mean)) + print("{:<40} {:<10.2f}".format(f"Median {metric_name} (ms):", median)) + result[f"mean_{metric_attribute_name}_ms"] = mean + result[f"median_{metric_attribute_name}_ms"] = median + result[f"std_{metric_attribute_name}_ms"] = std + for p, value in percentiles: p_word = str(int(p)) if int(p) == p else str(p) print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value)) result[f"p{p_word}_{metric_attribute_name}_ms"] = value @@ -1376,6 +1396,20 @@ async def benchmark( process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)") process_one_metric("itl", "ITL", "Inter-token Latency") process_one_metric("e2el", "E2EL", "End-to-end Latency") + if queue_times is not None: + process_one_metric( + "client_queue_time", + "Client Queue Time", + "Client-side Queueing", + queue_times, + ) + if e2els_including_queue is not None: + process_one_metric( + "e2el_including_client_queue", + "E2EL incl. Client Queue", + "Queue-inclusive End-to-end Latency", + e2els_including_queue, + ) if diffusion_stats is not None: print("{s:{c}^{n}}".format(s="Diffusion Decoding", n=50, c="-")) @@ -1776,7 +1810,8 @@ def add_cli_args(parser: FlexibleArgumentParser): default=None, help="Comma-separated list of selected metrics to report percentiles. " "This argument specifies the metrics to report percentiles. " - 'Allowed metric names are "ttft", "tpot", "itl", "e2el". ' + 'Allowed metric names are "ttft", "tpot", "itl", "e2el", ' + '"client_queue_time", "e2el_including_client_queue". ' 'If not specified, defaults to "ttft,tpot,itl" for generative models ' 'and "e2el" for pooling models.', )