[Benchmark] Add probe requests to vllm bench serve (#49611)

Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
Guan-Ming Chiu
2026-08-01 22:10:56 +08:00
committed by GitHub
parent ab06486175
commit 63e78ce365
3 changed files with 106 additions and 0 deletions
+80
View File
@@ -795,6 +795,7 @@ async def benchmark(
ready_check_timeout_sec: int = 600,
ssl_context: ssl.SSLContext | bool | None = None,
self_timed: bool = False,
probe_request_rate: float = 0.0,
):
try:
request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
@@ -971,6 +972,30 @@ async def benchmark(
request_func_input=request_func_input, session=session, pbar=pbar
)
probe_outputs: list[RequestFuncOutput] = []
probe_stop = asyncio.Event()
async def probe_loop():
probe_input = replace(
test_input,
prompt="Hi",
prompt_len=1,
output_len=1,
multi_modal_content=None,
chat_messages=None,
)
interval = 1 / probe_request_rate
while not probe_stop.is_set():
probe_outputs.append(
await request_func(request_func_input=probe_input, session=session)
)
await asyncio.sleep(interval)
probe_task: asyncio.Task | None = None
if probe_request_rate > 0:
print(f"Probe request rate: {probe_request_rate} req/s")
probe_task = asyncio.create_task(probe_loop())
benchmark_start_time = time.perf_counter()
tasks: list[asyncio.Task] = []
@@ -1042,6 +1067,10 @@ async def benchmark(
)
outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)
if probe_task is not None:
probe_stop.set()
await probe_task
if pbar is not None:
pbar.close()
@@ -1189,6 +1218,44 @@ async def benchmark(
)
)
probe_stats: dict[str, Any] | None = None
if probe_task is not None:
probe_lats = [o.latency for o in probe_outputs if o.success]
if probe_lats:
probe_stats = {
"probe_completed": len(probe_lats),
"probe_failed": len(probe_outputs) - len(probe_lats),
"probe_median_e2el_ms": float(np.median(probe_lats)) * 1000,
"probe_p99_e2el_ms": float(np.percentile(probe_lats, 99)) * 1000,
"probe_max_e2el_ms": float(max(probe_lats)) * 1000,
}
print("{s:{c}^{n}}".format(s="Probe Requests", n=50, c="-"))
print(
"{:<40} {:<10}".format(
"Probe requests completed:", probe_stats["probe_completed"]
)
)
print(
"{:<40} {:<10}".format(
"Probe requests failed:", probe_stats["probe_failed"]
)
)
print(
"{:<40} {:<10.2f}".format(
"Median probe E2EL (ms):", probe_stats["probe_median_e2el_ms"]
)
)
print(
"{:<40} {:<10.2f}".format(
"P99 probe E2EL (ms):", probe_stats["probe_p99_e2el_ms"]
)
)
print(
"{:<40} {:<10.2f}".format(
"Max probe E2EL (ms):", probe_stats["probe_max_e2el_ms"]
)
)
result: dict[str, Any]
if isinstance(metrics, BenchmarkMetrics):
result = {
@@ -1223,6 +1290,9 @@ async def benchmark(
"errors": [output.error for output in outputs],
}
if probe_stats is not None:
result.update(probe_stats)
if rps_change_events:
result["rps_change_events"] = rps_change_events
@@ -1605,6 +1675,15 @@ def add_cli_args(parser: FlexibleArgumentParser):
"bursty requests. A higher burstiness value (burstiness > 1) "
"results in a more uniform arrival of requests.",
)
parser.add_argument(
"--probe-request-rate",
type=float,
default=0.0,
help="If positive, send single-token text-only probe requests at "
"this rate (req/s) alongside the main workload, bypassing "
"--max-concurrency, and report their latency separately. Useful "
"for measuring how the main workload stalls unrelated requests.",
)
parser.add_argument(
"--disable-tqdm",
action="store_true",
@@ -2121,6 +2200,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
ready_check_timeout_sec=args.ready_check_timeout_sec,
ssl_context=ssl_context,
self_timed=args.self_timed,
probe_request_rate=args.probe_request_rate,
)
# Save config and results to json