[Benchmark] Add sweep warmup and failure recovery (#57305)

Signed-off-by: louie-tsai <louie.tsai@intel.com>
This commit is contained in:
Louie Tsai
2026-09-30 11:16:41 +08:00
committed by GitHub
parent 42d9f8ccdb
commit 1b77cc3d67
4 changed files with 299 additions and 34 deletions
+6
View File
@@ -88,6 +88,12 @@ vllm bench sweep serve \
By default, each parameter combination is benchmarked 3 times to make the results more reliable. You can adjust the number of runs by setting `--num-runs`.
You can optionally run an unmeasured warmup before each parameter combination by setting `--warmup-num-prompts`. The warmup result is saved as `warmup.json` for inspection, but is excluded from the measured summary.
You can use `--continue-on-error` to record a failed warmup, benchmark run, or server configuration and continue processing the remaining sweep combinations. Failure details are saved in `.failure.json` files.
Both options are also supported by `vllm bench sweep serve_workload`.
!!! important
If both `--serve-params` and `--bench-params` are passed, the script will iterate over the Cartesian product between them.
You can use `--dry-run` to preview the commands to be run.
+126
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import pytest
from vllm.benchmarks.sweep import serve as sweep_serve
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
@@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization:
# The prefix (compilation_config) gets converted to hyphens,
# but the suffix (some_nested_param) is preserved
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
def test_run_comb_excludes_warmup_from_measured_results(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[tuple[int, int, str]] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 320}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=32,
dry_run=False,
)
assert calls == [
(-1, 32, "warmup.json"),
(0, 320, "run=0.json"),
(1, 320, "run=1.json"),
]
assert measured == [{"run_number": 0}, {"run_number": 1}]
def test_run_comb_continue_on_error_keeps_later_runs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
if run_number == 0:
raise RuntimeError("synthetic run failure")
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 100}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=0,
dry_run=False,
continue_on_error=True,
)
assert calls == [0, 1]
assert measured == [{"run_number": 1}]
assert (base_path / "run=0.failure.json").exists()
assert (base_path / "summary.json").exists()
def test_run_comb_warmup_default_is_backward_compatible(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
"""Older callers can omit warmup_num_prompts."""
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 10}),
link_vars=[],
base_path=base_path,
num_runs=1,
dry_run=False,
)
assert calls == [0]
assert measured == [{"run_number": 0}]
+141 -30
View File
@@ -221,29 +221,80 @@ def run_comb(
base_path: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int = 0,
continue_on_error: bool = False,
):
if not _comb_is_valid(serve_comb, bench_comb, link_vars):
return None
comb_data = list[dict[str, object]]()
def run_benchmark_with_error_handling(
*,
bench_overrides: ParameterSweepItem,
run_number: int,
output_path: Path,
):
try:
return run_benchmark(
server,
bench_cmd,
serve_overrides=serve_comb,
bench_overrides=bench_overrides,
run_number=run_number,
output_path=output_path,
dry_run=dry_run,
)
except Exception as exc:
if not continue_on_error:
raise
failure_path = output_path.with_name(output_path.stem + ".failure.json")
failure_path.parent.mkdir(parents=True, exist_ok=True)
failure_path.write_text(
json.dumps(
{
"run_number": run_number,
"error_type": type(exc).__name__,
"error": str(exc),
"serve_overrides": dict(serve_comb),
"bench_overrides": dict(bench_overrides),
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}")
print(f"Failure details: {failure_path}")
if server is not None:
with contextlib.suppress(Exception):
server.after_bench()
return None
if warmup_num_prompts > 0:
warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts}
run_benchmark_with_error_handling(
bench_overrides=warmup_overrides,
run_number=-1,
output_path=base_path / "warmup.json",
)
for run_number in range(num_runs):
run_data = run_benchmark(
server,
bench_cmd,
serve_overrides=serve_comb,
run_data = run_benchmark_with_error_handling(
bench_overrides=bench_comb,
run_number=run_number,
output_path=_get_comb_run_path(base_path, run_number),
dry_run=dry_run,
)
if run_data is not None:
comb_data.append(run_data)
if dry_run:
return None
if not comb_data:
print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable")
return []
with _get_comb_run_path(base_path, run_number=None).open("w") as f:
json.dump(comb_data, f, indent=4)
@@ -262,36 +313,69 @@ def run_combs(
link_vars: list[tuple[str, str]],
experiment_dir: Path,
num_runs: int,
warmup_num_prompts: int,
dry_run: bool,
continue_on_error: bool = False,
):
all_data = list[dict[str, object]]()
for serve_comb in serve_params:
with server_ctx(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
serve_comb=serve_comb,
bench_params=bench_params,
experiment_dir=experiment_dir,
dry_run=dry_run,
server_ready_timeout=server_ready_timeout,
) as server:
for bench_comb in bench_params:
base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb)
try:
with server_ctx(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
serve_comb=serve_comb,
bench_params=bench_params,
experiment_dir=experiment_dir,
dry_run=dry_run,
server_ready_timeout=server_ready_timeout,
) as server:
for bench_comb in bench_params:
base_path = _get_comb_base_path(
experiment_dir, serve_comb, bench_comb
)
comb_data = run_comb(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
base_path=base_path,
num_runs=num_runs,
dry_run=dry_run,
comb_data = run_comb(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
base_path=base_path,
num_runs=num_runs,
warmup_num_prompts=warmup_num_prompts,
dry_run=dry_run,
continue_on_error=continue_on_error,
)
if comb_data is not None:
all_data.extend(comb_data)
except Exception as exc:
if not continue_on_error:
raise
failure_name = sanitize_filename(
f"SERVE-{serve_comb.name or 'default'}-failure.json"
)
failure_path = experiment_dir / failure_name
failure_path.parent.mkdir(parents=True, exist_ok=True)
failure_path.write_text(
json.dumps(
{
"error_type": type(exc).__name__,
"error": str(exc),
"serve_overrides": dict(serve_comb),
},
indent=2,
)
if comb_data is not None:
all_data.extend(comb_data)
+ "\n",
encoding="utf-8",
)
print(
"[FAILED SERVER COMBINATION] "
f"{serve_comb.name or '(default)'}: "
f"{type(exc).__name__}: {exc}"
)
print(f"Failure details: {failure_path}")
if dry_run:
return None
@@ -315,6 +399,8 @@ class SweepServeArgs:
output_dir: Path
experiment_name: str
num_runs: int
warmup_num_prompts: int
continue_on_error: bool
dry_run: bool
resume: bool
@@ -351,6 +437,9 @@ class SweepServeArgs:
num_runs = args.num_runs
if num_runs < 1:
raise ValueError("`num_runs` should be at least 1.")
warmup_num_prompts = args.warmup_num_prompts
if warmup_num_prompts < 0:
raise ValueError("`warmup_num_prompts` should be at least 0.")
return cls(
serve_cmd=serve_cmd,
@@ -363,6 +452,8 @@ class SweepServeArgs:
output_dir=Path(args.output_dir),
experiment_name=experiment_name,
num_runs=num_runs,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=args.continue_on_error,
dry_run=args.dry_run,
resume=args.resume,
server_ready_timeout=args.server_ready_timeout,
@@ -453,6 +544,24 @@ class SweepServeArgs:
default=3,
help="Number of runs per parameter combination.",
)
parser.add_argument(
"--warmup-num-prompts",
type=int,
default=0,
help=(
"Number of prompts in one unmeasured warmup run before each "
"parameter combination. The warmup is saved as warmup.json but "
"excluded from summary statistics."
),
)
parser.add_argument(
"--continue-on-error",
action="store_true",
help=(
"Record and skip a failed warmup or benchmark run instead of "
"terminating the entire sweep."
),
)
parser.add_argument(
"--dry-run",
action="store_true",
@@ -523,7 +632,9 @@ def run_main(args: SweepServeArgs):
bench_params=args.bench_params,
experiment_dir=experiment_dir,
num_runs=args.num_runs,
warmup_num_prompts=args.warmup_num_prompts,
dry_run=args.dry_run,
continue_on_error=args.continue_on_error,
)
+26 -4
View File
@@ -63,6 +63,8 @@ def run_comb_workload(
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
workload_var: WorkloadVariable,
workload_value: int,
) -> list[dict[str, object]] | None:
@@ -82,6 +84,8 @@ def run_comb_workload(
),
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
)
@@ -97,6 +101,8 @@ def explore_comb_workloads(
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
):
print("[WL START]")
print(f"Serve parameters: {serve_comb.as_text() or '(None)'}")
@@ -129,6 +135,8 @@ def explore_comb_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=1,
)
@@ -141,16 +149,22 @@ def explore_comb_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=dataset_size,
)
if serial_workload_data is None or batch_workload_data is None:
if not serial_workload_data or not batch_workload_data:
if dry_run:
print("Omitting intermediate Workload iterations.")
print("[WL END]")
return
else:
print(
"Unable to establish both Workload Explorer endpoints; "
"skipping intermediate workload levels."
)
print("[WL END]")
return []
serial_workload_value = math.ceil(
_estimate_workload_avg(serial_workload_data, workload_var)
@@ -181,6 +195,8 @@ def explore_comb_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=inter_workload_value,
)
@@ -207,6 +223,8 @@ def explore_combs_workloads(
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
):
if any(bench_comb.has_param(workload_var) for bench_comb in bench_params):
raise ValueError(
@@ -238,6 +256,8 @@ def explore_combs_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
)
if comb_data is not None:
@@ -315,6 +335,8 @@ def run_main(args: SweepServeWorkloadArgs):
experiment_dir=experiment_dir,
num_runs=args.num_runs,
dry_run=args.dry_run,
warmup_num_prompts=args.warmup_num_prompts,
continue_on_error=args.continue_on_error,
)