From 1b77cc3d67eddd98c7ecbad7024213a3eeaf7e93 Mon Sep 17 00:00:00 2001 From: Louie Tsai Date: Tue, 29 Sep 2026 20:16:41 -0700 Subject: [PATCH] [Benchmark] Add sweep warmup and failure recovery (#57305) Signed-off-by: louie-tsai --- docs/benchmarking/sweeps.md | 6 + tests/benchmarks/sweep/test_param_sweep.py | 126 +++++++++++++++ vllm/benchmarks/sweep/serve.py | 171 +++++++++++++++++---- vllm/benchmarks/sweep/serve_workload.py | 30 +++- 4 files changed, 299 insertions(+), 34 deletions(-) diff --git a/docs/benchmarking/sweeps.md b/docs/benchmarking/sweeps.md index 41a799cf21..8975f3a2c8 100644 --- a/docs/benchmarking/sweeps.md +++ b/docs/benchmarking/sweeps.md @@ -88,6 +88,12 @@ vllm bench sweep serve \ By default, each parameter combination is benchmarked 3 times to make the results more reliable. You can adjust the number of runs by setting `--num-runs`. +You can optionally run an unmeasured warmup before each parameter combination by setting `--warmup-num-prompts`. The warmup result is saved as `warmup.json` for inspection, but is excluded from the measured summary. + +You can use `--continue-on-error` to record a failed warmup, benchmark run, or server configuration and continue processing the remaining sweep combinations. Failure details are saved in `.failure.json` files. + +Both options are also supported by `vllm bench sweep serve_workload`. + !!! important If both `--serve-params` and `--bench-params` are passed, the script will iterate over the Cartesian product between them. You can use `--dry-run` to preview the commands to be run. diff --git a/tests/benchmarks/sweep/test_param_sweep.py b/tests/benchmarks/sweep/test_param_sweep.py index 467797d991..483eb13378 100644 --- a/tests/benchmarks/sweep/test_param_sweep.py +++ b/tests/benchmarks/sweep/test_param_sweep.py @@ -6,6 +6,7 @@ from pathlib import Path import pytest +from vllm.benchmarks.sweep import serve as sweep_serve from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem @@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization: # The prefix (compilation_config) gets converted to hyphens, # but the suffix (some_nested_param) is preserved assert any("compilation-config.some_nested_param" in arg for arg in cmd) + + +def test_run_comb_excludes_warmup_from_measured_results( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + calls: list[tuple[int, int, str]] = [] + + def fake_run_benchmark( + server, + bench_cmd, + *, + serve_overrides, + bench_overrides, + run_number, + output_path, + dry_run, + ): + calls.append( + (run_number, int(bench_overrides["num_prompts"]), output_path.name) + ) + return {"run_number": run_number} + + monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark) + base_path = tmp_path / "combination" + base_path.mkdir() + + measured = sweep_serve.run_comb( + None, + [], + serve_comb=ParameterSweepItem(), + bench_comb=ParameterSweepItem({"num_prompts": 320}), + link_vars=[], + base_path=base_path, + num_runs=2, + warmup_num_prompts=32, + dry_run=False, + ) + + assert calls == [ + (-1, 32, "warmup.json"), + (0, 320, "run=0.json"), + (1, 320, "run=1.json"), + ] + assert measured == [{"run_number": 0}, {"run_number": 1}] + + +def test_run_comb_continue_on_error_keeps_later_runs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + calls: list[int] = [] + + def fake_run_benchmark( + server, + bench_cmd, + *, + serve_overrides, + bench_overrides, + run_number, + output_path, + dry_run, + ): + calls.append(run_number) + if run_number == 0: + raise RuntimeError("synthetic run failure") + return {"run_number": run_number} + + monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark) + base_path = tmp_path / "combination" + base_path.mkdir() + + measured = sweep_serve.run_comb( + None, + [], + serve_comb=ParameterSweepItem(), + bench_comb=ParameterSweepItem({"num_prompts": 100}), + link_vars=[], + base_path=base_path, + num_runs=2, + warmup_num_prompts=0, + dry_run=False, + continue_on_error=True, + ) + + assert calls == [0, 1] + assert measured == [{"run_number": 1}] + assert (base_path / "run=0.failure.json").exists() + assert (base_path / "summary.json").exists() + + +def test_run_comb_warmup_default_is_backward_compatible( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +): + """Older callers can omit warmup_num_prompts.""" + calls: list[int] = [] + + def fake_run_benchmark( + server, + bench_cmd, + *, + serve_overrides, + bench_overrides, + run_number, + output_path, + dry_run, + ): + calls.append(run_number) + return {"run_number": run_number} + + monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark) + base_path = tmp_path / "combination" + base_path.mkdir() + + measured = sweep_serve.run_comb( + None, + [], + serve_comb=ParameterSweepItem(), + bench_comb=ParameterSweepItem({"num_prompts": 10}), + link_vars=[], + base_path=base_path, + num_runs=1, + dry_run=False, + ) + + assert calls == [0] + assert measured == [{"run_number": 0}] diff --git a/vllm/benchmarks/sweep/serve.py b/vllm/benchmarks/sweep/serve.py index 369eded5f6..9f16134720 100644 --- a/vllm/benchmarks/sweep/serve.py +++ b/vllm/benchmarks/sweep/serve.py @@ -221,29 +221,80 @@ def run_comb( base_path: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int = 0, + continue_on_error: bool = False, ): if not _comb_is_valid(serve_comb, bench_comb, link_vars): return None comb_data = list[dict[str, object]]() + def run_benchmark_with_error_handling( + *, + bench_overrides: ParameterSweepItem, + run_number: int, + output_path: Path, + ): + try: + return run_benchmark( + server, + bench_cmd, + serve_overrides=serve_comb, + bench_overrides=bench_overrides, + run_number=run_number, + output_path=output_path, + dry_run=dry_run, + ) + except Exception as exc: + if not continue_on_error: + raise + failure_path = output_path.with_name(output_path.stem + ".failure.json") + failure_path.parent.mkdir(parents=True, exist_ok=True) + failure_path.write_text( + json.dumps( + { + "run_number": run_number, + "error_type": type(exc).__name__, + "error": str(exc), + "serve_overrides": dict(serve_comb), + "bench_overrides": dict(bench_overrides), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}") + print(f"Failure details: {failure_path}") + if server is not None: + with contextlib.suppress(Exception): + server.after_bench() + return None + + if warmup_num_prompts > 0: + warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts} + run_benchmark_with_error_handling( + bench_overrides=warmup_overrides, + run_number=-1, + output_path=base_path / "warmup.json", + ) + for run_number in range(num_runs): - run_data = run_benchmark( - server, - bench_cmd, - serve_overrides=serve_comb, + run_data = run_benchmark_with_error_handling( bench_overrides=bench_comb, run_number=run_number, output_path=_get_comb_run_path(base_path, run_number), - dry_run=dry_run, ) - if run_data is not None: comb_data.append(run_data) if dry_run: return None + if not comb_data: + print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable") + return [] + with _get_comb_run_path(base_path, run_number=None).open("w") as f: json.dump(comb_data, f, indent=4) @@ -262,36 +313,69 @@ def run_combs( link_vars: list[tuple[str, str]], experiment_dir: Path, num_runs: int, + warmup_num_prompts: int, dry_run: bool, + continue_on_error: bool = False, ): all_data = list[dict[str, object]]() for serve_comb in serve_params: - with server_ctx( - serve_cmd, - after_bench_cmd, - show_stdout=show_stdout, - serve_comb=serve_comb, - bench_params=bench_params, - experiment_dir=experiment_dir, - dry_run=dry_run, - server_ready_timeout=server_ready_timeout, - ) as server: - for bench_comb in bench_params: - base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb) + try: + with server_ctx( + serve_cmd, + after_bench_cmd, + show_stdout=show_stdout, + serve_comb=serve_comb, + bench_params=bench_params, + experiment_dir=experiment_dir, + dry_run=dry_run, + server_ready_timeout=server_ready_timeout, + ) as server: + for bench_comb in bench_params: + base_path = _get_comb_base_path( + experiment_dir, serve_comb, bench_comb + ) - comb_data = run_comb( - server, - bench_cmd, - serve_comb=serve_comb, - bench_comb=bench_comb, - link_vars=link_vars, - base_path=base_path, - num_runs=num_runs, - dry_run=dry_run, + comb_data = run_comb( + server, + bench_cmd, + serve_comb=serve_comb, + bench_comb=bench_comb, + link_vars=link_vars, + base_path=base_path, + num_runs=num_runs, + warmup_num_prompts=warmup_num_prompts, + dry_run=dry_run, + continue_on_error=continue_on_error, + ) + + if comb_data is not None: + all_data.extend(comb_data) + except Exception as exc: + if not continue_on_error: + raise + failure_name = sanitize_filename( + f"SERVE-{serve_comb.name or 'default'}-failure.json" + ) + failure_path = experiment_dir / failure_name + failure_path.parent.mkdir(parents=True, exist_ok=True) + failure_path.write_text( + json.dumps( + { + "error_type": type(exc).__name__, + "error": str(exc), + "serve_overrides": dict(serve_comb), + }, + indent=2, ) - - if comb_data is not None: - all_data.extend(comb_data) + + "\n", + encoding="utf-8", + ) + print( + "[FAILED SERVER COMBINATION] " + f"{serve_comb.name or '(default)'}: " + f"{type(exc).__name__}: {exc}" + ) + print(f"Failure details: {failure_path}") if dry_run: return None @@ -315,6 +399,8 @@ class SweepServeArgs: output_dir: Path experiment_name: str num_runs: int + warmup_num_prompts: int + continue_on_error: bool dry_run: bool resume: bool @@ -351,6 +437,9 @@ class SweepServeArgs: num_runs = args.num_runs if num_runs < 1: raise ValueError("`num_runs` should be at least 1.") + warmup_num_prompts = args.warmup_num_prompts + if warmup_num_prompts < 0: + raise ValueError("`warmup_num_prompts` should be at least 0.") return cls( serve_cmd=serve_cmd, @@ -363,6 +452,8 @@ class SweepServeArgs: output_dir=Path(args.output_dir), experiment_name=experiment_name, num_runs=num_runs, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=args.continue_on_error, dry_run=args.dry_run, resume=args.resume, server_ready_timeout=args.server_ready_timeout, @@ -453,6 +544,24 @@ class SweepServeArgs: default=3, help="Number of runs per parameter combination.", ) + parser.add_argument( + "--warmup-num-prompts", + type=int, + default=0, + help=( + "Number of prompts in one unmeasured warmup run before each " + "parameter combination. The warmup is saved as warmup.json but " + "excluded from summary statistics." + ), + ) + parser.add_argument( + "--continue-on-error", + action="store_true", + help=( + "Record and skip a failed warmup or benchmark run instead of " + "terminating the entire sweep." + ), + ) parser.add_argument( "--dry-run", action="store_true", @@ -523,7 +632,9 @@ def run_main(args: SweepServeArgs): bench_params=args.bench_params, experiment_dir=experiment_dir, num_runs=args.num_runs, + warmup_num_prompts=args.warmup_num_prompts, dry_run=args.dry_run, + continue_on_error=args.continue_on_error, ) diff --git a/vllm/benchmarks/sweep/serve_workload.py b/vllm/benchmarks/sweep/serve_workload.py index d762ca2c64..a488837e2e 100644 --- a/vllm/benchmarks/sweep/serve_workload.py +++ b/vllm/benchmarks/sweep/serve_workload.py @@ -63,6 +63,8 @@ def run_comb_workload( experiment_dir: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int, + continue_on_error: bool, workload_var: WorkloadVariable, workload_value: int, ) -> list[dict[str, object]] | None: @@ -82,6 +84,8 @@ def run_comb_workload( ), num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, ) @@ -97,6 +101,8 @@ def explore_comb_workloads( experiment_dir: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int, + continue_on_error: bool, ): print("[WL START]") print(f"Serve parameters: {serve_comb.as_text() or '(None)'}") @@ -129,6 +135,8 @@ def explore_comb_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, workload_var=workload_var, workload_value=1, ) @@ -141,16 +149,22 @@ def explore_comb_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, workload_var=workload_var, workload_value=dataset_size, ) - if serial_workload_data is None or batch_workload_data is None: + if not serial_workload_data or not batch_workload_data: if dry_run: print("Omitting intermediate Workload iterations.") - print("[WL END]") - - return + else: + print( + "Unable to establish both Workload Explorer endpoints; " + "skipping intermediate workload levels." + ) + print("[WL END]") + return [] serial_workload_value = math.ceil( _estimate_workload_avg(serial_workload_data, workload_var) @@ -181,6 +195,8 @@ def explore_comb_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, workload_var=workload_var, workload_value=inter_workload_value, ) @@ -207,6 +223,8 @@ def explore_combs_workloads( experiment_dir: Path, num_runs: int, dry_run: bool, + warmup_num_prompts: int, + continue_on_error: bool, ): if any(bench_comb.has_param(workload_var) for bench_comb in bench_params): raise ValueError( @@ -238,6 +256,8 @@ def explore_combs_workloads( experiment_dir=experiment_dir, num_runs=num_runs, dry_run=dry_run, + warmup_num_prompts=warmup_num_prompts, + continue_on_error=continue_on_error, ) if comb_data is not None: @@ -315,6 +335,8 @@ def run_main(args: SweepServeWorkloadArgs): experiment_dir=experiment_dir, num_runs=args.num_runs, dry_run=args.dry_run, + warmup_num_prompts=args.warmup_num_prompts, + continue_on_error=args.continue_on_error, )