[Benchmark] Add sweep warmup and failure recovery (#57305)
Signed-off-by: louie-tsai <louie.tsai@intel.com>
This commit is contained in:
@@ -88,6 +88,12 @@ vllm bench sweep serve \
|
||||
|
||||
By default, each parameter combination is benchmarked 3 times to make the results more reliable. You can adjust the number of runs by setting `--num-runs`.
|
||||
|
||||
You can optionally run an unmeasured warmup before each parameter combination by setting `--warmup-num-prompts`. The warmup result is saved as `warmup.json` for inspection, but is excluded from the measured summary.
|
||||
|
||||
You can use `--continue-on-error` to record a failed warmup, benchmark run, or server configuration and continue processing the remaining sweep combinations. Failure details are saved in `.failure.json` files.
|
||||
|
||||
Both options are also supported by `vllm bench sweep serve_workload`.
|
||||
|
||||
!!! important
|
||||
If both `--serve-params` and `--bench-params` are passed, the script will iterate over the Cartesian product between them.
|
||||
You can use `--dry-run` to preview the commands to be run.
|
||||
|
||||
@@ -6,6 +6,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.benchmarks.sweep import serve as sweep_serve
|
||||
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
|
||||
|
||||
|
||||
@@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization:
|
||||
# The prefix (compilation_config) gets converted to hyphens,
|
||||
# but the suffix (some_nested_param) is preserved
|
||||
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
|
||||
|
||||
|
||||
def test_run_comb_excludes_warmup_from_measured_results(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[tuple[int, int, str]] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(
|
||||
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
|
||||
)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 320}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=32,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
(-1, 32, "warmup.json"),
|
||||
(0, 320, "run=0.json"),
|
||||
(1, 320, "run=1.json"),
|
||||
]
|
||||
assert measured == [{"run_number": 0}, {"run_number": 1}]
|
||||
|
||||
|
||||
def test_run_comb_continue_on_error_keeps_later_runs(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
if run_number == 0:
|
||||
raise RuntimeError("synthetic run failure")
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 100}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=2,
|
||||
warmup_num_prompts=0,
|
||||
dry_run=False,
|
||||
continue_on_error=True,
|
||||
)
|
||||
|
||||
assert calls == [0, 1]
|
||||
assert measured == [{"run_number": 1}]
|
||||
assert (base_path / "run=0.failure.json").exists()
|
||||
assert (base_path / "summary.json").exists()
|
||||
|
||||
|
||||
def test_run_comb_warmup_default_is_backward_compatible(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
):
|
||||
"""Older callers can omit warmup_num_prompts."""
|
||||
calls: list[int] = []
|
||||
|
||||
def fake_run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
*,
|
||||
serve_overrides,
|
||||
bench_overrides,
|
||||
run_number,
|
||||
output_path,
|
||||
dry_run,
|
||||
):
|
||||
calls.append(run_number)
|
||||
return {"run_number": run_number}
|
||||
|
||||
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
|
||||
base_path = tmp_path / "combination"
|
||||
base_path.mkdir()
|
||||
|
||||
measured = sweep_serve.run_comb(
|
||||
None,
|
||||
[],
|
||||
serve_comb=ParameterSweepItem(),
|
||||
bench_comb=ParameterSweepItem({"num_prompts": 10}),
|
||||
link_vars=[],
|
||||
base_path=base_path,
|
||||
num_runs=1,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
assert calls == [0]
|
||||
assert measured == [{"run_number": 0}]
|
||||
|
||||
+141
-30
@@ -221,29 +221,80 @@ def run_comb(
|
||||
base_path: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int = 0,
|
||||
continue_on_error: bool = False,
|
||||
):
|
||||
if not _comb_is_valid(serve_comb, bench_comb, link_vars):
|
||||
return None
|
||||
|
||||
comb_data = list[dict[str, object]]()
|
||||
|
||||
def run_benchmark_with_error_handling(
|
||||
*,
|
||||
bench_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
output_path: Path,
|
||||
):
|
||||
try:
|
||||
return run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_overrides=serve_comb,
|
||||
bench_overrides=bench_overrides,
|
||||
run_number=run_number,
|
||||
output_path=output_path,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failure_path = output_path.with_name(output_path.stem + ".failure.json")
|
||||
failure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
failure_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"run_number": run_number,
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"serve_overrides": dict(serve_comb),
|
||||
"bench_overrides": dict(bench_overrides),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}")
|
||||
print(f"Failure details: {failure_path}")
|
||||
if server is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
server.after_bench()
|
||||
return None
|
||||
|
||||
if warmup_num_prompts > 0:
|
||||
warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts}
|
||||
run_benchmark_with_error_handling(
|
||||
bench_overrides=warmup_overrides,
|
||||
run_number=-1,
|
||||
output_path=base_path / "warmup.json",
|
||||
)
|
||||
|
||||
for run_number in range(num_runs):
|
||||
run_data = run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_overrides=serve_comb,
|
||||
run_data = run_benchmark_with_error_handling(
|
||||
bench_overrides=bench_comb,
|
||||
run_number=run_number,
|
||||
output_path=_get_comb_run_path(base_path, run_number),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
if run_data is not None:
|
||||
comb_data.append(run_data)
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
if not comb_data:
|
||||
print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable")
|
||||
return []
|
||||
|
||||
with _get_comb_run_path(base_path, run_number=None).open("w") as f:
|
||||
json.dump(comb_data, f, indent=4)
|
||||
|
||||
@@ -262,36 +313,69 @@ def run_combs(
|
||||
link_vars: list[tuple[str, str]],
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
warmup_num_prompts: int,
|
||||
dry_run: bool,
|
||||
continue_on_error: bool = False,
|
||||
):
|
||||
all_data = list[dict[str, object]]()
|
||||
for serve_comb in serve_params:
|
||||
with server_ctx(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
serve_comb=serve_comb,
|
||||
bench_params=bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
dry_run=dry_run,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
) as server:
|
||||
for bench_comb in bench_params:
|
||||
base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb)
|
||||
try:
|
||||
with server_ctx(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
serve_comb=serve_comb,
|
||||
bench_params=bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
dry_run=dry_run,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
) as server:
|
||||
for bench_comb in bench_params:
|
||||
base_path = _get_comb_base_path(
|
||||
experiment_dir, serve_comb, bench_comb
|
||||
)
|
||||
|
||||
comb_data = run_comb(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
base_path=base_path,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
comb_data = run_comb(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
base_path=base_path,
|
||||
num_runs=num_runs,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
dry_run=dry_run,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failure_name = sanitize_filename(
|
||||
f"SERVE-{serve_comb.name or 'default'}-failure.json"
|
||||
)
|
||||
failure_path = experiment_dir / failure_name
|
||||
failure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
failure_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"serve_overrides": dict(serve_comb),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
"[FAILED SERVER COMBINATION] "
|
||||
f"{serve_comb.name or '(default)'}: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
print(f"Failure details: {failure_path}")
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
@@ -315,6 +399,8 @@ class SweepServeArgs:
|
||||
output_dir: Path
|
||||
experiment_name: str
|
||||
num_runs: int
|
||||
warmup_num_prompts: int
|
||||
continue_on_error: bool
|
||||
dry_run: bool
|
||||
resume: bool
|
||||
|
||||
@@ -351,6 +437,9 @@ class SweepServeArgs:
|
||||
num_runs = args.num_runs
|
||||
if num_runs < 1:
|
||||
raise ValueError("`num_runs` should be at least 1.")
|
||||
warmup_num_prompts = args.warmup_num_prompts
|
||||
if warmup_num_prompts < 0:
|
||||
raise ValueError("`warmup_num_prompts` should be at least 0.")
|
||||
|
||||
return cls(
|
||||
serve_cmd=serve_cmd,
|
||||
@@ -363,6 +452,8 @@ class SweepServeArgs:
|
||||
output_dir=Path(args.output_dir),
|
||||
experiment_name=experiment_name,
|
||||
num_runs=num_runs,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=args.continue_on_error,
|
||||
dry_run=args.dry_run,
|
||||
resume=args.resume,
|
||||
server_ready_timeout=args.server_ready_timeout,
|
||||
@@ -453,6 +544,24 @@ class SweepServeArgs:
|
||||
default=3,
|
||||
help="Number of runs per parameter combination.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup-num-prompts",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Number of prompts in one unmeasured warmup run before each "
|
||||
"parameter combination. The warmup is saved as warmup.json but "
|
||||
"excluded from summary statistics."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Record and skip a failed warmup or benchmark run instead of "
|
||||
"terminating the entire sweep."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
@@ -523,7 +632,9 @@ def run_main(args: SweepServeArgs):
|
||||
bench_params=args.bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
warmup_num_prompts=args.warmup_num_prompts,
|
||||
dry_run=args.dry_run,
|
||||
continue_on_error=args.continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ def run_comb_workload(
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
workload_var: WorkloadVariable,
|
||||
workload_value: int,
|
||||
) -> list[dict[str, object]] | None:
|
||||
@@ -82,6 +84,8 @@ def run_comb_workload(
|
||||
),
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
@@ -97,6 +101,8 @@ def explore_comb_workloads(
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
):
|
||||
print("[WL START]")
|
||||
print(f"Serve parameters: {serve_comb.as_text() or '(None)'}")
|
||||
@@ -129,6 +135,8 @@ def explore_comb_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=1,
|
||||
)
|
||||
@@ -141,16 +149,22 @@ def explore_comb_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=dataset_size,
|
||||
)
|
||||
|
||||
if serial_workload_data is None or batch_workload_data is None:
|
||||
if not serial_workload_data or not batch_workload_data:
|
||||
if dry_run:
|
||||
print("Omitting intermediate Workload iterations.")
|
||||
print("[WL END]")
|
||||
|
||||
return
|
||||
else:
|
||||
print(
|
||||
"Unable to establish both Workload Explorer endpoints; "
|
||||
"skipping intermediate workload levels."
|
||||
)
|
||||
print("[WL END]")
|
||||
return []
|
||||
|
||||
serial_workload_value = math.ceil(
|
||||
_estimate_workload_avg(serial_workload_data, workload_var)
|
||||
@@ -181,6 +195,8 @@ def explore_comb_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=inter_workload_value,
|
||||
)
|
||||
@@ -207,6 +223,8 @@ def explore_combs_workloads(
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
):
|
||||
if any(bench_comb.has_param(workload_var) for bench_comb in bench_params):
|
||||
raise ValueError(
|
||||
@@ -238,6 +256,8 @@ def explore_combs_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
@@ -315,6 +335,8 @@ def run_main(args: SweepServeWorkloadArgs):
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
dry_run=args.dry_run,
|
||||
warmup_num_prompts=args.warmup_num_prompts,
|
||||
continue_on_error=args.continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user