[Benchmark] Add sweep warmup and failure recovery (#57305)

Signed-off-by: louie-tsai <louie.tsai@intel.com>
This commit is contained in:
Louie Tsai
2026-09-30 11:16:41 +08:00
committed by GitHub
parent 42d9f8ccdb
commit 1b77cc3d67
4 changed files with 299 additions and 34 deletions
+126
View File
@@ -6,6 +6,7 @@ from pathlib import Path
import pytest
from vllm.benchmarks.sweep import serve as sweep_serve
from vllm.benchmarks.sweep.param_sweep import ParameterSweep, ParameterSweepItem
@@ -247,3 +248,128 @@ class TestParameterSweepItemKeyNormalization:
# The prefix (compilation_config) gets converted to hyphens,
# but the suffix (some_nested_param) is preserved
assert any("compilation-config.some_nested_param" in arg for arg in cmd)
def test_run_comb_excludes_warmup_from_measured_results(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[tuple[int, int, str]] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(
(run_number, int(bench_overrides["num_prompts"]), output_path.name)
)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 320}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=32,
dry_run=False,
)
assert calls == [
(-1, 32, "warmup.json"),
(0, 320, "run=0.json"),
(1, 320, "run=1.json"),
]
assert measured == [{"run_number": 0}, {"run_number": 1}]
def test_run_comb_continue_on_error_keeps_later_runs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
if run_number == 0:
raise RuntimeError("synthetic run failure")
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 100}),
link_vars=[],
base_path=base_path,
num_runs=2,
warmup_num_prompts=0,
dry_run=False,
continue_on_error=True,
)
assert calls == [0, 1]
assert measured == [{"run_number": 1}]
assert (base_path / "run=0.failure.json").exists()
assert (base_path / "summary.json").exists()
def test_run_comb_warmup_default_is_backward_compatible(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
"""Older callers can omit warmup_num_prompts."""
calls: list[int] = []
def fake_run_benchmark(
server,
bench_cmd,
*,
serve_overrides,
bench_overrides,
run_number,
output_path,
dry_run,
):
calls.append(run_number)
return {"run_number": run_number}
monkeypatch.setattr(sweep_serve, "run_benchmark", fake_run_benchmark)
base_path = tmp_path / "combination"
base_path.mkdir()
measured = sweep_serve.run_comb(
None,
[],
serve_comb=ParameterSweepItem(),
bench_comb=ParameterSweepItem({"num_prompts": 10}),
link_vars=[],
base_path=base_path,
num_runs=1,
dry_run=False,
)
assert calls == [0]
assert measured == [{"run_number": 0}]