[Benchmark] Add sweep warmup and failure recovery (#57305)

Signed-off-by: louie-tsai <louie.tsai@intel.com>
This commit is contained in:
Louie Tsai
2026-09-30 11:16:41 +08:00
committed by GitHub
parent 42d9f8ccdb
commit 1b77cc3d67
4 changed files with 299 additions and 34 deletions
+141 -30
View File
@@ -221,29 +221,80 @@ def run_comb(
base_path: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int = 0,
continue_on_error: bool = False,
):
if not _comb_is_valid(serve_comb, bench_comb, link_vars):
return None
comb_data = list[dict[str, object]]()
def run_benchmark_with_error_handling(
*,
bench_overrides: ParameterSweepItem,
run_number: int,
output_path: Path,
):
try:
return run_benchmark(
server,
bench_cmd,
serve_overrides=serve_comb,
bench_overrides=bench_overrides,
run_number=run_number,
output_path=output_path,
dry_run=dry_run,
)
except Exception as exc:
if not continue_on_error:
raise
failure_path = output_path.with_name(output_path.stem + ".failure.json")
failure_path.parent.mkdir(parents=True, exist_ok=True)
failure_path.write_text(
json.dumps(
{
"run_number": run_number,
"error_type": type(exc).__name__,
"error": str(exc),
"serve_overrides": dict(serve_comb),
"bench_overrides": dict(bench_overrides),
},
indent=2,
)
+ "\n",
encoding="utf-8",
)
print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}")
print(f"Failure details: {failure_path}")
if server is not None:
with contextlib.suppress(Exception):
server.after_bench()
return None
if warmup_num_prompts > 0:
warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts}
run_benchmark_with_error_handling(
bench_overrides=warmup_overrides,
run_number=-1,
output_path=base_path / "warmup.json",
)
for run_number in range(num_runs):
run_data = run_benchmark(
server,
bench_cmd,
serve_overrides=serve_comb,
run_data = run_benchmark_with_error_handling(
bench_overrides=bench_comb,
run_number=run_number,
output_path=_get_comb_run_path(base_path, run_number),
dry_run=dry_run,
)
if run_data is not None:
comb_data.append(run_data)
if dry_run:
return None
if not comb_data:
print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable")
return []
with _get_comb_run_path(base_path, run_number=None).open("w") as f:
json.dump(comb_data, f, indent=4)
@@ -262,36 +313,69 @@ def run_combs(
link_vars: list[tuple[str, str]],
experiment_dir: Path,
num_runs: int,
warmup_num_prompts: int,
dry_run: bool,
continue_on_error: bool = False,
):
all_data = list[dict[str, object]]()
for serve_comb in serve_params:
with server_ctx(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
serve_comb=serve_comb,
bench_params=bench_params,
experiment_dir=experiment_dir,
dry_run=dry_run,
server_ready_timeout=server_ready_timeout,
) as server:
for bench_comb in bench_params:
base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb)
try:
with server_ctx(
serve_cmd,
after_bench_cmd,
show_stdout=show_stdout,
serve_comb=serve_comb,
bench_params=bench_params,
experiment_dir=experiment_dir,
dry_run=dry_run,
server_ready_timeout=server_ready_timeout,
) as server:
for bench_comb in bench_params:
base_path = _get_comb_base_path(
experiment_dir, serve_comb, bench_comb
)
comb_data = run_comb(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
base_path=base_path,
num_runs=num_runs,
dry_run=dry_run,
comb_data = run_comb(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
link_vars=link_vars,
base_path=base_path,
num_runs=num_runs,
warmup_num_prompts=warmup_num_prompts,
dry_run=dry_run,
continue_on_error=continue_on_error,
)
if comb_data is not None:
all_data.extend(comb_data)
except Exception as exc:
if not continue_on_error:
raise
failure_name = sanitize_filename(
f"SERVE-{serve_comb.name or 'default'}-failure.json"
)
failure_path = experiment_dir / failure_name
failure_path.parent.mkdir(parents=True, exist_ok=True)
failure_path.write_text(
json.dumps(
{
"error_type": type(exc).__name__,
"error": str(exc),
"serve_overrides": dict(serve_comb),
},
indent=2,
)
if comb_data is not None:
all_data.extend(comb_data)
+ "\n",
encoding="utf-8",
)
print(
"[FAILED SERVER COMBINATION] "
f"{serve_comb.name or '(default)'}: "
f"{type(exc).__name__}: {exc}"
)
print(f"Failure details: {failure_path}")
if dry_run:
return None
@@ -315,6 +399,8 @@ class SweepServeArgs:
output_dir: Path
experiment_name: str
num_runs: int
warmup_num_prompts: int
continue_on_error: bool
dry_run: bool
resume: bool
@@ -351,6 +437,9 @@ class SweepServeArgs:
num_runs = args.num_runs
if num_runs < 1:
raise ValueError("`num_runs` should be at least 1.")
warmup_num_prompts = args.warmup_num_prompts
if warmup_num_prompts < 0:
raise ValueError("`warmup_num_prompts` should be at least 0.")
return cls(
serve_cmd=serve_cmd,
@@ -363,6 +452,8 @@ class SweepServeArgs:
output_dir=Path(args.output_dir),
experiment_name=experiment_name,
num_runs=num_runs,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=args.continue_on_error,
dry_run=args.dry_run,
resume=args.resume,
server_ready_timeout=args.server_ready_timeout,
@@ -453,6 +544,24 @@ class SweepServeArgs:
default=3,
help="Number of runs per parameter combination.",
)
parser.add_argument(
"--warmup-num-prompts",
type=int,
default=0,
help=(
"Number of prompts in one unmeasured warmup run before each "
"parameter combination. The warmup is saved as warmup.json but "
"excluded from summary statistics."
),
)
parser.add_argument(
"--continue-on-error",
action="store_true",
help=(
"Record and skip a failed warmup or benchmark run instead of "
"terminating the entire sweep."
),
)
parser.add_argument(
"--dry-run",
action="store_true",
@@ -523,7 +632,9 @@ def run_main(args: SweepServeArgs):
bench_params=args.bench_params,
experiment_dir=experiment_dir,
num_runs=args.num_runs,
warmup_num_prompts=args.warmup_num_prompts,
dry_run=args.dry_run,
continue_on_error=args.continue_on_error,
)
+26 -4
View File
@@ -63,6 +63,8 @@ def run_comb_workload(
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
workload_var: WorkloadVariable,
workload_value: int,
) -> list[dict[str, object]] | None:
@@ -82,6 +84,8 @@ def run_comb_workload(
),
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
)
@@ -97,6 +101,8 @@ def explore_comb_workloads(
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
):
print("[WL START]")
print(f"Serve parameters: {serve_comb.as_text() or '(None)'}")
@@ -129,6 +135,8 @@ def explore_comb_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=1,
)
@@ -141,16 +149,22 @@ def explore_comb_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=dataset_size,
)
if serial_workload_data is None or batch_workload_data is None:
if not serial_workload_data or not batch_workload_data:
if dry_run:
print("Omitting intermediate Workload iterations.")
print("[WL END]")
return
else:
print(
"Unable to establish both Workload Explorer endpoints; "
"skipping intermediate workload levels."
)
print("[WL END]")
return []
serial_workload_value = math.ceil(
_estimate_workload_avg(serial_workload_data, workload_var)
@@ -181,6 +195,8 @@ def explore_comb_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
workload_var=workload_var,
workload_value=inter_workload_value,
)
@@ -207,6 +223,8 @@ def explore_combs_workloads(
experiment_dir: Path,
num_runs: int,
dry_run: bool,
warmup_num_prompts: int,
continue_on_error: bool,
):
if any(bench_comb.has_param(workload_var) for bench_comb in bench_params):
raise ValueError(
@@ -238,6 +256,8 @@ def explore_combs_workloads(
experiment_dir=experiment_dir,
num_runs=num_runs,
dry_run=dry_run,
warmup_num_prompts=warmup_num_prompts,
continue_on_error=continue_on_error,
)
if comb_data is not None:
@@ -315,6 +335,8 @@ def run_main(args: SweepServeWorkloadArgs):
experiment_dir=experiment_dir,
num_runs=args.num_runs,
dry_run=args.dry_run,
warmup_num_prompts=args.warmup_num_prompts,
continue_on_error=args.continue_on_error,
)