[Benchmark] Add sweep warmup and failure recovery (#57305)
Signed-off-by: louie-tsai <louie.tsai@intel.com>
This commit is contained in:
+141
-30
@@ -221,29 +221,80 @@ def run_comb(
|
||||
base_path: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int = 0,
|
||||
continue_on_error: bool = False,
|
||||
):
|
||||
if not _comb_is_valid(serve_comb, bench_comb, link_vars):
|
||||
return None
|
||||
|
||||
comb_data = list[dict[str, object]]()
|
||||
|
||||
def run_benchmark_with_error_handling(
|
||||
*,
|
||||
bench_overrides: ParameterSweepItem,
|
||||
run_number: int,
|
||||
output_path: Path,
|
||||
):
|
||||
try:
|
||||
return run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_overrides=serve_comb,
|
||||
bench_overrides=bench_overrides,
|
||||
run_number=run_number,
|
||||
output_path=output_path,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failure_path = output_path.with_name(output_path.stem + ".failure.json")
|
||||
failure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
failure_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"run_number": run_number,
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"serve_overrides": dict(serve_comb),
|
||||
"bench_overrides": dict(bench_overrides),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}")
|
||||
print(f"Failure details: {failure_path}")
|
||||
if server is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
server.after_bench()
|
||||
return None
|
||||
|
||||
if warmup_num_prompts > 0:
|
||||
warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts}
|
||||
run_benchmark_with_error_handling(
|
||||
bench_overrides=warmup_overrides,
|
||||
run_number=-1,
|
||||
output_path=base_path / "warmup.json",
|
||||
)
|
||||
|
||||
for run_number in range(num_runs):
|
||||
run_data = run_benchmark(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_overrides=serve_comb,
|
||||
run_data = run_benchmark_with_error_handling(
|
||||
bench_overrides=bench_comb,
|
||||
run_number=run_number,
|
||||
output_path=_get_comb_run_path(base_path, run_number),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
if run_data is not None:
|
||||
comb_data.append(run_data)
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
|
||||
if not comb_data:
|
||||
print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable")
|
||||
return []
|
||||
|
||||
with _get_comb_run_path(base_path, run_number=None).open("w") as f:
|
||||
json.dump(comb_data, f, indent=4)
|
||||
|
||||
@@ -262,36 +313,69 @@ def run_combs(
|
||||
link_vars: list[tuple[str, str]],
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
warmup_num_prompts: int,
|
||||
dry_run: bool,
|
||||
continue_on_error: bool = False,
|
||||
):
|
||||
all_data = list[dict[str, object]]()
|
||||
for serve_comb in serve_params:
|
||||
with server_ctx(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
serve_comb=serve_comb,
|
||||
bench_params=bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
dry_run=dry_run,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
) as server:
|
||||
for bench_comb in bench_params:
|
||||
base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb)
|
||||
try:
|
||||
with server_ctx(
|
||||
serve_cmd,
|
||||
after_bench_cmd,
|
||||
show_stdout=show_stdout,
|
||||
serve_comb=serve_comb,
|
||||
bench_params=bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
dry_run=dry_run,
|
||||
server_ready_timeout=server_ready_timeout,
|
||||
) as server:
|
||||
for bench_comb in bench_params:
|
||||
base_path = _get_comb_base_path(
|
||||
experiment_dir, serve_comb, bench_comb
|
||||
)
|
||||
|
||||
comb_data = run_comb(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
base_path=base_path,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
comb_data = run_comb(
|
||||
server,
|
||||
bench_cmd,
|
||||
serve_comb=serve_comb,
|
||||
bench_comb=bench_comb,
|
||||
link_vars=link_vars,
|
||||
base_path=base_path,
|
||||
num_runs=num_runs,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
dry_run=dry_run,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
except Exception as exc:
|
||||
if not continue_on_error:
|
||||
raise
|
||||
failure_name = sanitize_filename(
|
||||
f"SERVE-{serve_comb.name or 'default'}-failure.json"
|
||||
)
|
||||
failure_path = experiment_dir / failure_name
|
||||
failure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
failure_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"error_type": type(exc).__name__,
|
||||
"error": str(exc),
|
||||
"serve_overrides": dict(serve_comb),
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
all_data.extend(comb_data)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
print(
|
||||
"[FAILED SERVER COMBINATION] "
|
||||
f"{serve_comb.name or '(default)'}: "
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
print(f"Failure details: {failure_path}")
|
||||
|
||||
if dry_run:
|
||||
return None
|
||||
@@ -315,6 +399,8 @@ class SweepServeArgs:
|
||||
output_dir: Path
|
||||
experiment_name: str
|
||||
num_runs: int
|
||||
warmup_num_prompts: int
|
||||
continue_on_error: bool
|
||||
dry_run: bool
|
||||
resume: bool
|
||||
|
||||
@@ -351,6 +437,9 @@ class SweepServeArgs:
|
||||
num_runs = args.num_runs
|
||||
if num_runs < 1:
|
||||
raise ValueError("`num_runs` should be at least 1.")
|
||||
warmup_num_prompts = args.warmup_num_prompts
|
||||
if warmup_num_prompts < 0:
|
||||
raise ValueError("`warmup_num_prompts` should be at least 0.")
|
||||
|
||||
return cls(
|
||||
serve_cmd=serve_cmd,
|
||||
@@ -363,6 +452,8 @@ class SweepServeArgs:
|
||||
output_dir=Path(args.output_dir),
|
||||
experiment_name=experiment_name,
|
||||
num_runs=num_runs,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=args.continue_on_error,
|
||||
dry_run=args.dry_run,
|
||||
resume=args.resume,
|
||||
server_ready_timeout=args.server_ready_timeout,
|
||||
@@ -453,6 +544,24 @@ class SweepServeArgs:
|
||||
default=3,
|
||||
help="Number of runs per parameter combination.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup-num-prompts",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Number of prompts in one unmeasured warmup run before each "
|
||||
"parameter combination. The warmup is saved as warmup.json but "
|
||||
"excluded from summary statistics."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--continue-on-error",
|
||||
action="store_true",
|
||||
help=(
|
||||
"Record and skip a failed warmup or benchmark run instead of "
|
||||
"terminating the entire sweep."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
@@ -523,7 +632,9 @@ def run_main(args: SweepServeArgs):
|
||||
bench_params=args.bench_params,
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
warmup_num_prompts=args.warmup_num_prompts,
|
||||
dry_run=args.dry_run,
|
||||
continue_on_error=args.continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ def run_comb_workload(
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
workload_var: WorkloadVariable,
|
||||
workload_value: int,
|
||||
) -> list[dict[str, object]] | None:
|
||||
@@ -82,6 +84,8 @@ def run_comb_workload(
|
||||
),
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
@@ -97,6 +101,8 @@ def explore_comb_workloads(
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
):
|
||||
print("[WL START]")
|
||||
print(f"Serve parameters: {serve_comb.as_text() or '(None)'}")
|
||||
@@ -129,6 +135,8 @@ def explore_comb_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=1,
|
||||
)
|
||||
@@ -141,16 +149,22 @@ def explore_comb_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=dataset_size,
|
||||
)
|
||||
|
||||
if serial_workload_data is None or batch_workload_data is None:
|
||||
if not serial_workload_data or not batch_workload_data:
|
||||
if dry_run:
|
||||
print("Omitting intermediate Workload iterations.")
|
||||
print("[WL END]")
|
||||
|
||||
return
|
||||
else:
|
||||
print(
|
||||
"Unable to establish both Workload Explorer endpoints; "
|
||||
"skipping intermediate workload levels."
|
||||
)
|
||||
print("[WL END]")
|
||||
return []
|
||||
|
||||
serial_workload_value = math.ceil(
|
||||
_estimate_workload_avg(serial_workload_data, workload_var)
|
||||
@@ -181,6 +195,8 @@ def explore_comb_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
workload_var=workload_var,
|
||||
workload_value=inter_workload_value,
|
||||
)
|
||||
@@ -207,6 +223,8 @@ def explore_combs_workloads(
|
||||
experiment_dir: Path,
|
||||
num_runs: int,
|
||||
dry_run: bool,
|
||||
warmup_num_prompts: int,
|
||||
continue_on_error: bool,
|
||||
):
|
||||
if any(bench_comb.has_param(workload_var) for bench_comb in bench_params):
|
||||
raise ValueError(
|
||||
@@ -238,6 +256,8 @@ def explore_combs_workloads(
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=num_runs,
|
||||
dry_run=dry_run,
|
||||
warmup_num_prompts=warmup_num_prompts,
|
||||
continue_on_error=continue_on_error,
|
||||
)
|
||||
|
||||
if comb_data is not None:
|
||||
@@ -315,6 +335,8 @@ def run_main(args: SweepServeWorkloadArgs):
|
||||
experiment_dir=experiment_dir,
|
||||
num_runs=args.num_runs,
|
||||
dry_run=args.dry_run,
|
||||
warmup_num_prompts=args.warmup_num_prompts,
|
||||
continue_on_error=args.continue_on_error,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user