[Benchmark][2/2] Use spline interpolation to tune SLA variables (#32095)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-01-10 20:27:27 -08:00
committed by GitHub
parent 2a4dbe24ea
commit ef96fa3f1f
3 changed files with 232 additions and 264 deletions
+73 -143
View File
@@ -3,14 +3,11 @@
import argparse
import contextlib
import json
import math
from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path
from typing import ClassVar, Literal, get_args
from typing_extensions import assert_never
from vllm.utils.import_utils import PlaceholderModule
from .param_sweep import ParameterSweep, ParameterSweepItem
@@ -24,6 +21,15 @@ try:
except ImportError:
pd = PlaceholderModule("pandas")
try:
from scipy.interpolate import PchipInterpolator
except ImportError:
PchipInterpolator = (
PlaceholderModule("scipy")
.placeholder_attr("interpolate")
.placeholder_attr("PchipInterpolator")
)
def _get_sla_base_path(
output_dir: Path,
@@ -118,18 +124,36 @@ def run_sla(
SLAVariable = Literal["request_rate", "max_concurrency"]
def _estimate_sla_value(run_data: dict[str, object], sla_variable: SLAVariable):
request_throughput = float(run_data["request_throughput"]) # type: ignore
if sla_variable == "request_rate":
return request_throughput
if sla_variable == "max_concurrency":
mean_latency_ms = float(run_data["mean_e2el_ms"]) # type: ignore
return request_throughput * mean_latency_ms / 1000
class SLAHistory(dict[int, float]):
def __init__(self, min_value: int, max_value: int) -> None:
super().__init__()
assert_never(sla_variable)
self.min_value = min_value
self.max_value = max_value
def get_xy(self) -> tuple[list[int], list[float]]:
xs = list[int]()
ys = list[float]()
for x, y in sorted(self.items()):
xs.append(x)
ys.append(y)
return xs, ys
def get_max_passing(self) -> float:
return max(
(val for val, margin in self.items() if margin <= 0),
default=self.min_value,
)
def get_min_failing(self) -> float:
return min(
(val for val, margin in self.items() if margin > 0),
default=self.max_value,
)
def _estimate_sla_bounds(
def solve_sla(
server: ServerProcess | None,
bench_cmd: list[str],
*,
@@ -140,17 +164,33 @@ def _estimate_sla_bounds(
num_runs: int,
dry_run: bool,
sla_variable: SLAVariable,
init_value: int,
max_value: int,
sla_min_value: int = 1,
sla_max_value: int = 8192, # The value that represents infinite QPS
):
sla_data = list[dict[str, object]]()
history = SLAHistory(min_value=sla_min_value, max_value=sla_max_value)
val: int = init_value
assert val > 0
# NOTE: We don't use equality here to be more robust against noisy results
while history.get_max_passing() + 1 < history.get_min_failing():
if len(history) == 0:
val = sla_max_value
elif len(history) == 1:
val = sla_min_value
else:
spl = PchipInterpolator(*history.get_xy(), extrapolate=False)
spl_roots = spl.solve()
if len(spl_roots) == 0:
# Fallback to binary search
val = int((history.get_max_passing() + history.get_min_failing()) / 2)
else:
val = int(spl_roots[0])
history = dict[int, float]()
if val in history:
# Cover both sides (floor and ceil) of the root to be sure
# that it is indeed the target value
val += 1
while True:
val = max(sla_min_value, min(val, sla_max_value))
print(f"Testing {sla_variable}: {val} req/s")
iter_data = run_sla(
@@ -162,8 +202,9 @@ def _estimate_sla_bounds(
num_runs=num_runs,
dry_run=dry_run,
)
if iter_data is None:
return None
assert iter_data is not None
sla_data.extend(iter_data)
iter_data_mean = {
@@ -175,92 +216,14 @@ def _estimate_sla_bounds(
criterion.print_and_compute_margin(iter_data_mean, k)
for k, criterion in sla_comb.items()
]
margin = max(sla_margins)
history[val] = margin
history[val] = margin = max(sla_margins)
if margin <= 0:
print("SLA criteria are met.")
val *= 2
print(f"SLA criteria are met. ({margin=:.2f})")
else:
print("SLA criteria are not met.")
break
print(f"SLA criteria are not met. ({margin=:.2f})")
if val >= max_value:
break
max_passing = max(
(val for val, margin in history.items() if margin <= 0),
default=0,
)
min_failing = min(
(val for val, margin in history.items() if margin > 0),
default=max_value,
)
return sla_data, (max_passing, min_failing), history
def _find_sla_value(
server: ServerProcess | None,
bench_cmd: list[str],
*,
serve_comb: ParameterSweepItem,
bench_comb: ParameterSweepItem,
sla_comb: SLASweepItem,
base_path: Path,
num_runs: int,
dry_run: bool,
sla_variable: SLAVariable,
min_value: int,
max_value: int,
):
sla_data = list[dict[str, object]]()
left: int = min_value
right: int = max_value
history = dict[int, float]()
while True:
val = (left + right) // 2
print(f"Testing {sla_variable}: {val} req/s")
iter_data = run_sla(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb | {sla_variable: val},
iter_path=_get_sla_iter_path(base_path, sla_comb, sla_variable, val),
num_runs=num_runs,
dry_run=dry_run,
)
assert iter_data is not None
sla_data.extend(iter_data)
iter_data_mean = {
k: sum(float(run_data[k]) for run_data in iter_data) / len(iter_data) # type: ignore
for k in sla_comb
}
sla_margins = [
criterion.print_and_compute_margin(iter_data_mean, k)
for k, criterion in sla_comb.items()
]
margin = max(sla_margins)
history[val] = margin
if margin <= 0:
print("SLA criteria are met.")
left = val
else:
print("SLA criteria are not met.")
right = val
if right - left <= 1 and left in history:
break
return sla_data, left, history
return sla_data, history
def search_sla(
@@ -271,7 +234,6 @@ def search_sla(
bench_comb: ParameterSweepItem,
sla_comb: SLASweepItem,
sla_variable: SLAVariable,
sla_inf_value: int = 65536, # The value that represents infinite QPS
base_path: Path,
num_runs: int,
dry_run: bool,
@@ -279,57 +241,25 @@ def search_sla(
print("[SLA START]")
print(f"SLA criteria: {sla_comb.as_text()}")
sla_data_0 = run_sla(
result = solve_sla(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb | {sla_variable: sla_inf_value},
iter_path=_get_sla_iter_path(base_path, sla_comb, sla_variable, sla_inf_value),
bench_comb=bench_comb,
sla_comb=sla_comb,
base_path=base_path,
num_runs=num_runs,
dry_run=dry_run,
sla_variable=sla_variable,
)
if sla_data_0 is None:
if result is None:
assert dry_run
print("Omitting SLA search.")
print("[SLA END]")
return None
return
sla_init_value = math.ceil(
sum(_estimate_sla_value(item, sla_variable) for item in sla_data_0)
/ len(sla_data_0)
)
print(f"Initial {sla_variable} to search: {sla_init_value} req/s.")
sla_data_1, (sla_min, sla_max), _ = _estimate_sla_bounds(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
sla_comb=sla_comb,
base_path=base_path,
num_runs=num_runs,
dry_run=dry_run,
sla_variable=sla_variable,
init_value=sla_init_value,
max_value=sla_inf_value,
)
print(f"Range of {sla_variable} to search: [{sla_min}, {sla_max}] req/s.")
sla_data_2, sla_value, _ = _find_sla_value(
server,
bench_cmd,
serve_comb=serve_comb,
bench_comb=bench_comb,
sla_comb=sla_comb,
base_path=base_path,
num_runs=num_runs,
dry_run=dry_run,
sla_variable=sla_variable,
min_value=sla_min,
max_value=sla_max,
)
sla_data = sla_data_0 + sla_data_1 + sla_data_2
sla_data, sla_history = result
sla_value = sla_history.get_max_passing()
print(f"Maximum {sla_variable} for SLA: {sla_value} req/s.")
with _get_sla_iter_path(