Measure encoder compile time seperate from llm backbone (#39240)

Signed-off-by: Lucas Kabela <lucaskabela@meta.com>
This commit is contained in:
Lucas Kabela
2026-04-14 10:52:49 -04:00
committed by GitHub
parent f02b3269e7
commit c08f3b2a62
9 changed files with 186 additions and 147 deletions
+113 -128
View File
@@ -16,7 +16,7 @@ import shutil
import tempfile
import time
from contextlib import contextmanager
from typing import Any
from typing import Any, NamedTuple
import numpy as np
from tqdm import tqdm
@@ -27,6 +27,82 @@ from vllm.benchmarks.lib.utils import (
)
from vllm.engine.arg_utils import EngineArgs
PERCENTAGES = [10, 25, 50, 75, 90, 99]
class MetricDesc(NamedTuple):
"""Descriptor for a metric to collect from each iteration."""
iter_key: str # key in the iteration result dict
suffix: str # result key suffix, e.g. "startup", "compilation"
display_name: str
class MetricStats(NamedTuple):
"""Aggregated statistics for a single benchmark metric."""
key: str # e.g. "cold_startup", "warm_encoder_compilation"
display_name: str
values: list[float]
avg: float
percentiles: dict[int, float]
_BASE_METRICS = [
MetricDesc("total_startup_time", "startup", "Startup time"),
MetricDesc("compilation_time", "compilation", "Compilation time"),
]
_ENCODER_METRIC = MetricDesc(
"encoder_compilation_time",
"encoder_compilation",
"Encoder compilation time",
)
def _compute_metric(
phase: str,
desc: MetricDesc,
iterations: list[dict[str, float]],
) -> MetricStats:
values = [m[desc.iter_key] for m in iterations]
arr = np.array(values)
return MetricStats(
key=f"{phase}_{desc.suffix}",
display_name=desc.display_name,
values=values,
avg=float(np.mean(arr)),
percentiles=dict(zip(PERCENTAGES, np.percentile(arr, PERCENTAGES).tolist())),
)
def _collect_phase_metrics(
phase: str,
iterations: list[dict[str, float]],
has_encoder: bool,
) -> list[MetricStats]:
metrics = [_compute_metric(phase, desc, iterations) for desc in _BASE_METRICS]
if has_encoder:
metrics.append(_compute_metric(phase, _ENCODER_METRIC, iterations))
return metrics
def _print_phase(phase_name: str, metrics: list[MetricStats]) -> None:
print(f"\n{phase_name}:")
for m in metrics:
print(f"Avg {m.display_name.lower()}: {m.avg:.2f} seconds")
for m in metrics:
print(f"{m.display_name} percentiles:")
for pct, val in m.percentiles.items():
print(f" {pct}%: {val:.2f} seconds")
def _metric_to_json(m: MetricStats) -> dict[str, Any]:
return {
f"avg_{m.key}_time": m.avg,
f"{m.key}_times": m.values,
f"{m.key}_percentiles": m.percentiles,
}
@contextmanager
def cold_startup():
@@ -72,6 +148,7 @@ def run_startup_in_subprocess(engine_args, result_queue):
# Extract compilation time if available
compilation_time = 0.0
encoder_compilation_time = 0.0
if hasattr(llm.llm_engine, "vllm_config"):
vllm_config = llm.llm_engine.vllm_config
if (
@@ -79,11 +156,15 @@ def run_startup_in_subprocess(engine_args, result_queue):
and vllm_config.compilation_config is not None
):
compilation_time = vllm_config.compilation_config.compilation_time
encoder_compilation_time = (
vllm_config.compilation_config.encoder_compilation_time
)
result_queue.put(
{
"total_startup_time": total_startup_time,
"compilation_time": compilation_time,
"encoder_compilation_time": encoder_compilation_time,
}
)
@@ -93,65 +174,20 @@ def run_startup_in_subprocess(engine_args, result_queue):
def save_to_pytorch_benchmark_format(
args: argparse.Namespace, results: dict[str, Any]
args: argparse.Namespace, metrics: list[MetricStats]
) -> None:
base_name = os.path.splitext(args.output_json)[0]
cold_startup_records = convert_to_pytorch_benchmark_format(
args=args,
metrics={
"avg_cold_startup_time": [results["avg_cold_startup_time"]],
},
extra_info={
"cold_startup_times": results["cold_startup_times"],
"cold_startup_percentiles": results["cold_startup_percentiles"],
},
)
if cold_startup_records:
write_to_json(f"{base_name}.cold_startup.pytorch.json", cold_startup_records)
cold_compilation_records = convert_to_pytorch_benchmark_format(
args=args,
metrics={
"avg_cold_compilation_time": [results["avg_cold_compilation_time"]],
},
extra_info={
"cold_compilation_times": results["cold_compilation_times"],
"cold_compilation_percentiles": results["cold_compilation_percentiles"],
},
)
if cold_compilation_records:
write_to_json(
f"{base_name}.cold_compilation.pytorch.json", cold_compilation_records
)
warm_startup_records = convert_to_pytorch_benchmark_format(
args=args,
metrics={
"avg_warm_startup_time": [results["avg_warm_startup_time"]],
},
extra_info={
"warm_startup_times": results["warm_startup_times"],
"warm_startup_percentiles": results["warm_startup_percentiles"],
},
)
if warm_startup_records:
write_to_json(f"{base_name}.warm_startup.pytorch.json", warm_startup_records)
warm_compilation_records = convert_to_pytorch_benchmark_format(
args=args,
metrics={
"avg_warm_compilation_time": [results["avg_warm_compilation_time"]],
},
extra_info={
"warm_compilation_times": results["warm_compilation_times"],
"warm_compilation_percentiles": results["warm_compilation_percentiles"],
},
)
if warm_compilation_records:
write_to_json(
f"{base_name}.warm_compilation.pytorch.json", warm_compilation_records
for m in metrics:
records = convert_to_pytorch_benchmark_format(
args=args,
metrics={f"avg_{m.key}_time": [m.avg]},
extra_info={
f"{m.key}_times": m.values,
f"{m.key}_percentiles": m.percentiles,
},
)
if records:
write_to_json(f"{base_name}.{m.key}.pytorch.json", records)
def add_cli_args(parser: argparse.ArgumentParser):
@@ -224,97 +260,46 @@ def main(args: argparse.Namespace):
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
print("Setting VLLM_ENABLE_V1_MULTIPROCESSING=0 to collect startup metrics.\n")
# Collect cold startup iterations
print("Measuring cold startup time...\n")
cold_startup_times = []
cold_compilation_times = []
cold_iterations = []
for i in tqdm(range(args.num_iters_cold), desc="Cold startup iterations"):
with cold_startup():
metrics = create_llm_and_measure_startup()
cold_startup_times.append(metrics["total_startup_time"])
cold_compilation_times.append(metrics["compilation_time"])
cold_iterations.append(create_llm_and_measure_startup())
# Warmup for warm startup
print("\nWarming up for warm startup measurement...\n")
for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
create_llm_and_measure_startup()
# Collect warm startup iterations
print("\nMeasuring warm startup time...\n")
warm_startup_times = []
warm_compilation_times = []
warm_iterations = []
for i in tqdm(range(args.num_iters_warm), desc="Warm startup iterations"):
metrics = create_llm_and_measure_startup()
warm_startup_times.append(metrics["total_startup_time"])
warm_compilation_times.append(metrics["compilation_time"])
warm_iterations.append(create_llm_and_measure_startup())
# Calculate statistics
cold_startup_array = np.array(cold_startup_times)
cold_compilation_array = np.array(cold_compilation_times)
warm_startup_array = np.array(warm_startup_times)
warm_compilation_array = np.array(warm_compilation_times)
# Determine if encoder compilation occurred in any iteration
has_encoder = any(
m["encoder_compilation_time"] > 0 for m in cold_iterations + warm_iterations
)
avg_cold_startup = np.mean(cold_startup_array)
avg_cold_compilation = np.mean(cold_compilation_array)
avg_warm_startup = np.mean(warm_startup_array)
avg_warm_compilation = np.mean(warm_compilation_array)
percentages = [10, 25, 50, 75, 90, 99]
cold_startup_percentiles = np.percentile(cold_startup_array, percentages)
cold_compilation_percentiles = np.percentile(cold_compilation_array, percentages)
warm_startup_percentiles = np.percentile(warm_startup_array, percentages)
warm_compilation_percentiles = np.percentile(warm_compilation_array, percentages)
cold_metrics = _collect_phase_metrics("cold", cold_iterations, has_encoder)
warm_metrics = _collect_phase_metrics("warm", warm_iterations, has_encoder)
all_metrics = cold_metrics + warm_metrics
# Print results
print("\n" + "=" * 60)
print("STARTUP TIME BENCHMARK RESULTS")
print("=" * 60)
# Cold startup statistics
print("\nCOLD STARTUP:")
print(f"Avg total startup time: {avg_cold_startup:.2f} seconds")
print(f"Avg compilation time: {avg_cold_compilation:.2f} seconds")
print("Startup time percentiles:")
for percentage, percentile in zip(percentages, cold_startup_percentiles):
print(f" {percentage}%: {percentile:.2f} seconds")
print("Compilation time percentiles:")
for percentage, percentile in zip(percentages, cold_compilation_percentiles):
print(f" {percentage}%: {percentile:.2f} seconds")
# Warm startup statistics
print("\nWARM STARTUP:")
print(f"Avg total startup time: {avg_warm_startup:.2f} seconds")
print(f"Avg compilation time: {avg_warm_compilation:.2f} seconds")
print("Startup time percentiles:")
for percentage, percentile in zip(percentages, warm_startup_percentiles):
print(f" {percentage}%: {percentile:.2f} seconds")
print("Compilation time percentiles:")
for percentage, percentile in zip(percentages, warm_compilation_percentiles):
print(f" {percentage}%: {percentile:.2f} seconds")
_print_phase("COLD STARTUP", cold_metrics)
_print_phase("WARM STARTUP", warm_metrics)
print("=" * 60)
# Output JSON results if specified
if args.output_json:
results = {
"avg_cold_startup_time": float(avg_cold_startup),
"avg_cold_compilation_time": float(avg_cold_compilation),
"cold_startup_times": cold_startup_times,
"cold_compilation_times": cold_compilation_times,
"cold_startup_percentiles": dict(
zip(percentages, cold_startup_percentiles.tolist())
),
"cold_compilation_percentiles": dict(
zip(percentages, cold_compilation_percentiles.tolist())
),
"avg_warm_startup_time": float(avg_warm_startup),
"avg_warm_compilation_time": float(avg_warm_compilation),
"warm_startup_times": warm_startup_times,
"warm_compilation_times": warm_compilation_times,
"warm_startup_percentiles": dict(
zip(percentages, warm_startup_percentiles.tolist())
),
"warm_compilation_percentiles": dict(
zip(percentages, warm_compilation_percentiles.tolist())
),
}
results: dict[str, Any] = {}
for m in all_metrics:
results.update(_metric_to_json(m))
with open(args.output_json, "w") as f:
json.dump(results, f, indent=4)
save_to_pytorch_benchmark_format(args, results)
save_to_pytorch_benchmark_format(args, all_metrics)