feat(benchmark): add encoder forward pass benchmarking to mm-processor (#31655)
Signed-off-by: Reagan <reaganjlee@gmail.com> Signed-off-by: Reagan Lee <96998476+reaganjlee@users.noreply.github.com> Co-authored-by: Hiroken. <105287758+HirokenOvo@users.noreply.github.com>
This commit is contained in:
@@ -22,6 +22,10 @@ from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.benchmarks.datasets import (
|
||||
MultiModalConversationDataset,
|
||||
VisionArenaDataset,
|
||||
)
|
||||
from vllm.benchmarks.throughput import get_requests
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.multimodal.processing.context import (
|
||||
@@ -45,26 +49,21 @@ def collect_mm_processor_stats(
|
||||
"""
|
||||
all_stats = get_timing_stats_from_engine_client(llm_engine)
|
||||
|
||||
stats_by_stage = {
|
||||
"hf_processor_time": [],
|
||||
"hashing_time": [],
|
||||
"cache_lookup_time": [],
|
||||
"prompt_update_time": [],
|
||||
"total_time": [],
|
||||
}
|
||||
stat_keys = [
|
||||
"hf_processor_time",
|
||||
"hashing_time",
|
||||
"cache_lookup_time",
|
||||
"prompt_update_time",
|
||||
"preprocessor_total_time",
|
||||
"encoder_forward_time",
|
||||
"num_encoder_calls",
|
||||
]
|
||||
stats_by_stage = {key: [] for key in stat_keys}
|
||||
|
||||
for stats_dict in all_stats.values():
|
||||
stats_by_stage["hf_processor_time"].append(
|
||||
stats_dict.get("hf_processor_time", 0.0)
|
||||
)
|
||||
stats_by_stage["hashing_time"].append(stats_dict.get("hashing_time", 0.0))
|
||||
stats_by_stage["cache_lookup_time"].append(
|
||||
stats_dict.get("cache_lookup_time", 0.0)
|
||||
)
|
||||
stats_by_stage["prompt_update_time"].append(
|
||||
stats_dict.get("prompt_update_time", 0.0)
|
||||
)
|
||||
stats_by_stage["total_time"].append(stats_dict.get("total_time", 0.0))
|
||||
for key in stat_keys:
|
||||
if key in stats_dict:
|
||||
stats_by_stage[key].append(stats_dict[key])
|
||||
|
||||
return stats_by_stage
|
||||
|
||||
@@ -88,14 +87,14 @@ def calculate_mm_processor_metrics(
|
||||
}
|
||||
continue
|
||||
|
||||
times_ms = [t * 1000 for t in times]
|
||||
is_count_metric = stage_name == "num_encoder_calls"
|
||||
values = times if is_count_metric else [t * 1000 for t in times]
|
||||
|
||||
metrics[stage_name] = {
|
||||
"mean": float(np.mean(times_ms)),
|
||||
"median": float(np.median(times_ms)),
|
||||
"std": float(np.std(times_ms)),
|
||||
**{
|
||||
f"p{p}": float(np.percentile(times_ms, p)) for p in selected_percentiles
|
||||
},
|
||||
"mean": float(np.mean(values)),
|
||||
"median": float(np.median(values)),
|
||||
"std": float(np.std(values)),
|
||||
**{f"p{p}": float(np.percentile(values, p)) for p in selected_percentiles},
|
||||
}
|
||||
|
||||
return metrics
|
||||
@@ -114,6 +113,23 @@ def validate_args(args):
|
||||
if not hasattr(args, "max_loras"):
|
||||
args.max_loras = None
|
||||
|
||||
if args.dataset_name == "hf" and not args.dataset_path:
|
||||
raise ValueError(
|
||||
"--dataset-path is required when using --dataset-name hf. "
|
||||
"For multimodal benchmarking, specify a dataset like "
|
||||
"'lmarena-ai/VisionArena-Chat'."
|
||||
)
|
||||
if args.dataset_name == "hf":
|
||||
supported_mm_datasets = (
|
||||
VisionArenaDataset.SUPPORTED_DATASET_PATHS.keys()
|
||||
| MultiModalConversationDataset.SUPPORTED_DATASET_PATHS
|
||||
)
|
||||
if args.dataset_path not in supported_mm_datasets:
|
||||
raise ValueError(
|
||||
f"{args.dataset_path} is not a supported multimodal dataset. "
|
||||
f"Supported multimodal datasets are: {sorted(supported_mm_datasets)}"
|
||||
)
|
||||
|
||||
|
||||
def benchmark_multimodal_processor(
|
||||
args: argparse.Namespace,
|
||||
@@ -223,6 +239,17 @@ def benchmark_multimodal_processor(
|
||||
std_e2el_ms = 0.0
|
||||
percentiles_e2el_ms = [(p, 0.0) for p in selected_percentiles]
|
||||
|
||||
encoder_summary = {}
|
||||
if (
|
||||
"num_encoder_calls" in mm_stats_by_stage
|
||||
and mm_stats_by_stage["num_encoder_calls"]
|
||||
):
|
||||
encoder_calls = mm_stats_by_stage["num_encoder_calls"]
|
||||
encoder_summary = {
|
||||
"total_encoder_calls": int(sum(encoder_calls)),
|
||||
"num_requests_with_encoder_calls": len(encoder_calls),
|
||||
}
|
||||
|
||||
benchmark_result = {
|
||||
"completed": completed,
|
||||
"failed": failed,
|
||||
@@ -231,6 +258,7 @@ def benchmark_multimodal_processor(
|
||||
"std_e2el_ms": std_e2el_ms,
|
||||
"percentiles_e2el_ms": percentiles_e2el_ms,
|
||||
"mm_processor_stats": mm_processor_metrics,
|
||||
"encoder_summary": encoder_summary,
|
||||
}
|
||||
|
||||
return benchmark_result
|
||||
@@ -248,7 +276,7 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None:
|
||||
"--dataset-name",
|
||||
type=str,
|
||||
default="random-mm",
|
||||
choices=["random-mm", "random-rerank"],
|
||||
choices=["random-mm", "hf"],
|
||||
help="Name of the dataset to benchmark on. Defaults to 'random-mm'.",
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -266,6 +294,34 @@ def add_cli_args(parser: argparse.ArgumentParser) -> None:
|
||||
add_random_dataset_base_args(parser)
|
||||
add_random_multimodal_dataset_args(parser)
|
||||
|
||||
# HuggingFace dataset arguments
|
||||
parser.add_argument(
|
||||
"--dataset-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to the dataset file or HuggingFace dataset name "
|
||||
"(e.g., 'yale-nlp/MMVU', 'lmarena-ai/VisionArena-Chat').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hf-subset",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Subset of the HuggingFace dataset (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hf-split",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Split of the HuggingFace dataset (e.g., 'train', 'test', 'validation').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Output length for each request. "
|
||||
"Overrides the default output lengths from the dataset.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=str,
|
||||
@@ -296,14 +352,17 @@ def main(args: argparse.Namespace) -> None:
|
||||
print("=" * 80)
|
||||
|
||||
if "mm_processor_stats" in result:
|
||||
print("\nMM Processor Timing (ms):")
|
||||
print("\nMM Processor Metrics:")
|
||||
selected_percentiles = [
|
||||
float(p) for p in getattr(args, "metric_percentiles", "99").split(",")
|
||||
]
|
||||
mm_data = []
|
||||
for stage, metrics in result["mm_processor_stats"].items():
|
||||
is_count = stage == "num_encoder_calls"
|
||||
unit = "" if is_count else " (ms)"
|
||||
|
||||
row = {
|
||||
"Stage": stage,
|
||||
"Stage": stage + unit,
|
||||
"Mean": f"{metrics['mean']:.2f}",
|
||||
"Median": f"{metrics['median']:.2f}",
|
||||
"Std": f"{metrics['std']:.2f}",
|
||||
@@ -315,6 +374,14 @@ def main(args: argparse.Namespace) -> None:
|
||||
mm_df = pd.DataFrame(mm_data)
|
||||
print(mm_df.to_string(index=False))
|
||||
|
||||
if "encoder_summary" in result and result["encoder_summary"]:
|
||||
total_calls = result["encoder_summary"]["total_encoder_calls"]
|
||||
num_requests = result["encoder_summary"]["num_requests_with_encoder_calls"]
|
||||
print(
|
||||
f"\nSummary: {total_calls} total encoder calls "
|
||||
f"across {num_requests} requests."
|
||||
)
|
||||
|
||||
if "mean_e2el_ms" in result:
|
||||
print("\nEnd-to-End Latency (ms):")
|
||||
selected_percentiles = [
|
||||
|
||||
Reference in New Issue
Block a user