[Attention] Improve attention benchmarks: configs and profiling (#39336)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Matthew Bonanni
2026-06-12 16:24:25 -04:00
committed by GitHub
parent 39cb9bf292
commit cf567cbc71
15 changed files with 574 additions and 168 deletions
+9 -13
View File
@@ -108,7 +108,6 @@ python benchmark.py \
--backends flash triton flashinfer \
--batch-specs "q2k" "8q1s1k" "2q2k_32q1s1k" \
--num-layers 10 \
--repeats 5 \
--output-csv results.csv
```
@@ -164,14 +163,17 @@ python benchmark.py \
# Model configuration
--num-layers N # Number of layers
--head-dim N # Head dimension
--v-head-dim N # Value head dimension (defaults to --head-dim)
--num-q-heads N # Query heads
--num-kv-heads N # KV heads
--block-size N # Block size
--kv-lora-rank N # MLA KV LoRA rank
--qk-nope-head-dim N # MLA non-RoPE QK head dim
--qk-rope-head-dim N # MLA RoPE QK head dim
# Benchmark settings
--device DEVICE # Device (default: cuda:0)
--repeats N # Repetitions
--warmup-iters N # Warmup iterations
--warmup-ms N # Warmup window in ms for triton do_bench
--profile-memory # Profile memory usage
# Parameter sweeps
@@ -211,8 +213,6 @@ config = BenchmarkConfig(
num_kv_heads=1,
block_size=128,
device="cuda:0",
repeats=5,
warmup_iters=3,
)
# CUTLASS MLA with specific num_kv_splits
@@ -253,14 +253,10 @@ formatter.save_json(results, "output.json")
## Tips
**1. Warmup matters** - Use `--warmup-iters 10` for stable results
**1. Save results** - Always use `--output-csv` or `--output-json`
**2. Multiple repeats** - Use `--repeats 20` for low variance
**2. Test incrementally** - Start with `--num-layers 1`
**3. Save results** - Always use `--output-csv` or `--output-json`
**3. Extended grammar** - Leverage spec decode, chunked prefill patterns
**4. Test incrementally** - Start with `--num-layers 1 --repeats 1`
**5. Extended grammar** - Leverage spec decode, chunked prefill patterns
**6. Parameter sweeps** - Use `--sweep-param` and `--sweep-values` to find optimal values
**4. Parameter sweeps** - Use `--sweep-param` and `--sweep-values` to find optimal values
+179 -39
View File
@@ -26,6 +26,9 @@ Examples:
"""
import argparse
import os
import shutil
import subprocess
import sys
from dataclasses import replace
from pathlib import Path
@@ -83,13 +86,15 @@ def run_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
else:
return run_standard_attention_benchmark(config)
except Exception as e:
error_msg = str(e) or repr(e)
return BenchmarkResult(
config=config,
mean_time=float("inf"),
median_time=float("inf"),
std_time=0,
min_time=float("inf"),
max_time=float("inf"),
error=str(e),
error=error_msg,
)
@@ -115,9 +120,12 @@ def run_model_parameter_sweep(
"""
all_results = []
console.print(
f"[yellow]Model sweep mode: testing {sweep.param_name} = {sweep.values}[/]"
sweep_desc = (
f"{sweep.param_name} = {sweep.values}"
if sweep.param_name
else f"{len(sweep.values)} configurations"
)
console.print(f"[yellow]Model sweep mode: testing {sweep_desc}[/]")
total = len(backends) * len(batch_specs) * len(sweep.values)
@@ -125,9 +133,9 @@ def run_model_parameter_sweep(
for backend in backends:
for spec in batch_specs:
for value in sweep.values:
# Create config with modified model parameter
# Create config with modified model parameter(s)
config_args = base_config_args.copy()
config_args[sweep.param_name] = value
sweep.apply(config_args, value)
# Create config with original backend for running
clean_config = BenchmarkConfig(
@@ -144,13 +152,21 @@ def run_model_parameter_sweep(
all_results.append(result)
if not result.success:
err_label = (
f"{sweep.param_name}={value}"
if sweep.param_name
else f"{value}"
)
console.print(
f"[red]Error {backend} {spec} {sweep.param_name}="
f"{value}: {result.error}[/]"
f"[red]Error {backend} {spec} {err_label}"
f": {result.error}[/]"
)
pbar.update(1)
if base_config_args.get("ncu_profile"):
return all_results
# Display sweep results - create separate table for each parameter value
console.print("\n[bold green]Model Parameter Sweep Results:[/]")
formatter = ResultsFormatter(console)
@@ -184,7 +200,10 @@ def run_model_parameter_sweep(
)
for param_value in sorted_param_values:
console.print(f"\n[bold cyan]{sweep.param_name} = {param_value}[/]")
label = (
f"{sweep.param_name} = {param_value}" if sweep.param_name else param_value
)
console.print(f"\n[bold cyan]{label}[/]")
param_results = by_param_value[param_value]
# Create modified results with original backend names
@@ -200,8 +219,9 @@ def run_model_parameter_sweep(
formatter.print_table(modified_results, backends, compare_to_fastest=True)
# Show optimal backend for each (param_value, batch_spec) combination
sweep_name = sweep.param_name or "config"
console.print(
f"\n[bold cyan]Optimal backend for each ({sweep.param_name}, batch_spec):[/]"
f"\n[bold cyan]Optimal backend for each ({sweep_name}, batch_spec):[/]"
)
# Group by (param_value, batch_spec)
@@ -236,7 +256,10 @@ def run_model_parameter_sweep(
for param_value, spec in sorted_keys:
# Print header when param value changes
if param_value != current_param_value:
console.print(f"\n [bold]{sweep.param_name}={param_value}:[/]")
header = (
f"{sweep.param_name}={param_value}" if sweep.param_name else param_value
)
console.print(f"\n [bold]{header}:[/]")
current_param_value = param_value
results = by_param_and_spec[(param_value, spec)]
@@ -322,6 +345,9 @@ def run_parameter_sweep(
pbar.update(1)
if base_config_args.get("ncu_profile"):
return all_results
# Display sweep results
console.print("\n[bold green]Sweep Results:[/]")
backend_labels = [sweep.get_label(b, v) for b in backends for v in sweep_values]
@@ -474,11 +500,35 @@ def main():
parser.add_argument("--num-q-heads", type=int, default=32, help="Query heads")
parser.add_argument("--num-kv-heads", type=int, default=8, help="KV heads")
parser.add_argument("--block-size", type=int, default=16, help="Block size")
parser.add_argument(
"--v-head-dim",
type=int,
default=None,
help="Value head dimension (defaults to --head-dim if unset)",
)
# MLA-specific model dimensions
parser.add_argument(
"--kv-lora-rank", type=int, default=None, help="MLA KV LoRA rank"
)
parser.add_argument(
"--qk-nope-head-dim", type=int, default=None, help="MLA non-RoPE QK head dim"
)
parser.add_argument(
"--qk-rope-head-dim", type=int, default=None, help="MLA RoPE QK head dim"
)
# Benchmark settings
parser.add_argument("--device", default="cuda:0", help="Device")
parser.add_argument("--repeats", type=int, default=1, help="Repetitions")
parser.add_argument("--warmup-iters", type=int, default=3, help="Warmup iterations")
parser.add_argument(
"--warmup-ms",
type=int,
default=None,
help=(
"Warmup window in ms for triton's do_bench (default: triton's own). "
"Has no effect with CUDA graphs; pass --no-cuda-graphs to use it."
),
)
parser.add_argument("--profile-memory", action="store_true", help="Profile memory")
parser.add_argument(
"--kv-cache-dtype",
@@ -491,10 +541,33 @@ def main():
action=argparse.BooleanOptionalAction,
default=True,
help=(
"Launch kernels with CUDA graphs to eliminate CPU overhead"
"in measurements (default: True)"
"Use triton do_bench_cudagraph (True) or do_bench (False) "
"for timing. CUDA graphs eliminate CPU launch overhead "
"(default: True)"
),
)
parser.add_argument(
"--num-splits",
type=int,
default=None,
help="FlashAttention split-K factor (0=auto heuristic, 1=disabled, >1=force N)",
)
parser.add_argument(
"--ncu-profile",
action="store_true",
default=False,
help=(
"Enable Nsight Compute profiling mode. Automatically wraps the "
"script with ncu, capturing a profile with source correlation. "
"Use --ncu-output to set the output file name."
),
)
parser.add_argument(
"--ncu-output",
type=str,
default="profile",
help="Output file name for ncu profile (default: 'profile').",
)
# Parameter sweep (use YAML config for advanced sweeps)
parser.add_argument(
@@ -576,23 +649,28 @@ def main():
model = yaml_config["model"]
args.num_layers = model.get("num_layers", args.num_layers)
args.head_dim = model.get("head_dim", args.head_dim)
args.v_head_dim = model.get("v_head_dim", args.v_head_dim)
args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
args.block_size = model.get("block_size", args.block_size)
# MLA-specific dimensions
args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
args.qk_rope_head_dim = model.get("qk_rope_head_dim", args.qk_rope_head_dim)
# Benchmark settings (top-level keys)
if "device" in yaml_config:
args.device = yaml_config["device"]
if "repeats" in yaml_config:
args.repeats = yaml_config["repeats"]
if "warmup_iters" in yaml_config:
args.warmup_iters = yaml_config["warmup_iters"]
if "warmup_ms" in yaml_config:
args.warmup_ms = yaml_config["warmup_ms"]
if "profile_memory" in yaml_config:
args.profile_memory = yaml_config["profile_memory"]
if "kv_cache_dtype" in yaml_config:
args.kv_cache_dtype = yaml_config["kv_cache_dtype"]
if "cuda_graphs" in yaml_config:
args.cuda_graphs = yaml_config["cuda_graphs"]
if "ncu_profile" in yaml_config:
args.ncu_profile = yaml_config["ncu_profile"]
# Parameter sweep configuration
if "parameter_sweep" in yaml_config:
@@ -612,7 +690,7 @@ def main():
if "model_parameter_sweep" in yaml_config:
sweep_config = yaml_config["model_parameter_sweep"]
args.model_parameter_sweep = ModelParameterSweep(
param_name=sweep_config["param_name"],
param_name=sweep_config.get("param_name"),
values=sweep_config["values"],
label_format=sweep_config.get(
"label_format", "{backend}_{param_name}_{value}"
@@ -631,6 +709,32 @@ def main():
console.print()
# Re-exec under ncu if --ncu-profile and not already inside ncu. This runs
# after YAML processing so ncu_profile set via config file is honored.
if args.ncu_profile and "_NCU_INNER" not in os.environ:
ncu = shutil.which("ncu")
if ncu is None:
print("Error: 'ncu' not found in PATH", file=sys.stderr)
sys.exit(1)
cmd = [
ncu,
"--profile-from-start",
"off",
"--set",
"full",
"--import-source",
"yes",
"-o",
args.ncu_output,
sys.executable,
*sys.argv,
]
env = os.environ.copy()
env["CUTE_DSL_LINEINFO"] = "1"
env["_NCU_INNER"] = "1"
print(f"Launching: {' '.join(cmd)}")
sys.exit(subprocess.call(cmd, env=env))
# Handle CLI-based parameter sweep (if not from YAML)
if (
(not hasattr(args, "parameter_sweep") or args.parameter_sweep is None)
@@ -655,6 +759,18 @@ def main():
console.print(f"Batch specs: {', '.join(args.batch_specs)}")
console.print(f"KV cache dtype: {args.kv_cache_dtype}")
console.print(f"CUDA graphs: {args.cuda_graphs}")
if args.warmup_ms is not None and args.cuda_graphs:
console.print(
"[yellow]Warning: --warmup-ms is ignored with CUDA graphs "
"(do_bench_cudagraph warms up internally). Pass --no-cuda-graphs "
"to use it.[/]"
)
if args.num_splits == 0 and args.cuda_graphs:
console.print(
"[yellow]Warning: --num-splits 0 (FA3 heuristic) is not CUDA-graph "
"compatible and may fail or fall back. Pass --no-cuda-graphs or use "
"--num-splits >=1.[/]"
)
console.print()
init_workspace_manager(args.device)
@@ -662,6 +778,15 @@ def main():
# Run benchmarks
all_results = []
# Under ncu profiling the kernels run only to be captured by the profiler;
# timings are placeholder zeros, so the result tables and saved metrics are
# skipped. The Nsight Compute report (--ncu-output) holds the real data.
if args.ncu_profile:
console.print(
"[dim]ncu profiling enabled: result tables and saved metrics are "
"skipped (timings are placeholder zeros).[/]"
)
# Handle special mode: decode_vs_prefill comparison
if hasattr(args, "mode") and args.mode == "decode_vs_prefill":
console.print("[yellow]Mode: Decode vs Prefill pipeline comparison[/]")
@@ -708,11 +833,11 @@ def main():
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
repeats=args.repeats,
warmup_iters=args.warmup_iters,
profile_memory=args.profile_memory,
kv_cache_dtype=args.kv_cache_dtype,
use_cuda_graphs=args.cuda_graphs,
ncu_profile=args.ncu_profile,
warmup_ms=args.warmup_ms,
)
# Add decode pipeline config
@@ -749,6 +874,7 @@ def main():
result = BenchmarkResult(
config=config,
mean_time=timing["mean"],
median_time=timing.get("median", timing["mean"]),
std_time=timing["std"],
min_time=timing["min"],
max_time=timing["max"],
@@ -770,6 +896,7 @@ def main():
result = BenchmarkResult(
config=config,
mean_time=float("inf"),
median_time=float("inf"),
std_time=0,
min_time=float("inf"),
max_time=float("inf"),
@@ -779,6 +906,9 @@ def main():
pbar.update(1)
if args.ncu_profile:
return
# Display decode vs prefill results
console.print("\n[bold green]Decode vs Prefill Results:[/]")
@@ -858,15 +988,20 @@ def main():
base_config_args = {
"num_layers": args.num_layers,
"head_dim": args.head_dim,
"v_head_dim": args.v_head_dim,
"num_q_heads": args.num_q_heads,
"num_kv_heads": args.num_kv_heads,
"block_size": args.block_size,
"device": args.device,
"repeats": args.repeats,
"warmup_iters": args.warmup_iters,
"profile_memory": args.profile_memory,
"kv_cache_dtype": args.kv_cache_dtype,
"use_cuda_graphs": args.cuda_graphs,
"ncu_profile": args.ncu_profile,
"warmup_ms": args.warmup_ms,
"num_splits": args.num_splits,
"kv_lora_rank": args.kv_lora_rank,
"qk_nope_head_dim": args.qk_nope_head_dim,
"qk_rope_head_dim": args.qk_rope_head_dim,
}
all_results = run_model_parameter_sweep(
backends,
@@ -882,15 +1017,17 @@ def main():
base_config_args = {
"num_layers": args.num_layers,
"head_dim": args.head_dim,
"v_head_dim": args.v_head_dim,
"num_q_heads": args.num_q_heads,
"num_kv_heads": args.num_kv_heads,
"block_size": args.block_size,
"device": args.device,
"repeats": args.repeats,
"warmup_iters": args.warmup_iters,
"profile_memory": args.profile_memory,
"kv_cache_dtype": args.kv_cache_dtype,
"use_cuda_graphs": args.cuda_graphs,
"ncu_profile": args.ncu_profile,
"warmup_ms": args.warmup_ms,
"num_splits": args.num_splits,
}
all_results = run_parameter_sweep(
backends, args.batch_specs, base_config_args, args.parameter_sweep, console
@@ -914,15 +1051,17 @@ def main():
batch_spec=spec,
num_layers=args.num_layers,
head_dim=args.head_dim,
v_head_dim=getattr(args, "v_head_dim", None),
num_q_heads=args.num_q_heads,
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
repeats=args.repeats,
warmup_iters=args.warmup_iters,
profile_memory=args.profile_memory,
kv_cache_dtype=args.kv_cache_dtype,
use_cuda_graphs=args.cuda_graphs,
ncu_profile=args.ncu_profile,
warmup_ms=args.warmup_ms,
num_splits=args.num_splits,
)
result = run_benchmark(config)
@@ -935,9 +1074,10 @@ def main():
pbar.update(1)
console.print("\n[bold green]Results:[/]")
formatter = ResultsFormatter(console)
formatter.print_table(decode_results, backends)
if not args.ncu_profile:
console.print("\n[bold green]Results:[/]")
formatter = ResultsFormatter(console)
formatter.print_table(decode_results, backends)
# Run prefill backend comparison
if prefill_backends:
@@ -962,9 +1102,8 @@ def main():
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
repeats=args.repeats,
warmup_iters=args.warmup_iters,
profile_memory=args.profile_memory,
warmup_ms=args.warmup_ms,
prefill_backend=pb,
)
@@ -980,16 +1119,17 @@ def main():
pbar.update(1)
console.print("\n[bold green]Prefill Backend Results:[/]")
formatter = ResultsFormatter(console)
formatter.print_table(
prefill_results, prefill_backends, compare_to_fastest=True
)
if not args.ncu_profile:
console.print("\n[bold green]Prefill Backend Results:[/]")
formatter = ResultsFormatter(console)
formatter.print_table(
prefill_results, prefill_backends, compare_to_fastest=True
)
all_results = decode_results + prefill_results
# Save results
if all_results:
# Save results (skip ncu profiling runs: timings are placeholder zeros)
if all_results and not args.ncu_profile:
formatter = ResultsFormatter(console)
if args.output_csv:
formatter.save_csv(all_results, args.output_csv)
+54 -6
View File
@@ -15,6 +15,8 @@ from batch_spec import get_batch_type, parse_batch_spec
from rich.console import Console
from rich.table import Table
from vllm.triton_utils import triton
def batch_spec_sort_key(spec: str) -> tuple[int, int, int]:
"""
@@ -34,6 +36,30 @@ def batch_spec_sort_key(spec: str) -> tuple[int, int, int]:
return (0, 0, 0)
def run_do_bench(
benchmark_fn,
use_cuda_graphs: bool,
warmup_ms: int | None = None,
) -> list[float]:
kwargs: dict[str, Any] = {"return_mode": "all"}
if use_cuda_graphs:
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
else:
if warmup_ms is not None:
kwargs["warmup"] = warmup_ms
result = triton.testing.do_bench(benchmark_fn, **kwargs)
return result
def run_ncu_profile(benchmark_fn) -> None:
benchmark_fn()
torch.accelerator.synchronize()
torch.cuda.cudart().cudaProfilerStart()
benchmark_fn()
torch.accelerator.synchronize()
torch.cuda.cudart().cudaProfilerStop()
# Mock classes for vLLM attention infrastructure
@@ -182,18 +208,37 @@ class ParameterSweep:
@dataclass
class ModelParameterSweep:
"""Configuration for sweeping a model configuration parameter."""
"""Configuration for sweeping model configuration parameter(s).
param_name: str # Name of the model config parameter to sweep (e.g., "num_q_heads")
values: list[Any] # List of values to test
label_format: str = "{backend}_{param_name}_{value}" # Result label template
Supports two modes:
- Single param: param_name="head_dim", values=[128, 256, 512]
- Multi param: values=[{head_dim: 192, v_head_dim: 128}, {head_dim: 256}]
When values are dicts, each dict's keys are applied as config overrides.
"""
param_name: str | None = None
values: list[Any] | None = None
label_format: str = "{backend}_{param_name}_{value}"
def get_label(self, backend: str, value: Any) -> str:
"""Generate a label for a specific parameter value."""
if isinstance(value, dict):
return self.label_format.format(
backend=backend, param_name=self.param_name, value=value, **value
)
return self.label_format.format(
backend=backend, param_name=self.param_name, value=value
)
def apply(self, config_args: dict, value: Any) -> None:
"""Apply a sweep value to config args."""
if isinstance(value, dict):
config_args.update(value)
elif self.param_name is not None:
config_args[self.param_name] = value
else:
raise ValueError("param_name must be set if sweep values are not dicts")
@dataclass
class BenchmarkConfig:
@@ -208,10 +253,10 @@ class BenchmarkConfig:
block_size: int
device: str
dtype: torch.dtype = torch.float16
repeats: int = 1
warmup_iters: int = 3
profile_memory: bool = False
use_cuda_graphs: bool = False
ncu_profile: bool = False
warmup_ms: int | None = None
# "auto" or "fp8"
kv_cache_dtype: str = "auto"
@@ -226,6 +271,7 @@ class BenchmarkConfig:
# Backend-specific tuning
num_kv_splits: int | None = None # CUTLASS MLA
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
@dataclass
@@ -234,6 +280,7 @@ class BenchmarkResult:
config: BenchmarkConfig
mean_time: float # seconds
median_time: float # seconds
std_time: float # seconds
min_time: float # seconds
max_time: float # seconds
@@ -252,6 +299,7 @@ class BenchmarkResult:
return {
"config": asdict(self.config),
"mean_time": self.mean_time,
"median_time": self.median_time,
"std_time": self.std_time,
"min_time": self.min_time,
"max_time": self.max_time,
@@ -56,8 +56,6 @@ backends:
- TOKENSPEED_MLA # Blackwell + R1 dims + FP8 KV (use --kv-cache-dtype fp8)
device: "cuda:0"
repeats: 100
warmup_iters: 10
profile_memory: true
# Backend-specific tuning
@@ -51,8 +51,6 @@ backends:
- FLASHMLA # Hopper only
device: "cuda:0"
repeats: 5
warmup_iters: 3
profile_memory: true
# Analyze chunked prefill workspace size impact
@@ -124,5 +124,3 @@ prefill_backends:
- tokenspeed
device: "cuda:0"
repeats: 20
warmup_iters: 5
@@ -53,6 +53,4 @@ backends:
- FLASHINFER_MLA_SPARSE
device: "cuda:0"
repeats: 100
warmup_iters: 10
profile_memory: true
@@ -57,6 +57,4 @@ backends:
- FLASHINFER_MLA_SPARSE
device: "cuda:0"
repeats: 10
warmup_iters: 3
profile_memory: true
@@ -63,8 +63,6 @@ model:
# Benchmark settings
device: "cuda:0"
repeats: 15 # More repeats for spec decode variance
warmup_iters: 5
profile_memory: false
# Output
@@ -49,8 +49,6 @@ backends:
# Benchmark settings
device: "cuda:0"
repeats: 10 # More repeats for statistical significance
warmup_iters: 5
profile_memory: false
# Test these threshold values for optimization
@@ -43,6 +43,4 @@ backends:
- FLASHINFER
device: "cuda:0"
repeats: 5
warmup_iters: 3
profile_memory: false
@@ -0,0 +1,142 @@
# Standard attention decode benchmark configuration
# Sweeps num_q_heads and num_kv_heads to isolate effects of:
# 1. GQA ratio (fixed num_q_heads=32, vary num_kv_heads)
# 2. Absolute head count (fixed 4:1 ratio, vary scale)
model:
num_layers: 32
num_q_heads: 32 # Base value, overridden by sweep
num_kv_heads: 8 # Base value, overridden by sweep
head_dim: 128
block_size: 16
# Head count sweep: each entry overrides num_q_heads, num_kv_heads, and
# head_dim where it differs from the base (128). Head counts are per-GPU
# (i.e. after TP sharding).
#
# Group A — vary GQA ratio (fixed q=32, head_dim=128):
# 32:32 (MHA), 32:8 (GQA 4:1), 32:4 (GQA 8:1), 32:1 (MQA)
#
# Groups B-E — real model configs at various TP degrees:
# Model head_dim Full TP2 TP4 TP8
# Llama 3 8B 128 32:8 16:4 8:2 4:1
# Llama 3 70B 128 64:8 32:4 16:2 8:1
# GPT-OSS 120B 64 64:8 32:4 16:2 8:1
# Llama 3 405B 128 128:8 64:4 32:2 16:1
model_parameter_sweep:
values:
# --- head_dim=128 (Llama 3 family) ---
- { num_q_heads: 32, num_kv_heads: 32, head_dim: 128 } # MHA 1:1
- { num_q_heads: 32, num_kv_heads: 1, head_dim: 128 } # MQA 32:1
- { num_q_heads: 4, num_kv_heads: 1, head_dim: 128 } # Llama 3 8B TP8
- { num_q_heads: 8, num_kv_heads: 2, head_dim: 128 } # Llama 3 8B TP4
- { num_q_heads: 16, num_kv_heads: 4, head_dim: 128 } # Llama 3 8B TP2
- { num_q_heads: 32, num_kv_heads: 8, head_dim: 128 } # Llama 3 8B TP1 / GQA 4:1
- { num_q_heads: 8, num_kv_heads: 1, head_dim: 128 } # Llama 3 70B TP8
- { num_q_heads: 16, num_kv_heads: 2, head_dim: 128 } # Llama 3 70B TP4
- { num_q_heads: 32, num_kv_heads: 4, head_dim: 128 } # Llama 3 70B TP2 / GQA 8:1
- { num_q_heads: 64, num_kv_heads: 8, head_dim: 128 } # Llama 3 70B TP1
- { num_q_heads: 16, num_kv_heads: 1, head_dim: 128 } # Llama 3 405B TP8
- { num_q_heads: 32, num_kv_heads: 2, head_dim: 128 } # Llama 3 405B TP4
- { num_q_heads: 64, num_kv_heads: 4, head_dim: 128 } # Llama 3 405B TP2
- { num_q_heads: 128, num_kv_heads: 8, head_dim: 128 } # Llama 3 405B TP1
# --- head_dim=64 (GPT-OSS 120B) ---
- { num_q_heads: 8, num_kv_heads: 1, head_dim: 64 } # GPT-OSS 120B TP8
- { num_q_heads: 16, num_kv_heads: 2, head_dim: 64 } # GPT-OSS 120B TP4
- { num_q_heads: 32, num_kv_heads: 4, head_dim: 64 } # GPT-OSS 120B TP2
- { num_q_heads: 64, num_kv_heads: 8, head_dim: 64 } # GPT-OSS 120B TP1
label_format: "{backend}_q{num_q_heads}kv{num_kv_heads}d{head_dim}"
batch_specs:
# ---- batch_size x seq_len grid (decode: q_len=1) ----
# Small grid for quick iteration. Uncomment for full sweep.
# Batch size 1
- "q1s1k"
- "q1s512"
- "q1s2k"
- "q1s4k"
- "q1s8k"
- "q1s16k"
- "q1s32k"
# Batch size 2
- "2q1s512"
- "2q1s1k"
- "2q1s2k"
- "2q1s4k"
- "2q1s8k"
- "2q1s16k"
- "2q1s32k"
# Batch size 4
- "4q1s512"
- "4q1s1k"
- "4q1s2k"
- "4q1s4k"
- "4q1s8k"
- "4q1s16k"
- "4q1s32k"
# Batch size 8
- "8q1s1k"
- "8q1s512"
- "8q1s2k"
- "8q1s4k"
- "8q1s8k"
- "8q1s16k"
- "8q1s32k"
# Batch size 16
- "16q1s512"
- "16q1s1k"
- "16q1s2k"
- "16q1s4k"
- "16q1s8k"
- "16q1s16k"
- "16q1s32k"
# Batch size 32
- "32q1s512"
- "32q1s1k"
- "32q1s2k"
- "32q1s4k"
- "32q1s8k"
- "32q1s16k"
- "32q1s32k"
# Batch size 64
- "64q1s1k"
- "64q1s512"
- "64q1s2k"
- "64q1s4k"
- "64q1s8k"
- "64q1s16k"
- "64q1s32k"
# Batch size 128
- "128q1s512"
- "128q1s1k"
- "128q1s2k"
- "128q1s4k"
- "128q1s8k"
- "128q1s16k"
- "128q1s32k"
# Batch size 256
- "256q1s1k"
- "256q1s512"
- "256q1s2k"
- "256q1s4k"
- "256q1s8k"
- "256q1s16k"
- "256q1s32k"
# Available backends: FLASH_ATTN, TRITON_ATTN, FLASHINFER
backends:
- FLASH_ATTN
- TRITON_ATTN
- FLASHINFER
device: "cuda:0"
profile_memory: false
@@ -0,0 +1,108 @@
# Standard attention prefill benchmark configuration
# Sweeps num_q_heads and num_kv_heads to isolate effects of:
# 1. GQA ratio (fixed num_q_heads=32, vary num_kv_heads)
# 2. Absolute head count (fixed 4:1 ratio, vary scale)
model:
num_layers: 32
num_q_heads: 32 # Base value, overridden by sweep
num_kv_heads: 8 # Base value, overridden by sweep
head_dim: 128
block_size: 16
# Head count sweep: each entry overrides num_q_heads, num_kv_heads, and
# head_dim where it differs from the base (128). Head counts are per-GPU
# (i.e. after TP sharding).
#
# Group A — vary GQA ratio (fixed q=32, head_dim=128):
# 32:32 (MHA), 32:8 (GQA 4:1), 32:4 (GQA 8:1), 32:1 (MQA)
#
# Groups B-E — real model configs at various TP degrees:
# Model head_dim Full TP2 TP4 TP8
# Llama 3 8B 128 32:8 16:4 8:2 4:1
# Llama 3 70B 128 64:8 32:4 16:2 8:1
# GPT-OSS 120B 64 64:8 32:4 16:2 8:1
# Llama 3 405B 128 128:8 64:4 32:2 16:1
model_parameter_sweep:
values:
# --- head_dim=128 (Llama 3 family) ---
- { num_q_heads: 32, num_kv_heads: 32, head_dim: 128 } # MHA 1:1
- { num_q_heads: 32, num_kv_heads: 1, head_dim: 128 } # MQA 32:1
- { num_q_heads: 4, num_kv_heads: 1, head_dim: 128 } # Llama 3 8B TP8
- { num_q_heads: 8, num_kv_heads: 2, head_dim: 128 } # Llama 3 8B TP4
- { num_q_heads: 16, num_kv_heads: 4, head_dim: 128 } # Llama 3 8B TP2
- { num_q_heads: 32, num_kv_heads: 8, head_dim: 128 } # Llama 3 8B TP1 / GQA 4:1
- { num_q_heads: 8, num_kv_heads: 1, head_dim: 128 } # Llama 3 70B TP8
- { num_q_heads: 16, num_kv_heads: 2, head_dim: 128 } # Llama 3 70B TP4
- { num_q_heads: 32, num_kv_heads: 4, head_dim: 128 } # Llama 3 70B TP2 / GQA 8:1
- { num_q_heads: 64, num_kv_heads: 8, head_dim: 128 } # Llama 3 70B TP1
- { num_q_heads: 16, num_kv_heads: 1, head_dim: 128 } # Llama 3 405B TP8
- { num_q_heads: 32, num_kv_heads: 2, head_dim: 128 } # Llama 3 405B TP4
- { num_q_heads: 64, num_kv_heads: 4, head_dim: 128 } # Llama 3 405B TP2
- { num_q_heads: 128, num_kv_heads: 8, head_dim: 128 } # Llama 3 405B TP1
# --- head_dim=64 (GPT-OSS 120B) ---
- { num_q_heads: 8, num_kv_heads: 1, head_dim: 64 } # GPT-OSS 120B TP8
- { num_q_heads: 16, num_kv_heads: 2, head_dim: 64 } # GPT-OSS 120B TP4
- { num_q_heads: 32, num_kv_heads: 4, head_dim: 64 } # GPT-OSS 120B TP2
- { num_q_heads: 64, num_kv_heads: 8, head_dim: 64 } # GPT-OSS 120B TP1
label_format: "{backend}_q{num_q_heads}kv{num_kv_heads}d{head_dim}"
batch_specs:
# ---- batch_size x prefill_len grid (prefill: q_len == seq_len) ----
# Total tokens = batch_size * prefill_len, and prefill compute scales with
# prefill_len^2, so the largest cells are expensive. Trim batch sizes or
# lengths for quick iteration.
# Batch size 1
- "q512"
- "q1k"
- "q2k"
- "q4k"
- "q8k"
- "q16k"
- "q32k"
# Batch size 2
- "2q512"
- "2q1k"
- "2q2k"
- "2q4k"
- "2q8k"
- "2q16k"
- "2q32k"
# Batch size 4
- "4q512"
- "4q1k"
- "4q2k"
- "4q4k"
- "4q8k"
- "4q16k"
- "4q32k"
# Batch size 8
- "8q512"
- "8q1k"
- "8q2k"
- "8q4k"
- "8q8k"
- "8q16k"
- "8q32k"
# Batch size 16
- "16q512"
- "16q1k"
- "16q2k"
- "16q4k"
- "16q8k"
- "16q16k"
- "16q32k"
# Available backends: FLASH_ATTN, TRITON_ATTN, FLASHINFER
backends:
- FLASH_ATTN
- TRITON_ATTN
- FLASHINFER
device: "cuda:0"
profile_memory: false
+28 -33
View File
@@ -8,6 +8,8 @@ This module provides helpers for running MLA backends without
needing full VllmConfig integration.
"""
import statistics
import numpy as np
import torch
from batch_spec import parse_batch_spec
@@ -17,6 +19,8 @@ from common import (
MockIndexer,
MockKVBProj,
MockLayer,
run_do_bench,
run_ncu_profile,
setup_mla_dims,
)
@@ -820,7 +824,7 @@ def _run_single_benchmark(
num_prefill, mla_dims, query_fmt, device, torch.bfloat16
)
# Build forward function
# Build forward function (runs a single decode/prefill pass)
def forward_fn():
results = []
if has_decode:
@@ -839,44 +843,35 @@ def _run_single_benchmark(
)
return results[0] if len(results) == 1 else tuple(results)
# Warmup
for _ in range(config.warmup_iters):
forward_fn()
torch.accelerator.synchronize()
# Optionally capture a CUDA graph after warmup.
# Graph replay eliminates CPU launch overhead so timings reflect pure
# kernel time.
if config.use_cuda_graphs:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
forward_fn()
benchmark_fn = graph.replay
else:
benchmark_fn = forward_fn
# Benchmark
times = []
for _ in range(config.repeats):
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
def benchmark_fn():
for _ in range(config.num_layers):
benchmark_fn()
end.record()
forward_fn()
torch.accelerator.synchronize()
elapsed_ms = start.elapsed_time(end)
times.append(elapsed_ms / 1000.0 / config.num_layers)
if config.ncu_profile:
run_ncu_profile(benchmark_fn)
return BenchmarkResult(
config=config,
mean_time=0.0,
median_time=0.0,
std_time=0.0,
min_time=0.0,
max_time=0.0,
throughput_tokens_per_sec=0.0,
)
all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
# Convert ms to seconds per layer
times = [t / 1000.0 / config.num_layers for t in all_ms]
mean_time = statistics.mean(times)
mean_time = float(np.mean(times))
return BenchmarkResult(
config=config,
mean_time=mean_time,
std_time=float(np.std(times)),
min_time=float(np.min(times)),
max_time=float(np.max(times)),
median_time=statistics.median(times),
std_time=statistics.stdev(times) if len(times) > 1 else 0.0,
min_time=min(times),
max_time=max(times),
throughput_tokens_per_sec=total_q / mean_time if mean_time > 0 else 0,
)
+54 -61
View File
@@ -9,13 +9,20 @@ This module provides helpers for running standard attention backends
"""
import logging
import statistics
import types
from contextlib import contextmanager
import numpy as np
import torch
from batch_spec import parse_batch_spec, reorder_for_flashinfer
from common import BenchmarkConfig, BenchmarkResult, MockLayer, get_attention_scale
from common import (
BenchmarkConfig,
BenchmarkResult,
MockLayer,
get_attention_scale,
run_do_bench,
run_ncu_profile,
)
from vllm.config import (
CacheConfig,
@@ -208,6 +215,13 @@ def _create_backend_impl(
scale = get_attention_scale(config.head_dim)
# Set v_head_dim for diff-headdim backends. Always reset (defaulting to
# head_dim) so a prior run's value doesn't leak into this one via the
# backend's class-level state.
if hasattr(backend_class, "set_head_size_v"):
v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim
backend_class.set_head_size_v(v_dim)
impl = backend_class.get_impl_cls()(
num_heads=config.num_q_heads,
head_size=config.head_dim,
@@ -300,6 +314,7 @@ def _create_input_tensors(
from vllm.platforms import current_platform
q_dtype = current_platform.fp8_dtype()
v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim
q_list = [
torch.randn(
total_q, config.num_q_heads, config.head_dim, device=device, dtype=dtype
@@ -313,9 +328,7 @@ def _create_input_tensors(
for _ in range(config.num_layers)
]
v_list = [
torch.randn(
total_q, config.num_kv_heads, config.head_dim, device=device, dtype=dtype
)
torch.randn(total_q, config.num_kv_heads, v_dim, device=device, dtype=dtype)
for _ in range(config.num_layers)
]
return q_list, k_list, v_list
@@ -389,14 +402,17 @@ def _run_single_benchmark(
device: torch.device,
dtype: torch.dtype,
) -> tuple:
"""Run single benchmark iteration with warmup and timing loop."""
total_q = q_list[0].shape[0]
out = torch.empty(
total_q, config.num_q_heads, config.head_dim, device=device, dtype=dtype
)
"""Run single benchmark using triton's do_bench_cudagraph/do_bench.
# Warmup
for _ in range(config.warmup_iters):
Returns:
(timing_stats, mem_stats) where timing_stats is a dict with
mean/std/min/max in seconds per layer.
"""
total_q = q_list[0].shape[0]
v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim
out = torch.empty(total_q, config.num_q_heads, v_dim, device=device, dtype=dtype)
def benchmark_fn():
for i in range(config.num_layers):
impl.forward(
layer,
@@ -407,52 +423,22 @@ def _run_single_benchmark(
attn_metadata,
output=out,
)
torch.accelerator.synchronize()
# Optionally capture a CUDA graph after warmup.
# Graph replay eliminates CPU launch overhead so timings reflect pure
# kernel time.
if config.use_cuda_graphs:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
for i in range(config.num_layers):
impl.forward(
layer,
q_list[i],
k_list[i],
v_list[i],
cache_list[i],
attn_metadata,
output=out,
)
benchmark_fn = graph.replay
if config.ncu_profile:
run_ncu_profile(benchmark_fn)
timing_stats = dict.fromkeys(("mean", "median", "std", "min", "max"), 0.0)
else:
all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
def benchmark_fn():
for i in range(config.num_layers):
impl.forward(
layer,
q_list[i],
k_list[i],
v_list[i],
cache_list[i],
attn_metadata,
output=out,
)
# Benchmark
times = []
for _ in range(config.repeats):
start = torch.cuda.Event(enable_timing=True)
end = torch.cuda.Event(enable_timing=True)
start.record()
benchmark_fn()
end.record()
torch.accelerator.synchronize()
elapsed_ms = start.elapsed_time(end)
times.append(elapsed_ms / 1000.0 / config.num_layers) # seconds per layer
# Convert ms to seconds per layer
times = [t / 1000.0 / config.num_layers for t in all_ms]
timing_stats = {
"mean": statistics.mean(times),
"std": statistics.stdev(times) if len(times) > 1 else 0.0,
"min": min(times),
"max": max(times),
"median": statistics.median(times),
}
mem_stats = {}
if config.profile_memory:
@@ -461,7 +447,7 @@ def _run_single_benchmark(
"reserved_mb": torch.accelerator.memory_reserved(device) / 1024**2,
}
return times, mem_stats
return timing_stats, mem_stats
# ============================================================================
@@ -541,6 +527,12 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
common_attn_metadata=common_metadata,
)
# Override num_splits for split-K testing (FlashAttention only)
if config.num_splits is not None and hasattr(
attn_metadata, "max_num_splits"
):
attn_metadata.max_num_splits = config.num_splits
# Only quantize queries when the impl supports it
quantize_query = config.kv_cache_dtype.startswith("fp8") and getattr(
impl, "supports_quant_query_input", False
@@ -553,7 +545,7 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
config, max_num_blocks, backend_class, device, dtype
)
times, mem_stats = _run_single_benchmark(
timing_stats, mem_stats = _run_single_benchmark(
config,
impl,
layer,
@@ -566,15 +558,16 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
dtype,
)
mean_time = np.mean(times)
mean_time = timing_stats["mean"]
throughput = total_q / mean_time if mean_time > 0 else 0
return BenchmarkResult(
config=config,
mean_time=mean_time,
std_time=np.std(times),
min_time=np.min(times),
max_time=np.max(times),
median_time=timing_stats["median"],
std_time=timing_stats["std"],
min_time=timing_stats["min"],
max_time=timing_stats["max"],
throughput_tokens_per_sec=throughput,
memory_allocated_mb=mem_stats.get("allocated_mb"),
memory_reserved_mb=mem_stats.get("reserved_mb"),