[1/N] Add dense MHA path for sparse MLA short sequences (#47327)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-07-14 00:29:56 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 0b0ef8d7eb
commit c4f5cd60da
21 changed files with 1990 additions and 646 deletions
+175 -3
View File
@@ -75,7 +75,11 @@ def run_mla_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
from mla_runner import run_mla_benchmark as run_mla
return run_mla(
config.backend, config, prefill_backend=config.prefill_backend, **kwargs
config.backend,
config,
prefill_backend=config.prefill_backend,
sparse_mla_force_mqa=config.sparse_mla_force_mqa,
**kwargs,
)
@@ -592,6 +596,30 @@ def main():
default="profile",
help="Output file name for ncu profile (default: 'profile').",
)
parser.add_argument(
"--torch-profile",
action="store_true",
default=False,
help="Collect a PyTorch profiler Chrome trace for each benchmark run.",
)
parser.add_argument(
"--torch-profile-dir",
default=None,
help="Directory for PyTorch profiler traces.",
)
parser.add_argument(
"--torch-profile-iters",
type=int,
default=3,
help="Number of forward passes to record per PyTorch profiler trace.",
)
parser.add_argument(
"--sparse-mla-mha-variants",
nargs="+",
default=None,
choices=["dense_mha", "mqa"],
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
)
# Parameter sweep (use YAML config for advanced sweeps)
parser.add_argument(
@@ -641,6 +669,7 @@ def main():
# Prefill backends (e.g., ["fa3", "fa4"])
args.prefill_backends = yaml_config.get("prefill_backends", None)
args.prefill_backend = yaml_config.get("prefill_backend", None)
# FP8 output benchmark knobs; CLI wins.
if args.fp8_output_scale is None:
@@ -683,6 +712,9 @@ def main():
args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
args.block_size = model.get("block_size", args.block_size)
args.max_model_len = model.get(
"max_model_len", getattr(args, "max_model_len", None)
)
# MLA-specific dimensions
args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
@@ -701,6 +733,21 @@ def main():
args.cuda_graphs = yaml_config["cuda_graphs"]
if "ncu_profile" in yaml_config:
args.ncu_profile = yaml_config["ncu_profile"]
if "torch_profile" in yaml_config:
args.torch_profile = yaml_config["torch_profile"]
if "torch_profile_dir" in yaml_config:
args.torch_profile_dir = yaml_config["torch_profile_dir"]
if "torch_profile_iters" in yaml_config:
args.torch_profile_iters = yaml_config["torch_profile_iters"]
args.sparse_mla_topk_pattern = yaml_config.get(
"sparse_mla_topk_pattern", "random"
)
args.sparse_mla_dense_mha_max_seq_len = yaml_config.get(
"sparse_mla_dense_mha_max_seq_len", None
)
args.sparse_mla_mha_variants = yaml_config.get(
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
)
# Parameter sweep configuration
if "parameter_sweep" in yaml_config:
@@ -842,8 +889,6 @@ def main():
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
repeats=args.repeats,
warmup_iters=args.warmup_iters,
profile_memory=args.profile_memory,
kv_cache_dtype=args.kv_cache_dtype,
use_cuda_graphs=args.cuda_graphs,
@@ -1063,6 +1108,133 @@ def main():
f"\n [yellow]Prefill always faster for batch_size={bs}[/]"
)
# Handle MHA vs MQA comparison mode for sparse MLA
elif hasattr(args, "mode") and args.mode == "mha_vs_mqa":
console.print("[yellow]Mode: MHA vs MQA comparison for sparse MLA[/]")
sparse_mla_topk_pattern = getattr(args, "sparse_mla_topk_pattern", "random")
dense_mha_max_seq_len = getattr(args, "sparse_mla_dense_mha_max_seq_len", None)
prefill_backend = getattr(args, "prefill_backend", None)
if prefill_backend:
console.print(f"Prefill backend: {prefill_backend}")
available_variants = [
("dense_mha", False, "dense"),
("mqa", True, "auto"),
]
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
if requested_variants is not None:
valid_variants = {label for label, _, _ in available_variants}
invalid_variants = sorted(set(requested_variants) - valid_variants)
if invalid_variants:
raise ValueError(
"Invalid sparse_mla_mha_variants entries: "
f"{invalid_variants}. Valid variants are: "
f"{sorted(valid_variants)}"
)
requested_variant_set = set(requested_variants)
variants = [
variant
for variant in available_variants
if variant[0] in requested_variant_set
]
else:
variants = available_variants
formatter = ResultsFormatter(console)
total = 0
for spec in args.batch_specs:
q_len = max(request.q_len for request in parse_batch_spec(spec))
for variant_label, _, _ in variants:
if (
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
):
continue
total += len(backends)
with tqdm(total=total, desc="Benchmarking") as pbar:
for spec in args.batch_specs:
q_len = max(request.q_len for request in parse_batch_spec(spec))
for backend in backends:
for variant_label, force_mqa, mha_mode in variants:
if (
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
):
continue
config = BenchmarkConfig(
backend=f"{backend}_{variant_label}",
batch_spec=spec,
num_layers=args.num_layers,
head_dim=args.head_dim,
num_q_heads=args.num_q_heads,
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
max_model_len=getattr(args, "max_model_len", None),
kv_cache_dtype=args.kv_cache_dtype,
profile_memory=args.profile_memory,
use_cuda_graphs=args.cuda_graphs,
ncu_profile=args.ncu_profile,
torch_profile=args.torch_profile,
torch_profile_dir=args.torch_profile_dir,
torch_profile_iters=args.torch_profile_iters,
warmup_ms=args.warmup_ms,
kv_lora_rank=getattr(args, "kv_lora_rank", None),
qk_nope_head_dim=getattr(args, "qk_nope_head_dim", None),
qk_rope_head_dim=getattr(args, "qk_rope_head_dim", None),
v_head_dim=getattr(args, "v_head_dim", None),
sparse_mla_force_mqa=force_mqa,
sparse_mla_mha_mode=mha_mode,
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
prefill_backend=prefill_backend,
)
# run_mla_benchmark needs the real backend name
from mla_runner import run_mla_benchmark as run_mla
run_label = f"{backend}_{variant_label} {spec}"
pbar.set_postfix_str(run_label)
try:
result = run_mla(
backend,
config,
prefill_backend=prefill_backend,
sparse_mla_force_mqa=force_mqa,
)
except Exception as e:
result = BenchmarkResult(
config=config,
mean_time=float("inf"),
median_time=float("inf"),
std_time=0,
min_time=float("inf"),
max_time=float("inf"),
error=str(e),
)
all_results.append(result)
if args.output_csv:
formatter.save_csv(all_results, args.output_csv)
if args.output_json:
formatter.save_json(all_results, args.output_json)
if not result.success:
console.print(
f"[red]Error {backend}_{variant_label} "
f"{spec}: {result.error}[/]"
)
pbar.update(1)
# Display results with variant labels as separate "backends"
console.print("\n[bold green]MHA vs MQA Results:[/]")
variant_backends = [f"{b}_{v}" for b in backends for v, _, _ in variants]
formatter.print_table(all_results, variant_backends)
# Handle model parameter sweep mode
elif hasattr(args, "model_parameter_sweep") and args.model_parameter_sweep:
# Model parameter sweep
+68 -37
View File
@@ -4,8 +4,10 @@
"""Common utilities for attention benchmarking."""
import csv
import gc
import json
import math
from collections.abc import Sequence
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any
@@ -44,10 +46,13 @@ def run_do_bench(
kwargs: dict[str, Any] = {"return_mode": "all"}
if use_cuda_graphs:
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
gc.collect()
torch.accelerator.empty_cache()
else:
if warmup_ms is not None:
kwargs["warmup"] = warmup_ms
result = triton.testing.do_bench(benchmark_fn, **kwargs)
torch.accelerator.synchronize()
return result
@@ -91,42 +96,6 @@ except ImportError:
AttentionLayerBase = object # Fallback
class MockKVBProj:
"""Mock KV projection layer for MLA prefill mode.
Mimics ColumnParallelLinear behavior for kv_b_proj in MLA backends.
Projects kv_c_normed to [qk_nope_head_dim + v_head_dim] per head.
"""
def __init__(self, num_heads: int, qk_nope_head_dim: int, v_head_dim: int):
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
self.v_head_dim = v_head_dim
self.out_dim = qk_nope_head_dim + v_head_dim
self.weight = torch.empty(0, dtype=torch.bfloat16)
def __call__(self, x: torch.Tensor) -> tuple[torch.Tensor]:
"""
Project kv_c_normed to output space.
Args:
x: Input tensor [num_tokens, kv_lora_rank]
Returns:
Tuple containing output tensor
[num_tokens, num_heads, qk_nope_head_dim + v_head_dim]
"""
num_tokens = x.shape[0]
result = torch.randn(
num_tokens,
self.num_heads,
self.out_dim,
device=x.device,
dtype=x.dtype,
)
return (result,) # Return as tuple to match ColumnParallelLinear API
class MockIndexer:
"""Mock Indexer for sparse MLA backends.
@@ -158,6 +127,60 @@ class MockIndexer:
)
self.topk_indices_buffer[:num_tokens] = indices
def fill_indices(
self,
num_tokens: int,
max_kv_len: int,
pattern: str = "random",
requests: Sequence[Any] | None = None,
):
if pattern == "random":
self.fill_random_indices(num_tokens, max_kv_len)
return
if pattern == "prefix":
indices = torch.arange(
self.topk_tokens,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
)
indices = (indices % max_kv_len).expand(num_tokens, -1)
self.topk_indices_buffer[:num_tokens] = indices
return
if pattern == "sliding_window":
if requests is None:
start = max(max_kv_len - self.topk_tokens, 0)
indices = torch.arange(
start,
start + self.topk_tokens,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
)
indices = indices.clamp(max=max_kv_len - 1).expand(num_tokens, -1)
self.topk_indices_buffer[:num_tokens] = indices
return
rows = []
offsets = torch.arange(
self.topk_tokens,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
) - (self.topk_tokens - 1)
for request in requests:
q_len = request.q_len
kv_len = request.kv_len
context_len = kv_len - q_len
positions = torch.arange(
context_len,
kv_len,
dtype=torch.int32,
device=self.topk_indices_buffer.device,
)
row_indices = positions[:, None] + offsets[None, :]
rows.append(row_indices.clamp(min=0, max=kv_len - 1))
self.topk_indices_buffer[:num_tokens] = torch.cat(rows, dim=0)
return
raise ValueError(f"Unknown sparse MLA topk pattern: {pattern}")
class MockLayer(AttentionLayerBase):
"""Mock attention layer with scale parameters and impl.
@@ -252,10 +275,14 @@ class BenchmarkConfig:
num_kv_heads: int
block_size: int
device: str
max_model_len: int | None = None
dtype: torch.dtype = torch.float16
profile_memory: bool = False
use_cuda_graphs: bool = False
use_cuda_graphs: bool = True
ncu_profile: bool = False
torch_profile: bool = False
torch_profile_dir: str | None = None
torch_profile_iters: int = 3
warmup_ms: int | None = None
# "auto" or "fp8"
@@ -271,6 +298,10 @@ class BenchmarkConfig:
# Backend-specific tuning
num_kv_splits: int | None = None # CUTLASS MLA
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
sparse_mla_dense_mha_max_seq_len: int | None = None
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
@@ -0,0 +1,474 @@
# Sparse MLA benchmark: forward_mha vs forward_mqa
#
# Usage:
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml
#
# Heatmap grid:
# - batch_size: 1, 2, 4, 8, 16, 32
# - seq_len: 32, 64, 128, 256, 512, 1024, 2048
# - q_len: powers of two through seq_len
#
# Specs with q_len < seq_len include context; the q_len == seq_len diagonal
# covers pure prefill.
# The model shape below is the DP case. For the TP8 run, manually change
# model.num_q_heads from 128 to 16 before rerunning this benchmark.
mode: mha_vs_mqa
model:
name: "deepseek-v3"
num_layers: 60
num_q_heads: 128
num_kv_heads: 1
head_dim: 576
kv_lora_rank: 512
qk_nope_head_dim: 128
qk_rope_head_dim: 64
v_head_dim: 128
block_size: 128
max_model_len: 2048
batch_specs:
# Batch size 1
# seq_len = 32
- "1q1s32"
- "1q2s32"
- "1q4s32"
- "1q8s32"
- "1q16s32"
- "1q32"
# seq_len = 64
- "1q1s64"
- "1q2s64"
- "1q4s64"
- "1q8s64"
- "1q16s64"
- "1q32s64"
- "1q64"
# seq_len = 128
- "1q1s128"
- "1q2s128"
- "1q4s128"
- "1q8s128"
- "1q16s128"
- "1q32s128"
- "1q64s128"
- "1q128"
# seq_len = 256
- "1q1s256"
- "1q2s256"
- "1q4s256"
- "1q8s256"
- "1q16s256"
- "1q32s256"
- "1q64s256"
- "1q128s256"
- "1q256"
# seq_len = 512
- "1q1s512"
- "1q2s512"
- "1q4s512"
- "1q8s512"
- "1q16s512"
- "1q32s512"
- "1q64s512"
- "1q128s512"
- "1q256s512"
- "1q512"
# seq_len = 1024
- "1q1s1024"
- "1q2s1024"
- "1q4s1024"
- "1q8s1024"
- "1q16s1024"
- "1q32s1024"
- "1q64s1024"
- "1q128s1024"
- "1q256s1024"
- "1q512s1024"
- "1q1024"
# seq_len = 2048
- "1q1s2048"
- "1q2s2048"
- "1q4s2048"
- "1q8s2048"
- "1q16s2048"
- "1q32s2048"
- "1q64s2048"
- "1q128s2048"
- "1q256s2048"
- "1q512s2048"
- "1q1024s2048"
- "1q2048"
# Batch size 2
# seq_len = 32
- "2q1s32"
- "2q2s32"
- "2q4s32"
- "2q8s32"
- "2q16s32"
- "2q32"
# seq_len = 64
- "2q1s64"
- "2q2s64"
- "2q4s64"
- "2q8s64"
- "2q16s64"
- "2q32s64"
- "2q64"
# seq_len = 128
- "2q1s128"
- "2q2s128"
- "2q4s128"
- "2q8s128"
- "2q16s128"
- "2q32s128"
- "2q64s128"
- "2q128"
# seq_len = 256
- "2q1s256"
- "2q2s256"
- "2q4s256"
- "2q8s256"
- "2q16s256"
- "2q32s256"
- "2q64s256"
- "2q128s256"
- "2q256"
# seq_len = 512
- "2q1s512"
- "2q2s512"
- "2q4s512"
- "2q8s512"
- "2q16s512"
- "2q32s512"
- "2q64s512"
- "2q128s512"
- "2q256s512"
- "2q512"
# seq_len = 1024
- "2q1s1024"
- "2q2s1024"
- "2q4s1024"
- "2q8s1024"
- "2q16s1024"
- "2q32s1024"
- "2q64s1024"
- "2q128s1024"
- "2q256s1024"
- "2q512s1024"
- "2q1024"
# seq_len = 2048
- "2q1s2048"
- "2q2s2048"
- "2q4s2048"
- "2q8s2048"
- "2q16s2048"
- "2q32s2048"
- "2q64s2048"
- "2q128s2048"
- "2q256s2048"
- "2q512s2048"
- "2q1024s2048"
- "2q2048"
# Batch size 4
# seq_len = 32
- "4q1s32"
- "4q2s32"
- "4q4s32"
- "4q8s32"
- "4q16s32"
- "4q32"
# seq_len = 64
- "4q1s64"
- "4q2s64"
- "4q4s64"
- "4q8s64"
- "4q16s64"
- "4q32s64"
- "4q64"
# seq_len = 128
- "4q1s128"
- "4q2s128"
- "4q4s128"
- "4q8s128"
- "4q16s128"
- "4q32s128"
- "4q64s128"
- "4q128"
# seq_len = 256
- "4q1s256"
- "4q2s256"
- "4q4s256"
- "4q8s256"
- "4q16s256"
- "4q32s256"
- "4q64s256"
- "4q128s256"
- "4q256"
# seq_len = 512
- "4q1s512"
- "4q2s512"
- "4q4s512"
- "4q8s512"
- "4q16s512"
- "4q32s512"
- "4q64s512"
- "4q128s512"
- "4q256s512"
- "4q512"
# seq_len = 1024
- "4q1s1024"
- "4q2s1024"
- "4q4s1024"
- "4q8s1024"
- "4q16s1024"
- "4q32s1024"
- "4q64s1024"
- "4q128s1024"
- "4q256s1024"
- "4q512s1024"
- "4q1024"
# seq_len = 2048
- "4q1s2048"
- "4q2s2048"
- "4q4s2048"
- "4q8s2048"
- "4q16s2048"
- "4q32s2048"
- "4q64s2048"
- "4q128s2048"
- "4q256s2048"
- "4q512s2048"
- "4q1024s2048"
- "4q2048"
# Batch size 8
# seq_len = 32
- "8q1s32"
- "8q2s32"
- "8q4s32"
- "8q8s32"
- "8q16s32"
- "8q32"
# seq_len = 64
- "8q1s64"
- "8q2s64"
- "8q4s64"
- "8q8s64"
- "8q16s64"
- "8q32s64"
- "8q64"
# seq_len = 128
- "8q1s128"
- "8q2s128"
- "8q4s128"
- "8q8s128"
- "8q16s128"
- "8q32s128"
- "8q64s128"
- "8q128"
# seq_len = 256
- "8q1s256"
- "8q2s256"
- "8q4s256"
- "8q8s256"
- "8q16s256"
- "8q32s256"
- "8q64s256"
- "8q128s256"
- "8q256"
# seq_len = 512
- "8q1s512"
- "8q2s512"
- "8q4s512"
- "8q8s512"
- "8q16s512"
- "8q32s512"
- "8q64s512"
- "8q128s512"
- "8q256s512"
- "8q512"
# seq_len = 1024
- "8q1s1024"
- "8q2s1024"
- "8q4s1024"
- "8q8s1024"
- "8q16s1024"
- "8q32s1024"
- "8q64s1024"
- "8q128s1024"
- "8q256s1024"
- "8q512s1024"
- "8q1024"
# seq_len = 2048
- "8q1s2048"
- "8q2s2048"
- "8q4s2048"
- "8q8s2048"
- "8q16s2048"
- "8q32s2048"
- "8q64s2048"
- "8q128s2048"
- "8q256s2048"
- "8q512s2048"
- "8q1024s2048"
- "8q2048"
# Batch size 16
# seq_len = 32
- "16q1s32"
- "16q2s32"
- "16q4s32"
- "16q8s32"
- "16q16s32"
- "16q32"
# seq_len = 64
- "16q1s64"
- "16q2s64"
- "16q4s64"
- "16q8s64"
- "16q16s64"
- "16q32s64"
- "16q64"
# seq_len = 128
- "16q1s128"
- "16q2s128"
- "16q4s128"
- "16q8s128"
- "16q16s128"
- "16q32s128"
- "16q64s128"
- "16q128"
# seq_len = 256
- "16q1s256"
- "16q2s256"
- "16q4s256"
- "16q8s256"
- "16q16s256"
- "16q32s256"
- "16q64s256"
- "16q128s256"
- "16q256"
# seq_len = 512
- "16q1s512"
- "16q2s512"
- "16q4s512"
- "16q8s512"
- "16q16s512"
- "16q32s512"
- "16q64s512"
- "16q128s512"
- "16q256s512"
- "16q512"
# seq_len = 1024
- "16q1s1024"
- "16q2s1024"
- "16q4s1024"
- "16q8s1024"
- "16q16s1024"
- "16q32s1024"
- "16q64s1024"
- "16q128s1024"
- "16q256s1024"
- "16q512s1024"
- "16q1024"
# seq_len = 2048
- "16q1s2048"
- "16q2s2048"
- "16q4s2048"
- "16q8s2048"
- "16q16s2048"
- "16q32s2048"
- "16q64s2048"
- "16q128s2048"
- "16q256s2048"
- "16q512s2048"
- "16q1024s2048"
- "16q2048"
# Batch size 32
# seq_len = 32
- "32q1s32"
- "32q2s32"
- "32q4s32"
- "32q8s32"
- "32q16s32"
- "32q32"
# seq_len = 64
- "32q1s64"
- "32q2s64"
- "32q4s64"
- "32q8s64"
- "32q16s64"
- "32q32s64"
- "32q64"
# seq_len = 128
- "32q1s128"
- "32q2s128"
- "32q4s128"
- "32q8s128"
- "32q16s128"
- "32q32s128"
- "32q64s128"
- "32q128"
# seq_len = 256
- "32q1s256"
- "32q2s256"
- "32q4s256"
- "32q8s256"
- "32q16s256"
- "32q32s256"
- "32q64s256"
- "32q128s256"
- "32q256"
# seq_len = 512
- "32q1s512"
- "32q2s512"
- "32q4s512"
- "32q8s512"
- "32q16s512"
- "32q32s512"
- "32q64s512"
- "32q128s512"
- "32q256s512"
- "32q512"
# seq_len = 1024
- "32q1s1024"
- "32q2s1024"
- "32q4s1024"
- "32q8s1024"
- "32q16s1024"
- "32q32s1024"
- "32q64s1024"
- "32q128s1024"
- "32q256s1024"
- "32q512s1024"
- "32q1024"
# seq_len = 2048
- "32q1s2048"
- "32q2s2048"
- "32q4s2048"
- "32q8s2048"
- "32q16s2048"
- "32q32s2048"
- "32q64s2048"
- "32q128s2048"
- "32q256s2048"
- "32q512s2048"
- "32q1024s2048"
- "32q2048"
backends:
- FLASHMLA_SPARSE
device: "cuda:0"
profile_memory: false
sparse_mla_dense_mha_max_seq_len: 2048
sparse_mla_topk_pattern: "random"
output:
csv: "benchmark_output/mla_sparse_mha_vs_mqa.csv"
json: "benchmark_output/mla_sparse_mha_vs_mqa.json"
+171 -17
View File
@@ -9,6 +9,8 @@ needing full VllmConfig integration.
"""
import statistics
import tempfile
from pathlib import Path
import numpy as np
import torch
@@ -17,7 +19,6 @@ from common import (
BenchmarkResult,
MockHfConfig,
MockIndexer,
MockKVBProj,
MockLayer,
run_do_bench,
run_ncu_profile,
@@ -33,8 +34,59 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
def _safe_profile_name(value: str) -> str:
return "".join(c if c.isalnum() or c in "._-" else "_" for c in value)
def _create_kv_b_proj(
mla_dims: dict,
device: torch.device,
):
kv_b_proj = ColumnParallelLinear(
mla_dims["kv_lora_rank"],
mla_dims["num_q_heads"]
* (mla_dims["qk_nope_head_dim"] + mla_dims["v_head_dim"]),
bias=False,
params_dtype=torch.bfloat16,
quant_config=None,
prefix="benchmark.kv_b_proj",
).to(device)
with torch.no_grad():
kv_b_proj.weight.copy_(torch.randn_like(kv_b_proj.weight))
return kv_b_proj
def _ensure_single_rank_model_parallel() -> None:
import torch.distributed as dist
from vllm.distributed import (
ensure_model_parallel_initialized,
init_distributed_environment,
model_parallel_is_initialized,
)
if not dist.is_available():
return
if not dist.is_initialized():
with tempfile.NamedTemporaryFile(
prefix="vllm_bench_dist_", delete=False
) as init_file:
distributed_init_method = f"file://{init_file.name}"
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=distributed_init_method,
local_rank=0,
backend="nccl",
)
if not model_parallel_is_initialized():
ensure_model_parallel_initialized(1, 1)
# ============================================================================
# VllmConfig Creation
# ============================================================================
@@ -66,10 +118,12 @@ def create_minimal_vllm_config(
block_size: int = 128,
max_num_seqs: int = 256,
max_num_batched_tokens: int = 8192,
max_model_len: int = 32768,
mla_dims: dict | None = None,
index_topk: int | None = None,
prefill_backend: str | None = None,
kv_cache_dtype: str = "auto",
sparse_mla_force_mqa: bool = False,
) -> VllmConfig:
"""
Create minimal VllmConfig for MLA benchmarks.
@@ -86,6 +140,8 @@ def create_minimal_vllm_config(
prefill_backend: Prefill backend name (e.g., "fa3", "fa4", "flashinfer",
"trtllm"). Configures the attention config to force
the specified prefill backend.
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
forward_mqa (even prefill tokens).
Returns:
VllmConfig for benchmarking
@@ -131,7 +187,7 @@ def create_minimal_vllm_config(
trust_remote_code=True,
dtype="bfloat16",
seed=0,
max_model_len=32768,
max_model_len=max_model_len,
quantization=None,
enforce_eager=False,
max_logprobs=20,
@@ -163,7 +219,7 @@ def create_minimal_vllm_config(
scheduler_config = SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=max(max_num_batched_tokens, max_num_seqs),
max_model_len=32768,
max_model_len=max_model_len,
is_encoder_decoder=False,
enable_chunked_prefill=True,
)
@@ -192,6 +248,9 @@ def create_minimal_vllm_config(
"flash_attn_version"
]
if sparse_mla_force_mqa:
vllm_config.attention_config.sparse_mla_force_mqa = True
return vllm_config
@@ -548,12 +607,7 @@ def _create_backend_impl(
# Calculate scale
scale = 1.0 / np.sqrt(mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"])
# Create mock kv_b_proj layer for prefill mode
mock_kv_b_proj = MockKVBProj(
num_heads=mla_dims["num_q_heads"],
qk_nope_head_dim=mla_dims["qk_nope_head_dim"],
v_head_dim=mla_dims["v_head_dim"],
)
kv_b_proj = _create_kv_b_proj(mla_dims, device)
# Create indexer for sparse backends
indexer = None
@@ -584,7 +638,7 @@ def _create_backend_impl(
"qk_rope_head_dim": mla_dims["qk_rope_head_dim"],
"qk_head_dim": mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"],
"v_head_dim": mla_dims["v_head_dim"],
"kv_b_proj": mock_kv_b_proj,
"kv_b_proj": kv_b_proj,
}
# Add indexer for sparse backends
@@ -785,14 +839,35 @@ def _run_single_benchmark(
# Fill indexer with random indices for sparse backends
is_sparse = backend_cfg.get("is_sparse", False)
if is_sparse and indexer is not None:
indexer.fill_random_indices(total_q, max_kv_len)
indexer.fill_indices(
total_q,
max_kv_len,
getattr(config, "sparse_mla_topk_pattern", "random"),
)
# Determine which forward methods to use based on metadata.
# Sparse MLA backends always use forward_mqa
has_decode = is_sparse or getattr(metadata, "decode", None) is not None
has_prefill = not is_sparse and getattr(metadata, "prefill", None) is not None
# Non-sparse backends use .decode/.prefill sub-objects.
# Sparse backends use num_decode_tokens/num_prefills directly.
#
# sparse_mla_force_mqa overrides: even for prefill metadata, use MQA.
force_mqa = getattr(config, "sparse_mla_force_mqa", False)
force_dense_mha = getattr(config, "sparse_mla_mha_mode", "auto") == "dense"
if force_mqa:
has_decode = True
has_prefill = False
elif is_sparse:
has_decode = metadata.num_decode_tokens > 0
has_prefill = metadata.num_prefills > 0
else:
has_decode = metadata.decode is not None
has_prefill = metadata.prefill is not None
if not has_decode and not has_prefill:
raise RuntimeError("Metadata has neither decode nor prefill metadata")
if is_sparse and force_dense_mha and not has_prefill:
raise RuntimeError(
"Sparse MLA dense_mha benchmark did not produce prefill metadata. "
"Check reorder_batch_threshold/path forcing."
)
num_decode = (
metadata.num_decode_tokens
@@ -871,7 +946,6 @@ def _run_single_benchmark(
metadata,
prefill_inputs["k_scale"],
prefill_fp8_output if fused_output else prefill_inputs["output"],
prefill_output_scale if fused_output else None,
)
if fused_output:
out = prefill_fp8_output
@@ -898,6 +972,48 @@ def _run_single_benchmark(
throughput_tokens_per_sec=0.0,
)
if config.torch_profile:
profile_dir = Path(
config.torch_profile_dir or "benchmark_outputs/torch_profiles"
)
profile_dir.mkdir(parents=True, exist_ok=True)
trace_name = _safe_profile_name(f"{config.backend}_{config.batch_spec}")
trace_path = profile_dir / f"{trace_name}.json"
iters = max(config.torch_profile_iters, 1)
forward_fn()
torch.accelerator.synchronize()
with torch.profiler.profile(
activities=[
torch.profiler.ProfilerActivity.CPU,
torch.profiler.ProfilerActivity.CUDA,
],
record_shapes=True,
profile_memory=True,
with_stack=False,
) as prof:
for _ in range(iters):
forward_fn()
torch.accelerator.synchronize()
prof.step()
prof.export_chrome_trace(str(trace_path))
print(f"Saved PyTorch profiler trace to {trace_path}")
print(
prof.key_averages().table(
sort_by="cuda_time_total",
row_limit=25,
)
)
return BenchmarkResult(
config=config,
mean_time=0.0,
median_time=0.0,
std_time=0.0,
min_time=0.0,
max_time=0.0,
throughput_tokens_per_sec=0.0,
)
all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
# Convert ms to seconds per layer
@@ -920,6 +1036,7 @@ def _run_mla_benchmark_batched(
configs_with_params: list[tuple], # [(config, threshold, num_splits), ...]
index_topk: int = 2048,
prefill_backend: str | None = None,
sparse_mla_force_mqa: bool = False,
output_scale: float | None = None,
fuse_quant_op: bool = False,
) -> list[BenchmarkResult]:
@@ -940,6 +1057,8 @@ def _run_mla_benchmark_batched(
index_topk: Topk value for sparse MLA backends (default 2048)
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
When set, forces the specified FlashAttention version for prefill.
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
forward_mqa (even prefill tokens).
Returns:
List of BenchmarkResult objects
@@ -980,21 +1099,41 @@ def _run_mla_benchmark_batched(
sum(r.q_len for r in parse_batch_spec(cfg.batch_spec))
for cfg, *_ in configs_with_params
)
max_model_len = max(
max_total_q,
max(
getattr(cfg, "max_model_len", None) or 32768
for cfg, *_ in configs_with_params
),
)
# Create and set vLLM config for MLA (reused across all benchmarks)
vllm_config = create_minimal_vllm_config(
model_name="deepseek-v3", # Used only for model path
block_size=block_size,
max_num_batched_tokens=max_total_q,
max_model_len=max_model_len,
mla_dims=mla_dims, # Use custom dims from config or default
index_topk=index_topk if is_sparse else None,
prefill_backend=prefill_backend,
kv_cache_dtype=kv_cache_dtype,
sparse_mla_force_mqa=sparse_mla_force_mqa,
)
results = []
# Initialize workspace manager (needed by metadata builders)
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
if not is_workspace_manager_initialized():
init_workspace_manager(device)
with set_current_vllm_config(vllm_config):
_ensure_single_rank_model_parallel()
# Create backend impl, layer, builder, and indexer (reused across benchmarks)
impl, layer, builder_instance, indexer = _create_backend_impl(
backend_cfg,
@@ -1040,9 +1179,20 @@ def _run_mla_benchmark_batched(
for config, threshold, num_splits in configs_with_params:
# Set threshold for this benchmark (FlashAttn/FlashMLA only)
original_threshold = None
if threshold is not None and builder_instance:
effective_threshold = threshold
force_dense_mha = (
is_sparse
and getattr(config, "sparse_mla_mha_mode", "auto") == "dense"
and not getattr(config, "sparse_mla_force_mqa", False)
)
if force_dense_mha:
# Sparse MLA normally treats q_len <= 1 as decode. Use an
# impossible threshold so dense_mha benchmarks actually run
# the prefill/MHA path, including q_len=1 short extends.
effective_threshold = -1
if effective_threshold is not None and builder_instance:
original_threshold = builder_instance.reorder_batch_threshold
builder_instance.reorder_batch_threshold = threshold
builder_instance.reorder_batch_threshold = effective_threshold
# Set num_splits for CUTLASS
original_num_splits = None
@@ -1090,6 +1240,7 @@ def run_mla_benchmark(
num_kv_splits: int | None = None,
index_topk: int = 2048,
prefill_backend: str | None = None,
sparse_mla_force_mqa: bool = False,
output_scale: float | None = None,
fuse_quant_op: bool = False,
) -> BenchmarkResult | list[BenchmarkResult]:
@@ -1111,6 +1262,8 @@ def run_mla_benchmark(
index_topk: Topk value for sparse MLA backends (default 2048)
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
When set, forces the specified FlashAttention version for prefill.
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
forward_mqa (even prefill tokens).
output_scale: Static per-tensor FP8 scale for prefill output (None = bf16).
fuse_quant_op: With output_scale set, fuse the FP8 write into the prefill
kernel vs a standalone post-quant kernel. See _run_single_benchmark.
@@ -1142,6 +1295,7 @@ def run_mla_benchmark(
configs_with_params,
index_topk,
prefill_backend=prefill_backend,
sparse_mla_force_mqa=sparse_mla_force_mqa,
output_scale=output_scale,
fuse_quant_op=fuse_quant_op,
)