[1/N] Add dense MHA path for sparse MLA short sequences (#47327)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-07-14 00:29:56 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 0b0ef8d7eb
commit c4f5cd60da
21 changed files with 1990 additions and 646 deletions
+175 -3
View File
@@ -75,7 +75,11 @@ def run_mla_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
from mla_runner import run_mla_benchmark as run_mla
return run_mla(
config.backend, config, prefill_backend=config.prefill_backend, **kwargs
config.backend,
config,
prefill_backend=config.prefill_backend,
sparse_mla_force_mqa=config.sparse_mla_force_mqa,
**kwargs,
)
@@ -592,6 +596,30 @@ def main():
default="profile",
help="Output file name for ncu profile (default: 'profile').",
)
parser.add_argument(
"--torch-profile",
action="store_true",
default=False,
help="Collect a PyTorch profiler Chrome trace for each benchmark run.",
)
parser.add_argument(
"--torch-profile-dir",
default=None,
help="Directory for PyTorch profiler traces.",
)
parser.add_argument(
"--torch-profile-iters",
type=int,
default=3,
help="Number of forward passes to record per PyTorch profiler trace.",
)
parser.add_argument(
"--sparse-mla-mha-variants",
nargs="+",
default=None,
choices=["dense_mha", "mqa"],
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
)
# Parameter sweep (use YAML config for advanced sweeps)
parser.add_argument(
@@ -641,6 +669,7 @@ def main():
# Prefill backends (e.g., ["fa3", "fa4"])
args.prefill_backends = yaml_config.get("prefill_backends", None)
args.prefill_backend = yaml_config.get("prefill_backend", None)
# FP8 output benchmark knobs; CLI wins.
if args.fp8_output_scale is None:
@@ -683,6 +712,9 @@ def main():
args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
args.block_size = model.get("block_size", args.block_size)
args.max_model_len = model.get(
"max_model_len", getattr(args, "max_model_len", None)
)
# MLA-specific dimensions
args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
@@ -701,6 +733,21 @@ def main():
args.cuda_graphs = yaml_config["cuda_graphs"]
if "ncu_profile" in yaml_config:
args.ncu_profile = yaml_config["ncu_profile"]
if "torch_profile" in yaml_config:
args.torch_profile = yaml_config["torch_profile"]
if "torch_profile_dir" in yaml_config:
args.torch_profile_dir = yaml_config["torch_profile_dir"]
if "torch_profile_iters" in yaml_config:
args.torch_profile_iters = yaml_config["torch_profile_iters"]
args.sparse_mla_topk_pattern = yaml_config.get(
"sparse_mla_topk_pattern", "random"
)
args.sparse_mla_dense_mha_max_seq_len = yaml_config.get(
"sparse_mla_dense_mha_max_seq_len", None
)
args.sparse_mla_mha_variants = yaml_config.get(
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
)
# Parameter sweep configuration
if "parameter_sweep" in yaml_config:
@@ -842,8 +889,6 @@ def main():
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
repeats=args.repeats,
warmup_iters=args.warmup_iters,
profile_memory=args.profile_memory,
kv_cache_dtype=args.kv_cache_dtype,
use_cuda_graphs=args.cuda_graphs,
@@ -1063,6 +1108,133 @@ def main():
f"\n [yellow]Prefill always faster for batch_size={bs}[/]"
)
# Handle MHA vs MQA comparison mode for sparse MLA
elif hasattr(args, "mode") and args.mode == "mha_vs_mqa":
console.print("[yellow]Mode: MHA vs MQA comparison for sparse MLA[/]")
sparse_mla_topk_pattern = getattr(args, "sparse_mla_topk_pattern", "random")
dense_mha_max_seq_len = getattr(args, "sparse_mla_dense_mha_max_seq_len", None)
prefill_backend = getattr(args, "prefill_backend", None)
if prefill_backend:
console.print(f"Prefill backend: {prefill_backend}")
available_variants = [
("dense_mha", False, "dense"),
("mqa", True, "auto"),
]
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
if requested_variants is not None:
valid_variants = {label for label, _, _ in available_variants}
invalid_variants = sorted(set(requested_variants) - valid_variants)
if invalid_variants:
raise ValueError(
"Invalid sparse_mla_mha_variants entries: "
f"{invalid_variants}. Valid variants are: "
f"{sorted(valid_variants)}"
)
requested_variant_set = set(requested_variants)
variants = [
variant
for variant in available_variants
if variant[0] in requested_variant_set
]
else:
variants = available_variants
formatter = ResultsFormatter(console)
total = 0
for spec in args.batch_specs:
q_len = max(request.q_len for request in parse_batch_spec(spec))
for variant_label, _, _ in variants:
if (
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
):
continue
total += len(backends)
with tqdm(total=total, desc="Benchmarking") as pbar:
for spec in args.batch_specs:
q_len = max(request.q_len for request in parse_batch_spec(spec))
for backend in backends:
for variant_label, force_mqa, mha_mode in variants:
if (
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
):
continue
config = BenchmarkConfig(
backend=f"{backend}_{variant_label}",
batch_spec=spec,
num_layers=args.num_layers,
head_dim=args.head_dim,
num_q_heads=args.num_q_heads,
num_kv_heads=args.num_kv_heads,
block_size=args.block_size,
device=args.device,
max_model_len=getattr(args, "max_model_len", None),
kv_cache_dtype=args.kv_cache_dtype,
profile_memory=args.profile_memory,
use_cuda_graphs=args.cuda_graphs,
ncu_profile=args.ncu_profile,
torch_profile=args.torch_profile,
torch_profile_dir=args.torch_profile_dir,
torch_profile_iters=args.torch_profile_iters,
warmup_ms=args.warmup_ms,
kv_lora_rank=getattr(args, "kv_lora_rank", None),
qk_nope_head_dim=getattr(args, "qk_nope_head_dim", None),
qk_rope_head_dim=getattr(args, "qk_rope_head_dim", None),
v_head_dim=getattr(args, "v_head_dim", None),
sparse_mla_force_mqa=force_mqa,
sparse_mla_mha_mode=mha_mode,
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
prefill_backend=prefill_backend,
)
# run_mla_benchmark needs the real backend name
from mla_runner import run_mla_benchmark as run_mla
run_label = f"{backend}_{variant_label} {spec}"
pbar.set_postfix_str(run_label)
try:
result = run_mla(
backend,
config,
prefill_backend=prefill_backend,
sparse_mla_force_mqa=force_mqa,
)
except Exception as e:
result = BenchmarkResult(
config=config,
mean_time=float("inf"),
median_time=float("inf"),
std_time=0,
min_time=float("inf"),
max_time=float("inf"),
error=str(e),
)
all_results.append(result)
if args.output_csv:
formatter.save_csv(all_results, args.output_csv)
if args.output_json:
formatter.save_json(all_results, args.output_json)
if not result.success:
console.print(
f"[red]Error {backend}_{variant_label} "
f"{spec}: {result.error}[/]"
)
pbar.update(1)
# Display results with variant labels as separate "backends"
console.print("\n[bold green]MHA vs MQA Results:[/]")
variant_backends = [f"{b}_{v}" for b in backends for v, _, _ in variants]
formatter.print_table(all_results, variant_backends)
# Handle model parameter sweep mode
elif hasattr(args, "model_parameter_sweep") and args.model_parameter_sweep:
# Model parameter sweep