[1/N] Add dense MHA path for sparse MLA short sequences (#47327)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
0b0ef8d7eb
commit
c4f5cd60da
@@ -75,7 +75,11 @@ def run_mla_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
|
||||
from mla_runner import run_mla_benchmark as run_mla
|
||||
|
||||
return run_mla(
|
||||
config.backend, config, prefill_backend=config.prefill_backend, **kwargs
|
||||
config.backend,
|
||||
config,
|
||||
prefill_backend=config.prefill_backend,
|
||||
sparse_mla_force_mqa=config.sparse_mla_force_mqa,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -592,6 +596,30 @@ def main():
|
||||
default="profile",
|
||||
help="Output file name for ncu profile (default: 'profile').",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--torch-profile",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Collect a PyTorch profiler Chrome trace for each benchmark run.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--torch-profile-dir",
|
||||
default=None,
|
||||
help="Directory for PyTorch profiler traces.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--torch-profile-iters",
|
||||
type=int,
|
||||
default=3,
|
||||
help="Number of forward passes to record per PyTorch profiler trace.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sparse-mla-mha-variants",
|
||||
nargs="+",
|
||||
default=None,
|
||||
choices=["dense_mha", "mqa"],
|
||||
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
|
||||
)
|
||||
|
||||
# Parameter sweep (use YAML config for advanced sweeps)
|
||||
parser.add_argument(
|
||||
@@ -641,6 +669,7 @@ def main():
|
||||
|
||||
# Prefill backends (e.g., ["fa3", "fa4"])
|
||||
args.prefill_backends = yaml_config.get("prefill_backends", None)
|
||||
args.prefill_backend = yaml_config.get("prefill_backend", None)
|
||||
|
||||
# FP8 output benchmark knobs; CLI wins.
|
||||
if args.fp8_output_scale is None:
|
||||
@@ -683,6 +712,9 @@ def main():
|
||||
args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
|
||||
args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
|
||||
args.block_size = model.get("block_size", args.block_size)
|
||||
args.max_model_len = model.get(
|
||||
"max_model_len", getattr(args, "max_model_len", None)
|
||||
)
|
||||
# MLA-specific dimensions
|
||||
args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
|
||||
args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
|
||||
@@ -701,6 +733,21 @@ def main():
|
||||
args.cuda_graphs = yaml_config["cuda_graphs"]
|
||||
if "ncu_profile" in yaml_config:
|
||||
args.ncu_profile = yaml_config["ncu_profile"]
|
||||
if "torch_profile" in yaml_config:
|
||||
args.torch_profile = yaml_config["torch_profile"]
|
||||
if "torch_profile_dir" in yaml_config:
|
||||
args.torch_profile_dir = yaml_config["torch_profile_dir"]
|
||||
if "torch_profile_iters" in yaml_config:
|
||||
args.torch_profile_iters = yaml_config["torch_profile_iters"]
|
||||
args.sparse_mla_topk_pattern = yaml_config.get(
|
||||
"sparse_mla_topk_pattern", "random"
|
||||
)
|
||||
args.sparse_mla_dense_mha_max_seq_len = yaml_config.get(
|
||||
"sparse_mla_dense_mha_max_seq_len", None
|
||||
)
|
||||
args.sparse_mla_mha_variants = yaml_config.get(
|
||||
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
|
||||
)
|
||||
|
||||
# Parameter sweep configuration
|
||||
if "parameter_sweep" in yaml_config:
|
||||
@@ -842,8 +889,6 @@ def main():
|
||||
num_kv_heads=args.num_kv_heads,
|
||||
block_size=args.block_size,
|
||||
device=args.device,
|
||||
repeats=args.repeats,
|
||||
warmup_iters=args.warmup_iters,
|
||||
profile_memory=args.profile_memory,
|
||||
kv_cache_dtype=args.kv_cache_dtype,
|
||||
use_cuda_graphs=args.cuda_graphs,
|
||||
@@ -1063,6 +1108,133 @@ def main():
|
||||
f"\n [yellow]Prefill always faster for batch_size={bs}[/]"
|
||||
)
|
||||
|
||||
# Handle MHA vs MQA comparison mode for sparse MLA
|
||||
elif hasattr(args, "mode") and args.mode == "mha_vs_mqa":
|
||||
console.print("[yellow]Mode: MHA vs MQA comparison for sparse MLA[/]")
|
||||
|
||||
sparse_mla_topk_pattern = getattr(args, "sparse_mla_topk_pattern", "random")
|
||||
dense_mha_max_seq_len = getattr(args, "sparse_mla_dense_mha_max_seq_len", None)
|
||||
prefill_backend = getattr(args, "prefill_backend", None)
|
||||
if prefill_backend:
|
||||
console.print(f"Prefill backend: {prefill_backend}")
|
||||
available_variants = [
|
||||
("dense_mha", False, "dense"),
|
||||
("mqa", True, "auto"),
|
||||
]
|
||||
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
|
||||
if requested_variants is not None:
|
||||
valid_variants = {label for label, _, _ in available_variants}
|
||||
invalid_variants = sorted(set(requested_variants) - valid_variants)
|
||||
if invalid_variants:
|
||||
raise ValueError(
|
||||
"Invalid sparse_mla_mha_variants entries: "
|
||||
f"{invalid_variants}. Valid variants are: "
|
||||
f"{sorted(valid_variants)}"
|
||||
)
|
||||
requested_variant_set = set(requested_variants)
|
||||
variants = [
|
||||
variant
|
||||
for variant in available_variants
|
||||
if variant[0] in requested_variant_set
|
||||
]
|
||||
else:
|
||||
variants = available_variants
|
||||
formatter = ResultsFormatter(console)
|
||||
total = 0
|
||||
for spec in args.batch_specs:
|
||||
q_len = max(request.q_len for request in parse_batch_spec(spec))
|
||||
for variant_label, _, _ in variants:
|
||||
if (
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
total += len(backends)
|
||||
|
||||
with tqdm(total=total, desc="Benchmarking") as pbar:
|
||||
for spec in args.batch_specs:
|
||||
q_len = max(request.q_len for request in parse_batch_spec(spec))
|
||||
for backend in backends:
|
||||
for variant_label, force_mqa, mha_mode in variants:
|
||||
if (
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
config = BenchmarkConfig(
|
||||
backend=f"{backend}_{variant_label}",
|
||||
batch_spec=spec,
|
||||
num_layers=args.num_layers,
|
||||
head_dim=args.head_dim,
|
||||
num_q_heads=args.num_q_heads,
|
||||
num_kv_heads=args.num_kv_heads,
|
||||
block_size=args.block_size,
|
||||
device=args.device,
|
||||
max_model_len=getattr(args, "max_model_len", None),
|
||||
kv_cache_dtype=args.kv_cache_dtype,
|
||||
profile_memory=args.profile_memory,
|
||||
use_cuda_graphs=args.cuda_graphs,
|
||||
ncu_profile=args.ncu_profile,
|
||||
torch_profile=args.torch_profile,
|
||||
torch_profile_dir=args.torch_profile_dir,
|
||||
torch_profile_iters=args.torch_profile_iters,
|
||||
warmup_ms=args.warmup_ms,
|
||||
kv_lora_rank=getattr(args, "kv_lora_rank", None),
|
||||
qk_nope_head_dim=getattr(args, "qk_nope_head_dim", None),
|
||||
qk_rope_head_dim=getattr(args, "qk_rope_head_dim", None),
|
||||
v_head_dim=getattr(args, "v_head_dim", None),
|
||||
sparse_mla_force_mqa=force_mqa,
|
||||
sparse_mla_mha_mode=mha_mode,
|
||||
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
|
||||
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
|
||||
prefill_backend=prefill_backend,
|
||||
)
|
||||
|
||||
# run_mla_benchmark needs the real backend name
|
||||
from mla_runner import run_mla_benchmark as run_mla
|
||||
|
||||
run_label = f"{backend}_{variant_label} {spec}"
|
||||
pbar.set_postfix_str(run_label)
|
||||
|
||||
try:
|
||||
result = run_mla(
|
||||
backend,
|
||||
config,
|
||||
prefill_backend=prefill_backend,
|
||||
sparse_mla_force_mqa=force_mqa,
|
||||
)
|
||||
except Exception as e:
|
||||
result = BenchmarkResult(
|
||||
config=config,
|
||||
mean_time=float("inf"),
|
||||
median_time=float("inf"),
|
||||
std_time=0,
|
||||
min_time=float("inf"),
|
||||
max_time=float("inf"),
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
all_results.append(result)
|
||||
if args.output_csv:
|
||||
formatter.save_csv(all_results, args.output_csv)
|
||||
if args.output_json:
|
||||
formatter.save_json(all_results, args.output_json)
|
||||
|
||||
if not result.success:
|
||||
console.print(
|
||||
f"[red]Error {backend}_{variant_label} "
|
||||
f"{spec}: {result.error}[/]"
|
||||
)
|
||||
|
||||
pbar.update(1)
|
||||
|
||||
# Display results with variant labels as separate "backends"
|
||||
console.print("\n[bold green]MHA vs MQA Results:[/]")
|
||||
variant_backends = [f"{b}_{v}" for b in backends for v, _, _ in variants]
|
||||
formatter.print_table(all_results, variant_backends)
|
||||
|
||||
# Handle model parameter sweep mode
|
||||
elif hasattr(args, "model_parameter_sweep") and args.model_parameter_sweep:
|
||||
# Model parameter sweep
|
||||
|
||||
@@ -4,8 +4,10 @@
|
||||
"""Common utilities for attention benchmarking."""
|
||||
|
||||
import csv
|
||||
import gc
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -44,10 +46,13 @@ def run_do_bench(
|
||||
kwargs: dict[str, Any] = {"return_mode": "all"}
|
||||
if use_cuda_graphs:
|
||||
result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
else:
|
||||
if warmup_ms is not None:
|
||||
kwargs["warmup"] = warmup_ms
|
||||
result = triton.testing.do_bench(benchmark_fn, **kwargs)
|
||||
torch.accelerator.synchronize()
|
||||
return result
|
||||
|
||||
|
||||
@@ -91,42 +96,6 @@ except ImportError:
|
||||
AttentionLayerBase = object # Fallback
|
||||
|
||||
|
||||
class MockKVBProj:
|
||||
"""Mock KV projection layer for MLA prefill mode.
|
||||
|
||||
Mimics ColumnParallelLinear behavior for kv_b_proj in MLA backends.
|
||||
Projects kv_c_normed to [qk_nope_head_dim + v_head_dim] per head.
|
||||
"""
|
||||
|
||||
def __init__(self, num_heads: int, qk_nope_head_dim: int, v_head_dim: int):
|
||||
self.num_heads = num_heads
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.out_dim = qk_nope_head_dim + v_head_dim
|
||||
self.weight = torch.empty(0, dtype=torch.bfloat16)
|
||||
|
||||
def __call__(self, x: torch.Tensor) -> tuple[torch.Tensor]:
|
||||
"""
|
||||
Project kv_c_normed to output space.
|
||||
|
||||
Args:
|
||||
x: Input tensor [num_tokens, kv_lora_rank]
|
||||
|
||||
Returns:
|
||||
Tuple containing output tensor
|
||||
[num_tokens, num_heads, qk_nope_head_dim + v_head_dim]
|
||||
"""
|
||||
num_tokens = x.shape[0]
|
||||
result = torch.randn(
|
||||
num_tokens,
|
||||
self.num_heads,
|
||||
self.out_dim,
|
||||
device=x.device,
|
||||
dtype=x.dtype,
|
||||
)
|
||||
return (result,) # Return as tuple to match ColumnParallelLinear API
|
||||
|
||||
|
||||
class MockIndexer:
|
||||
"""Mock Indexer for sparse MLA backends.
|
||||
|
||||
@@ -158,6 +127,60 @@ class MockIndexer:
|
||||
)
|
||||
self.topk_indices_buffer[:num_tokens] = indices
|
||||
|
||||
def fill_indices(
|
||||
self,
|
||||
num_tokens: int,
|
||||
max_kv_len: int,
|
||||
pattern: str = "random",
|
||||
requests: Sequence[Any] | None = None,
|
||||
):
|
||||
if pattern == "random":
|
||||
self.fill_random_indices(num_tokens, max_kv_len)
|
||||
return
|
||||
if pattern == "prefix":
|
||||
indices = torch.arange(
|
||||
self.topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
)
|
||||
indices = (indices % max_kv_len).expand(num_tokens, -1)
|
||||
self.topk_indices_buffer[:num_tokens] = indices
|
||||
return
|
||||
if pattern == "sliding_window":
|
||||
if requests is None:
|
||||
start = max(max_kv_len - self.topk_tokens, 0)
|
||||
indices = torch.arange(
|
||||
start,
|
||||
start + self.topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
)
|
||||
indices = indices.clamp(max=max_kv_len - 1).expand(num_tokens, -1)
|
||||
self.topk_indices_buffer[:num_tokens] = indices
|
||||
return
|
||||
|
||||
rows = []
|
||||
offsets = torch.arange(
|
||||
self.topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
) - (self.topk_tokens - 1)
|
||||
for request in requests:
|
||||
q_len = request.q_len
|
||||
kv_len = request.kv_len
|
||||
context_len = kv_len - q_len
|
||||
positions = torch.arange(
|
||||
context_len,
|
||||
kv_len,
|
||||
dtype=torch.int32,
|
||||
device=self.topk_indices_buffer.device,
|
||||
)
|
||||
row_indices = positions[:, None] + offsets[None, :]
|
||||
rows.append(row_indices.clamp(min=0, max=kv_len - 1))
|
||||
self.topk_indices_buffer[:num_tokens] = torch.cat(rows, dim=0)
|
||||
return
|
||||
raise ValueError(f"Unknown sparse MLA topk pattern: {pattern}")
|
||||
|
||||
|
||||
class MockLayer(AttentionLayerBase):
|
||||
"""Mock attention layer with scale parameters and impl.
|
||||
@@ -252,10 +275,14 @@ class BenchmarkConfig:
|
||||
num_kv_heads: int
|
||||
block_size: int
|
||||
device: str
|
||||
max_model_len: int | None = None
|
||||
dtype: torch.dtype = torch.float16
|
||||
profile_memory: bool = False
|
||||
use_cuda_graphs: bool = False
|
||||
use_cuda_graphs: bool = True
|
||||
ncu_profile: bool = False
|
||||
torch_profile: bool = False
|
||||
torch_profile_dir: str | None = None
|
||||
torch_profile_iters: int = 3
|
||||
warmup_ms: int | None = None
|
||||
|
||||
# "auto" or "fp8"
|
||||
@@ -271,6 +298,10 @@ class BenchmarkConfig:
|
||||
# Backend-specific tuning
|
||||
num_kv_splits: int | None = None # CUTLASS MLA
|
||||
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
|
||||
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
|
||||
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
|
||||
sparse_mla_dense_mha_max_seq_len: int | None = None
|
||||
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
|
||||
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,474 @@
|
||||
# Sparse MLA benchmark: forward_mha vs forward_mqa
|
||||
#
|
||||
# Usage:
|
||||
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml
|
||||
#
|
||||
# Heatmap grid:
|
||||
# - batch_size: 1, 2, 4, 8, 16, 32
|
||||
# - seq_len: 32, 64, 128, 256, 512, 1024, 2048
|
||||
# - q_len: powers of two through seq_len
|
||||
#
|
||||
# Specs with q_len < seq_len include context; the q_len == seq_len diagonal
|
||||
# covers pure prefill.
|
||||
# The model shape below is the DP case. For the TP8 run, manually change
|
||||
# model.num_q_heads from 128 to 16 before rerunning this benchmark.
|
||||
|
||||
mode: mha_vs_mqa
|
||||
|
||||
model:
|
||||
name: "deepseek-v3"
|
||||
num_layers: 60
|
||||
num_q_heads: 128
|
||||
num_kv_heads: 1
|
||||
head_dim: 576
|
||||
kv_lora_rank: 512
|
||||
qk_nope_head_dim: 128
|
||||
qk_rope_head_dim: 64
|
||||
v_head_dim: 128
|
||||
block_size: 128
|
||||
max_model_len: 2048
|
||||
|
||||
batch_specs:
|
||||
# Batch size 1
|
||||
# seq_len = 32
|
||||
- "1q1s32"
|
||||
- "1q2s32"
|
||||
- "1q4s32"
|
||||
- "1q8s32"
|
||||
- "1q16s32"
|
||||
- "1q32"
|
||||
# seq_len = 64
|
||||
- "1q1s64"
|
||||
- "1q2s64"
|
||||
- "1q4s64"
|
||||
- "1q8s64"
|
||||
- "1q16s64"
|
||||
- "1q32s64"
|
||||
- "1q64"
|
||||
# seq_len = 128
|
||||
- "1q1s128"
|
||||
- "1q2s128"
|
||||
- "1q4s128"
|
||||
- "1q8s128"
|
||||
- "1q16s128"
|
||||
- "1q32s128"
|
||||
- "1q64s128"
|
||||
- "1q128"
|
||||
# seq_len = 256
|
||||
- "1q1s256"
|
||||
- "1q2s256"
|
||||
- "1q4s256"
|
||||
- "1q8s256"
|
||||
- "1q16s256"
|
||||
- "1q32s256"
|
||||
- "1q64s256"
|
||||
- "1q128s256"
|
||||
- "1q256"
|
||||
# seq_len = 512
|
||||
- "1q1s512"
|
||||
- "1q2s512"
|
||||
- "1q4s512"
|
||||
- "1q8s512"
|
||||
- "1q16s512"
|
||||
- "1q32s512"
|
||||
- "1q64s512"
|
||||
- "1q128s512"
|
||||
- "1q256s512"
|
||||
- "1q512"
|
||||
# seq_len = 1024
|
||||
- "1q1s1024"
|
||||
- "1q2s1024"
|
||||
- "1q4s1024"
|
||||
- "1q8s1024"
|
||||
- "1q16s1024"
|
||||
- "1q32s1024"
|
||||
- "1q64s1024"
|
||||
- "1q128s1024"
|
||||
- "1q256s1024"
|
||||
- "1q512s1024"
|
||||
- "1q1024"
|
||||
# seq_len = 2048
|
||||
- "1q1s2048"
|
||||
- "1q2s2048"
|
||||
- "1q4s2048"
|
||||
- "1q8s2048"
|
||||
- "1q16s2048"
|
||||
- "1q32s2048"
|
||||
- "1q64s2048"
|
||||
- "1q128s2048"
|
||||
- "1q256s2048"
|
||||
- "1q512s2048"
|
||||
- "1q1024s2048"
|
||||
- "1q2048"
|
||||
|
||||
# Batch size 2
|
||||
# seq_len = 32
|
||||
- "2q1s32"
|
||||
- "2q2s32"
|
||||
- "2q4s32"
|
||||
- "2q8s32"
|
||||
- "2q16s32"
|
||||
- "2q32"
|
||||
# seq_len = 64
|
||||
- "2q1s64"
|
||||
- "2q2s64"
|
||||
- "2q4s64"
|
||||
- "2q8s64"
|
||||
- "2q16s64"
|
||||
- "2q32s64"
|
||||
- "2q64"
|
||||
# seq_len = 128
|
||||
- "2q1s128"
|
||||
- "2q2s128"
|
||||
- "2q4s128"
|
||||
- "2q8s128"
|
||||
- "2q16s128"
|
||||
- "2q32s128"
|
||||
- "2q64s128"
|
||||
- "2q128"
|
||||
# seq_len = 256
|
||||
- "2q1s256"
|
||||
- "2q2s256"
|
||||
- "2q4s256"
|
||||
- "2q8s256"
|
||||
- "2q16s256"
|
||||
- "2q32s256"
|
||||
- "2q64s256"
|
||||
- "2q128s256"
|
||||
- "2q256"
|
||||
# seq_len = 512
|
||||
- "2q1s512"
|
||||
- "2q2s512"
|
||||
- "2q4s512"
|
||||
- "2q8s512"
|
||||
- "2q16s512"
|
||||
- "2q32s512"
|
||||
- "2q64s512"
|
||||
- "2q128s512"
|
||||
- "2q256s512"
|
||||
- "2q512"
|
||||
# seq_len = 1024
|
||||
- "2q1s1024"
|
||||
- "2q2s1024"
|
||||
- "2q4s1024"
|
||||
- "2q8s1024"
|
||||
- "2q16s1024"
|
||||
- "2q32s1024"
|
||||
- "2q64s1024"
|
||||
- "2q128s1024"
|
||||
- "2q256s1024"
|
||||
- "2q512s1024"
|
||||
- "2q1024"
|
||||
# seq_len = 2048
|
||||
- "2q1s2048"
|
||||
- "2q2s2048"
|
||||
- "2q4s2048"
|
||||
- "2q8s2048"
|
||||
- "2q16s2048"
|
||||
- "2q32s2048"
|
||||
- "2q64s2048"
|
||||
- "2q128s2048"
|
||||
- "2q256s2048"
|
||||
- "2q512s2048"
|
||||
- "2q1024s2048"
|
||||
- "2q2048"
|
||||
|
||||
# Batch size 4
|
||||
# seq_len = 32
|
||||
- "4q1s32"
|
||||
- "4q2s32"
|
||||
- "4q4s32"
|
||||
- "4q8s32"
|
||||
- "4q16s32"
|
||||
- "4q32"
|
||||
# seq_len = 64
|
||||
- "4q1s64"
|
||||
- "4q2s64"
|
||||
- "4q4s64"
|
||||
- "4q8s64"
|
||||
- "4q16s64"
|
||||
- "4q32s64"
|
||||
- "4q64"
|
||||
# seq_len = 128
|
||||
- "4q1s128"
|
||||
- "4q2s128"
|
||||
- "4q4s128"
|
||||
- "4q8s128"
|
||||
- "4q16s128"
|
||||
- "4q32s128"
|
||||
- "4q64s128"
|
||||
- "4q128"
|
||||
# seq_len = 256
|
||||
- "4q1s256"
|
||||
- "4q2s256"
|
||||
- "4q4s256"
|
||||
- "4q8s256"
|
||||
- "4q16s256"
|
||||
- "4q32s256"
|
||||
- "4q64s256"
|
||||
- "4q128s256"
|
||||
- "4q256"
|
||||
# seq_len = 512
|
||||
- "4q1s512"
|
||||
- "4q2s512"
|
||||
- "4q4s512"
|
||||
- "4q8s512"
|
||||
- "4q16s512"
|
||||
- "4q32s512"
|
||||
- "4q64s512"
|
||||
- "4q128s512"
|
||||
- "4q256s512"
|
||||
- "4q512"
|
||||
# seq_len = 1024
|
||||
- "4q1s1024"
|
||||
- "4q2s1024"
|
||||
- "4q4s1024"
|
||||
- "4q8s1024"
|
||||
- "4q16s1024"
|
||||
- "4q32s1024"
|
||||
- "4q64s1024"
|
||||
- "4q128s1024"
|
||||
- "4q256s1024"
|
||||
- "4q512s1024"
|
||||
- "4q1024"
|
||||
# seq_len = 2048
|
||||
- "4q1s2048"
|
||||
- "4q2s2048"
|
||||
- "4q4s2048"
|
||||
- "4q8s2048"
|
||||
- "4q16s2048"
|
||||
- "4q32s2048"
|
||||
- "4q64s2048"
|
||||
- "4q128s2048"
|
||||
- "4q256s2048"
|
||||
- "4q512s2048"
|
||||
- "4q1024s2048"
|
||||
- "4q2048"
|
||||
|
||||
# Batch size 8
|
||||
# seq_len = 32
|
||||
- "8q1s32"
|
||||
- "8q2s32"
|
||||
- "8q4s32"
|
||||
- "8q8s32"
|
||||
- "8q16s32"
|
||||
- "8q32"
|
||||
# seq_len = 64
|
||||
- "8q1s64"
|
||||
- "8q2s64"
|
||||
- "8q4s64"
|
||||
- "8q8s64"
|
||||
- "8q16s64"
|
||||
- "8q32s64"
|
||||
- "8q64"
|
||||
# seq_len = 128
|
||||
- "8q1s128"
|
||||
- "8q2s128"
|
||||
- "8q4s128"
|
||||
- "8q8s128"
|
||||
- "8q16s128"
|
||||
- "8q32s128"
|
||||
- "8q64s128"
|
||||
- "8q128"
|
||||
# seq_len = 256
|
||||
- "8q1s256"
|
||||
- "8q2s256"
|
||||
- "8q4s256"
|
||||
- "8q8s256"
|
||||
- "8q16s256"
|
||||
- "8q32s256"
|
||||
- "8q64s256"
|
||||
- "8q128s256"
|
||||
- "8q256"
|
||||
# seq_len = 512
|
||||
- "8q1s512"
|
||||
- "8q2s512"
|
||||
- "8q4s512"
|
||||
- "8q8s512"
|
||||
- "8q16s512"
|
||||
- "8q32s512"
|
||||
- "8q64s512"
|
||||
- "8q128s512"
|
||||
- "8q256s512"
|
||||
- "8q512"
|
||||
# seq_len = 1024
|
||||
- "8q1s1024"
|
||||
- "8q2s1024"
|
||||
- "8q4s1024"
|
||||
- "8q8s1024"
|
||||
- "8q16s1024"
|
||||
- "8q32s1024"
|
||||
- "8q64s1024"
|
||||
- "8q128s1024"
|
||||
- "8q256s1024"
|
||||
- "8q512s1024"
|
||||
- "8q1024"
|
||||
# seq_len = 2048
|
||||
- "8q1s2048"
|
||||
- "8q2s2048"
|
||||
- "8q4s2048"
|
||||
- "8q8s2048"
|
||||
- "8q16s2048"
|
||||
- "8q32s2048"
|
||||
- "8q64s2048"
|
||||
- "8q128s2048"
|
||||
- "8q256s2048"
|
||||
- "8q512s2048"
|
||||
- "8q1024s2048"
|
||||
- "8q2048"
|
||||
|
||||
# Batch size 16
|
||||
# seq_len = 32
|
||||
- "16q1s32"
|
||||
- "16q2s32"
|
||||
- "16q4s32"
|
||||
- "16q8s32"
|
||||
- "16q16s32"
|
||||
- "16q32"
|
||||
# seq_len = 64
|
||||
- "16q1s64"
|
||||
- "16q2s64"
|
||||
- "16q4s64"
|
||||
- "16q8s64"
|
||||
- "16q16s64"
|
||||
- "16q32s64"
|
||||
- "16q64"
|
||||
# seq_len = 128
|
||||
- "16q1s128"
|
||||
- "16q2s128"
|
||||
- "16q4s128"
|
||||
- "16q8s128"
|
||||
- "16q16s128"
|
||||
- "16q32s128"
|
||||
- "16q64s128"
|
||||
- "16q128"
|
||||
# seq_len = 256
|
||||
- "16q1s256"
|
||||
- "16q2s256"
|
||||
- "16q4s256"
|
||||
- "16q8s256"
|
||||
- "16q16s256"
|
||||
- "16q32s256"
|
||||
- "16q64s256"
|
||||
- "16q128s256"
|
||||
- "16q256"
|
||||
# seq_len = 512
|
||||
- "16q1s512"
|
||||
- "16q2s512"
|
||||
- "16q4s512"
|
||||
- "16q8s512"
|
||||
- "16q16s512"
|
||||
- "16q32s512"
|
||||
- "16q64s512"
|
||||
- "16q128s512"
|
||||
- "16q256s512"
|
||||
- "16q512"
|
||||
# seq_len = 1024
|
||||
- "16q1s1024"
|
||||
- "16q2s1024"
|
||||
- "16q4s1024"
|
||||
- "16q8s1024"
|
||||
- "16q16s1024"
|
||||
- "16q32s1024"
|
||||
- "16q64s1024"
|
||||
- "16q128s1024"
|
||||
- "16q256s1024"
|
||||
- "16q512s1024"
|
||||
- "16q1024"
|
||||
# seq_len = 2048
|
||||
- "16q1s2048"
|
||||
- "16q2s2048"
|
||||
- "16q4s2048"
|
||||
- "16q8s2048"
|
||||
- "16q16s2048"
|
||||
- "16q32s2048"
|
||||
- "16q64s2048"
|
||||
- "16q128s2048"
|
||||
- "16q256s2048"
|
||||
- "16q512s2048"
|
||||
- "16q1024s2048"
|
||||
- "16q2048"
|
||||
|
||||
# Batch size 32
|
||||
# seq_len = 32
|
||||
- "32q1s32"
|
||||
- "32q2s32"
|
||||
- "32q4s32"
|
||||
- "32q8s32"
|
||||
- "32q16s32"
|
||||
- "32q32"
|
||||
# seq_len = 64
|
||||
- "32q1s64"
|
||||
- "32q2s64"
|
||||
- "32q4s64"
|
||||
- "32q8s64"
|
||||
- "32q16s64"
|
||||
- "32q32s64"
|
||||
- "32q64"
|
||||
# seq_len = 128
|
||||
- "32q1s128"
|
||||
- "32q2s128"
|
||||
- "32q4s128"
|
||||
- "32q8s128"
|
||||
- "32q16s128"
|
||||
- "32q32s128"
|
||||
- "32q64s128"
|
||||
- "32q128"
|
||||
# seq_len = 256
|
||||
- "32q1s256"
|
||||
- "32q2s256"
|
||||
- "32q4s256"
|
||||
- "32q8s256"
|
||||
- "32q16s256"
|
||||
- "32q32s256"
|
||||
- "32q64s256"
|
||||
- "32q128s256"
|
||||
- "32q256"
|
||||
# seq_len = 512
|
||||
- "32q1s512"
|
||||
- "32q2s512"
|
||||
- "32q4s512"
|
||||
- "32q8s512"
|
||||
- "32q16s512"
|
||||
- "32q32s512"
|
||||
- "32q64s512"
|
||||
- "32q128s512"
|
||||
- "32q256s512"
|
||||
- "32q512"
|
||||
# seq_len = 1024
|
||||
- "32q1s1024"
|
||||
- "32q2s1024"
|
||||
- "32q4s1024"
|
||||
- "32q8s1024"
|
||||
- "32q16s1024"
|
||||
- "32q32s1024"
|
||||
- "32q64s1024"
|
||||
- "32q128s1024"
|
||||
- "32q256s1024"
|
||||
- "32q512s1024"
|
||||
- "32q1024"
|
||||
# seq_len = 2048
|
||||
- "32q1s2048"
|
||||
- "32q2s2048"
|
||||
- "32q4s2048"
|
||||
- "32q8s2048"
|
||||
- "32q16s2048"
|
||||
- "32q32s2048"
|
||||
- "32q64s2048"
|
||||
- "32q128s2048"
|
||||
- "32q256s2048"
|
||||
- "32q512s2048"
|
||||
- "32q1024s2048"
|
||||
- "32q2048"
|
||||
|
||||
backends:
|
||||
- FLASHMLA_SPARSE
|
||||
|
||||
device: "cuda:0"
|
||||
profile_memory: false
|
||||
sparse_mla_dense_mha_max_seq_len: 2048
|
||||
sparse_mla_topk_pattern: "random"
|
||||
|
||||
output:
|
||||
csv: "benchmark_output/mla_sparse_mha_vs_mqa.csv"
|
||||
json: "benchmark_output/mla_sparse_mha_vs_mqa.json"
|
||||
@@ -9,6 +9,8 @@ needing full VllmConfig integration.
|
||||
"""
|
||||
|
||||
import statistics
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -17,7 +19,6 @@ from common import (
|
||||
BenchmarkResult,
|
||||
MockHfConfig,
|
||||
MockIndexer,
|
||||
MockKVBProj,
|
||||
MockLayer,
|
||||
run_do_bench,
|
||||
run_ncu_profile,
|
||||
@@ -33,8 +34,59 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import MLAPrefillBackendEnum
|
||||
|
||||
|
||||
def _safe_profile_name(value: str) -> str:
|
||||
return "".join(c if c.isalnum() or c in "._-" else "_" for c in value)
|
||||
|
||||
|
||||
def _create_kv_b_proj(
|
||||
mla_dims: dict,
|
||||
device: torch.device,
|
||||
):
|
||||
kv_b_proj = ColumnParallelLinear(
|
||||
mla_dims["kv_lora_rank"],
|
||||
mla_dims["num_q_heads"]
|
||||
* (mla_dims["qk_nope_head_dim"] + mla_dims["v_head_dim"]),
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prefix="benchmark.kv_b_proj",
|
||||
).to(device)
|
||||
with torch.no_grad():
|
||||
kv_b_proj.weight.copy_(torch.randn_like(kv_b_proj.weight))
|
||||
return kv_b_proj
|
||||
|
||||
|
||||
def _ensure_single_rank_model_parallel() -> None:
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
|
||||
if not dist.is_available():
|
||||
return
|
||||
if not dist.is_initialized():
|
||||
with tempfile.NamedTemporaryFile(
|
||||
prefix="vllm_bench_dist_", delete=False
|
||||
) as init_file:
|
||||
distributed_init_method = f"file://{init_file.name}"
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=distributed_init_method,
|
||||
local_rank=0,
|
||||
backend="nccl",
|
||||
)
|
||||
if not model_parallel_is_initialized():
|
||||
ensure_model_parallel_initialized(1, 1)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# VllmConfig Creation
|
||||
# ============================================================================
|
||||
@@ -66,10 +118,12 @@ def create_minimal_vllm_config(
|
||||
block_size: int = 128,
|
||||
max_num_seqs: int = 256,
|
||||
max_num_batched_tokens: int = 8192,
|
||||
max_model_len: int = 32768,
|
||||
mla_dims: dict | None = None,
|
||||
index_topk: int | None = None,
|
||||
prefill_backend: str | None = None,
|
||||
kv_cache_dtype: str = "auto",
|
||||
sparse_mla_force_mqa: bool = False,
|
||||
) -> VllmConfig:
|
||||
"""
|
||||
Create minimal VllmConfig for MLA benchmarks.
|
||||
@@ -86,6 +140,8 @@ def create_minimal_vllm_config(
|
||||
prefill_backend: Prefill backend name (e.g., "fa3", "fa4", "flashinfer",
|
||||
"trtllm"). Configures the attention config to force
|
||||
the specified prefill backend.
|
||||
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
|
||||
forward_mqa (even prefill tokens).
|
||||
|
||||
Returns:
|
||||
VllmConfig for benchmarking
|
||||
@@ -131,7 +187,7 @@ def create_minimal_vllm_config(
|
||||
trust_remote_code=True,
|
||||
dtype="bfloat16",
|
||||
seed=0,
|
||||
max_model_len=32768,
|
||||
max_model_len=max_model_len,
|
||||
quantization=None,
|
||||
enforce_eager=False,
|
||||
max_logprobs=20,
|
||||
@@ -163,7 +219,7 @@ def create_minimal_vllm_config(
|
||||
scheduler_config = SchedulerConfig(
|
||||
max_num_seqs=max_num_seqs,
|
||||
max_num_batched_tokens=max(max_num_batched_tokens, max_num_seqs),
|
||||
max_model_len=32768,
|
||||
max_model_len=max_model_len,
|
||||
is_encoder_decoder=False,
|
||||
enable_chunked_prefill=True,
|
||||
)
|
||||
@@ -192,6 +248,9 @@ def create_minimal_vllm_config(
|
||||
"flash_attn_version"
|
||||
]
|
||||
|
||||
if sparse_mla_force_mqa:
|
||||
vllm_config.attention_config.sparse_mla_force_mqa = True
|
||||
|
||||
return vllm_config
|
||||
|
||||
|
||||
@@ -548,12 +607,7 @@ def _create_backend_impl(
|
||||
# Calculate scale
|
||||
scale = 1.0 / np.sqrt(mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"])
|
||||
|
||||
# Create mock kv_b_proj layer for prefill mode
|
||||
mock_kv_b_proj = MockKVBProj(
|
||||
num_heads=mla_dims["num_q_heads"],
|
||||
qk_nope_head_dim=mla_dims["qk_nope_head_dim"],
|
||||
v_head_dim=mla_dims["v_head_dim"],
|
||||
)
|
||||
kv_b_proj = _create_kv_b_proj(mla_dims, device)
|
||||
|
||||
# Create indexer for sparse backends
|
||||
indexer = None
|
||||
@@ -584,7 +638,7 @@ def _create_backend_impl(
|
||||
"qk_rope_head_dim": mla_dims["qk_rope_head_dim"],
|
||||
"qk_head_dim": mla_dims["qk_nope_head_dim"] + mla_dims["qk_rope_head_dim"],
|
||||
"v_head_dim": mla_dims["v_head_dim"],
|
||||
"kv_b_proj": mock_kv_b_proj,
|
||||
"kv_b_proj": kv_b_proj,
|
||||
}
|
||||
|
||||
# Add indexer for sparse backends
|
||||
@@ -785,14 +839,35 @@ def _run_single_benchmark(
|
||||
# Fill indexer with random indices for sparse backends
|
||||
is_sparse = backend_cfg.get("is_sparse", False)
|
||||
if is_sparse and indexer is not None:
|
||||
indexer.fill_random_indices(total_q, max_kv_len)
|
||||
indexer.fill_indices(
|
||||
total_q,
|
||||
max_kv_len,
|
||||
getattr(config, "sparse_mla_topk_pattern", "random"),
|
||||
)
|
||||
|
||||
# Determine which forward methods to use based on metadata.
|
||||
# Sparse MLA backends always use forward_mqa
|
||||
has_decode = is_sparse or getattr(metadata, "decode", None) is not None
|
||||
has_prefill = not is_sparse and getattr(metadata, "prefill", None) is not None
|
||||
# Non-sparse backends use .decode/.prefill sub-objects.
|
||||
# Sparse backends use num_decode_tokens/num_prefills directly.
|
||||
#
|
||||
# sparse_mla_force_mqa overrides: even for prefill metadata, use MQA.
|
||||
force_mqa = getattr(config, "sparse_mla_force_mqa", False)
|
||||
force_dense_mha = getattr(config, "sparse_mla_mha_mode", "auto") == "dense"
|
||||
if force_mqa:
|
||||
has_decode = True
|
||||
has_prefill = False
|
||||
elif is_sparse:
|
||||
has_decode = metadata.num_decode_tokens > 0
|
||||
has_prefill = metadata.num_prefills > 0
|
||||
else:
|
||||
has_decode = metadata.decode is not None
|
||||
has_prefill = metadata.prefill is not None
|
||||
if not has_decode and not has_prefill:
|
||||
raise RuntimeError("Metadata has neither decode nor prefill metadata")
|
||||
if is_sparse and force_dense_mha and not has_prefill:
|
||||
raise RuntimeError(
|
||||
"Sparse MLA dense_mha benchmark did not produce prefill metadata. "
|
||||
"Check reorder_batch_threshold/path forcing."
|
||||
)
|
||||
|
||||
num_decode = (
|
||||
metadata.num_decode_tokens
|
||||
@@ -871,7 +946,6 @@ def _run_single_benchmark(
|
||||
metadata,
|
||||
prefill_inputs["k_scale"],
|
||||
prefill_fp8_output if fused_output else prefill_inputs["output"],
|
||||
prefill_output_scale if fused_output else None,
|
||||
)
|
||||
if fused_output:
|
||||
out = prefill_fp8_output
|
||||
@@ -898,6 +972,48 @@ def _run_single_benchmark(
|
||||
throughput_tokens_per_sec=0.0,
|
||||
)
|
||||
|
||||
if config.torch_profile:
|
||||
profile_dir = Path(
|
||||
config.torch_profile_dir or "benchmark_outputs/torch_profiles"
|
||||
)
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
trace_name = _safe_profile_name(f"{config.backend}_{config.batch_spec}")
|
||||
trace_path = profile_dir / f"{trace_name}.json"
|
||||
iters = max(config.torch_profile_iters, 1)
|
||||
|
||||
forward_fn()
|
||||
torch.accelerator.synchronize()
|
||||
with torch.profiler.profile(
|
||||
activities=[
|
||||
torch.profiler.ProfilerActivity.CPU,
|
||||
torch.profiler.ProfilerActivity.CUDA,
|
||||
],
|
||||
record_shapes=True,
|
||||
profile_memory=True,
|
||||
with_stack=False,
|
||||
) as prof:
|
||||
for _ in range(iters):
|
||||
forward_fn()
|
||||
torch.accelerator.synchronize()
|
||||
prof.step()
|
||||
prof.export_chrome_trace(str(trace_path))
|
||||
print(f"Saved PyTorch profiler trace to {trace_path}")
|
||||
print(
|
||||
prof.key_averages().table(
|
||||
sort_by="cuda_time_total",
|
||||
row_limit=25,
|
||||
)
|
||||
)
|
||||
return BenchmarkResult(
|
||||
config=config,
|
||||
mean_time=0.0,
|
||||
median_time=0.0,
|
||||
std_time=0.0,
|
||||
min_time=0.0,
|
||||
max_time=0.0,
|
||||
throughput_tokens_per_sec=0.0,
|
||||
)
|
||||
|
||||
all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
|
||||
|
||||
# Convert ms to seconds per layer
|
||||
@@ -920,6 +1036,7 @@ def _run_mla_benchmark_batched(
|
||||
configs_with_params: list[tuple], # [(config, threshold, num_splits), ...]
|
||||
index_topk: int = 2048,
|
||||
prefill_backend: str | None = None,
|
||||
sparse_mla_force_mqa: bool = False,
|
||||
output_scale: float | None = None,
|
||||
fuse_quant_op: bool = False,
|
||||
) -> list[BenchmarkResult]:
|
||||
@@ -940,6 +1057,8 @@ def _run_mla_benchmark_batched(
|
||||
index_topk: Topk value for sparse MLA backends (default 2048)
|
||||
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
|
||||
When set, forces the specified FlashAttention version for prefill.
|
||||
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
|
||||
forward_mqa (even prefill tokens).
|
||||
|
||||
Returns:
|
||||
List of BenchmarkResult objects
|
||||
@@ -980,21 +1099,41 @@ def _run_mla_benchmark_batched(
|
||||
sum(r.q_len for r in parse_batch_spec(cfg.batch_spec))
|
||||
for cfg, *_ in configs_with_params
|
||||
)
|
||||
max_model_len = max(
|
||||
max_total_q,
|
||||
max(
|
||||
getattr(cfg, "max_model_len", None) or 32768
|
||||
for cfg, *_ in configs_with_params
|
||||
),
|
||||
)
|
||||
|
||||
# Create and set vLLM config for MLA (reused across all benchmarks)
|
||||
vllm_config = create_minimal_vllm_config(
|
||||
model_name="deepseek-v3", # Used only for model path
|
||||
block_size=block_size,
|
||||
max_num_batched_tokens=max_total_q,
|
||||
max_model_len=max_model_len,
|
||||
mla_dims=mla_dims, # Use custom dims from config or default
|
||||
index_topk=index_topk if is_sparse else None,
|
||||
prefill_backend=prefill_backend,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
sparse_mla_force_mqa=sparse_mla_force_mqa,
|
||||
)
|
||||
|
||||
results = []
|
||||
|
||||
# Initialize workspace manager (needed by metadata builders)
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
if not is_workspace_manager_initialized():
|
||||
init_workspace_manager(device)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
_ensure_single_rank_model_parallel()
|
||||
|
||||
# Create backend impl, layer, builder, and indexer (reused across benchmarks)
|
||||
impl, layer, builder_instance, indexer = _create_backend_impl(
|
||||
backend_cfg,
|
||||
@@ -1040,9 +1179,20 @@ def _run_mla_benchmark_batched(
|
||||
for config, threshold, num_splits in configs_with_params:
|
||||
# Set threshold for this benchmark (FlashAttn/FlashMLA only)
|
||||
original_threshold = None
|
||||
if threshold is not None and builder_instance:
|
||||
effective_threshold = threshold
|
||||
force_dense_mha = (
|
||||
is_sparse
|
||||
and getattr(config, "sparse_mla_mha_mode", "auto") == "dense"
|
||||
and not getattr(config, "sparse_mla_force_mqa", False)
|
||||
)
|
||||
if force_dense_mha:
|
||||
# Sparse MLA normally treats q_len <= 1 as decode. Use an
|
||||
# impossible threshold so dense_mha benchmarks actually run
|
||||
# the prefill/MHA path, including q_len=1 short extends.
|
||||
effective_threshold = -1
|
||||
if effective_threshold is not None and builder_instance:
|
||||
original_threshold = builder_instance.reorder_batch_threshold
|
||||
builder_instance.reorder_batch_threshold = threshold
|
||||
builder_instance.reorder_batch_threshold = effective_threshold
|
||||
|
||||
# Set num_splits for CUTLASS
|
||||
original_num_splits = None
|
||||
@@ -1090,6 +1240,7 @@ def run_mla_benchmark(
|
||||
num_kv_splits: int | None = None,
|
||||
index_topk: int = 2048,
|
||||
prefill_backend: str | None = None,
|
||||
sparse_mla_force_mqa: bool = False,
|
||||
output_scale: float | None = None,
|
||||
fuse_quant_op: bool = False,
|
||||
) -> BenchmarkResult | list[BenchmarkResult]:
|
||||
@@ -1111,6 +1262,8 @@ def run_mla_benchmark(
|
||||
index_topk: Topk value for sparse MLA backends (default 2048)
|
||||
prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
|
||||
When set, forces the specified FlashAttention version for prefill.
|
||||
sparse_mla_force_mqa: If True, forces all sparse MLA tokens through
|
||||
forward_mqa (even prefill tokens).
|
||||
output_scale: Static per-tensor FP8 scale for prefill output (None = bf16).
|
||||
fuse_quant_op: With output_scale set, fuse the FP8 write into the prefill
|
||||
kernel vs a standalone post-quant kernel. See _run_single_benchmark.
|
||||
@@ -1142,6 +1295,7 @@ def run_mla_benchmark(
|
||||
configs_with_params,
|
||||
index_topk,
|
||||
prefill_backend=prefill_backend,
|
||||
sparse_mla_force_mqa=sparse_mla_force_mqa,
|
||||
output_scale=output_scale,
|
||||
fuse_quant_op=fuse_quant_op,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user