[2/N][Attention] Enable masked MHA for sparse MLA prefills (#48770)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
864a87febb
commit
82ae4164ee
@@ -617,7 +617,7 @@ def main():
|
||||
"--sparse-mla-mha-variants",
|
||||
nargs="+",
|
||||
default=None,
|
||||
choices=["dense_mha", "mqa"],
|
||||
choices=["dense_mha", "masked_mha", "mqa"],
|
||||
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
|
||||
)
|
||||
|
||||
@@ -748,6 +748,9 @@ def main():
|
||||
args.sparse_mla_mha_variants = yaml_config.get(
|
||||
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
|
||||
)
|
||||
args.sparse_mla_masked_mha_max_seq_len = yaml_config.get(
|
||||
"sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
|
||||
# Parameter sweep configuration
|
||||
if "parameter_sweep" in yaml_config:
|
||||
@@ -1119,6 +1122,7 @@ def main():
|
||||
console.print(f"Prefill backend: {prefill_backend}")
|
||||
available_variants = [
|
||||
("dense_mha", False, "dense"),
|
||||
("masked_mha", False, "masked"),
|
||||
("mqa", True, "auto"),
|
||||
]
|
||||
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
|
||||
@@ -1139,6 +1143,9 @@ def main():
|
||||
]
|
||||
else:
|
||||
variants = available_variants
|
||||
masked_mha_max_seq_len = getattr(
|
||||
args, "sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
formatter = ResultsFormatter(console)
|
||||
total = 0
|
||||
for spec in args.batch_specs:
|
||||
@@ -1148,6 +1155,10 @@ def main():
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
) or (
|
||||
variant_label == "masked_mha"
|
||||
and masked_mha_max_seq_len is not None
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
total += len(backends)
|
||||
@@ -1161,6 +1172,10 @@ def main():
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
) or (
|
||||
variant_label == "masked_mha"
|
||||
and masked_mha_max_seq_len is not None
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
config = BenchmarkConfig(
|
||||
@@ -1190,6 +1205,7 @@ def main():
|
||||
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
|
||||
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
|
||||
prefill_backend=prefill_backend,
|
||||
sparse_mla_masked_mha_max_seq_len=masked_mha_max_seq_len,
|
||||
)
|
||||
|
||||
# run_mla_benchmark needs the real backend name
|
||||
|
||||
@@ -299,8 +299,9 @@ class BenchmarkConfig:
|
||||
num_kv_splits: int | None = None # CUTLASS MLA
|
||||
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
|
||||
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
|
||||
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
|
||||
sparse_mla_mha_mode: str = "auto" # "auto", "dense", or "masked"
|
||||
sparse_mla_dense_mha_max_seq_len: int | None = None
|
||||
sparse_mla_masked_mha_max_seq_len: int | None = None
|
||||
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
|
||||
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)
|
||||
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
# Sparse MLA benchmark: forward_mha vs forward_mqa crossover plot
|
||||
#
|
||||
# Dense pure-prefill sweep to generate a smooth speedup curve. The 100 points
|
||||
# run from q32 through q262144, approximately even on the log2 scale, and include
|
||||
# routing boundaries plus extra samples around the 6k-8k crossover region.
|
||||
# Dense MHA is only benchmarked through q32768 because it dominates runtime above
|
||||
# that range.
|
||||
#
|
||||
# Usage:
|
||||
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml
|
||||
|
||||
mode: mha_vs_mqa
|
||||
|
||||
model:
|
||||
name: "deepseek-v3"
|
||||
num_layers: 60
|
||||
num_q_heads: 16
|
||||
num_kv_heads: 1
|
||||
head_dim: 576
|
||||
kv_lora_rank: 512
|
||||
qk_nope_head_dim: 128
|
||||
qk_rope_head_dim: 64
|
||||
v_head_dim: 128
|
||||
block_size: 128
|
||||
max_model_len: 262144
|
||||
|
||||
batch_specs:
|
||||
- "q32"
|
||||
- "q36"
|
||||
- "q40"
|
||||
- "q45"
|
||||
- "q50"
|
||||
- "q56"
|
||||
- "q62"
|
||||
- "q70"
|
||||
- "q78"
|
||||
- "q87"
|
||||
- "q97"
|
||||
- "q109"
|
||||
- "q122"
|
||||
- "q136"
|
||||
- "q152"
|
||||
- "q170"
|
||||
- "q190"
|
||||
- "q212"
|
||||
- "q237"
|
||||
- "q255"
|
||||
- "q256"
|
||||
- "q257"
|
||||
- "q265"
|
||||
- "q296"
|
||||
- "q331"
|
||||
- "q370"
|
||||
- "q413"
|
||||
- "q462"
|
||||
- "q516"
|
||||
- "q577"
|
||||
- "q645"
|
||||
- "q721"
|
||||
- "q806"
|
||||
- "q901"
|
||||
- "q1007"
|
||||
- "q1125"
|
||||
- "q1257"
|
||||
- "q1405"
|
||||
- "q1571"
|
||||
- "q1756"
|
||||
- "q1962"
|
||||
- "q2048"
|
||||
- "q2049"
|
||||
- "q2193"
|
||||
- "q2451"
|
||||
- "q2740"
|
||||
- "q3062"
|
||||
- "q3422"
|
||||
- "q3825"
|
||||
- "q4275"
|
||||
- "q4778"
|
||||
- "q5340"
|
||||
- "q5854"
|
||||
- "q5969"
|
||||
- "q6000"
|
||||
- "q6144"
|
||||
- "q6367"
|
||||
- "q6671"
|
||||
- "q6925"
|
||||
- "q7168"
|
||||
- "q7456"
|
||||
- "q7532"
|
||||
- "q8192"
|
||||
- "q8193"
|
||||
- "q8333"
|
||||
- "q9314"
|
||||
- "q10410"
|
||||
- "q11635"
|
||||
- "q13004"
|
||||
- "q14534"
|
||||
- "q16244"
|
||||
- "q18156"
|
||||
- "q20292"
|
||||
- "q22680"
|
||||
- "q25349"
|
||||
- "q28332"
|
||||
- "q31665"
|
||||
- "q32768"
|
||||
- "q32769"
|
||||
- "q35391"
|
||||
- "q39556"
|
||||
- "q44210"
|
||||
- "q49413"
|
||||
- "q55227"
|
||||
- "q61726"
|
||||
- "q65536"
|
||||
|
||||
backends:
|
||||
- FLASHMLA_SPARSE
|
||||
|
||||
device: "cuda:0"
|
||||
profile_memory: true
|
||||
sparse_mla_dense_mha_max_seq_len: 25349
|
||||
sparse_mla_masked_mha_max_seq_len: 8192
|
||||
sparse_mla_mha_variants: ["dense_mha", "masked_mha", "mqa"]
|
||||
sparse_mla_topk_pattern: "random"
|
||||
@@ -931,10 +931,17 @@ def _run_single_benchmark(
|
||||
prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
|
||||
|
||||
fused_output = output_scale is not None and fuse_quant_op
|
||||
mha_mode = getattr(config, "sparse_mla_mha_mode", "auto")
|
||||
|
||||
# Build forward function (runs a single decode/prefill pass)
|
||||
def forward_fn():
|
||||
results = []
|
||||
impl._sparse_mla_force_masked_mha = ( # type: ignore[attr-defined]
|
||||
mha_mode == "masked"
|
||||
)
|
||||
impl._sparse_mla_force_dense_mha = ( # type: ignore[attr-defined]
|
||||
mha_mode == "dense"
|
||||
)
|
||||
if has_decode:
|
||||
results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
|
||||
if has_prefill:
|
||||
|
||||
Reference in New Issue
Block a user