[2/N][Attention] Enable masked MHA for sparse MLA prefills (#48770)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
864a87febb
commit
82ae4164ee
@@ -617,7 +617,7 @@ def main():
|
||||
"--sparse-mla-mha-variants",
|
||||
nargs="+",
|
||||
default=None,
|
||||
choices=["dense_mha", "mqa"],
|
||||
choices=["dense_mha", "masked_mha", "mqa"],
|
||||
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
|
||||
)
|
||||
|
||||
@@ -748,6 +748,9 @@ def main():
|
||||
args.sparse_mla_mha_variants = yaml_config.get(
|
||||
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
|
||||
)
|
||||
args.sparse_mla_masked_mha_max_seq_len = yaml_config.get(
|
||||
"sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
|
||||
# Parameter sweep configuration
|
||||
if "parameter_sweep" in yaml_config:
|
||||
@@ -1119,6 +1122,7 @@ def main():
|
||||
console.print(f"Prefill backend: {prefill_backend}")
|
||||
available_variants = [
|
||||
("dense_mha", False, "dense"),
|
||||
("masked_mha", False, "masked"),
|
||||
("mqa", True, "auto"),
|
||||
]
|
||||
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
|
||||
@@ -1139,6 +1143,9 @@ def main():
|
||||
]
|
||||
else:
|
||||
variants = available_variants
|
||||
masked_mha_max_seq_len = getattr(
|
||||
args, "sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
formatter = ResultsFormatter(console)
|
||||
total = 0
|
||||
for spec in args.batch_specs:
|
||||
@@ -1148,6 +1155,10 @@ def main():
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
) or (
|
||||
variant_label == "masked_mha"
|
||||
and masked_mha_max_seq_len is not None
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
total += len(backends)
|
||||
@@ -1161,6 +1172,10 @@ def main():
|
||||
variant_label == "dense_mha"
|
||||
and dense_mha_max_seq_len is not None
|
||||
and q_len > dense_mha_max_seq_len
|
||||
) or (
|
||||
variant_label == "masked_mha"
|
||||
and masked_mha_max_seq_len is not None
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
config = BenchmarkConfig(
|
||||
@@ -1190,6 +1205,7 @@ def main():
|
||||
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
|
||||
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
|
||||
prefill_backend=prefill_backend,
|
||||
sparse_mla_masked_mha_max_seq_len=masked_mha_max_seq_len,
|
||||
)
|
||||
|
||||
# run_mla_benchmark needs the real backend name
|
||||
|
||||
Reference in New Issue
Block a user