[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
3a9bfc209f
commit
de9250ac9e
@@ -1143,12 +1143,26 @@ def main():
|
||||
]
|
||||
else:
|
||||
variants = available_variants
|
||||
model_sweep = getattr(args, "model_parameter_sweep", None)
|
||||
if model_sweep is None:
|
||||
num_q_heads_values = [args.num_q_heads]
|
||||
elif model_sweep.param_name == "num_q_heads":
|
||||
num_q_heads_values = model_sweep.values
|
||||
else:
|
||||
raise ValueError(
|
||||
"mha_vs_mqa mode only supports a num_q_heads model parameter sweep"
|
||||
)
|
||||
benchmark_specs = [
|
||||
(spec, num_q_heads)
|
||||
for spec in args.batch_specs
|
||||
for num_q_heads in num_q_heads_values
|
||||
]
|
||||
masked_mha_max_seq_len = getattr(
|
||||
args, "sparse_mla_masked_mha_max_seq_len", None
|
||||
)
|
||||
formatter = ResultsFormatter(console)
|
||||
total = 0
|
||||
for spec in args.batch_specs:
|
||||
for spec, _ in benchmark_specs:
|
||||
q_len = max(request.q_len for request in parse_batch_spec(spec))
|
||||
for variant_label, _, _ in variants:
|
||||
if (
|
||||
@@ -1164,7 +1178,7 @@ def main():
|
||||
total += len(backends)
|
||||
|
||||
with tqdm(total=total, desc="Benchmarking") as pbar:
|
||||
for spec in args.batch_specs:
|
||||
for spec, num_q_heads in benchmark_specs:
|
||||
q_len = max(request.q_len for request in parse_batch_spec(spec))
|
||||
for backend in backends:
|
||||
for variant_label, force_mqa, mha_mode in variants:
|
||||
@@ -1178,12 +1192,17 @@ def main():
|
||||
and q_len > masked_mha_max_seq_len
|
||||
):
|
||||
continue
|
||||
result_backend = f"{backend}_{variant_label}"
|
||||
if model_sweep is not None:
|
||||
result_backend = model_sweep.get_label(
|
||||
result_backend, num_q_heads
|
||||
)
|
||||
config = BenchmarkConfig(
|
||||
backend=f"{backend}_{variant_label}",
|
||||
backend=result_backend,
|
||||
batch_spec=spec,
|
||||
num_layers=args.num_layers,
|
||||
head_dim=args.head_dim,
|
||||
num_q_heads=args.num_q_heads,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=args.num_kv_heads,
|
||||
block_size=args.block_size,
|
||||
device=args.device,
|
||||
@@ -1211,7 +1230,7 @@ def main():
|
||||
# run_mla_benchmark needs the real backend name
|
||||
from mla_runner import run_mla_benchmark as run_mla
|
||||
|
||||
run_label = f"{backend}_{variant_label} {spec}"
|
||||
run_label = f"{backend}_{variant_label} {num_q_heads}h {spec}"
|
||||
pbar.set_postfix_str(run_label)
|
||||
|
||||
try:
|
||||
@@ -1248,7 +1267,14 @@ def main():
|
||||
|
||||
# Display results with variant labels as separate "backends"
|
||||
console.print("\n[bold green]MHA vs MQA Results:[/]")
|
||||
variant_backends = [f"{b}_{v}" for b in backends for v, _, _ in variants]
|
||||
variant_backends = []
|
||||
for num_q_heads in num_q_heads_values:
|
||||
for backend in backends:
|
||||
for variant, _, _ in variants:
|
||||
label = f"{backend}_{variant}"
|
||||
if model_sweep is not None:
|
||||
label = model_sweep.get_label(label, num_q_heads)
|
||||
variant_backends.append(label)
|
||||
formatter.print_table(all_results, variant_backends)
|
||||
|
||||
# Handle model parameter sweep mode
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
mode: mha_vs_mqa
|
||||
model:
|
||||
name: "glm-5.x"
|
||||
num_layers: 78
|
||||
num_q_heads: 64
|
||||
num_kv_heads: 1
|
||||
head_dim: 576
|
||||
kv_lora_rank: 512
|
||||
qk_nope_head_dim: 192
|
||||
qk_rope_head_dim: 64
|
||||
v_head_dim: 256
|
||||
block_size: 128
|
||||
max_model_len: 131072
|
||||
model_parameter_sweep:
|
||||
param_name: "num_q_heads"
|
||||
values: [64, 32, 16, 8]
|
||||
label_format: "{backend}_{value}h"
|
||||
batch_specs:
|
||||
- "q256s2048"
|
||||
- "q512s2048"
|
||||
- "q1024s2048"
|
||||
- "q1536s2048"
|
||||
- "q2048"
|
||||
- "q256s4096"
|
||||
- "q512s4096"
|
||||
- "q1024s4096"
|
||||
- "q1536s4096"
|
||||
- "q2048s4096"
|
||||
- "q4096"
|
||||
- "q256s8192"
|
||||
- "q512s8192"
|
||||
- "q1024s8192"
|
||||
- "q1536s8192"
|
||||
- "q2048s8192"
|
||||
- "q4096s8192"
|
||||
- "q8192"
|
||||
- "q256s16384"
|
||||
- "q512s16384"
|
||||
- "q1024s16384"
|
||||
- "q1536s16384"
|
||||
- "q2048s16384"
|
||||
- "q4096s16384"
|
||||
- "q8192s16384"
|
||||
- "q16384"
|
||||
- "q256s32768"
|
||||
- "q512s32768"
|
||||
- "q1024s32768"
|
||||
- "q1536s32768"
|
||||
- "q2048s32768"
|
||||
- "q4096s32768"
|
||||
- "q8192s32768"
|
||||
- "q16384s32768"
|
||||
- "q32768"
|
||||
backends: ["FLASHMLA_SPARSE", "FLASHINFER_MLA_SPARSE"]
|
||||
device: "cuda:0"
|
||||
profile_memory: false
|
||||
sparse_mla_masked_mha_max_seq_len: 131072
|
||||
sparse_mla_mha_variants: ["masked_mha", "mqa"]
|
||||
sparse_mla_topk_pattern: "random"
|
||||
@@ -313,6 +313,9 @@ _BACKEND_PROPERTIES = {
|
||||
"FLASHMLA_SPARSE": {
|
||||
"query_format": "concat", # Single concatenated tensor (vs tuple)
|
||||
},
|
||||
"FLASHINFER_MLA_SPARSE": {
|
||||
"query_format": "concat", # Single concatenated tensor (vs tuple)
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1184,6 +1187,35 @@ def _run_mla_benchmark_batched(
|
||||
)
|
||||
# Run each benchmark with the shared impl
|
||||
for config, threshold, num_splits in configs_with_params:
|
||||
force_masked_mha = (
|
||||
is_sparse and getattr(config, "sparse_mla_mha_mode", "auto") == "masked"
|
||||
)
|
||||
if force_masked_mha:
|
||||
impl.masked_mha_available = True
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES,
|
||||
_topk_mask_shape,
|
||||
)
|
||||
|
||||
requests = parse_batch_spec(config.batch_spec)
|
||||
max_query_len = max(request.q_len for request in requests)
|
||||
mask_shape = _topk_mask_shape(
|
||||
len(requests), max_query_len, max_query_len
|
||||
)
|
||||
mask_words = max(
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
|
||||
mask_shape[0] * mask_shape[1] * mask_shape[2],
|
||||
)
|
||||
if (
|
||||
builder_instance.topk_mask_workspace is None
|
||||
or builder_instance.topk_mask_workspace.numel() != mask_words
|
||||
):
|
||||
builder_instance.topk_mask_workspace = torch.zeros(
|
||||
mask_words,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Set threshold for this benchmark (FlashAttn/FlashMLA only)
|
||||
original_threshold = None
|
||||
effective_threshold = threshold
|
||||
|
||||
Reference in New Issue
Block a user