[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 21:17:12 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 3a9bfc209f
commit de9250ac9e
10 changed files with 527 additions and 74 deletions
@@ -827,25 +827,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
use_mha = True
if self.impl.is_sparse and num_mha_tokens > 0:
prefill = getattr(attn_metadata, "prefill", None)
use_dense_mha = getattr(prefill, "use_dense_mha", False)
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
use_masked_mha = (
self.prefill_backend is not None
and self.impl.masked_mha_available # type: ignore[attr-defined]
and self.impl.dcp_world_size <= 1
and prefill is not None
and _use_masked_mha(
backend_name=self.attn_backend.get_name(),
tensor_parallel_size=self._vllm_config.parallel_config.tensor_parallel_size,
query_len=prefill.max_query_len,
seq_len=prefill_max_seq_len,
)
and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined]
)
use_mha = (use_dense_mha or use_masked_mha) and not (
self._vllm_config.attention_config.sparse_mla_force_mqa
)
use_mha = self._use_sparse_mha(attn_metadata)
if not use_mha:
num_mqa_tokens = q.size(0)
num_mha_tokens = 0
@@ -1063,6 +1045,31 @@ class MLAAttention(nn.Module, AttentionLayerBase):
output_padded[num_actual_toks:].zero_()
return output_padded
def _use_sparse_mha(self, attn_metadata: "MLACommonMetadata") -> bool:
prefill = attn_metadata.prefill
if prefill is None:
return False
use_masked_mha = (
self.prefill_backend is not None
and self.impl.masked_mha_available # type: ignore[attr-defined]
and self.impl.dcp_world_size <= 1
and _use_masked_mha(
backend_name=self.attn_backend.get_name(),
tensor_parallel_size=(
self._vllm_config.parallel_config.tensor_parallel_size
),
qk_head_dim=self.qk_nope_head_dim + self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
query_len=prefill.max_query_len,
seq_len=attn_metadata.prefill_max_seq_len, # type: ignore[attr-defined]
has_context=prefill.chunked_context is not None,
)
and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined]
)
return (prefill.use_dense_mha or use_masked_mha) and not (
self._vllm_config.attention_config.sparse_mla_force_mqa
)
def process_weights_after_loading(self, act_dtype: torch.dtype):
# Let per-backend impls do their own weight packing first (no-op
# unless overridden), mirroring Attention.process_weights_after_loading.
@@ -1619,36 +1626,121 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
)
_DSV32_MASKED_MHA_THRESHOLDS: dict[str, dict[int, tuple[int | None, ...]]] = {
"FLASHMLA_SPARSE": {
1: (1536, 4096, None, None, None),
2: (512, 1024, 4096, None, None),
4: (512, 1024, 1536, 8192, None),
8: (512, 512, 1024, 2048, 16384),
_MaskedMHARanges = tuple[tuple[int, int], ...]
_MASKED_MHA_THRESHOLDS: dict[
tuple[int, int],
dict[str, dict[int, tuple[_MaskedMHARanges, _MaskedMHARanges | None]]],
] = {
(192, 128): {
"FLASHMLA_SPARSE": {
1: (((2048, 1536), (4096, 4096)), None),
2: (((2048, 512), (4096, 1024), (8192, 4096)), None),
4: (
((2048, 512), (4096, 1024), (8192, 1536), (16384, 8192)),
None,
),
8: (
(
(2048, 512),
(4096, 512),
(8192, 1024),
(16384, 2048),
(32768, 16384),
),
None,
),
},
"FLASHINFER_MLA_SPARSE": {
8: (
(
(2048, 512),
(4096, 1024),
(8192, 1024),
(16384, 2048),
(32768, 32768),
),
None,
),
},
},
"FLASHINFER_MLA_SPARSE": {
8: (512, 1024, 1024, 2048, 32768),
(256, 256): {
"FLASHMLA_SPARSE": {
1: (((8 * 1024, 0),), ()),
2: (
((20 * 1024, 0),),
((6 * 1024, 4 * 1024), (10 * 1024, 8 * 1024)),
),
4: (
((48 * 1024, 0),),
(
(8 * 1024, 4 * 1024),
(16 * 1024, 8 * 1024),
(24 * 1024, 16 * 1024),
(34 * 1024, 32 * 1024),
),
),
8: (
((112 * 1024, 0),),
(
(8 * 1024, 4 * 1024),
(16 * 1024, 8 * 1024),
(32 * 1024, 16 * 1024),
(48 * 1024, 32 * 1024),
),
),
},
"FLASHINFER_MLA_SPARSE": {
4: (
((36 * 1024, 0),),
(
(12 * 1024, 4 * 1024),
(16 * 1024, 8 * 1024),
(24 * 1024, 16 * 1024),
(28 * 1024, 24 * 1024),
),
),
8: (
((64 * 1024, 0),),
(
(20 * 1024, 4 * 1024),
(24 * 1024, 8 * 1024),
(32 * 1024, 16 * 1024),
(48 * 1024, 32 * 1024),
(56 * 1024, 40 * 1024),
(64 * 1024, 48 * 1024),
(72 * 1024, 56 * 1024),
),
),
},
},
}
_DSV32_SEQ_LEN_BUCKETS = (2048, 4096, 8192, 16384, 32768)
def _use_masked_mha(
*,
backend_name: str,
tensor_parallel_size: int,
qk_head_dim: int,
v_head_dim: int,
query_len: int,
seq_len: int,
has_context: bool,
) -> bool:
thresholds = _DSV32_MASKED_MHA_THRESHOLDS.get(backend_name, {}).get(
tensor_parallel_size
thresholds = (
_MASKED_MHA_THRESHOLDS.get((qk_head_dim, v_head_dim), {})
.get(backend_name, {})
.get(tensor_parallel_size)
)
if thresholds is None:
return False
for bucket_idx, bucket_seq_len in enumerate(_DSV32_SEQ_LEN_BUCKETS):
if seq_len <= bucket_seq_len:
min_query_len = thresholds[bucket_idx]
return min_query_len is not None and query_len >= min_query_len
pure_prefill_ranges, context_ranges = thresholds
ranges = pure_prefill_ranges
if has_context and context_ranges is not None:
ranges = context_ranges
for max_seq_len, min_query_len in ranges:
if seq_len <= max_seq_len:
return query_len >= min_query_len
return False
@@ -88,16 +88,27 @@ def _is_masked_mha_available(
"""Check if masked MHA can ever fire for this model configuration."""
if not current_platform.is_device_capability_family(100):
return False
if (
num_heads_total != 128
or kv_lora_rank != 512
or qk_nope_head_dim != 128
or qk_rope_head_dim != 64
or v_head_dim != 128
model_dims = (
num_heads_total,
kv_lora_rank,
qk_nope_head_dim,
qk_rope_head_dim,
v_head_dim,
)
if model_dims not in (
(128, 512, 128, 64, 128),
(64, 512, 192, 64, 256),
):
return False
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
fa_version = get_flash_attn_version(head_size=qk_head_dim, head_size_v=v_head_dim)
if qk_head_dim == 256 and v_head_dim == 256:
# This path uses contiguous K/V, so it does not need the paged-KV
# features that keep head-dim 256 disabled in the general FA selector.
fa_version = get_flash_attn_version()
else:
fa_version = get_flash_attn_version(
head_size=qk_head_dim, head_size_v=v_head_dim
)
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)