[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
3a9bfc209f
commit
de9250ac9e
@@ -827,25 +827,7 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
use_mha = True
|
||||
|
||||
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||
prefill = getattr(attn_metadata, "prefill", None)
|
||||
use_dense_mha = getattr(prefill, "use_dense_mha", False)
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
use_masked_mha = (
|
||||
self.prefill_backend is not None
|
||||
and self.impl.masked_mha_available # type: ignore[attr-defined]
|
||||
and self.impl.dcp_world_size <= 1
|
||||
and prefill is not None
|
||||
and _use_masked_mha(
|
||||
backend_name=self.attn_backend.get_name(),
|
||||
tensor_parallel_size=self._vllm_config.parallel_config.tensor_parallel_size,
|
||||
query_len=prefill.max_query_len,
|
||||
seq_len=prefill_max_seq_len,
|
||||
)
|
||||
and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined]
|
||||
)
|
||||
use_mha = (use_dense_mha or use_masked_mha) and not (
|
||||
self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
)
|
||||
use_mha = self._use_sparse_mha(attn_metadata)
|
||||
if not use_mha:
|
||||
num_mqa_tokens = q.size(0)
|
||||
num_mha_tokens = 0
|
||||
@@ -1063,6 +1045,31 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
output_padded[num_actual_toks:].zero_()
|
||||
return output_padded
|
||||
|
||||
def _use_sparse_mha(self, attn_metadata: "MLACommonMetadata") -> bool:
|
||||
prefill = attn_metadata.prefill
|
||||
if prefill is None:
|
||||
return False
|
||||
use_masked_mha = (
|
||||
self.prefill_backend is not None
|
||||
and self.impl.masked_mha_available # type: ignore[attr-defined]
|
||||
and self.impl.dcp_world_size <= 1
|
||||
and _use_masked_mha(
|
||||
backend_name=self.attn_backend.get_name(),
|
||||
tensor_parallel_size=(
|
||||
self._vllm_config.parallel_config.tensor_parallel_size
|
||||
),
|
||||
qk_head_dim=self.qk_nope_head_dim + self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
query_len=prefill.max_query_len,
|
||||
seq_len=attn_metadata.prefill_max_seq_len, # type: ignore[attr-defined]
|
||||
has_context=prefill.chunked_context is not None,
|
||||
)
|
||||
and self.impl.masked_mha_workspace_fits(prefill) # type: ignore[attr-defined]
|
||||
)
|
||||
return (prefill.use_dense_mha or use_masked_mha) and not (
|
||||
self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, act_dtype: torch.dtype):
|
||||
# Let per-backend impls do their own weight packing first (no-op
|
||||
# unless overridden), mirroring Attention.process_weights_after_loading.
|
||||
@@ -1619,36 +1626,121 @@ def get_mla_dims(model_config: ModelConfig) -> MLADims:
|
||||
)
|
||||
|
||||
|
||||
_DSV32_MASKED_MHA_THRESHOLDS: dict[str, dict[int, tuple[int | None, ...]]] = {
|
||||
"FLASHMLA_SPARSE": {
|
||||
1: (1536, 4096, None, None, None),
|
||||
2: (512, 1024, 4096, None, None),
|
||||
4: (512, 1024, 1536, 8192, None),
|
||||
8: (512, 512, 1024, 2048, 16384),
|
||||
_MaskedMHARanges = tuple[tuple[int, int], ...]
|
||||
|
||||
_MASKED_MHA_THRESHOLDS: dict[
|
||||
tuple[int, int],
|
||||
dict[str, dict[int, tuple[_MaskedMHARanges, _MaskedMHARanges | None]]],
|
||||
] = {
|
||||
(192, 128): {
|
||||
"FLASHMLA_SPARSE": {
|
||||
1: (((2048, 1536), (4096, 4096)), None),
|
||||
2: (((2048, 512), (4096, 1024), (8192, 4096)), None),
|
||||
4: (
|
||||
((2048, 512), (4096, 1024), (8192, 1536), (16384, 8192)),
|
||||
None,
|
||||
),
|
||||
8: (
|
||||
(
|
||||
(2048, 512),
|
||||
(4096, 512),
|
||||
(8192, 1024),
|
||||
(16384, 2048),
|
||||
(32768, 16384),
|
||||
),
|
||||
None,
|
||||
),
|
||||
},
|
||||
"FLASHINFER_MLA_SPARSE": {
|
||||
8: (
|
||||
(
|
||||
(2048, 512),
|
||||
(4096, 1024),
|
||||
(8192, 1024),
|
||||
(16384, 2048),
|
||||
(32768, 32768),
|
||||
),
|
||||
None,
|
||||
),
|
||||
},
|
||||
},
|
||||
"FLASHINFER_MLA_SPARSE": {
|
||||
8: (512, 1024, 1024, 2048, 32768),
|
||||
(256, 256): {
|
||||
"FLASHMLA_SPARSE": {
|
||||
1: (((8 * 1024, 0),), ()),
|
||||
2: (
|
||||
((20 * 1024, 0),),
|
||||
((6 * 1024, 4 * 1024), (10 * 1024, 8 * 1024)),
|
||||
),
|
||||
4: (
|
||||
((48 * 1024, 0),),
|
||||
(
|
||||
(8 * 1024, 4 * 1024),
|
||||
(16 * 1024, 8 * 1024),
|
||||
(24 * 1024, 16 * 1024),
|
||||
(34 * 1024, 32 * 1024),
|
||||
),
|
||||
),
|
||||
8: (
|
||||
((112 * 1024, 0),),
|
||||
(
|
||||
(8 * 1024, 4 * 1024),
|
||||
(16 * 1024, 8 * 1024),
|
||||
(32 * 1024, 16 * 1024),
|
||||
(48 * 1024, 32 * 1024),
|
||||
),
|
||||
),
|
||||
},
|
||||
"FLASHINFER_MLA_SPARSE": {
|
||||
4: (
|
||||
((36 * 1024, 0),),
|
||||
(
|
||||
(12 * 1024, 4 * 1024),
|
||||
(16 * 1024, 8 * 1024),
|
||||
(24 * 1024, 16 * 1024),
|
||||
(28 * 1024, 24 * 1024),
|
||||
),
|
||||
),
|
||||
8: (
|
||||
((64 * 1024, 0),),
|
||||
(
|
||||
(20 * 1024, 4 * 1024),
|
||||
(24 * 1024, 8 * 1024),
|
||||
(32 * 1024, 16 * 1024),
|
||||
(48 * 1024, 32 * 1024),
|
||||
(56 * 1024, 40 * 1024),
|
||||
(64 * 1024, 48 * 1024),
|
||||
(72 * 1024, 56 * 1024),
|
||||
),
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
_DSV32_SEQ_LEN_BUCKETS = (2048, 4096, 8192, 16384, 32768)
|
||||
|
||||
|
||||
def _use_masked_mha(
|
||||
*,
|
||||
backend_name: str,
|
||||
tensor_parallel_size: int,
|
||||
qk_head_dim: int,
|
||||
v_head_dim: int,
|
||||
query_len: int,
|
||||
seq_len: int,
|
||||
has_context: bool,
|
||||
) -> bool:
|
||||
thresholds = _DSV32_MASKED_MHA_THRESHOLDS.get(backend_name, {}).get(
|
||||
tensor_parallel_size
|
||||
thresholds = (
|
||||
_MASKED_MHA_THRESHOLDS.get((qk_head_dim, v_head_dim), {})
|
||||
.get(backend_name, {})
|
||||
.get(tensor_parallel_size)
|
||||
)
|
||||
if thresholds is None:
|
||||
return False
|
||||
for bucket_idx, bucket_seq_len in enumerate(_DSV32_SEQ_LEN_BUCKETS):
|
||||
if seq_len <= bucket_seq_len:
|
||||
min_query_len = thresholds[bucket_idx]
|
||||
return min_query_len is not None and query_len >= min_query_len
|
||||
pure_prefill_ranges, context_ranges = thresholds
|
||||
ranges = pure_prefill_ranges
|
||||
if has_context and context_ranges is not None:
|
||||
ranges = context_ranges
|
||||
for max_seq_len, min_query_len in ranges:
|
||||
if seq_len <= max_seq_len:
|
||||
return query_len >= min_query_len
|
||||
return False
|
||||
|
||||
|
||||
|
||||
@@ -88,16 +88,27 @@ def _is_masked_mha_available(
|
||||
"""Check if masked MHA can ever fire for this model configuration."""
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
return False
|
||||
if (
|
||||
num_heads_total != 128
|
||||
or kv_lora_rank != 512
|
||||
or qk_nope_head_dim != 128
|
||||
or qk_rope_head_dim != 64
|
||||
or v_head_dim != 128
|
||||
model_dims = (
|
||||
num_heads_total,
|
||||
kv_lora_rank,
|
||||
qk_nope_head_dim,
|
||||
qk_rope_head_dim,
|
||||
v_head_dim,
|
||||
)
|
||||
if model_dims not in (
|
||||
(128, 512, 128, 64, 128),
|
||||
(64, 512, 192, 64, 256),
|
||||
):
|
||||
return False
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
fa_version = get_flash_attn_version(head_size=qk_head_dim, head_size_v=v_head_dim)
|
||||
if qk_head_dim == 256 and v_head_dim == 256:
|
||||
# This path uses contiguous K/V, so it does not need the paged-KV
|
||||
# features that keep head-dim 256 disabled in the general FA selector.
|
||||
fa_version = get_flash_attn_version()
|
||||
else:
|
||||
fa_version = get_flash_attn_version(
|
||||
head_size=qk_head_dim, head_size_v=v_head_dim
|
||||
)
|
||||
return fa_version == 4 and not is_quantized_kv_cache(kv_cache_dtype)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user