[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 21:17:12 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 3a9bfc209f
commit de9250ac9e
10 changed files with 527 additions and 74 deletions
+23 -14
View File
@@ -119,8 +119,6 @@ class DeepseekV32Attention(MLAAttention):
indexer: "DeepseekV32Indexer | None"
indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer
supports_dense_mha_prefill = False
def __init__(
self,
vllm_config: VllmConfig,
@@ -303,15 +301,6 @@ class DeepseekV32Attention(MLAAttention):
device=hidden_states.device,
)
forward_context = get_forward_context()
attn_metadata_raw = forward_context.attn_metadata
if isinstance(attn_metadata_raw, dict):
attn_metadata = attn_metadata_raw.get(self.layer_name)
elif isinstance(attn_metadata_raw, list):
attn_metadata = attn_metadata_raw[0].get(self.layer_name)
else:
attn_metadata = attn_metadata_raw
attn_metadata = cast("MLACommonMetadata | None", attn_metadata)
slot_mapping = forward_context.slot_mapping
assert isinstance(slot_mapping, dict)
mla_slot = slot_mapping.get(self.layer_name)
@@ -337,7 +326,7 @@ class DeepseekV32Attention(MLAAttention):
indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0
if attn_metadata is None or self.use_pcp:
if forward_context.attn_metadata is None or self.use_pcp:
mla_kv_cache = None
mla_k_scale = None
indexer_k_cache = None
@@ -346,8 +335,8 @@ class DeepseekV32Attention(MLAAttention):
mla_kv_cache = self.kv_cache
mla_k_scale = self._k_scale
kv_c_out = torch.empty_like(kv_c) if self.use_pcp else None
k_pe_out = torch.empty_like(k_pe) if self.use_pcp else None
kv_c_out = torch.empty_like(kv_c)
k_pe_out = torch.empty_like(k_pe)
q_c = fused_norm_rope(
positions,
q_c,
@@ -403,7 +392,10 @@ class DeepseekV32Attention(MLAAttention):
)
self._sparse_indexer_and_attn(
positions,
q_c,
q_nope,
q_pe,
index_q_fp8,
index_k_out,
index_weights_out,
@@ -418,7 +410,10 @@ class DeepseekV32Attention(MLAAttention):
@eager_break_during_capture
def _sparse_indexer_and_attn(
self,
positions: torch.Tensor,
q_c: torch.Tensor,
q_nope: torch.Tensor,
q_pe: torch.Tensor,
index_q_fp8: torch.Tensor | None,
index_k: torch.Tensor | None,
index_weights_out: torch.Tensor | None,
@@ -498,6 +493,20 @@ class DeepseekV32Attention(MLAAttention):
output.zero_()
return
if self._use_sparse_mha(attn_metadata):
assert kv_c is not None and k_pe is not None
mha_q_pe = self.rotary_emb(positions, q_pe)[0] if self._fp8_query else mqa_q
mha_q = torch.cat((q_nope, mha_q_pe), dim=-1)
self.forward_impl(
mha_q,
kv_c,
k_pe.unsqueeze(1),
kv_cache,
attn_metadata,
output,
)
return
if self._fp8_kv_needs_view:
kv_cache = kv_cache.view(torch.float8_e4m3fn)
if self._fp8_query: