[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 21:17:12 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 3a9bfc209f
commit de9250ac9e
10 changed files with 527 additions and 74 deletions
@@ -8,6 +8,7 @@ import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
@@ -49,7 +50,6 @@ def make_mla_metadata(*, use_dense_mha: bool = True, num_decode_tokens: int = 0)
[
"short",
"threshold_mismatch",
"mqa_only_layer",
"force_mqa",
"mla_decode",
"capture",
@@ -141,11 +141,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
topk_indices = torch.full((7, 2048), 17, dtype=torch.int32)
def run_indexer():
if batch_kind == "mqa_only_layer":
assert not DeepseekV32Attention.supports_dense_mha_prefill
dense_mha_layer = ""
else:
dense_mha_layer = MLA_LAYER
assert DeepseekV32Attention.supports_dense_mha_prefill
return sparse_indexer.sparse_attn_indexer(
hidden_states,
INDEXER_LAYER,
@@ -163,7 +159,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
topk_indices,
False,
False,
dense_mha_layer,
MLA_LAYER,
)
if should_skip:
@@ -222,3 +218,75 @@ def test_skipped_k_cache_insert_accepts_no_k(
assert result is topk_indices
assert torch.all(topk_indices == -1)
@pytest.mark.parametrize("fp8_query", [False, True])
def test_deepseek_v32_dispatches_selected_mha(
monkeypatch: pytest.MonkeyPatch,
fp8_query: bool,
) -> None:
attn_metadata = SimpleNamespace(num_actual_tokens=2)
kv_cache = torch.empty(1)
monkeypatch.setattr(
deepseek_v32_attention,
"get_attention_context",
lambda _: (attn_metadata, None, kv_cache, None),
)
observed = {}
def record_forward_impl(*args):
observed["args"] = args
layer = SimpleNamespace(
indexer=None,
skip_topk=False,
layer_name=MLA_LAYER,
use_pcp=False,
_fp8_query=fp8_query,
_use_sparse_mha=lambda _: True,
rotary_emb=lambda _positions, q: (q + 1, None),
forward_impl=record_forward_impl,
)
q_nope = torch.randn(2, 1, 2)
q_pe = torch.randn(2, 1, 2)
mqa_q = torch.randn(2, 1, 2)
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
kv_c = torch.empty(2, 2)
k_pe = torch.empty(2, 2)
output = torch.empty(2, 2)
DeepseekV32Attention._sparse_indexer_and_attn(
layer,
torch.arange(2),
torch.empty(2, 2),
q_nope,
q_pe,
None,
None,
None,
kv_c,
k_pe,
torch.empty(2, 1, 2),
mqa_q,
output,
)
expected_args = (
mha_q,
kv_c,
kv_cache,
attn_metadata,
output,
)
actual_args = observed["args"]
torch.testing.assert_close(actual_args[0], mha_q)
assert actual_args[2].shape == (2, 1, 2)
assert actual_args[2].data_ptr() == k_pe.data_ptr()
assert all(
actual is expected
for actual, expected in zip(
actual_args[1:2] + actual_args[3:],
expected_args[1:],
)
)