[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
3a9bfc209f
commit
de9250ac9e
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
|
||||
|
||||
@@ -49,7 +50,6 @@ def make_mla_metadata(*, use_dense_mha: bool = True, num_decode_tokens: int = 0)
|
||||
[
|
||||
"short",
|
||||
"threshold_mismatch",
|
||||
"mqa_only_layer",
|
||||
"force_mqa",
|
||||
"mla_decode",
|
||||
"capture",
|
||||
@@ -141,11 +141,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices = torch.full((7, 2048), 17, dtype=torch.int32)
|
||||
|
||||
def run_indexer():
|
||||
if batch_kind == "mqa_only_layer":
|
||||
assert not DeepseekV32Attention.supports_dense_mha_prefill
|
||||
dense_mha_layer = ""
|
||||
else:
|
||||
dense_mha_layer = MLA_LAYER
|
||||
assert DeepseekV32Attention.supports_dense_mha_prefill
|
||||
return sparse_indexer.sparse_attn_indexer(
|
||||
hidden_states,
|
||||
INDEXER_LAYER,
|
||||
@@ -163,7 +159,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices,
|
||||
False,
|
||||
False,
|
||||
dense_mha_layer,
|
||||
MLA_LAYER,
|
||||
)
|
||||
|
||||
if should_skip:
|
||||
@@ -222,3 +218,75 @@ def test_skipped_k_cache_insert_accepts_no_k(
|
||||
|
||||
assert result is topk_indices
|
||||
assert torch.all(topk_indices == -1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fp8_query", [False, True])
|
||||
def test_deepseek_v32_dispatches_selected_mha(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_query: bool,
|
||||
) -> None:
|
||||
attn_metadata = SimpleNamespace(num_actual_tokens=2)
|
||||
kv_cache = torch.empty(1)
|
||||
monkeypatch.setattr(
|
||||
deepseek_v32_attention,
|
||||
"get_attention_context",
|
||||
lambda _: (attn_metadata, None, kv_cache, None),
|
||||
)
|
||||
|
||||
observed = {}
|
||||
|
||||
def record_forward_impl(*args):
|
||||
observed["args"] = args
|
||||
|
||||
layer = SimpleNamespace(
|
||||
indexer=None,
|
||||
skip_topk=False,
|
||||
layer_name=MLA_LAYER,
|
||||
use_pcp=False,
|
||||
_fp8_query=fp8_query,
|
||||
_use_sparse_mha=lambda _: True,
|
||||
rotary_emb=lambda _positions, q: (q + 1, None),
|
||||
forward_impl=record_forward_impl,
|
||||
)
|
||||
q_nope = torch.randn(2, 1, 2)
|
||||
q_pe = torch.randn(2, 1, 2)
|
||||
mqa_q = torch.randn(2, 1, 2)
|
||||
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
|
||||
kv_c = torch.empty(2, 2)
|
||||
k_pe = torch.empty(2, 2)
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
DeepseekV32Attention._sparse_indexer_and_attn(
|
||||
layer,
|
||||
torch.arange(2),
|
||||
torch.empty(2, 2),
|
||||
q_nope,
|
||||
q_pe,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
kv_c,
|
||||
k_pe,
|
||||
torch.empty(2, 1, 2),
|
||||
mqa_q,
|
||||
output,
|
||||
)
|
||||
|
||||
expected_args = (
|
||||
mha_q,
|
||||
kv_c,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
actual_args = observed["args"]
|
||||
torch.testing.assert_close(actual_args[0], mha_q)
|
||||
assert actual_args[2].shape == (2, 1, 2)
|
||||
assert actual_args[2].data_ptr() == k_pe.data_ptr()
|
||||
assert all(
|
||||
actual is expected
|
||||
for actual, expected in zip(
|
||||
actual_args[1:2] + actual_args[3:],
|
||||
expected_args[1:],
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user