[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Matthew Bonanni
2026-08-27 21:17:12 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 3a9bfc209f
commit de9250ac9e
10 changed files with 527 additions and 74 deletions
@@ -8,6 +8,7 @@ import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
@@ -49,7 +50,6 @@ def make_mla_metadata(*, use_dense_mha: bool = True, num_decode_tokens: int = 0)
[
"short",
"threshold_mismatch",
"mqa_only_layer",
"force_mqa",
"mla_decode",
"capture",
@@ -141,11 +141,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
topk_indices = torch.full((7, 2048), 17, dtype=torch.int32)
def run_indexer():
if batch_kind == "mqa_only_layer":
assert not DeepseekV32Attention.supports_dense_mha_prefill
dense_mha_layer = ""
else:
dense_mha_layer = MLA_LAYER
assert DeepseekV32Attention.supports_dense_mha_prefill
return sparse_indexer.sparse_attn_indexer(
hidden_states,
INDEXER_LAYER,
@@ -163,7 +159,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
topk_indices,
False,
False,
dense_mha_layer,
MLA_LAYER,
)
if should_skip:
@@ -222,3 +218,75 @@ def test_skipped_k_cache_insert_accepts_no_k(
assert result is topk_indices
assert torch.all(topk_indices == -1)
@pytest.mark.parametrize("fp8_query", [False, True])
def test_deepseek_v32_dispatches_selected_mha(
monkeypatch: pytest.MonkeyPatch,
fp8_query: bool,
) -> None:
attn_metadata = SimpleNamespace(num_actual_tokens=2)
kv_cache = torch.empty(1)
monkeypatch.setattr(
deepseek_v32_attention,
"get_attention_context",
lambda _: (attn_metadata, None, kv_cache, None),
)
observed = {}
def record_forward_impl(*args):
observed["args"] = args
layer = SimpleNamespace(
indexer=None,
skip_topk=False,
layer_name=MLA_LAYER,
use_pcp=False,
_fp8_query=fp8_query,
_use_sparse_mha=lambda _: True,
rotary_emb=lambda _positions, q: (q + 1, None),
forward_impl=record_forward_impl,
)
q_nope = torch.randn(2, 1, 2)
q_pe = torch.randn(2, 1, 2)
mqa_q = torch.randn(2, 1, 2)
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
kv_c = torch.empty(2, 2)
k_pe = torch.empty(2, 2)
output = torch.empty(2, 2)
DeepseekV32Attention._sparse_indexer_and_attn(
layer,
torch.arange(2),
torch.empty(2, 2),
q_nope,
q_pe,
None,
None,
None,
kv_c,
k_pe,
torch.empty(2, 1, 2),
mqa_q,
output,
)
expected_args = (
mha_q,
kv_c,
kv_cache,
attn_metadata,
output,
)
actual_args = observed["args"]
torch.testing.assert_close(actual_args[0], mha_q)
assert actual_args[2].shape == (2, 1, 2)
assert actual_args[2].data_ptr() == k_pe.data_ptr()
assert all(
actual is expected
for actual, expected in zip(
actual_args[1:2] + actual_args[3:],
expected_args[1:],
)
)
+119
View File
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
MLAAttention,
QueryLenSupport,
_DecodeConcatQuantFP8,
_use_masked_mha,
build_mla_chunked_context_metadata,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
@@ -68,6 +69,124 @@ if current_platform.is_rocm():
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize(
("tensor_parallel_size", "query_len", "expected"),
[
(1, 8192, True),
(1, 9216, False),
(2, 20480, True),
(2, 24576, False),
(4, 48 * 1024, True),
(4, 56 * 1024, False),
(8, 112 * 1024, True),
(8, 128 * 1024, False),
],
)
def test_glm5_masked_mha_pure_prefill_routing(
tensor_parallel_size, query_len, expected
):
assert (
_use_masked_mha(
backend_name="FLASHMLA_SPARSE",
tensor_parallel_size=tensor_parallel_size,
qk_head_dim=256,
v_head_dim=256,
query_len=query_len,
seq_len=query_len,
has_context=False,
)
is expected
)
@pytest.mark.parametrize(
("tensor_parallel_size", "query_len", "seq_len", "expected"),
[
(1, 8192, 8192, False),
(2, 4096, 6144, True),
(2, 4096, 8192, False),
(2, 8192, 10240, True),
(4, 16384, 24576, True),
(4, 16384, 32768, False),
(4, 32768, 34 * 1024, True),
(4, 32768, 36 * 1024, False),
(8, 32768, 49152, True),
(8, 32768, 65536, False),
],
)
def test_glm5_masked_mha_context_routing(
tensor_parallel_size, query_len, seq_len, expected
):
assert (
_use_masked_mha(
backend_name="FLASHMLA_SPARSE",
tensor_parallel_size=tensor_parallel_size,
qk_head_dim=256,
v_head_dim=256,
query_len=query_len,
seq_len=seq_len,
has_context=True,
)
is expected
)
@pytest.mark.parametrize(
("tensor_parallel_size", "query_len", "seq_len", "has_context", "expected"),
[
(4, 36 * 1024, 36 * 1024, False, True),
(4, 40 * 1024, 40 * 1024, False, False),
(4, 4 * 1024, 12 * 1024, True, True),
(4, 4 * 1024, 16 * 1024, True, False),
(4, 24 * 1024, 28 * 1024, True, True),
(4, 32 * 1024, 36 * 1024, True, False),
(8, 64 * 1024, 64 * 1024, False, True),
(8, 68 * 1024, 68 * 1024, False, False),
(8, 4 * 1024, 20 * 1024, True, True),
(8, 16 * 1024, 32 * 1024, True, True),
(8, 48 * 1024, 64 * 1024, True, True),
(8, 56 * 1024, 72 * 1024, True, True),
(8, 63 * 1024, 79 * 1024, True, False),
],
)
def test_glm5_flashinfer_masked_mha_routing(
tensor_parallel_size, query_len, seq_len, has_context, expected
):
assert (
_use_masked_mha(
backend_name="FLASHINFER_MLA_SPARSE",
tensor_parallel_size=tensor_parallel_size,
qk_head_dim=256,
v_head_dim=256,
query_len=query_len,
seq_len=seq_len,
has_context=has_context,
)
is expected
)
def test_masked_mha_routing_is_dimension_specific():
assert _use_masked_mha(
backend_name="FLASHMLA_SPARSE",
tensor_parallel_size=1,
qk_head_dim=192,
v_head_dim=128,
query_len=1536,
seq_len=2048,
has_context=True,
)
assert not _use_masked_mha(
backend_name="FLASHMLA_SPARSE",
tensor_parallel_size=1,
qk_head_dim=128,
v_head_dim=128,
query_len=1536,
seq_len=2048,
has_context=True,
)
@pytest.mark.parametrize(
("cache_dtype", "expected_quant_mode"),
[
+42 -5
View File
@@ -21,6 +21,7 @@ from tests.v1.attention.utils import (
)
from vllm import _custom_ops as ops
from vllm.config import set_current_vllm_config
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
from vllm.model_executor.layers.attention.sparse_mla_attention import (
GLOBAL_TOPK_MASK_MAX_BYTES,
_masked_mha_workspace_fits,
@@ -884,6 +885,35 @@ def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expect
)
@pytest.mark.parametrize(
("backend_name", "tensor_parallel_size", "query_len"),
[
("FLASHMLA_SPARSE", 4, 48 * 1024),
("FLASHMLA_SPARSE", 8, 112 * 1024),
("FLASHINFER_MLA_SPARSE", 4, 36 * 1024),
("FLASHINFER_MLA_SPARSE", 8, 64 * 1024),
],
)
def test_masked_mha_workspace_guards_long_routing_policy(
backend_name, tensor_parallel_size, query_len
):
assert _use_masked_mha(
backend_name=backend_name,
tensor_parallel_size=tensor_parallel_size,
qk_head_dim=256,
v_head_dim=256,
query_len=query_len,
seq_len=query_len,
has_context=False,
)
assert not _masked_mha_workspace_fits(
batch_size=1,
max_query_len=query_len,
max_context_chunk_seq_len=0,
workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
)
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
"""Request count and context chunk length are independent multipliers."""
base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
@@ -912,14 +942,25 @@ PREFILL_BATCH_SPECS = {
)
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
@pytest.mark.parametrize(
("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
[
pytest.param(128, 128, 64, 128, id="deepseek_hd192_v128"),
pytest.param(64, 192, 64, 256, id="glm5_hd256_v256"),
],
)
def test_sparse_backend_prefill_correctness(
default_vllm_config,
dist_init,
batch_name,
kv_cache_dtype,
num_heads,
qk_nope_head_dim,
qk_rope_head_dim,
v_head_dim,
workspace_init,
):
"""Test single-pass dense and masked MHA for sparse MLA prefill."""
"""Test dense and masked MHA across supported sparse MLA dimensions."""
backend_cls = FlashMLASparseBackend
batch_spec = PREFILL_BATCH_SPECS[batch_name]
@@ -927,11 +968,7 @@ def test_sparse_backend_prefill_correctness(
dtype = torch.bfloat16
block_size = 64
num_heads = 128
kv_lora_rank = 512
qk_nope_head_dim = 128
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
masked_mha = batch_name.startswith("masked_mha")
topk_tokens = 200 if masked_mha else 512