[Attention] Enable masked MHA for GLM-5 head dimensions (#53785)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
3a9bfc209f
commit
de9250ac9e
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
|
||||
|
||||
@@ -49,7 +50,6 @@ def make_mla_metadata(*, use_dense_mha: bool = True, num_decode_tokens: int = 0)
|
||||
[
|
||||
"short",
|
||||
"threshold_mismatch",
|
||||
"mqa_only_layer",
|
||||
"force_mqa",
|
||||
"mla_decode",
|
||||
"capture",
|
||||
@@ -141,11 +141,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices = torch.full((7, 2048), 17, dtype=torch.int32)
|
||||
|
||||
def run_indexer():
|
||||
if batch_kind == "mqa_only_layer":
|
||||
assert not DeepseekV32Attention.supports_dense_mha_prefill
|
||||
dense_mha_layer = ""
|
||||
else:
|
||||
dense_mha_layer = MLA_LAYER
|
||||
assert DeepseekV32Attention.supports_dense_mha_prefill
|
||||
return sparse_indexer.sparse_attn_indexer(
|
||||
hidden_states,
|
||||
INDEXER_LAYER,
|
||||
@@ -163,7 +159,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices,
|
||||
False,
|
||||
False,
|
||||
dense_mha_layer,
|
||||
MLA_LAYER,
|
||||
)
|
||||
|
||||
if should_skip:
|
||||
@@ -222,3 +218,75 @@ def test_skipped_k_cache_insert_accepts_no_k(
|
||||
|
||||
assert result is topk_indices
|
||||
assert torch.all(topk_indices == -1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fp8_query", [False, True])
|
||||
def test_deepseek_v32_dispatches_selected_mha(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
fp8_query: bool,
|
||||
) -> None:
|
||||
attn_metadata = SimpleNamespace(num_actual_tokens=2)
|
||||
kv_cache = torch.empty(1)
|
||||
monkeypatch.setattr(
|
||||
deepseek_v32_attention,
|
||||
"get_attention_context",
|
||||
lambda _: (attn_metadata, None, kv_cache, None),
|
||||
)
|
||||
|
||||
observed = {}
|
||||
|
||||
def record_forward_impl(*args):
|
||||
observed["args"] = args
|
||||
|
||||
layer = SimpleNamespace(
|
||||
indexer=None,
|
||||
skip_topk=False,
|
||||
layer_name=MLA_LAYER,
|
||||
use_pcp=False,
|
||||
_fp8_query=fp8_query,
|
||||
_use_sparse_mha=lambda _: True,
|
||||
rotary_emb=lambda _positions, q: (q + 1, None),
|
||||
forward_impl=record_forward_impl,
|
||||
)
|
||||
q_nope = torch.randn(2, 1, 2)
|
||||
q_pe = torch.randn(2, 1, 2)
|
||||
mqa_q = torch.randn(2, 1, 2)
|
||||
mha_q = torch.cat((q_nope, q_pe + 1 if fp8_query else mqa_q), dim=-1)
|
||||
kv_c = torch.empty(2, 2)
|
||||
k_pe = torch.empty(2, 2)
|
||||
output = torch.empty(2, 2)
|
||||
|
||||
DeepseekV32Attention._sparse_indexer_and_attn(
|
||||
layer,
|
||||
torch.arange(2),
|
||||
torch.empty(2, 2),
|
||||
q_nope,
|
||||
q_pe,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
kv_c,
|
||||
k_pe,
|
||||
torch.empty(2, 1, 2),
|
||||
mqa_q,
|
||||
output,
|
||||
)
|
||||
|
||||
expected_args = (
|
||||
mha_q,
|
||||
kv_c,
|
||||
kv_cache,
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
actual_args = observed["args"]
|
||||
torch.testing.assert_close(actual_args[0], mha_q)
|
||||
assert actual_args[2].shape == (2, 1, 2)
|
||||
assert actual_args[2].data_ptr() == k_pe.data_ptr()
|
||||
assert all(
|
||||
actual is expected
|
||||
for actual, expected in zip(
|
||||
actual_args[1:2] + actual_args[3:],
|
||||
expected_args[1:],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.attention.mla_attention import (
|
||||
MLAAttention,
|
||||
QueryLenSupport,
|
||||
_DecodeConcatQuantFP8,
|
||||
_use_masked_mha,
|
||||
build_mla_chunked_context_metadata,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
@@ -68,6 +69,124 @@ if current_platform.is_rocm():
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tensor_parallel_size", "query_len", "expected"),
|
||||
[
|
||||
(1, 8192, True),
|
||||
(1, 9216, False),
|
||||
(2, 20480, True),
|
||||
(2, 24576, False),
|
||||
(4, 48 * 1024, True),
|
||||
(4, 56 * 1024, False),
|
||||
(8, 112 * 1024, True),
|
||||
(8, 128 * 1024, False),
|
||||
],
|
||||
)
|
||||
def test_glm5_masked_mha_pure_prefill_routing(
|
||||
tensor_parallel_size, query_len, expected
|
||||
):
|
||||
assert (
|
||||
_use_masked_mha(
|
||||
backend_name="FLASHMLA_SPARSE",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
qk_head_dim=256,
|
||||
v_head_dim=256,
|
||||
query_len=query_len,
|
||||
seq_len=query_len,
|
||||
has_context=False,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tensor_parallel_size", "query_len", "seq_len", "expected"),
|
||||
[
|
||||
(1, 8192, 8192, False),
|
||||
(2, 4096, 6144, True),
|
||||
(2, 4096, 8192, False),
|
||||
(2, 8192, 10240, True),
|
||||
(4, 16384, 24576, True),
|
||||
(4, 16384, 32768, False),
|
||||
(4, 32768, 34 * 1024, True),
|
||||
(4, 32768, 36 * 1024, False),
|
||||
(8, 32768, 49152, True),
|
||||
(8, 32768, 65536, False),
|
||||
],
|
||||
)
|
||||
def test_glm5_masked_mha_context_routing(
|
||||
tensor_parallel_size, query_len, seq_len, expected
|
||||
):
|
||||
assert (
|
||||
_use_masked_mha(
|
||||
backend_name="FLASHMLA_SPARSE",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
qk_head_dim=256,
|
||||
v_head_dim=256,
|
||||
query_len=query_len,
|
||||
seq_len=seq_len,
|
||||
has_context=True,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("tensor_parallel_size", "query_len", "seq_len", "has_context", "expected"),
|
||||
[
|
||||
(4, 36 * 1024, 36 * 1024, False, True),
|
||||
(4, 40 * 1024, 40 * 1024, False, False),
|
||||
(4, 4 * 1024, 12 * 1024, True, True),
|
||||
(4, 4 * 1024, 16 * 1024, True, False),
|
||||
(4, 24 * 1024, 28 * 1024, True, True),
|
||||
(4, 32 * 1024, 36 * 1024, True, False),
|
||||
(8, 64 * 1024, 64 * 1024, False, True),
|
||||
(8, 68 * 1024, 68 * 1024, False, False),
|
||||
(8, 4 * 1024, 20 * 1024, True, True),
|
||||
(8, 16 * 1024, 32 * 1024, True, True),
|
||||
(8, 48 * 1024, 64 * 1024, True, True),
|
||||
(8, 56 * 1024, 72 * 1024, True, True),
|
||||
(8, 63 * 1024, 79 * 1024, True, False),
|
||||
],
|
||||
)
|
||||
def test_glm5_flashinfer_masked_mha_routing(
|
||||
tensor_parallel_size, query_len, seq_len, has_context, expected
|
||||
):
|
||||
assert (
|
||||
_use_masked_mha(
|
||||
backend_name="FLASHINFER_MLA_SPARSE",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
qk_head_dim=256,
|
||||
v_head_dim=256,
|
||||
query_len=query_len,
|
||||
seq_len=seq_len,
|
||||
has_context=has_context,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_masked_mha_routing_is_dimension_specific():
|
||||
assert _use_masked_mha(
|
||||
backend_name="FLASHMLA_SPARSE",
|
||||
tensor_parallel_size=1,
|
||||
qk_head_dim=192,
|
||||
v_head_dim=128,
|
||||
query_len=1536,
|
||||
seq_len=2048,
|
||||
has_context=True,
|
||||
)
|
||||
assert not _use_masked_mha(
|
||||
backend_name="FLASHMLA_SPARSE",
|
||||
tensor_parallel_size=1,
|
||||
qk_head_dim=128,
|
||||
v_head_dim=128,
|
||||
query_len=1536,
|
||||
seq_len=2048,
|
||||
has_context=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cache_dtype", "expected_quant_mode"),
|
||||
[
|
||||
|
||||
@@ -21,6 +21,7 @@ from tests.v1.attention.utils import (
|
||||
)
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.model_executor.layers.attention.mla_attention import _use_masked_mha
|
||||
from vllm.model_executor.layers.attention.sparse_mla_attention import (
|
||||
GLOBAL_TOPK_MASK_MAX_BYTES,
|
||||
_masked_mha_workspace_fits,
|
||||
@@ -884,6 +885,35 @@ def test_masked_mha_workspace_fits_single_request_boundary(max_query_len, expect
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend_name", "tensor_parallel_size", "query_len"),
|
||||
[
|
||||
("FLASHMLA_SPARSE", 4, 48 * 1024),
|
||||
("FLASHMLA_SPARSE", 8, 112 * 1024),
|
||||
("FLASHINFER_MLA_SPARSE", 4, 36 * 1024),
|
||||
("FLASHINFER_MLA_SPARSE", 8, 64 * 1024),
|
||||
],
|
||||
)
|
||||
def test_masked_mha_workspace_guards_long_routing_policy(
|
||||
backend_name, tensor_parallel_size, query_len
|
||||
):
|
||||
assert _use_masked_mha(
|
||||
backend_name=backend_name,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
qk_head_dim=256,
|
||||
v_head_dim=256,
|
||||
query_len=query_len,
|
||||
seq_len=query_len,
|
||||
has_context=False,
|
||||
)
|
||||
assert not _masked_mha_workspace_fits(
|
||||
batch_size=1,
|
||||
max_query_len=query_len,
|
||||
max_context_chunk_seq_len=0,
|
||||
workspace_numel=GLOBAL_TOPK_MASK_MAX_BYTES // torch.int32.itemsize,
|
||||
)
|
||||
|
||||
|
||||
def test_masked_mha_workspace_fits_accounts_for_batch_and_context():
|
||||
"""Request count and context chunk length are independent multipliers."""
|
||||
base = dict(batch_size=2, max_query_len=2048, max_context_chunk_seq_len=2048)
|
||||
@@ -912,14 +942,25 @@ PREFILL_BATCH_SPECS = {
|
||||
)
|
||||
@pytest.mark.parametrize("batch_name", list(PREFILL_BATCH_SPECS.keys()))
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto"])
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "qk_nope_head_dim", "qk_rope_head_dim", "v_head_dim"),
|
||||
[
|
||||
pytest.param(128, 128, 64, 128, id="deepseek_hd192_v128"),
|
||||
pytest.param(64, 192, 64, 256, id="glm5_hd256_v256"),
|
||||
],
|
||||
)
|
||||
def test_sparse_backend_prefill_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
batch_name,
|
||||
kv_cache_dtype,
|
||||
num_heads,
|
||||
qk_nope_head_dim,
|
||||
qk_rope_head_dim,
|
||||
v_head_dim,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test single-pass dense and masked MHA for sparse MLA prefill."""
|
||||
"""Test dense and masked MHA across supported sparse MLA dimensions."""
|
||||
backend_cls = FlashMLASparseBackend
|
||||
batch_spec = PREFILL_BATCH_SPECS[batch_name]
|
||||
|
||||
@@ -927,11 +968,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
dtype = torch.bfloat16
|
||||
block_size = 64
|
||||
|
||||
num_heads = 128
|
||||
kv_lora_rank = 512
|
||||
qk_nope_head_dim = 128
|
||||
qk_rope_head_dim = 64
|
||||
v_head_dim = 128
|
||||
head_size = kv_lora_rank + qk_rope_head_dim
|
||||
masked_mha = batch_name.startswith("masked_mha")
|
||||
topk_tokens = 200 if masked_mha else 512
|
||||
|
||||
Reference in New Issue
Block a user