[Bugfix][MLA] Do not use Dense MHA for GLM-5.2 (#52512)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
ad5e71b276
commit
01e56caaf2
@@ -8,6 +8,7 @@ import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
|
||||
|
||||
INDEXER_LAYER = "model.layers.0.self_attn.indexer.k_cache"
|
||||
@@ -45,7 +46,15 @@ def make_mla_metadata(*, use_dense_mha: bool = True, num_decode_tokens: int = 0)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"batch_kind",
|
||||
["short", "threshold_mismatch", "force_mqa", "mla_decode", "capture", "full"],
|
||||
[
|
||||
"short",
|
||||
"threshold_mismatch",
|
||||
"mqa_only_layer",
|
||||
"force_mqa",
|
||||
"mla_decode",
|
||||
"capture",
|
||||
"full",
|
||||
],
|
||||
)
|
||||
def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
@@ -132,6 +141,11 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices = torch.full((7, 2048), 17, dtype=torch.int32)
|
||||
|
||||
def run_indexer():
|
||||
if batch_kind == "mqa_only_layer":
|
||||
assert not DeepseekV32Attention.supports_dense_mha_prefill
|
||||
dense_mha_layer = ""
|
||||
else:
|
||||
dense_mha_layer = MLA_LAYER
|
||||
return sparse_indexer.sparse_attn_indexer(
|
||||
hidden_states,
|
||||
INDEXER_LAYER,
|
||||
@@ -149,7 +163,7 @@ def test_short_prefill_updates_k_cache_before_scoring_decision(
|
||||
topk_indices,
|
||||
False,
|
||||
False,
|
||||
MLA_LAYER,
|
||||
dense_mha_layer,
|
||||
)
|
||||
|
||||
if should_skip:
|
||||
|
||||
Reference in New Issue
Block a user