[Attention] Skip sparse indexer scoring for dense short prefills (#48407)

Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Yiliu Dong
2026-07-28 16:17:22 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent 6453fc0b8c
commit ba702e978e
7 changed files with 230 additions and 12 deletions
@@ -769,13 +769,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
num_mha_tokens = q.size(0) - num_mqa_tokens
if self.impl.is_sparse and num_mha_tokens > 0:
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
use_mha = (
self.prefill_backend is not None
and prefill_max_seq_len <= attn_metadata.topk_tokens # type: ignore[attr-defined]
and not self._vllm_config.attention_config.sparse_mla_force_mqa
)
if not use_mha:
prefill_metadata = getattr(attn_metadata, "prefill", None)
if not getattr(prefill_metadata, "use_dense_mha", False):
num_mqa_tokens = q.size(0)
num_mha_tokens = 0
@@ -1409,6 +1404,10 @@ class MLACommonPrefillMetadata:
q_data_type: torch.dtype | None = None
output_dtype: torch.dtype | None = None
prefill_backend: MLAPrefillBackend | None = None
# Whether the prefill suffix is routed through dense MHA.
# Indexer scoring may be skipped only for a pure-prefill batch,
# since decode tokens still consume top-k indices.
use_dense_mha: bool = False
@dataclass
@@ -203,6 +203,10 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
q_data_type=self.model_config.dtype,
output_dtype=self.model_config.dtype,
prefill_backend=self._prefill_backend,
use_dense_mha=(
prefill_max_seq_len <= self.topk_tokens
and not self.vllm_config.attention_config.sparse_mla_force_mqa
),
)
self._prefill_backend.prepare_metadata(prefill)