[Attention] Skip sparse indexer scoring for dense short prefills (#48407)
Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6453fc0b8c
commit
ba702e978e
@@ -769,13 +769,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
use_mha = (
|
||||
self.prefill_backend is not None
|
||||
and prefill_max_seq_len <= attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
and not self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
)
|
||||
if not use_mha:
|
||||
prefill_metadata = getattr(attn_metadata, "prefill", None)
|
||||
if not getattr(prefill_metadata, "use_dense_mha", False):
|
||||
num_mqa_tokens = q.size(0)
|
||||
num_mha_tokens = 0
|
||||
|
||||
@@ -1409,6 +1404,10 @@ class MLACommonPrefillMetadata:
|
||||
q_data_type: torch.dtype | None = None
|
||||
output_dtype: torch.dtype | None = None
|
||||
prefill_backend: MLAPrefillBackend | None = None
|
||||
# Whether the prefill suffix is routed through dense MHA.
|
||||
# Indexer scoring may be skipped only for a pure-prefill batch,
|
||||
# since decode tokens still consume top-k indices.
|
||||
use_dense_mha: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -203,6 +203,10 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
q_data_type=self.model_config.dtype,
|
||||
output_dtype=self.model_config.dtype,
|
||||
prefill_backend=self._prefill_backend,
|
||||
use_dense_mha=(
|
||||
prefill_max_seq_len <= self.topk_tokens
|
||||
and not self.vllm_config.attention_config.sparse_mla_force_mqa
|
||||
),
|
||||
)
|
||||
self._prefill_backend.prepare_metadata(prefill)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user