[Attention] Skip sparse indexer scoring for dense short prefills (#48407)
Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6453fc0b8c
commit
ba702e978e
@@ -8,6 +8,7 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -65,6 +66,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
skip_topk: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -119,7 +121,26 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
)
|
||||
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
if indexer_op is not None and hasattr(
|
||||
indexer_op, "dense_mha_metadata_layer_name"
|
||||
):
|
||||
enable_short_prefill_scoring_skip = (
|
||||
allow_short_prefill_indexer_scoring_skip
|
||||
and not self.skip_topk
|
||||
and not getattr(indexer_op, "use_pcp", False)
|
||||
and current_platform.is_cuda()
|
||||
)
|
||||
# The indexer and main MLA use independent decode thresholds and
|
||||
# may classify the same short extend differently. Bind the main
|
||||
# MLA layer name so the eager indexer op can check whether the
|
||||
# batch's top-k indices will be consumed.
|
||||
# PCP is excluded because indexer cache/scoring ownership differs
|
||||
# across ranks and the no-consumer invariant has not been
|
||||
# established there.
|
||||
indexer_op.dense_mha_metadata_layer_name = (
|
||||
self.mla_attn.layer_name if enable_short_prefill_scoring_skip else ""
|
||||
)
|
||||
self.prefix = prefix
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user