[Attention] Skip sparse indexer scoring for dense short prefills (#48407)
Signed-off-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
6453fc0b8c
commit
ba702e978e
@@ -769,13 +769,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
num_mha_tokens = q.size(0) - num_mqa_tokens
|
||||
|
||||
if self.impl.is_sparse and num_mha_tokens > 0:
|
||||
prefill_max_seq_len = attn_metadata.prefill_max_seq_len # type: ignore[attr-defined]
|
||||
use_mha = (
|
||||
self.prefill_backend is not None
|
||||
and prefill_max_seq_len <= attn_metadata.topk_tokens # type: ignore[attr-defined]
|
||||
and not self._vllm_config.attention_config.sparse_mla_force_mqa
|
||||
)
|
||||
if not use_mha:
|
||||
prefill_metadata = getattr(attn_metadata, "prefill", None)
|
||||
if not getattr(prefill_metadata, "use_dense_mha", False):
|
||||
num_mqa_tokens = q.size(0)
|
||||
num_mha_tokens = 0
|
||||
|
||||
@@ -1409,6 +1404,10 @@ class MLACommonPrefillMetadata:
|
||||
q_data_type: torch.dtype | None = None
|
||||
output_dtype: torch.dtype | None = None
|
||||
prefill_backend: MLAPrefillBackend | None = None
|
||||
# Whether the prefill suffix is routed through dense MHA.
|
||||
# Indexer scoring may be skipped only for a pure-prefill batch,
|
||||
# since decode tokens still consume top-k indices.
|
||||
use_dense_mha: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -203,6 +203,10 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
q_data_type=self.model_config.dtype,
|
||||
output_dtype=self.model_config.dtype,
|
||||
prefill_backend=self._prefill_backend,
|
||||
use_dense_mha=(
|
||||
prefill_max_seq_len <= self.topk_tokens
|
||||
and not self.vllm_config.attention_config.sparse_mla_force_mqa
|
||||
),
|
||||
)
|
||||
self._prefill_backend.prepare_metadata(prefill)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -65,6 +66,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
skip_topk: bool = False,
|
||||
allow_short_prefill_indexer_scoring_skip: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
@@ -119,7 +121,26 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
|
||||
indexer=self.indexer,
|
||||
topk_indices_buffer=mla_modules.topk_indices_buffer,
|
||||
)
|
||||
|
||||
indexer_op = getattr(self.indexer, "indexer_op", None)
|
||||
if indexer_op is not None and hasattr(
|
||||
indexer_op, "dense_mha_metadata_layer_name"
|
||||
):
|
||||
enable_short_prefill_scoring_skip = (
|
||||
allow_short_prefill_indexer_scoring_skip
|
||||
and not self.skip_topk
|
||||
and not getattr(indexer_op, "use_pcp", False)
|
||||
and current_platform.is_cuda()
|
||||
)
|
||||
# The indexer and main MLA use independent decode thresholds and
|
||||
# may classify the same short extend differently. Bind the main
|
||||
# MLA layer name so the eager indexer op can check whether the
|
||||
# batch's top-k indices will be consumed.
|
||||
# PCP is excluded because indexer cache/scoring ownership differs
|
||||
# across ranks and the no-consumer invariant has not been
|
||||
# established there.
|
||||
indexer_op.dense_mha_metadata_layer_name = (
|
||||
self.mla_attn.layer_name if enable_short_prefill_scoring_skip else ""
|
||||
)
|
||||
self.prefix = prefix
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -8,7 +8,7 @@ import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config import CUDAGraphMode, get_current_vllm_config
|
||||
from vllm.distributed import get_dcp_group, get_pcp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
@@ -310,6 +310,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer: torch.Tensor,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
dense_mha_metadata_layer_name: LayerNameType,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
@@ -317,7 +318,8 @@ def sparse_attn_indexer(
|
||||
skip_topk_buffer_clear: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# careful! this will be None in dummy run
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = forward_context.attn_metadata
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
|
||||
|
||||
@@ -357,6 +359,7 @@ def sparse_attn_indexer(
|
||||
topk_indices_buffer,
|
||||
skip_k_cache_insert,
|
||||
use_pcp,
|
||||
dense_mha_metadata_layer_name,
|
||||
use_fp4_cache,
|
||||
)
|
||||
attn_metadata_narrowed = attn_metadata[k_cache_prefix]
|
||||
@@ -402,6 +405,24 @@ def sparse_attn_indexer(
|
||||
scale_fmt,
|
||||
)
|
||||
|
||||
# The indexer and main MLA may classify the same short extend differently
|
||||
# because they use independent decode thresholds. Only the main MLA route
|
||||
# can determine whether the top-k indices will be consumed.
|
||||
if forward_context.cudagraph_runtime_mode != CUDAGraphMode.FULL:
|
||||
dense_mha_layer = _resolve_layer_name(dense_mha_metadata_layer_name)
|
||||
if dense_mha_layer:
|
||||
mla_metadata = attn_metadata.get(dense_mha_layer)
|
||||
prefill_metadata = getattr(mla_metadata, "prefill", None)
|
||||
if (
|
||||
getattr(prefill_metadata, "use_dense_mha", False)
|
||||
and getattr(mla_metadata, "num_decode_tokens", -1) == 0
|
||||
and not torch.cuda.is_current_stream_capturing()
|
||||
):
|
||||
# Deliberately leave the buffer untouched. Dense MHA does not
|
||||
# consume top-k indices for this batch; clearing it would be
|
||||
# unnecessary work.
|
||||
return topk_indices_buffer
|
||||
|
||||
# The buffer must be pre-filled with -1 (the "no token" sentinel) before the
|
||||
# top-k kernels scatter valid indices into it. On the fused deepseek_v32
|
||||
# nvidia path, _fused_norm_rope_kernel already cleared the same
|
||||
@@ -684,6 +705,7 @@ def sparse_attn_indexer_fake(
|
||||
topk_indices_buffer: torch.Tensor | None,
|
||||
skip_k_cache_insert: bool,
|
||||
use_pcp: bool,
|
||||
dense_mha_metadata_layer_name: LayerNameType,
|
||||
use_fp4_cache: bool = False,
|
||||
dcp_rank: int = 0,
|
||||
dcp_world_size: int = 1,
|
||||
@@ -739,6 +761,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer = topk_indices_buffer
|
||||
self.skip_k_cache_insert = skip_k_cache_insert
|
||||
self.use_fp4_cache = use_fp4_cache
|
||||
self.dense_mha_metadata_layer_name = ""
|
||||
# DCP scalars are constant for the run; resolve them here (config is set
|
||||
# during model construction) and pass them into the custom op, rather
|
||||
# than threading them through per-step metadata.
|
||||
@@ -800,6 +823,7 @@ class SparseAttnIndexer(CustomOp):
|
||||
self.topk_indices_buffer,
|
||||
self.skip_k_cache_insert,
|
||||
self.use_pcp,
|
||||
_encode_layer_name(self.dense_mha_metadata_layer_name),
|
||||
self.use_fp4_cache,
|
||||
self.dcp_rank,
|
||||
self.dcp_world_size,
|
||||
|
||||
@@ -1178,6 +1178,9 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
# the V1 proposer. A frozen True would leave the draft reading a
|
||||
# never-written topk buffer.
|
||||
skip_topk=_skip_topk and not is_mtp_layer,
|
||||
# Do not skip scoring for MTP layers: their top-k buffer may be
|
||||
# reused by later draft iterations through index sharing.
|
||||
allow_short_prefill_indexer_scoring_skip=not is_mtp_layer,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user