[Feature]: IndexCache support for DSA models (#37735)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chauncey
2026-04-29 15:15:35 -04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 6f20f81cbf
commit faab189554
3 changed files with 83 additions and 5 deletions
+8 -4
View File
@@ -64,6 +64,7 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
skip_topk: bool = False,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -87,6 +88,11 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
self.indexer_rope_emb = mla_modules.indexer_rotary_emb
self.is_sparse = mla_modules.is_sparse
# Whether to skip top-k token selection computation in this layer.
# When True, the indexer will not be called, and the layer will reuse
# the topk_tokens buffer written by a previous layer in the same pass.
# Refer: https://arxiv.org/abs/2603.12201 for more details.
self.skip_topk = skip_topk
if self.indexer is not None:
assert hasattr(self.indexer, "topk_tokens")
self.topk_tokens = self.indexer.topk_tokens
@@ -159,10 +165,8 @@ class MultiHeadLatentAttentionWrapper(PluggableLayer):
positions, q[..., self.qk_nope_head_dim :], k_pe
)
if self.indexer and self.is_sparse:
_topk_indices = self.indexer(
hidden_states, q_c, positions, self.indexer_rope_emb
)
if self.indexer and self.is_sparse and not self.skip_topk:
self.indexer(hidden_states, q_c, positions, self.indexer_rope_emb)
if llama_4_scaling is not None:
q *= llama_4_scaling
+21 -1
View File
@@ -82,7 +82,10 @@ from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.model_executor.models.utils import (
extract_layer_index,
sequence_parallel_chunk,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import direct_register_custom_op
@@ -963,6 +966,7 @@ class DeepseekV2MLAAttention(nn.Module):
self.is_v32 = hasattr(config, "index_topk")
_skip_topk = False
if self.is_v32:
self.indexer_rope_emb = get_rope(
qk_rope_head_dim,
@@ -980,6 +984,21 @@ class DeepseekV2MLAAttention(nn.Module):
topk_indices_buffer,
f"{prefix}.indexer",
)
# Enable IndexCache for DeepSeek models to reduce redundant top-k
# token selection computations in sparse attention.
use_index_cache = getattr(config, "use_index_cache", False)
if use_index_cache:
# IndexCache config
# Refer: https://arxiv.org/abs/2603.12201 for more details.
_index_topk_freq = getattr(config, "index_topk_freq", 1)
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
layer_id = extract_layer_index(prefix)
if _index_topk_pattern is None:
_skip_topk = max(layer_id - 1, 0) % _index_topk_freq != 0
elif 0 <= layer_id < len(_index_topk_pattern):
_skip_topk = _index_topk_pattern[layer_id] == "S"
else:
self.indexer_rope_emb = None
self.indexer = None
@@ -1017,6 +1036,7 @@ class DeepseekV2MLAAttention(nn.Module):
cache_config,
quant_config,
prefix,
skip_topk=_skip_topk,
)
def forward(