[feature] add index share feature for DSA MTP (#44420)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jared Wen
2026-06-06 22:04:14 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c7f7741d4
commit 32f34d3935
5 changed files with 114 additions and 25 deletions
+24 -2
View File
@@ -115,7 +115,9 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
)
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
positions=positions,
hidden_states=hidden_states,
residual=None,
)
hidden_states = residual + hidden_states
return hidden_states
@@ -147,6 +149,22 @@ class DeepSeekMultiTokenPredictor(nn.Module):
)
self.logits_processor = LogitsProcessor(config.vocab_size)
def set_skip_topk(self, skip: bool):
"""Toggle skip_topk on all MTP layers with sparse attention.
Called by the proposer to implement index_share_for_mtp_iteration:
step 0 sets skip=False (compute own indices), steps 1+ set skip=True
(reuse step 0's indices).
"""
for layer in self.layers.values():
mtp_block = getattr(layer, "mtp_block", None)
if mtp_block is not None:
self_attn = getattr(mtp_block, "self_attn", None)
if self_attn is not None:
mla_attn = getattr(self_attn, "mla_attn", None)
if mla_attn is not None and hasattr(mla_attn, "skip_topk"):
mla_attn.skip_topk = skip
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@@ -225,7 +243,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
input_ids,
positions,
hidden_states,
inputs_embeds,
spec_step_idx,
)
return hidden_states
+16 -15
View File
@@ -1018,19 +1018,20 @@ class DeepseekV2MLAAttention(nn.Module):
is_inplace_rope=self.indexer_rope_emb.enabled(),
)
# Enable IndexCache for DeepSeek models to reduce redundant top-k
# token selection computations in sparse attention.
use_index_cache = getattr(config, "use_index_cache", False)
if use_index_cache:
# IndexCache config
# Refer: https://arxiv.org/abs/2603.12201 for more details.
_index_topk_freq = getattr(config, "index_topk_freq", 1)
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
layer_id = extract_layer_index(prefix)
if _index_topk_pattern is None:
_skip_topk = max(layer_id - 1, 0) % _index_topk_freq != 0
elif 0 <= layer_id < len(_index_topk_pattern):
_skip_topk = _index_topk_pattern[layer_id] == "S"
# IndexCache config
# Refer: https://arxiv.org/abs/2603.12201 for more details.
_index_topk_freq = getattr(config, "index_topk_freq", 1)
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
layer_id = extract_layer_index(prefix)
if _index_topk_pattern is None:
_skip_topk = (
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq
!= 0
)
elif 0 <= layer_id < len(_index_topk_pattern):
_skip_topk = _index_topk_pattern[layer_id] == "S"
else:
self.indexer_rope_emb = None
@@ -1252,8 +1253,8 @@ class DeepseekV2Model(nn.Module):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: DeepseekV2DecoderLayer(
vllm_config,
prefix,
vllm_config=vllm_config,
prefix=prefix,
topk_indices_buffer=topk_indices_buffer,
),
prefix=f"{prefix}.layers",