[feature] add index share feature for DSA MTP (#44420)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c7f7741d4
commit
32f34d3935
@@ -115,7 +115,9 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
|
||||
)
|
||||
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
@@ -147,6 +149,22 @@ class DeepSeekMultiTokenPredictor(nn.Module):
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def set_skip_topk(self, skip: bool):
|
||||
"""Toggle skip_topk on all MTP layers with sparse attention.
|
||||
|
||||
Called by the proposer to implement index_share_for_mtp_iteration:
|
||||
step 0 sets skip=False (compute own indices), steps 1+ set skip=True
|
||||
(reuse step 0's indices).
|
||||
"""
|
||||
for layer in self.layers.values():
|
||||
mtp_block = getattr(layer, "mtp_block", None)
|
||||
if mtp_block is not None:
|
||||
self_attn = getattr(mtp_block, "self_attn", None)
|
||||
if self_attn is not None:
|
||||
mla_attn = getattr(self_attn, "mla_attn", None)
|
||||
if mla_attn is not None and hasattr(mla_attn, "skip_topk"):
|
||||
mla_attn.skip_topk = skip
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
@@ -225,7 +243,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
inputs_embeds,
|
||||
spec_step_idx,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
@@ -1018,19 +1018,20 @@ class DeepseekV2MLAAttention(nn.Module):
|
||||
is_inplace_rope=self.indexer_rope_emb.enabled(),
|
||||
)
|
||||
|
||||
# Enable IndexCache for DeepSeek models to reduce redundant top-k
|
||||
# token selection computations in sparse attention.
|
||||
use_index_cache = getattr(config, "use_index_cache", False)
|
||||
if use_index_cache:
|
||||
# IndexCache config
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
_index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
layer_id = extract_layer_index(prefix)
|
||||
if _index_topk_pattern is None:
|
||||
_skip_topk = max(layer_id - 1, 0) % _index_topk_freq != 0
|
||||
elif 0 <= layer_id < len(_index_topk_pattern):
|
||||
_skip_topk = _index_topk_pattern[layer_id] == "S"
|
||||
# IndexCache config
|
||||
# Refer: https://arxiv.org/abs/2603.12201 for more details.
|
||||
_index_topk_freq = getattr(config, "index_topk_freq", 1)
|
||||
_index_topk_pattern = getattr(config, "index_topk_pattern", None)
|
||||
_index_skip_topk_offset = getattr(config, "index_skip_topk_offset", 2)
|
||||
layer_id = extract_layer_index(prefix)
|
||||
|
||||
if _index_topk_pattern is None:
|
||||
_skip_topk = (
|
||||
max(layer_id - _index_skip_topk_offset + 1, 0) % _index_topk_freq
|
||||
!= 0
|
||||
)
|
||||
elif 0 <= layer_id < len(_index_topk_pattern):
|
||||
_skip_topk = _index_topk_pattern[layer_id] == "S"
|
||||
|
||||
else:
|
||||
self.indexer_rope_emb = None
|
||||
@@ -1252,8 +1253,8 @@ class DeepseekV2Model(nn.Module):
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: DeepseekV2DecoderLayer(
|
||||
vllm_config,
|
||||
prefix,
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
|
||||
Reference in New Issue
Block a user