[feature] add index share feature for DSA MTP (#44420)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c7f7741d4
commit
32f34d3935
@@ -115,7 +115,9 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
|
||||
)
|
||||
|
||||
hidden_states, residual = self.mtp_block(
|
||||
positions=positions, hidden_states=hidden_states, residual=None
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
residual=None,
|
||||
)
|
||||
hidden_states = residual + hidden_states
|
||||
return hidden_states
|
||||
@@ -147,6 +149,22 @@ class DeepSeekMultiTokenPredictor(nn.Module):
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
def set_skip_topk(self, skip: bool):
|
||||
"""Toggle skip_topk on all MTP layers with sparse attention.
|
||||
|
||||
Called by the proposer to implement index_share_for_mtp_iteration:
|
||||
step 0 sets skip=False (compute own indices), steps 1+ set skip=True
|
||||
(reuse step 0's indices).
|
||||
"""
|
||||
for layer in self.layers.values():
|
||||
mtp_block = getattr(layer, "mtp_block", None)
|
||||
if mtp_block is not None:
|
||||
self_attn = getattr(mtp_block, "self_attn", None)
|
||||
if self_attn is not None:
|
||||
mla_attn = getattr(self_attn, "mla_attn", None)
|
||||
if mla_attn is not None and hasattr(mla_attn, "skip_topk"):
|
||||
mla_attn.skip_topk = skip
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
@@ -225,7 +243,11 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts):
|
||||
spec_step_idx: int = 0,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, hidden_states, inputs_embeds, spec_step_idx
|
||||
input_ids,
|
||||
positions,
|
||||
hidden_states,
|
||||
inputs_embeds,
|
||||
spec_step_idx,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
|
||||
Reference in New Issue
Block a user