[BugFix][Spec Decode] Compact shared topk indices buffer after first MTP draft step (#47238)

This commit is contained in:
Giancarlo Delfin
2026-07-01 21:38:51 -07:00
committed by GitHub
parent 1360c42fe6
commit d63c8e9444
3 changed files with 27 additions and 0 deletions
@@ -169,6 +169,21 @@ class DeepSeekMultiTokenPredictor(nn.Module):
if mla_attn is not None and hasattr(mla_attn, "skip_topk"):
mla_attn.skip_topk = skip
def compact_topk_indices(self, slot_ids: torch.Tensor):
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
num_slots = slot_ids.numel()
for layer in self.layers.values():
mtp_block = getattr(layer, "mtp_block", None)
if mtp_block is not None:
self_attn = getattr(mtp_block, "self_attn", None)
if self_attn is not None:
mla_attn = getattr(self_attn, "mla_attn", None)
if mla_attn is not None and hasattr(
mla_attn, "topk_indices_buffer"
):
topk_indices_buffer = mla_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)