[BugFix][Spec Decode] Compact shared topk indices buffer after first MTP draft step (#47238)

This commit is contained in:
Giancarlo Delfin
2026-07-01 21:38:51 -07:00
committed by GitHub
parent 1360c42fe6
commit d63c8e9444
3 changed files with 27 additions and 0 deletions
+9
View File
@@ -130,6 +130,15 @@ class DeepseekV32MultiTokenPredictor(nn.Module):
if self_attn is not None and hasattr(self_attn, "skip_topk"):
self_attn.skip_topk = skip
def compact_topk_indices(self, slot_ids: torch.Tensor):
"""Gather the top-k index rows at ``slot_ids`` to the front of the buffer."""
num_slots = slot_ids.numel()
for layer in self.layers.values():
self_attn = getattr(layer.mtp_block, "self_attn", None)
if self_attn is not None and hasattr(self_attn, "topk_indices_buffer"):
topk_indices_buffer = self_attn.topk_indices_buffer
topk_indices_buffer[:num_slots] = topk_indices_buffer[slot_ids]
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)