[BugFix][Spec Decode] Compact shared topk indices buffer after first MTP draft step (#47238)

This commit is contained in:
Giancarlo Delfin
2026-07-01 21:38:51 -07:00
committed by GitHub
parent 1360c42fe6
commit d63c8e9444
3 changed files with 27 additions and 0 deletions
+3
View File
@@ -560,6 +560,9 @@ class SpecDecodeBaseProposer:
# and read the indices that step 0 just wrote into the shared buffer.
if self._share_mtp_indices and hasattr(self.model.model, "set_skip_topk"):
self.model.model.set_skip_topk(True)
# The topk indices were written for each query token in the multi-token
# batch. Compact the topk indices for each request's last token.
self.model.model.compact_topk_indices(token_indices_to_sample)
sample_hidden_states = last_hidden_states[token_indices_to_sample]