[BugFix][Spec Decode] Compact shared topk indices buffer after first MTP draft step (#47238)
This commit is contained in:
@@ -560,6 +560,9 @@ class SpecDecodeBaseProposer:
|
||||
# and read the indices that step 0 just wrote into the shared buffer.
|
||||
if self._share_mtp_indices and hasattr(self.model.model, "set_skip_topk"):
|
||||
self.model.model.set_skip_topk(True)
|
||||
# The topk indices were written for each query token in the multi-token
|
||||
# batch. Compact the topk indices for each request's last token.
|
||||
self.model.model.compact_topk_indices(token_indices_to_sample)
|
||||
|
||||
sample_hidden_states = last_hidden_states[token_indices_to_sample]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user