[Spec Decode] Avoid redundant hidden-states gather in draft prefill (#46968)

This commit is contained in:
Woosuk Kwon
2026-06-28 17:04:01 -07:00
committed by GitHub
parent 4dfbf1503b
commit 0472436541
@@ -360,7 +360,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
self.current_draft_step,
self.draft_logits,
)
self.hidden_states[:num_reqs] = hidden_states[last_token_indices]
if last_hidden_states is hidden_states:
self.hidden_states[:num_reqs] = sample_hidden_states
else:
self.hidden_states[:num_reqs] = hidden_states[last_token_indices]
self.input_buffers.positions[:num_reqs] = positions
def _multi_step_decode(