From 0472436541c842ecda6d249411f1d35649291a79 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Sun, 28 Jun 2026 17:04:01 -0700 Subject: [PATCH] [Spec Decode] Avoid redundant hidden-states gather in draft prefill (#46968) --- vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py index 422d3ac690..f1ab8677f7 100644 --- a/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py @@ -360,7 +360,10 @@ class AutoRegressiveSpeculator(DraftModelSpeculator): self.current_draft_step, self.draft_logits, ) - self.hidden_states[:num_reqs] = hidden_states[last_token_indices] + if last_hidden_states is hidden_states: + self.hidden_states[:num_reqs] = sample_hidden_states + else: + self.hidden_states[:num_reqs] = hidden_states[last_token_indices] self.input_buffers.positions[:num_reqs] = positions def _multi_step_decode(