From b58669cb427effa49928b7be5b6e0f4fd707bce5 Mon Sep 17 00:00:00 2001 From: Luochao Wang Date: Wed, 29 Apr 2026 14:20:13 -0700 Subject: [PATCH] =?UTF-8?q?[Perf][Spec=20Decode]=20Avoid=20per-step=20nump?= =?UTF-8?q?y=20allocation=20in=20prepare=5Fnext=5Ft=E2=80=A6=20(#41043)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: wangluochao902 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/v1/spec_decode/llm_base_proposer.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 1c59704de6..8d38038734 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -883,16 +883,12 @@ class SpecDecodeBaseProposer: is not sampled and comes from `request.get_token_id()` instead. This is denoted the "backup" token id. It also counts rejected tokens via `sampled_token_ids`. """ - # Precompute get_token_id for when there is no valid next token + # Precompute backup token IDs for discarded requests. num_reqs = gpu_input_batch.num_reqs - seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist() - self.backup_next_token_ids.np[:num_reqs] = np.array( - [ - requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i]) - for i in range(num_reqs) - ], - dtype=np.int32, - ) + for i in range(num_reqs): + self.backup_next_token_ids.np[i] = requests[ + gpu_input_batch.req_ids[i] + ].get_token_id(gpu_input_batch.num_tokens_no_spec[i] - 1) self.backup_next_token_ids.copy_to_gpu(num_reqs) backup_tokens_gpu = self.backup_next_token_ids.gpu