[Perf][Spec Decode] Avoid per-step numpy allocation in prepare_next_t… (#41043)

Signed-off-by: wangluochao902 <wangluochao902@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Luochao Wang
2026-04-29 14:20:13 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 1628239eb2
commit b58669cb42
+5 -9
View File
@@ -883,16 +883,12 @@ class SpecDecodeBaseProposer:
is not sampled and comes from `request.get_token_id()` instead. This is denoted
the "backup" token id. It also counts rejected tokens via `sampled_token_ids`.
"""
# Precompute get_token_id for when there is no valid next token
# Precompute backup token IDs for discarded requests.
num_reqs = gpu_input_batch.num_reqs
seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist()
self.backup_next_token_ids.np[:num_reqs] = np.array(
[
requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i])
for i in range(num_reqs)
],
dtype=np.int32,
)
for i in range(num_reqs):
self.backup_next_token_ids.np[i] = requests[
gpu_input_batch.req_ids[i]
].get_token_id(gpu_input_batch.num_tokens_no_spec[i] - 1)
self.backup_next_token_ids.copy_to_gpu(num_reqs)
backup_tokens_gpu = self.backup_next_token_ids.gpu