[Perf] Avoid forward scan for async output placeholders (#42938)

This commit is contained in:
Izik Golan
2026-05-19 20:16:07 -07:00
committed by GitHub
parent fadf5d332c
commit 2ae910ed88
+6 -1
View File
@@ -1050,7 +1050,12 @@ class InputBatch:
# output placeholders (tokens can be discarded after kv-load
# failure) or a larger number (async spec decode adds optimistic
# placeholders that may exceed the actual acceptance count).
first_placeholder = req_output_token_ids.index(-1)
first_placeholder = len(req_output_token_ids)
while (
first_placeholder > 0
and req_output_token_ids[first_placeholder - 1] == -1
):
first_placeholder -= 1
num_placeholders = len(req_output_token_ids) - first_placeholder
num_to_replace = min(num_sampled_ids, num_placeholders)
del new_ids[num_to_replace:]