fix: improve token_ids_cpu swap to copy only valid indices (#36255)

Signed-off-by: Jaebok Lee <jaebok9541@naver.com>
This commit is contained in:
zzaebok
2026-08-28 17:22:39 -07:00
committed by GitHub
parent 085e9bb07e
commit 68c52b5a93
+4 -4
View File
@@ -377,10 +377,10 @@ class InputBatch:
# self.token_ids_cpu[i1, ...], self.token_ids_cpu[i2, ...], =\
# self.token_ids_cpu[i2, ...], self.token_ids_cpu[i1, ...]
# instead, we need to temporarily copy the data for one of the indices
# TODO(lucas): optimize this by only copying valid indices
tmp = self.token_ids_cpu[i1, ...].copy()
self.token_ids_cpu[i1, ...] = self.token_ids_cpu[i2, ...]
self.token_ids_cpu[i2, ...] = tmp
n = max(self.num_tokens_no_spec[i1], self.num_tokens_no_spec[i2])
tmp = self.token_ids_cpu[i1, :n].copy()
self.token_ids_cpu[i1, :n] = self.token_ids_cpu[i2, :n]
self.token_ids_cpu[i2, :n] = tmp
swap_dict_values(self.generators, i1, i2)
swap_dict_values(self.min_tokens, i1, i2)