fix: improve token_ids_cpu swap to copy only valid indices (#36255)
Signed-off-by: Jaebok Lee <jaebok9541@naver.com>
This commit is contained in:
@@ -377,10 +377,10 @@ class InputBatch:
|
||||
# self.token_ids_cpu[i1, ...], self.token_ids_cpu[i2, ...], =\
|
||||
# self.token_ids_cpu[i2, ...], self.token_ids_cpu[i1, ...]
|
||||
# instead, we need to temporarily copy the data for one of the indices
|
||||
# TODO(lucas): optimize this by only copying valid indices
|
||||
tmp = self.token_ids_cpu[i1, ...].copy()
|
||||
self.token_ids_cpu[i1, ...] = self.token_ids_cpu[i2, ...]
|
||||
self.token_ids_cpu[i2, ...] = tmp
|
||||
n = max(self.num_tokens_no_spec[i1], self.num_tokens_no_spec[i2])
|
||||
tmp = self.token_ids_cpu[i1, :n].copy()
|
||||
self.token_ids_cpu[i1, :n] = self.token_ids_cpu[i2, :n]
|
||||
self.token_ids_cpu[i2, :n] = tmp
|
||||
|
||||
swap_dict_values(self.generators, i1, i2)
|
||||
swap_dict_values(self.min_tokens, i1, i2)
|
||||
|
||||
Reference in New Issue
Block a user