diff --git a/vllm/v1/worker/tpu_input_batch.py b/vllm/v1/worker/tpu_input_batch.py index 245ac3c902..c16156a5e3 100644 --- a/vllm/v1/worker/tpu_input_batch.py +++ b/vllm/v1/worker/tpu_input_batch.py @@ -377,10 +377,10 @@ class InputBatch: # self.token_ids_cpu[i1, ...], self.token_ids_cpu[i2, ...], =\ # self.token_ids_cpu[i2, ...], self.token_ids_cpu[i1, ...] # instead, we need to temporarily copy the data for one of the indices - # TODO(lucas): optimize this by only copying valid indices - tmp = self.token_ids_cpu[i1, ...].copy() - self.token_ids_cpu[i1, ...] = self.token_ids_cpu[i2, ...] - self.token_ids_cpu[i2, ...] = tmp + n = max(self.num_tokens_no_spec[i1], self.num_tokens_no_spec[i2]) + tmp = self.token_ids_cpu[i1, :n].copy() + self.token_ids_cpu[i1, :n] = self.token_ids_cpu[i2, :n] + self.token_ids_cpu[i2, :n] = tmp swap_dict_values(self.generators, i1, i2) swap_dict_values(self.min_tokens, i1, i2)