Implement prompt logprobs & Batched topk for computing logprobs (#1328)

Co-authored-by: Yunmo Chen <16273544+wanmok@users.noreply.github.com>
This commit is contained in:
Zhuohan Li
2023-10-16 10:56:50 -07:00
committed by GitHub
co-authored by Yunmo Chen
parent 928de46888
commit 9d9072a069
14 changed files with 369 additions and 130 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ def split_tensor_along_last_dim(
last_dim_size = divide(tensor.size()[last_dim], num_partitions)
# Split.
tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
# Note: torch.split does not create contiguous tensors by default.
# NOTE: torch.split does not create contiguous tensors by default.
if contiguous_split_chunks:
return tuple(chunk.contiguous() for chunk in tensor_list)