Implement prompt logprobs & Batched topk for computing logprobs (#1328)
Co-authored-by: Yunmo Chen <16273544+wanmok@users.noreply.github.com>
This commit is contained in:
@@ -9,7 +9,7 @@ from vllm.model_executor.parallel_utils.parallel_state import (
|
||||
def tensor_model_parallel_all_reduce(input_):
|
||||
"""All-reduce the input tensor across model parallel group.
|
||||
|
||||
Note: This operation is applied in-place on the input tensor.
|
||||
NOTE: This operation is applied in-place on the input tensor.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if get_tensor_model_parallel_world_size() == 1:
|
||||
|
||||
@@ -133,7 +133,7 @@ class ColumnParallelLinear(torch.nn.Module):
|
||||
params_dtype = torch.get_default_dtype()
|
||||
|
||||
# Parameters.
|
||||
# Note: torch.nn.functional.linear performs XA^T + b and as a result
|
||||
# NOTE: torch.nn.functional.linear performs XA^T + b and as a result
|
||||
# we allocate the transpose.
|
||||
self.create_weights(params_dtype)
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ def split_tensor_along_last_dim(
|
||||
last_dim_size = divide(tensor.size()[last_dim], num_partitions)
|
||||
# Split.
|
||||
tensor_list = torch.split(tensor, last_dim_size, dim=last_dim)
|
||||
# Note: torch.split does not create contiguous tensors by default.
|
||||
# NOTE: torch.split does not create contiguous tensors by default.
|
||||
if contiguous_split_chunks:
|
||||
return tuple(chunk.contiguous() for chunk in tensor_list)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user