[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -13,6 +13,7 @@ from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
logger = init_logger(__name__)
|
||||
is_batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
@@ -49,7 +50,9 @@ def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
|
||||
lora_strides_d1.append(lora_a_weight.stride(1))
|
||||
lora_strides_d2.append(lora_a_weight.stride(2))
|
||||
if len(lora_a_weights) > 1:
|
||||
lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64)
|
||||
lora_ptr_tensor = async_tensor_h2d(
|
||||
tensor_ptrs, dtype=torch.uint64, device=device
|
||||
)
|
||||
else:
|
||||
lora_ptr_tensor = lora_a_weights[0]
|
||||
|
||||
@@ -106,10 +109,11 @@ def _get_lora_b_ptr(
|
||||
hidden_sizes.append(lora_b_weight.size(1))
|
||||
|
||||
if len(lora_weights) > 1:
|
||||
# note these are device tensors
|
||||
lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64)
|
||||
slice_start_tensor = torch.tensor(
|
||||
slice_offset_lst, device=device, dtype=torch.uint64
|
||||
lora_ptr_tensor = async_tensor_h2d(
|
||||
tensor_ptrs, dtype=torch.uint64, device=device
|
||||
)
|
||||
slice_start_tensor = async_tensor_h2d(
|
||||
slice_offset_lst, dtype=torch.uint64, device=device
|
||||
)
|
||||
else:
|
||||
slice_start_tensor = slice_offset_lst[0]
|
||||
@@ -129,10 +133,18 @@ def _get_lora_b_ptr(
|
||||
same_stride = True
|
||||
|
||||
else:
|
||||
lora_strides_d0_tensor = torch.tensor(lora_strides_d0, device=device)
|
||||
lora_strides_d1_tensor = torch.tensor(lora_strides_d1, device=device)
|
||||
lora_strides_d2_tensor = torch.tensor(lora_strides_d2, device=device)
|
||||
hidden_sizes_tensor = torch.tensor(hidden_sizes, device=device)
|
||||
lora_strides_d0_tensor = async_tensor_h2d(
|
||||
lora_strides_d0, dtype=torch.int64, device=device
|
||||
)
|
||||
lora_strides_d1_tensor = async_tensor_h2d(
|
||||
lora_strides_d1, dtype=torch.int64, device=device
|
||||
)
|
||||
lora_strides_d2_tensor = async_tensor_h2d(
|
||||
lora_strides_d2, dtype=torch.int64, device=device
|
||||
)
|
||||
hidden_sizes_tensor = async_tensor_h2d(
|
||||
hidden_sizes, dtype=torch.int64, device=device
|
||||
)
|
||||
same_stride = False
|
||||
# MAX_N is the maximum hidden size among all the lora_b weights
|
||||
MAX_N = max(hidden_sizes)
|
||||
|
||||
@@ -5,6 +5,8 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# avoid circuit import
|
||||
from vllm.lora.layers import LoRAMapping
|
||||
@@ -110,8 +112,8 @@ def convert_mapping(
|
||||
embedding_indices,
|
||||
]
|
||||
|
||||
indices = torch.tensor(indices_list, dtype=torch.long, device=device)
|
||||
prompt_mapping_tensor = torch.tensor(
|
||||
indices = async_tensor_h2d(indices_list, dtype=torch.long, device=device)
|
||||
prompt_mapping_tensor = async_tensor_h2d(
|
||||
prompt_mapping, dtype=torch.long, device=device
|
||||
)
|
||||
embeddings_indices = torch.stack(
|
||||
|
||||
Reference in New Issue
Block a user