[Perf][4/n] Eliminate various GPU<->CPU syncs (#42347)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-05-19 10:35:54 -04:00
committed by GitHub
parent a78b842d0e
commit b82e908b4c
23 changed files with 129 additions and 108 deletions
+21 -9
View File
@@ -13,6 +13,7 @@ from vllm import envs
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import async_tensor_h2d
logger = init_logger(__name__)
is_batch_invariant = envs.VLLM_BATCH_INVARIANT
@@ -49,7 +50,9 @@ def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
lora_strides_d1.append(lora_a_weight.stride(1))
lora_strides_d2.append(lora_a_weight.stride(2))
if len(lora_a_weights) > 1:
lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64)
lora_ptr_tensor = async_tensor_h2d(
tensor_ptrs, dtype=torch.uint64, device=device
)
else:
lora_ptr_tensor = lora_a_weights[0]
@@ -106,10 +109,11 @@ def _get_lora_b_ptr(
hidden_sizes.append(lora_b_weight.size(1))
if len(lora_weights) > 1:
# note these are device tensors
lora_ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64)
slice_start_tensor = torch.tensor(
slice_offset_lst, device=device, dtype=torch.uint64
lora_ptr_tensor = async_tensor_h2d(
tensor_ptrs, dtype=torch.uint64, device=device
)
slice_start_tensor = async_tensor_h2d(
slice_offset_lst, dtype=torch.uint64, device=device
)
else:
slice_start_tensor = slice_offset_lst[0]
@@ -129,10 +133,18 @@ def _get_lora_b_ptr(
same_stride = True
else:
lora_strides_d0_tensor = torch.tensor(lora_strides_d0, device=device)
lora_strides_d1_tensor = torch.tensor(lora_strides_d1, device=device)
lora_strides_d2_tensor = torch.tensor(lora_strides_d2, device=device)
hidden_sizes_tensor = torch.tensor(hidden_sizes, device=device)
lora_strides_d0_tensor = async_tensor_h2d(
lora_strides_d0, dtype=torch.int64, device=device
)
lora_strides_d1_tensor = async_tensor_h2d(
lora_strides_d1, dtype=torch.int64, device=device
)
lora_strides_d2_tensor = async_tensor_h2d(
lora_strides_d2, dtype=torch.int64, device=device
)
hidden_sizes_tensor = async_tensor_h2d(
hidden_sizes, dtype=torch.int64, device=device
)
same_stride = False
# MAX_N is the maximum hidden size among all the lora_b weights
MAX_N = max(hidden_sizes)
+4 -2
View File
@@ -5,6 +5,8 @@ from typing import TYPE_CHECKING
import torch
from vllm.utils.torch_utils import async_tensor_h2d
if TYPE_CHECKING:
# avoid circuit import
from vllm.lora.layers import LoRAMapping
@@ -110,8 +112,8 @@ def convert_mapping(
embedding_indices,
]
indices = torch.tensor(indices_list, dtype=torch.long, device=device)
prompt_mapping_tensor = torch.tensor(
indices = async_tensor_h2d(indices_list, dtype=torch.long, device=device)
prompt_mapping_tensor = async_tensor_h2d(
prompt_mapping, dtype=torch.long, device=device
)
embeddings_indices = torch.stack(