[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
|
||||
@@ -179,9 +180,13 @@ def _make_src_and_dst_indices(
|
||||
src_device: torch.device | str,
|
||||
dst_device: torch.device | str,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
src_indices = torch.tensor(src_block_ids, device=src_device, dtype=torch.int64)
|
||||
dst_indices = torch.tensor(dst_block_ids, device=dst_device, dtype=torch.int64)
|
||||
return src_indices, dst_indices
|
||||
def _to(block_ids: list[int], device: torch.device | str) -> torch.Tensor:
|
||||
device = torch.device(device) if isinstance(device, str) else device
|
||||
if device.type == "cpu":
|
||||
return torch.tensor(block_ids, dtype=torch.int64, device=device)
|
||||
return async_tensor_h2d(block_ids, dtype=torch.int64, device=device)
|
||||
|
||||
return _to(src_block_ids, src_device), _to(dst_block_ids, dst_device)
|
||||
|
||||
|
||||
def copy_kv_blocks(
|
||||
|
||||
@@ -135,6 +135,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
"""
|
||||
slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
dst_kv_cache_layer_shape = dst_kv_cache_layer.shape
|
||||
num_pages = dst_kv_cache_layer_shape[0]
|
||||
@@ -178,9 +179,8 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
kv_cache = safetensors.torch.load_file(
|
||||
filename, device=str(kv_cache_layer.device)
|
||||
)["kv_cache"]
|
||||
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
|
||||
kv_cache = kv_cache_cpu.to("cuda", non_blocking=True)
|
||||
if isinstance(attn_metadata, dict):
|
||||
inject_kv_into_layer(
|
||||
kv_cache_layer,
|
||||
@@ -227,6 +227,7 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
Assume the shape of the layer is (num_pages, page_size, xxx)
|
||||
for MLA, and (num_pages, 2, page_size, xxx) otherwise.
|
||||
"""
|
||||
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
num_pages, page_size = layer.shape[0], layer.shape[1]
|
||||
return layer.reshape(num_pages * page_size, -1)[slot_mapping, ...]
|
||||
|
||||
Reference in New Issue
Block a user