[Perf] Avoid some more unnecessary GPU<->CPU syncs (#51458)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-08 17:13:26 -07:00
committed by GitHub
parent d608dfabfd
commit 9b0afeb4f6
13 changed files with 99 additions and 49 deletions
@@ -20,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.factory import KVConnectorFactory
from vllm.logger import init_logger
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.platforms import current_platform
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
@@ -179,9 +180,13 @@ def _make_src_and_dst_indices(
src_device: torch.device | str,
dst_device: torch.device | str,
) -> tuple[torch.Tensor, torch.Tensor]:
src_indices = torch.tensor(src_block_ids, device=src_device, dtype=torch.int64)
dst_indices = torch.tensor(dst_block_ids, device=dst_device, dtype=torch.int64)
return src_indices, dst_indices
def _to(block_ids: list[int], device: torch.device | str) -> torch.Tensor:
device = torch.device(device) if isinstance(device, str) else device
if device.type == "cpu":
return torch.tensor(block_ids, dtype=torch.int64, device=device)
return async_tensor_h2d(block_ids, dtype=torch.int64, device=device)
return _to(src_block_ids, src_device), _to(dst_block_ids, dst_device)
def copy_kv_blocks(
@@ -135,6 +135,7 @@ class ExampleConnector(KVConnectorBase_V1):
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
"""
slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
dst_kv_cache_layer_shape = dst_kv_cache_layer.shape
num_pages = dst_kv_cache_layer_shape[0]
@@ -178,9 +179,8 @@ class ExampleConnector(KVConnectorBase_V1):
filename = self._generate_filename_debug(
layer_name, request.token_ids, request.mm_hashes
)
kv_cache = safetensors.torch.load_file(
filename, device=str(kv_cache_layer.device)
)["kv_cache"]
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
kv_cache = kv_cache_cpu.to("cuda", non_blocking=True)
if isinstance(attn_metadata, dict):
inject_kv_into_layer(
kv_cache_layer,
@@ -227,6 +227,7 @@ class ExampleConnector(KVConnectorBase_V1):
Assume the shape of the layer is (num_pages, page_size, xxx)
for MLA, and (num_pages, 2, page_size, xxx) otherwise.
"""
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
num_pages, page_size = layer.shape[0], layer.shape[1]
return layer.reshape(num_pages * page_size, -1)[slot_mapping, ...]