[Perf] Ensure async h2d copies are pinned in more places (#55202)

This commit is contained in:
Nick Hill
2026-09-04 17:17:01 -07:00
committed by GitHub
parent 874df9373d
commit 8277c42e4c
21 changed files with 206 additions and 73 deletions
+41 -16
View File
@@ -59,7 +59,7 @@ from vllm.multimodal.processing import (
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.utils.torch_utils import PIN_MEMORY, async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
@@ -1305,17 +1305,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
# These are all built on the host, so stage them over
# non-blocking rather than forcing a sync per transfer.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values.device, non_blocking=True
)
# These are all built on the host; concat straight into pinned
# buffers so the H2D copies stay non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values.device
)
sample_indices = torch.concat(sample_indices, dim=0).to(
pixel_values.device, non_blocking=True
)
sample_indices = torch.concat(
sample_indices,
dim=0,
out=torch.empty(
sum(t.numel() for t in sample_indices),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values.device, non_blocking=True)
image_embeds = self.visual(
pixel_values=pixel_values,
@@ -1359,16 +1371,29 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
)
else:
pixel_values_videos = pixel_values_videos.type(self.visual.dtype)
# These are host-built; stage them over non-blocking.
siglip_position_ids = torch.concat(siglip_position_ids, dim=0).to(
pixel_values_videos.device, non_blocking=True
)
# These are host-built; concat straight into pinned buffers so
# the H2D copies stay non-blocking.
siglip_position_ids = torch.concat(
siglip_position_ids,
dim=0,
out=torch.empty(
sum(t.numel() for t in siglip_position_ids),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values_videos.device, non_blocking=True)
cu_seqlens = async_tensor_h2d(
cu_seqlens, dtype=torch.int32, device=pixel_values_videos.device
)
sample_indices = torch.concat(sample_indices, dim=0).to(
pixel_values_videos.device, non_blocking=True
)
sample_indices = torch.concat(
sample_indices,
dim=0,
out=torch.empty(
sum(t.numel() for t in sample_indices),
dtype=torch.int64,
pin_memory=PIN_MEMORY,
),
).to(pixel_values_videos.device, non_blocking=True)
video_embeds = self.visual(
pixel_values=pixel_values_videos,