Avoid long-blocking H2D copies in ViT (#51841)

Signed-off-by: Max Hu <hyoung2991@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Max Hu
2026-08-12 22:54:04 +08:00
committed by GitHub
co-authored by Nick Hill
parent 7aa248fcfe
commit b1b752042f
2 changed files with 23 additions and 5 deletions
+9 -1
View File
@@ -705,7 +705,15 @@ class Qwen3_VisionTransformer(nn.Module):
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
for t, h, w in grid_thw
]
pos_ids = torch.cat(pos_ids, dim=0).to(self.device, non_blocking=True)
num_pos = sum(p.shape[0] for p in pos_ids)
pinned = torch.empty(
(num_pos, pos_ids[0].shape[1]),
dtype=pos_ids[0].dtype,
pin_memory=True,
)
pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to(
self.device, non_blocking=True
)
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
+14 -4
View File
@@ -2256,10 +2256,20 @@ class GPUModelRunner(
if self.uses_mrope:
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
self.mrope_positions.gpu[:, :total_num_scheduled_tokens].copy_(
self.mrope_positions.cpu[:, :total_num_scheduled_tokens],
non_blocking=True,
)
# Copy one row at a time. mrope_positions is allocated as
# [3, max_num_tokens + 1] with a dummy trailing column to keep it
# non-contiguous for torch.compile, so cpu[:, :N] is a strided view.
# copy_() cannot express a strided source as a single
# cudaMemcpyAsync, so it first gathers into a contiguous *pageable*
# temporary, and a pageable H2D ignores non_blocking=True and
# synchronizes the stream before the transfer starts. Each row is
# contiguous within the pinned allocation, so per-row copies stay on
# the pinned path and are genuinely asynchronous.
for row in range(self.mrope_positions.gpu.shape[0]):
self.mrope_positions.gpu[row, :total_num_scheduled_tokens].copy_(
self.mrope_positions.cpu[row, :total_num_scheduled_tokens],
non_blocking=True,
)
elif self.uses_xdrope_dim > 0:
# Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
self.xdrope_positions.gpu[:, :total_num_scheduled_tokens].copy_(