Avoid long-blocking H2D copies in ViT (#51841)
Signed-off-by: Max Hu <hyoung2991@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -705,7 +705,15 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
|
||||
for t, h, w in grid_thw
|
||||
]
|
||||
pos_ids = torch.cat(pos_ids, dim=0).to(self.device, non_blocking=True)
|
||||
num_pos = sum(p.shape[0] for p in pos_ids)
|
||||
pinned = torch.empty(
|
||||
(num_pos, pos_ids[0].shape[1]),
|
||||
dtype=pos_ids[0].dtype,
|
||||
pin_memory=True,
|
||||
)
|
||||
pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to(
|
||||
self.device, non_blocking=True
|
||||
)
|
||||
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
@@ -2256,10 +2256,20 @@ class GPUModelRunner(
|
||||
|
||||
if self.uses_mrope:
|
||||
# Only relevant for models using M-RoPE (e.g, Qwen2-VL)
|
||||
self.mrope_positions.gpu[:, :total_num_scheduled_tokens].copy_(
|
||||
self.mrope_positions.cpu[:, :total_num_scheduled_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
# Copy one row at a time. mrope_positions is allocated as
|
||||
# [3, max_num_tokens + 1] with a dummy trailing column to keep it
|
||||
# non-contiguous for torch.compile, so cpu[:, :N] is a strided view.
|
||||
# copy_() cannot express a strided source as a single
|
||||
# cudaMemcpyAsync, so it first gathers into a contiguous *pageable*
|
||||
# temporary, and a pageable H2D ignores non_blocking=True and
|
||||
# synchronizes the stream before the transfer starts. Each row is
|
||||
# contiguous within the pinned allocation, so per-row copies stay on
|
||||
# the pinned path and are genuinely asynchronous.
|
||||
for row in range(self.mrope_positions.gpu.shape[0]):
|
||||
self.mrope_positions.gpu[row, :total_num_scheduled_tokens].copy_(
|
||||
self.mrope_positions.cpu[row, :total_num_scheduled_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
elif self.uses_xdrope_dim > 0:
|
||||
# Only relevant for models using XD-RoPE (e.g, HunYuan-VL)
|
||||
self.xdrope_positions.gpu[:, :total_num_scheduled_tokens].copy_(
|
||||
|
||||
Reference in New Issue
Block a user