Avoid long-blocking H2D copies in ViT (#51841)

Signed-off-by: Max Hu <hyoung2991@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Max Hu
2026-08-12 22:54:04 +08:00
committed by GitHub
co-authored by Nick Hill
parent 7aa248fcfe
commit b1b752042f
2 changed files with 23 additions and 5 deletions
+9 -1
View File
@@ -705,7 +705,15 @@ class Qwen3_VisionTransformer(nn.Module):
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
for t, h, w in grid_thw
]
pos_ids = torch.cat(pos_ids, dim=0).to(self.device, non_blocking=True)
num_pos = sum(p.shape[0] for p in pos_ids)
pinned = torch.empty(
(num_pos, pos_ids[0].shape[1]),
dtype=pos_ids[0].dtype,
pin_memory=True,
)
pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to(
self.device, non_blocking=True
)
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)