Avoid long-blocking H2D copies in ViT (#51841)
Signed-off-by: Max Hu <hyoung2991@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -705,7 +705,15 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
|
||||
for t, h, w in grid_thw
|
||||
]
|
||||
pos_ids = torch.cat(pos_ids, dim=0).to(self.device, non_blocking=True)
|
||||
num_pos = sum(p.shape[0] for p in pos_ids)
|
||||
pinned = torch.empty(
|
||||
(num_pos, pos_ids[0].shape[1]),
|
||||
dtype=pos_ids[0].dtype,
|
||||
pin_memory=True,
|
||||
)
|
||||
pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to(
|
||||
self.device, non_blocking=True
|
||||
)
|
||||
|
||||
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
||||
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
||||
|
||||
Reference in New Issue
Block a user