From b1b752042f622c692d5019c3ea122f2f7ee9d6ac Mon Sep 17 00:00:00 2001 From: Max Hu Date: Wed, 12 Aug 2026 10:54:04 -0400 Subject: [PATCH] Avoid long-blocking H2D copies in ViT (#51841) Signed-off-by: Max Hu Co-authored-by: Nick Hill --- vllm/model_executor/models/qwen3_vl.py | 10 +++++++++- vllm/v1/worker/gpu_model_runner.py | 18 ++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 179912a6e6..75780140d7 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -705,7 +705,15 @@ class Qwen3_VisionTransformer(nn.Module): else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1) for t, h, w in grid_thw ] - pos_ids = torch.cat(pos_ids, dim=0).to(self.device, non_blocking=True) + num_pos = sum(p.shape[0] for p in pos_ids) + pinned = torch.empty( + (num_pos, pos_ids[0].shape[1]), + dtype=pos_ids[0].dtype, + pin_memory=True, + ) + pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to( + self.device, non_blocking=True + ) # Use pre-computed cos_sin_cache from RotaryEmbedding cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 8b16ae0b84..defaccacf6 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2256,10 +2256,20 @@ class GPUModelRunner( if self.uses_mrope: # Only relevant for models using M-RoPE (e.g, Qwen2-VL) - self.mrope_positions.gpu[:, :total_num_scheduled_tokens].copy_( - self.mrope_positions.cpu[:, :total_num_scheduled_tokens], - non_blocking=True, - ) + # Copy one row at a time. mrope_positions is allocated as + # [3, max_num_tokens + 1] with a dummy trailing column to keep it + # non-contiguous for torch.compile, so cpu[:, :N] is a strided view. + # copy_() cannot express a strided source as a single + # cudaMemcpyAsync, so it first gathers into a contiguous *pageable* + # temporary, and a pageable H2D ignores non_blocking=True and + # synchronizes the stream before the transfer starts. Each row is + # contiguous within the pinned allocation, so per-row copies stay on + # the pinned path and are genuinely asynchronous. + for row in range(self.mrope_positions.gpu.shape[0]): + self.mrope_positions.gpu[row, :total_num_scheduled_tokens].copy_( + self.mrope_positions.cpu[row, :total_num_scheduled_tokens], + non_blocking=True, + ) elif self.uses_xdrope_dim > 0: # Only relevant for models using XD-RoPE (e.g, HunYuan-VL) self.xdrope_positions.gpu[:, :total_num_scheduled_tokens].copy_(