diff --git a/vllm/models/deepseek_v4/common/vision.py b/vllm/models/deepseek_v4/common/vision.py index bfc02768c0..cbcf34ffc7 100644 --- a/vllm/models/deepseek_v4/common/vision.py +++ b/vllm/models/deepseek_v4/common/vision.py @@ -38,6 +38,7 @@ from vllm.model_executor.models.vision import ( get_load_balance_assignment, is_vit_use_data_parallel, ) +from vllm.utils.torch_utils import async_tensor_h2d def _compute_vision_cos_sin( @@ -219,8 +220,8 @@ class DeepseekV4ViT(nn.Module): ) -> torch.Tensor: x = self.patch_embed(patches) cos, sin = get_vision_cos_sin(n_vit_h, n_vit_w, self.rope_dim, self.rope_theta) - cos = cos.to(device=x.device) - sin = sin.to(device=x.device) + cos = async_tensor_h2d(cos, x.device) + sin = async_tensor_h2d(sin, x.device) for block in self.blocks: x = block(x, cos, sin) return self.norm(x) @@ -279,8 +280,8 @@ def build_packed_vit_metadata( cu_seqlens.append(cu_seqlens[-1] + n) max_seqlen = max(max_seqlen, n) if cos_list: - cos = torch.cat(cos_list).to(device) - sin = torch.cat(sin_list).to(device) + cos = async_tensor_h2d(torch.cat(cos_list), device) + sin = async_tensor_h2d(torch.cat(sin_list), device) else: cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device) sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device) @@ -289,7 +290,7 @@ def build_packed_vit_metadata( return { "vit_cos": cos, "vit_sin": sin, - "cu_seqlens": torch.tensor(cu_seqlens, dtype=torch.int32, device=device), + "cu_seqlens": async_tensor_h2d(cu_seqlens, device, dtype=torch.int32), # Read on the host by the attention wrapper; keep on CPU. "max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32), } @@ -327,8 +328,10 @@ def build_packed_merge_metadata( mask_list.append(valid) offset += n_h * n_w if idx_list: - merge_idx = torch.cat(idx_list).to(device) - merge_mask = torch.cat(mask_list).unsqueeze(-1).to(device=device, dtype=dtype) + merge_idx = async_tensor_h2d(torch.cat(idx_list), device) + merge_mask = async_tensor_h2d( + torch.cat(mask_list).unsqueeze(-1), device, dtype=dtype + ) else: merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device) merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device) diff --git a/vllm/models/deepseek_v41/common/vl_cudagraph.py b/vllm/models/deepseek_v41/common/vl_cudagraph.py index e2a718b745..bd8ed5a3fd 100644 --- a/vllm/models/deepseek_v41/common/vl_cudagraph.py +++ b/vllm/models/deepseek_v41/common/vl_cudagraph.py @@ -25,6 +25,7 @@ from vllm.models.deepseek_v4.common.vision import ( build_packed_merge_metadata, build_packed_vit_metadata, ) +from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.worker.encoder_cudagraph_defs import ( EncoderCudaGraphCaptureInputs, EncoderCudaGraphConfig, @@ -370,18 +371,23 @@ class DeepseekV4VLEncoderCudaGraphMixin: r = self.config.vision_downsample_ratio vit_grid = self._get_grid_list(batch_mm_kwargs, "vit_grid") llm_grid = self._get_grid_list(batch_mm_kwargs, "llm_grid") - types = batch_mm_kwargs["types"].to(aligner_out.device) + types = batch_mm_kwargs["types"] n_rows = sum(-(-h // r) * (-(-w // r)) for h, w in vit_grid) span_lens = [lh * (lw + 1) + 2 for lh, lw in llm_grid] - # Batched span assembly: one masked fill per role across all items. - dtype = aligner_out.dtype - span = aligner_out.new_empty(sum(span_lens), self.config.hidden_size) - span[types == IMAGE] = aligner_out[:n_rows] - span[types == IMAGE_START] = self.image_start.to(dtype) - span[types == IMAGE_END] = self.image_end.to(dtype) - span[types == IMAGE_NEW_LINE] = self.image_newline.to(dtype) + # Batched span assembly: one gather over [aligner rows, START, + # NEW_LINE, END] for all items. The index is built from the host-side + # ``types`` so that neither the copy nor the gather syncs. + is_image = types == IMAGE + src_idx = torch.where( + is_image, + is_image.cumsum(0) - 1, + n_rows + (types == IMAGE_NEW_LINE).long() + 2 * (types == IMAGE_END).long(), + ) + delims = torch.stack([self.image_start, self.image_newline, self.image_end]) + src = torch.cat([aligner_out[:n_rows], delims.to(aligner_out.dtype)]) + span = src[async_tensor_h2d(src_idx, aligner_out.device)] # Freshly allocated, so later replays cannot clobber the results. offset = 0