[Bugfix][DSV4.1] Avoid host sync in ViT CUDA graph replay metadata (#58499)

This commit is contained in:
Kevin H. Luu
2026-09-26 11:37:36 -04:00
committed by GitHub
parent 0239bd2549
commit 7d8c5fe9a9
2 changed files with 24 additions and 15 deletions
+10 -7
View File
@@ -38,6 +38,7 @@ from vllm.model_executor.models.vision import (
get_load_balance_assignment,
is_vit_use_data_parallel,
)
from vllm.utils.torch_utils import async_tensor_h2d
def _compute_vision_cos_sin(
@@ -219,8 +220,8 @@ class DeepseekV4ViT(nn.Module):
) -> torch.Tensor:
x = self.patch_embed(patches)
cos, sin = get_vision_cos_sin(n_vit_h, n_vit_w, self.rope_dim, self.rope_theta)
cos = cos.to(device=x.device)
sin = sin.to(device=x.device)
cos = async_tensor_h2d(cos, x.device)
sin = async_tensor_h2d(sin, x.device)
for block in self.blocks:
x = block(x, cos, sin)
return self.norm(x)
@@ -279,8 +280,8 @@ def build_packed_vit_metadata(
cu_seqlens.append(cu_seqlens[-1] + n)
max_seqlen = max(max_seqlen, n)
if cos_list:
cos = torch.cat(cos_list).to(device)
sin = torch.cat(sin_list).to(device)
cos = async_tensor_h2d(torch.cat(cos_list), device)
sin = async_tensor_h2d(torch.cat(sin_list), device)
else:
cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
@@ -289,7 +290,7 @@ def build_packed_vit_metadata(
return {
"vit_cos": cos,
"vit_sin": sin,
"cu_seqlens": torch.tensor(cu_seqlens, dtype=torch.int32, device=device),
"cu_seqlens": async_tensor_h2d(cu_seqlens, device, dtype=torch.int32),
# Read on the host by the attention wrapper; keep on CPU.
"max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32),
}
@@ -327,8 +328,10 @@ def build_packed_merge_metadata(
mask_list.append(valid)
offset += n_h * n_w
if idx_list:
merge_idx = torch.cat(idx_list).to(device)
merge_mask = torch.cat(mask_list).unsqueeze(-1).to(device=device, dtype=dtype)
merge_idx = async_tensor_h2d(torch.cat(idx_list), device)
merge_mask = async_tensor_h2d(
torch.cat(mask_list).unsqueeze(-1), device, dtype=dtype
)
else:
merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device)
merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device)
@@ -25,6 +25,7 @@ from vllm.models.deepseek_v4.common.vision import (
build_packed_merge_metadata,
build_packed_vit_metadata,
)
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
EncoderCudaGraphConfig,
@@ -370,18 +371,23 @@ class DeepseekV4VLEncoderCudaGraphMixin:
r = self.config.vision_downsample_ratio
vit_grid = self._get_grid_list(batch_mm_kwargs, "vit_grid")
llm_grid = self._get_grid_list(batch_mm_kwargs, "llm_grid")
types = batch_mm_kwargs["types"].to(aligner_out.device)
types = batch_mm_kwargs["types"]
n_rows = sum(-(-h // r) * (-(-w // r)) for h, w in vit_grid)
span_lens = [lh * (lw + 1) + 2 for lh, lw in llm_grid]
# Batched span assembly: one masked fill per role across all items.
dtype = aligner_out.dtype
span = aligner_out.new_empty(sum(span_lens), self.config.hidden_size)
span[types == IMAGE] = aligner_out[:n_rows]
span[types == IMAGE_START] = self.image_start.to(dtype)
span[types == IMAGE_END] = self.image_end.to(dtype)
span[types == IMAGE_NEW_LINE] = self.image_newline.to(dtype)
# Batched span assembly: one gather over [aligner rows, START,
# NEW_LINE, END] for all items. The index is built from the host-side
# ``types`` so that neither the copy nor the gather syncs.
is_image = types == IMAGE
src_idx = torch.where(
is_image,
is_image.cumsum(0) - 1,
n_rows + (types == IMAGE_NEW_LINE).long() + 2 * (types == IMAGE_END).long(),
)
delims = torch.stack([self.image_start, self.image_newline, self.image_end])
src = torch.cat([aligner_out[:n_rows], delims.to(aligner_out.dtype)])
span = src[async_tensor_h2d(src_idx, aligner_out.device)]
# Freshly allocated, so later replays cannot clobber the results.
offset = 0