[Bugfix][DSV4.1] Avoid host sync in ViT CUDA graph replay metadata (#58499)
This commit is contained in:
@@ -38,6 +38,7 @@ from vllm.model_executor.models.vision import (
|
||||
get_load_balance_assignment,
|
||||
is_vit_use_data_parallel,
|
||||
)
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
|
||||
|
||||
def _compute_vision_cos_sin(
|
||||
@@ -219,8 +220,8 @@ class DeepseekV4ViT(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
x = self.patch_embed(patches)
|
||||
cos, sin = get_vision_cos_sin(n_vit_h, n_vit_w, self.rope_dim, self.rope_theta)
|
||||
cos = cos.to(device=x.device)
|
||||
sin = sin.to(device=x.device)
|
||||
cos = async_tensor_h2d(cos, x.device)
|
||||
sin = async_tensor_h2d(sin, x.device)
|
||||
for block in self.blocks:
|
||||
x = block(x, cos, sin)
|
||||
return self.norm(x)
|
||||
@@ -279,8 +280,8 @@ def build_packed_vit_metadata(
|
||||
cu_seqlens.append(cu_seqlens[-1] + n)
|
||||
max_seqlen = max(max_seqlen, n)
|
||||
if cos_list:
|
||||
cos = torch.cat(cos_list).to(device)
|
||||
sin = torch.cat(sin_list).to(device)
|
||||
cos = async_tensor_h2d(torch.cat(cos_list), device)
|
||||
sin = async_tensor_h2d(torch.cat(sin_list), device)
|
||||
else:
|
||||
cos = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
|
||||
sin = torch.zeros((0, 1, rope_dim), dtype=torch.float32, device=device)
|
||||
@@ -289,7 +290,7 @@ def build_packed_vit_metadata(
|
||||
return {
|
||||
"vit_cos": cos,
|
||||
"vit_sin": sin,
|
||||
"cu_seqlens": torch.tensor(cu_seqlens, dtype=torch.int32, device=device),
|
||||
"cu_seqlens": async_tensor_h2d(cu_seqlens, device, dtype=torch.int32),
|
||||
# Read on the host by the attention wrapper; keep on CPU.
|
||||
"max_seqlen": torch.tensor(max_seqlen, dtype=torch.int32),
|
||||
}
|
||||
@@ -327,8 +328,10 @@ def build_packed_merge_metadata(
|
||||
mask_list.append(valid)
|
||||
offset += n_h * n_w
|
||||
if idx_list:
|
||||
merge_idx = torch.cat(idx_list).to(device)
|
||||
merge_mask = torch.cat(mask_list).unsqueeze(-1).to(device=device, dtype=dtype)
|
||||
merge_idx = async_tensor_h2d(torch.cat(idx_list), device)
|
||||
merge_mask = async_tensor_h2d(
|
||||
torch.cat(mask_list).unsqueeze(-1), device, dtype=dtype
|
||||
)
|
||||
else:
|
||||
merge_idx = torch.zeros((0, r * r), dtype=torch.int64, device=device)
|
||||
merge_mask = torch.zeros((0, r * r, 1), dtype=dtype, device=device)
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.models.deepseek_v4.common.vision import (
|
||||
build_packed_merge_metadata,
|
||||
build_packed_vit_metadata,
|
||||
)
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
@@ -370,18 +371,23 @@ class DeepseekV4VLEncoderCudaGraphMixin:
|
||||
r = self.config.vision_downsample_ratio
|
||||
vit_grid = self._get_grid_list(batch_mm_kwargs, "vit_grid")
|
||||
llm_grid = self._get_grid_list(batch_mm_kwargs, "llm_grid")
|
||||
types = batch_mm_kwargs["types"].to(aligner_out.device)
|
||||
types = batch_mm_kwargs["types"]
|
||||
|
||||
n_rows = sum(-(-h // r) * (-(-w // r)) for h, w in vit_grid)
|
||||
span_lens = [lh * (lw + 1) + 2 for lh, lw in llm_grid]
|
||||
|
||||
# Batched span assembly: one masked fill per role across all items.
|
||||
dtype = aligner_out.dtype
|
||||
span = aligner_out.new_empty(sum(span_lens), self.config.hidden_size)
|
||||
span[types == IMAGE] = aligner_out[:n_rows]
|
||||
span[types == IMAGE_START] = self.image_start.to(dtype)
|
||||
span[types == IMAGE_END] = self.image_end.to(dtype)
|
||||
span[types == IMAGE_NEW_LINE] = self.image_newline.to(dtype)
|
||||
# Batched span assembly: one gather over [aligner rows, START,
|
||||
# NEW_LINE, END] for all items. The index is built from the host-side
|
||||
# ``types`` so that neither the copy nor the gather syncs.
|
||||
is_image = types == IMAGE
|
||||
src_idx = torch.where(
|
||||
is_image,
|
||||
is_image.cumsum(0) - 1,
|
||||
n_rows + (types == IMAGE_NEW_LINE).long() + 2 * (types == IMAGE_END).long(),
|
||||
)
|
||||
delims = torch.stack([self.image_start, self.image_newline, self.image_end])
|
||||
src = torch.cat([aligner_out[:n_rows], delims.to(aligner_out.dtype)])
|
||||
span = src[async_tensor_h2d(src_idx, aligner_out.device)]
|
||||
|
||||
# Freshly allocated, so later replays cannot clobber the results.
|
||||
offset = 0
|
||||
|
||||
Reference in New Issue
Block a user