[MM][CG] Enable encoder CUDA Graph for MiniCPM-V (#42785)

Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Yunzhu Lu
2026-09-10 06:28:48 +00:00
committed by GitHub
co-authored by Isotr0py
parent c3ccc0e957
commit 442d36031c
26 changed files with 843 additions and 103 deletions
@@ -38,8 +38,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.utils.torch_utils import async_tensor_h2d
from .utils import AutoWeightsLoader, WeightsMapper
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
@@ -81,41 +80,61 @@ class Idefics2VisionEmbeddings(nn.Module):
patch_attention_mask: torch.BoolTensor,
tgt_sizes: torch.IntTensor | None = None,
) -> torch.Tensor:
batch_size, max_nb_patches_h, max_nb_patches_w = patch_attention_mask.shape
batch_size = patch_attention_mask.shape[0]
device = patch_attention_mask.device
boundaries = torch.arange(
1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side
)
position_ids = torch.full(
size=(batch_size, max_nb_patches_h * max_nb_patches_w),
fill_value=0,
pin_memory=PIN_MEMORY,
1 / self.num_patches_per_side,
1.0,
1 / self.num_patches_per_side,
device=device,
)
with gpu_sync_allowed():
patch_attention_mask_cpu = patch_attention_mask.cpu()
tgt_sizes_cpu = tgt_sizes.cpu() if tgt_sizes is not None else None
flat_patch_attention_mask = patch_attention_mask.view(batch_size, -1)
max_patches = flat_patch_attention_mask.shape[-1]
for batch_idx, p_attn_mask in enumerate(patch_attention_mask_cpu):
if tgt_sizes_cpu is not None:
nb_patches_h = tgt_sizes_cpu[batch_idx][0]
nb_patches_w = tgt_sizes_cpu[batch_idx][1]
if tgt_sizes is not None:
# tgt_sizes may come from CPU-side mm_kwargs; use a pinned async
# copy to stay clean under VLLM_GPU_SYNC_CHECK.
if tgt_sizes.device != device:
tgt_sizes = async_tensor_h2d(tgt_sizes, device)
nb_patches_h = tgt_sizes[:, 0]
nb_patches_w = tgt_sizes[:, 1]
else:
if patch_attention_mask.dim() == 3:
nb_patches_h = patch_attention_mask[:, :, 0].sum(dim=1)
nb_patches_w = patch_attention_mask[:, 0, :].sum(dim=1)
else:
nb_patches_h = p_attn_mask[:, 0].sum()
nb_patches_w = p_attn_mask[0].sum()
fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
bucket_coords_h = torch.bucketize(
fractional_coords_h, boundaries, right=True
)
bucket_coords_w = torch.bucketize(
fractional_coords_w, boundaries, right=True
)
pos_ids = (
bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w
).flatten()
position_ids[batch_idx][p_attn_mask.view(-1)] = pos_ids
# Fallback for unexpected 2D masks without tgt_sizes
nb_patches_h = torch.ones(batch_size, device=device)
nb_patches_w = patch_attention_mask.sum(dim=1)
return position_ids.to(self.position_embedding.weight.device, non_blocking=True)
nb_h = nb_patches_h.clamp(min=1).unsqueeze(1)
nb_w = nb_patches_w.clamp(min=1).unsqueeze(1)
i = torch.arange(max_patches, device=device)
if patch_attention_mask.dim() == 3 and patch_attention_mask.shape[1] > 1:
stride_w = patch_attention_mask.shape[2] # padded 2D grid width
else:
stride_w = nb_w # (batch, 1) — per-item actual patch width
h_idx = i.unsqueeze(0) // stride_w
w_idx = i.unsqueeze(0) % stride_w
fractional_h = h_idx / nb_h
fractional_w = w_idx / nb_w
bucket_h = torch.bucketize(fractional_h, boundaries, right=True)
bucket_w = torch.bucketize(fractional_w, boundaries, right=True)
pos_ids = bucket_h * self.num_patches_per_side + bucket_w
position_ids = torch.where(flat_patch_attention_mask, pos_ids.to(torch.long), 0)
target_device = self.position_embedding.weight.device
if position_ids.device != target_device:
position_ids = async_tensor_h2d(position_ids, target_device)
return position_ids
def forward(
self,
@@ -461,13 +480,7 @@ class Idefics2VisionTransformer(nn.Module):
# - if apply_encoder_attention_mask is False, skip (not all models
# sharing this encoder apply masking in attention, e.g. Aria, Phi4)
# - if patch_attention_mask was None, skip attention masking
# - if any padding exists, create an additive 4D mask and pass it
# to attention; else skip mask for performance.
if (
not self.apply_encoder_attention_mask
or flat_patch_mask is None
or not torch.any(~flat_patch_mask)
):
if not self.apply_encoder_attention_mask or flat_patch_mask is None:
attention_mask = None
else:
# Additive mask: masked positions receive a large negative value.