[MM][CG] Support ViT full CUDA graph for Idefics3 and SmolVLM (#47625)

Signed-off-by: Ryouichi Kurihara <ryouichi.kurihara.628@gmail.com>
This commit is contained in:
Chipmunk
2026-08-28 17:16:31 +00:00
committed by GitHub
parent 06569a8696
commit ffe690eca9
3 changed files with 240 additions and 33 deletions
@@ -75,20 +75,12 @@ class Idefics2VisionEmbeddings(nn.Module):
self.num_positions = self.num_patches
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
def forward(
def get_position_ids(
self,
pixel_values: torch.FloatTensor,
patch_attention_mask: torch.BoolTensor,
tgt_sizes: torch.IntTensor | None = None,
) -> torch.Tensor:
batch_size, _, max_im_h, max_im_w = pixel_values.shape
target_dtype = self.patch_embedding.weight.dtype
patch_embeds = self.patch_embedding(pixel_values.to(target_dtype))
embeddings = patch_embeds.flatten(2).transpose(1, 2)
max_nb_patches_h, max_nb_patches_w = (
max_im_h // self.patch_size,
max_im_w // self.patch_size,
)
batch_size, max_nb_patches_h, max_nb_patches_w = patch_attention_mask.shape
boundaries = torch.arange(
1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side
)
@@ -119,9 +111,21 @@ class Idefics2VisionEmbeddings(nn.Module):
bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w
).flatten()
position_ids[batch_idx][p_attn_mask.view(-1)] = pos_ids
position_ids = position_ids.to(
self.position_embedding.weight.device, non_blocking=True
)
return position_ids.to(self.position_embedding.weight.device, non_blocking=True)
def forward(
self,
pixel_values: torch.FloatTensor,
patch_attention_mask: torch.BoolTensor,
tgt_sizes: torch.IntTensor | None = None,
position_ids: torch.Tensor | None = None,
) -> torch.Tensor:
target_dtype = self.patch_embedding.weight.dtype
patch_embeds = self.patch_embedding(pixel_values.to(target_dtype))
embeddings = patch_embeds.flatten(2).transpose(1, 2)
if position_ids is None:
position_ids = self.get_position_ids(patch_attention_mask, tgt_sizes)
embeddings += self.position_embedding(position_ids)
return embeddings
@@ -423,6 +427,7 @@ class Idefics2VisionTransformer(nn.Module):
pixel_values,
patch_attention_mask: torch.BoolTensor | None = None,
tgt_sizes: torch.IntTensor | None = None,
position_ids: torch.Tensor | None = None,
) -> torch.Tensor:
batch_size = pixel_values.size(0)
@@ -446,6 +451,7 @@ class Idefics2VisionTransformer(nn.Module):
pixel_values=pixel_values,
patch_attention_mask=patch_attention_mask,
tgt_sizes=tgt_sizes,
position_ids=position_ids,
)
# Align with HuggingFace NaViT SigLIP in MiniCPMV/O: