[MM][CG] Support ViT full CUDA graph for Idefics3 and SmolVLM (#47625)

Signed-off-by: Ryouichi Kurihara <ryouichi.kurihara.628@gmail.com>
This commit is contained in:
Chipmunk
2026-08-28 17:16:31 +00:00
committed by GitHub
parent 06569a8696
commit ffe690eca9
3 changed files with 240 additions and 33 deletions
@@ -193,6 +193,20 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"idefics3": VitCudagraphTestConfig(
model="HuggingFaceTB/SmolVLM-256M-Instruct",
modalities=["image"],
image_prompt=(
"<|begin_of_text|>User:<image>What is in this image?"
"<end_of_utterance>\nAssistant:"
),
max_model_len=4096,
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [4096],
},
vllm_runner_kwargs={"gpu_memory_utilization": 0.80},
marks=[pytest.mark.core_model],
),
"step3_vl": VitCudagraphTestConfig(
model="stepfun-ai/Step3-VL-10B",
modalities=["image"],
@@ -75,20 +75,12 @@ class Idefics2VisionEmbeddings(nn.Module):
self.num_positions = self.num_patches
self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim)
def forward(
def get_position_ids(
self,
pixel_values: torch.FloatTensor,
patch_attention_mask: torch.BoolTensor,
tgt_sizes: torch.IntTensor | None = None,
) -> torch.Tensor:
batch_size, _, max_im_h, max_im_w = pixel_values.shape
target_dtype = self.patch_embedding.weight.dtype
patch_embeds = self.patch_embedding(pixel_values.to(target_dtype))
embeddings = patch_embeds.flatten(2).transpose(1, 2)
max_nb_patches_h, max_nb_patches_w = (
max_im_h // self.patch_size,
max_im_w // self.patch_size,
)
batch_size, max_nb_patches_h, max_nb_patches_w = patch_attention_mask.shape
boundaries = torch.arange(
1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side
)
@@ -119,9 +111,21 @@ class Idefics2VisionEmbeddings(nn.Module):
bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w
).flatten()
position_ids[batch_idx][p_attn_mask.view(-1)] = pos_ids
position_ids = position_ids.to(
self.position_embedding.weight.device, non_blocking=True
)
return position_ids.to(self.position_embedding.weight.device, non_blocking=True)
def forward(
self,
pixel_values: torch.FloatTensor,
patch_attention_mask: torch.BoolTensor,
tgt_sizes: torch.IntTensor | None = None,
position_ids: torch.Tensor | None = None,
) -> torch.Tensor:
target_dtype = self.patch_embedding.weight.dtype
patch_embeds = self.patch_embedding(pixel_values.to(target_dtype))
embeddings = patch_embeds.flatten(2).transpose(1, 2)
if position_ids is None:
position_ids = self.get_position_ids(patch_attention_mask, tgt_sizes)
embeddings += self.position_embedding(position_ids)
return embeddings
@@ -423,6 +427,7 @@ class Idefics2VisionTransformer(nn.Module):
pixel_values,
patch_attention_mask: torch.BoolTensor | None = None,
tgt_sizes: torch.IntTensor | None = None,
position_ids: torch.Tensor | None = None,
) -> torch.Tensor:
batch_size = pixel_values.size(0)
@@ -446,6 +451,7 @@ class Idefics2VisionTransformer(nn.Module):
pixel_values=pixel_values,
patch_attention_mask=patch_attention_mask,
tgt_sizes=tgt_sizes,
position_ids=position_ids,
)
# Align with HuggingFace NaViT SigLIP in MiniCPMV/O:
+207 -20
View File
@@ -17,7 +17,7 @@
"""Inference-only Idefics3 model compatible with HuggingFace weights."""
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias
import torch
from torch import nn
@@ -60,6 +60,7 @@ from .idefics2_vision_model import (
)
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMultiModal,
)
@@ -442,6 +443,9 @@ class Idefics3Model(nn.Module):
self,
pixel_values: torch.Tensor,
pixel_attention_mask: torch.Tensor,
*,
remove_padding: bool = True,
position_ids: torch.Tensor | None = None,
) -> torch.Tensor:
# NOTE: we skip the step to select the vision feature layer since
# this is already done inside the vision tower
@@ -449,18 +453,36 @@ class Idefics3Model(nn.Module):
dtype=self.vision_model.embeddings.patch_embedding.weight.dtype
) # fp16 compatibility
# Remove padding images - padding images are full 0.
nb_values_per_image = pixel_values.shape[1:].numel()
real_images_inds = (pixel_values == 0.0).sum(
dim=(-1, -2, -3)
) != nb_values_per_image
with gpu_sync_allowed():
pixel_values = pixel_values[real_images_inds].contiguous()
if remove_padding:
# Remove padding images - padding images are full 0.
nb_values_per_image = pixel_values.shape[1:].numel()
real_images_inds = (pixel_values == 0.0).sum(
dim=(-1, -2, -3)
) != nb_values_per_image
with gpu_sync_allowed():
pixel_values = pixel_values[real_images_inds].contiguous()
# Handle the vision attention mask
# Remove padding images from the mask
pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous()
# Handle the vision attention mask
# Remove padding images from the mask
pixel_attention_mask = pixel_attention_mask[
real_images_inds
].contiguous()
patch_attention_mask = self.get_patch_attention_mask(pixel_attention_mask)
# Get sequence from the vision encoder
image_hidden_states = self.vision_model(
pixel_values=pixel_values,
patch_attention_mask=patch_attention_mask,
position_ids=position_ids,
)
return image_hidden_states
def get_patch_attention_mask(
self,
pixel_attention_mask: torch.Tensor,
) -> torch.Tensor:
patch_size = self.config.vision_config.patch_size
patches_subgrid = pixel_attention_mask.unfold(
dimension=1, size=patch_size, step=patch_size
@@ -468,15 +490,14 @@ class Idefics3Model(nn.Module):
patches_subgrid = patches_subgrid.unfold(
dimension=2, size=patch_size, step=patch_size
)
patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool()
# Get sequence from the vision encoder
image_hidden_states = self.vision_model(
pixel_values=pixel_values,
patch_attention_mask=patch_attention_mask,
)
return image_hidden_states
def get_position_ids(
self,
pixel_attention_mask: torch.Tensor,
) -> torch.Tensor:
patch_attention_mask = self.get_patch_attention_mask(pixel_attention_mask)
return self.vision_model.embeddings.get_position_ids(patch_attention_mask)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.text_model.embed_input_ids(input_ids)
@@ -502,7 +523,11 @@ class Idefics3Model(nn.Module):
info=Idefics3ProcessingInfo,
dummy_inputs=Idefics3DummyInputsBuilder,
)
class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLoRA):
class Idefics3ForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsLoRA, SupportsEncoderCudaGraph
):
supports_encoder_cudagraph = True
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -606,6 +631,168 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo
num_patches = image_input["num_patches"]
return [e.flatten(0, 1) for e in image_features.split(num_patches.tolist())]
# -- SupportsEncoderCudaGraph protocol methods --
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=["pixel_values", "pixel_attention_mask", "position_ids"],
out_hidden_size=self.config.text_config.hidden_size,
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config: VllmConfig,
) -> tuple[int, int]:
min_budget = self.model.image_seq_len
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
vllm_config.model_config.max_model_len,
)
return (min_budget, max_budget)
def _get_num_patches_list(self, mm_kwargs: dict[str, Any]) -> list[int]:
num_patches = mm_kwargs["num_patches"]
if isinstance(num_patches, torch.Tensor):
return num_patches.tolist()
return [int(n) for n in num_patches]
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
return [
EncoderItemSpec(
input_size=num_patches,
output_tokens=num_patches * self.model.image_seq_len,
)
for num_patches in self._get_num_patches_list(mm_kwargs)
]
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
pixel_values = mm_kwargs["pixel_values"]
pixel_attention_mask = mm_kwargs["pixel_attention_mask"]
num_patches_list = self._get_num_patches_list(mm_kwargs)
if len(indices) == 0:
return {
"pixel_values": pixel_values[:0],
"pixel_attention_mask": pixel_attention_mask[:0],
"num_patches": torch.empty(
(0,), dtype=torch.long, device=pixel_values.device
),
}
cum_patches = [0]
for num_patches in num_patches_list:
cum_patches.append(cum_patches[-1] + num_patches)
selected_pixel_values = torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
selected_attention_mask = torch.cat(
[pixel_attention_mask[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
selected_num_patches = torch.tensor(
[num_patches_list[i] for i in indices],
dtype=torch.long,
device=pixel_values.device,
)
return {
"pixel_values": selected_pixel_values,
"pixel_attention_mask": selected_attention_mask,
"num_patches": selected_num_patches,
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
image_size = self.config.vision_config.image_size
num_tiles = max(
(token_budget + self.model.image_seq_len - 1) // self.model.image_seq_len,
1,
)
dummy_pixel_values = torch.randn(
num_tiles, 3, image_size, image_size, device=device, dtype=dtype
)
dummy_pixel_attention_mask = torch.ones(
num_tiles, image_size, image_size, device=device, dtype=torch.bool
)
dummy_position_ids = self.model.get_position_ids(dummy_pixel_attention_mask)
return EncoderCudaGraphCaptureInputs(
values={
"pixel_values": dummy_pixel_values,
"pixel_attention_mask": dummy_pixel_attention_mask,
"position_ids": dummy_position_ids,
}
)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphReplayBuffers,
)
return EncoderCudaGraphReplayBuffers(
values={
"pixel_values": mm_kwargs["pixel_values"],
"pixel_attention_mask": mm_kwargs["pixel_attention_mask"],
"position_ids": self.model.get_position_ids(
mm_kwargs["pixel_attention_mask"]
),
}
)
def encoder_cudagraph_forward(
self,
inputs: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
image_features = self.model.image_pixels_to_features(
inputs["pixel_values"],
pixel_attention_mask=inputs["pixel_attention_mask"],
remove_padding=False,
position_ids=inputs["position_ids"],
)
image_features = self.model.connector(image_features)
return image_features.flatten(0, 1)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
image_input = self._parse_and_validate_image_input(**mm_kwargs)
assert image_input is not None and image_input["type"] == "pixel_values"
image_features = self._process_image_pixels(image_input)
image_features = self.model.connector(image_features)
return image_features.flatten(0, 1)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None: