diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index 8e12790925..1db1134fb4 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -193,6 +193,20 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { vllm_runner_kwargs={"trust_remote_code": True}, marks=[pytest.mark.core_model], ), + "idefics3": VitCudagraphTestConfig( + model="HuggingFaceTB/SmolVLM-256M-Instruct", + modalities=["image"], + image_prompt=( + "<|begin_of_text|>User:What is in this image?" + "\nAssistant:" + ), + max_model_len=4096, + compilation_config_overrides={ + "encoder_cudagraph_token_budgets": [4096], + }, + vllm_runner_kwargs={"gpu_memory_utilization": 0.80}, + marks=[pytest.mark.core_model], + ), "step3_vl": VitCudagraphTestConfig( model="stepfun-ai/Step3-VL-10B", modalities=["image"], diff --git a/vllm/model_executor/models/idefics2_vision_model.py b/vllm/model_executor/models/idefics2_vision_model.py index e3f59138d0..09887c1ec3 100644 --- a/vllm/model_executor/models/idefics2_vision_model.py +++ b/vllm/model_executor/models/idefics2_vision_model.py @@ -75,20 +75,12 @@ class Idefics2VisionEmbeddings(nn.Module): self.num_positions = self.num_patches self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - def forward( + def get_position_ids( self, - pixel_values: torch.FloatTensor, patch_attention_mask: torch.BoolTensor, tgt_sizes: torch.IntTensor | None = None, ) -> torch.Tensor: - batch_size, _, max_im_h, max_im_w = pixel_values.shape - target_dtype = self.patch_embedding.weight.dtype - patch_embeds = self.patch_embedding(pixel_values.to(target_dtype)) - embeddings = patch_embeds.flatten(2).transpose(1, 2) - max_nb_patches_h, max_nb_patches_w = ( - max_im_h // self.patch_size, - max_im_w // self.patch_size, - ) + batch_size, max_nb_patches_h, max_nb_patches_w = patch_attention_mask.shape boundaries = torch.arange( 1 / self.num_patches_per_side, 1.0, 1 / self.num_patches_per_side ) @@ -119,9 +111,21 @@ class Idefics2VisionEmbeddings(nn.Module): bucket_coords_h[:, None] * self.num_patches_per_side + bucket_coords_w ).flatten() position_ids[batch_idx][p_attn_mask.view(-1)] = pos_ids - position_ids = position_ids.to( - self.position_embedding.weight.device, non_blocking=True - ) + + return position_ids.to(self.position_embedding.weight.device, non_blocking=True) + + def forward( + self, + pixel_values: torch.FloatTensor, + patch_attention_mask: torch.BoolTensor, + tgt_sizes: torch.IntTensor | None = None, + position_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to(target_dtype)) + embeddings = patch_embeds.flatten(2).transpose(1, 2) + if position_ids is None: + position_ids = self.get_position_ids(patch_attention_mask, tgt_sizes) embeddings += self.position_embedding(position_ids) return embeddings @@ -423,6 +427,7 @@ class Idefics2VisionTransformer(nn.Module): pixel_values, patch_attention_mask: torch.BoolTensor | None = None, tgt_sizes: torch.IntTensor | None = None, + position_ids: torch.Tensor | None = None, ) -> torch.Tensor: batch_size = pixel_values.size(0) @@ -446,6 +451,7 @@ class Idefics2VisionTransformer(nn.Module): pixel_values=pixel_values, patch_attention_mask=patch_attention_mask, tgt_sizes=tgt_sizes, + position_ids=position_ids, ) # Align with HuggingFace NaViT SigLIP in MiniCPMV/O: diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index 223b881196..9a55ac51d3 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -17,7 +17,7 @@ """Inference-only Idefics3 model compatible with HuggingFace weights.""" from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Any, Literal, TypeAlias import torch from torch import nn @@ -60,6 +60,7 @@ from .idefics2_vision_model import ( ) from .interfaces import ( MultiModalEmbeddings, + SupportsEncoderCudaGraph, SupportsLoRA, SupportsMultiModal, ) @@ -442,6 +443,9 @@ class Idefics3Model(nn.Module): self, pixel_values: torch.Tensor, pixel_attention_mask: torch.Tensor, + *, + remove_padding: bool = True, + position_ids: torch.Tensor | None = None, ) -> torch.Tensor: # NOTE: we skip the step to select the vision feature layer since # this is already done inside the vision tower @@ -449,18 +453,36 @@ class Idefics3Model(nn.Module): dtype=self.vision_model.embeddings.patch_embedding.weight.dtype ) # fp16 compatibility - # Remove padding images - padding images are full 0. - nb_values_per_image = pixel_values.shape[1:].numel() - real_images_inds = (pixel_values == 0.0).sum( - dim=(-1, -2, -3) - ) != nb_values_per_image - with gpu_sync_allowed(): - pixel_values = pixel_values[real_images_inds].contiguous() + if remove_padding: + # Remove padding images - padding images are full 0. + nb_values_per_image = pixel_values.shape[1:].numel() + real_images_inds = (pixel_values == 0.0).sum( + dim=(-1, -2, -3) + ) != nb_values_per_image + with gpu_sync_allowed(): + pixel_values = pixel_values[real_images_inds].contiguous() - # Handle the vision attention mask - # Remove padding images from the mask - pixel_attention_mask = pixel_attention_mask[real_images_inds].contiguous() + # Handle the vision attention mask + # Remove padding images from the mask + pixel_attention_mask = pixel_attention_mask[ + real_images_inds + ].contiguous() + patch_attention_mask = self.get_patch_attention_mask(pixel_attention_mask) + + # Get sequence from the vision encoder + image_hidden_states = self.vision_model( + pixel_values=pixel_values, + patch_attention_mask=patch_attention_mask, + position_ids=position_ids, + ) + + return image_hidden_states + + def get_patch_attention_mask( + self, + pixel_attention_mask: torch.Tensor, + ) -> torch.Tensor: patch_size = self.config.vision_config.patch_size patches_subgrid = pixel_attention_mask.unfold( dimension=1, size=patch_size, step=patch_size @@ -468,15 +490,14 @@ class Idefics3Model(nn.Module): patches_subgrid = patches_subgrid.unfold( dimension=2, size=patch_size, step=patch_size ) - patch_attention_mask = (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() + return (patches_subgrid.sum(dim=(-1, -2)) > 0).bool() - # Get sequence from the vision encoder - image_hidden_states = self.vision_model( - pixel_values=pixel_values, - patch_attention_mask=patch_attention_mask, - ) - - return image_hidden_states + def get_position_ids( + self, + pixel_attention_mask: torch.Tensor, + ) -> torch.Tensor: + patch_attention_mask = self.get_patch_attention_mask(pixel_attention_mask) + return self.vision_model.embeddings.get_position_ids(patch_attention_mask) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.text_model.embed_input_ids(input_ids) @@ -502,7 +523,11 @@ class Idefics3Model(nn.Module): info=Idefics3ProcessingInfo, dummy_inputs=Idefics3DummyInputsBuilder, ) -class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLoRA): +class Idefics3ForConditionalGeneration( + nn.Module, SupportsMultiModal, SupportsLoRA, SupportsEncoderCudaGraph +): + supports_encoder_cudagraph = True + packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -606,6 +631,168 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo num_patches = image_input["num_patches"] return [e.flatten(0, 1) for e in image_features.split(num_patches.tolist())] + # -- SupportsEncoderCudaGraph protocol methods -- + + def get_encoder_cudagraph_config(self): + from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig + + return EncoderCudaGraphConfig( + modalities=["image"], + buffer_keys=["pixel_values", "pixel_attention_mask", "position_ids"], + out_hidden_size=self.config.text_config.hidden_size, + ) + + def get_encoder_cudagraph_budget_range( + self, + vllm_config: VllmConfig, + ) -> tuple[int, int]: + min_budget = self.model.image_seq_len + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + vllm_config.model_config.max_model_len, + ) + return (min_budget, max_budget) + + def _get_num_patches_list(self, mm_kwargs: dict[str, Any]) -> list[int]: + num_patches = mm_kwargs["num_patches"] + if isinstance(num_patches, torch.Tensor): + return num_patches.tolist() + return [int(n) for n in num_patches] + + def get_encoder_cudagraph_item_specs( + self, + mm_kwargs: dict[str, Any], + ): + from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec + + return [ + EncoderItemSpec( + input_size=num_patches, + output_tokens=num_patches * self.model.image_seq_len, + ) + for num_patches in self._get_num_patches_list(mm_kwargs) + ] + + def select_encoder_cudagraph_items( + self, + mm_kwargs: dict[str, Any], + indices: list[int], + ) -> dict[str, Any]: + pixel_values = mm_kwargs["pixel_values"] + pixel_attention_mask = mm_kwargs["pixel_attention_mask"] + num_patches_list = self._get_num_patches_list(mm_kwargs) + + if len(indices) == 0: + return { + "pixel_values": pixel_values[:0], + "pixel_attention_mask": pixel_attention_mask[:0], + "num_patches": torch.empty( + (0,), dtype=torch.long, device=pixel_values.device + ), + } + + cum_patches = [0] + for num_patches in num_patches_list: + cum_patches.append(cum_patches[-1] + num_patches) + + selected_pixel_values = torch.cat( + [pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices] + ) + selected_attention_mask = torch.cat( + [pixel_attention_mask[cum_patches[i] : cum_patches[i + 1]] for i in indices] + ) + selected_num_patches = torch.tensor( + [num_patches_list[i] for i in indices], + dtype=torch.long, + device=pixel_values.device, + ) + + return { + "pixel_values": selected_pixel_values, + "pixel_attention_mask": selected_attention_mask, + "num_patches": selected_num_patches, + } + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + path: str = "default", + ): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + ) + + image_size = self.config.vision_config.image_size + num_tiles = max( + (token_budget + self.model.image_seq_len - 1) // self.model.image_seq_len, + 1, + ) + dummy_pixel_values = torch.randn( + num_tiles, 3, image_size, image_size, device=device, dtype=dtype + ) + dummy_pixel_attention_mask = torch.ones( + num_tiles, image_size, image_size, device=device, dtype=torch.bool + ) + dummy_position_ids = self.model.get_position_ids(dummy_pixel_attention_mask) + + return EncoderCudaGraphCaptureInputs( + values={ + "pixel_values": dummy_pixel_values, + "pixel_attention_mask": dummy_pixel_attention_mask, + "position_ids": dummy_position_ids, + } + ) + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + path: str = "default", + ): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphReplayBuffers, + ) + + return EncoderCudaGraphReplayBuffers( + values={ + "pixel_values": mm_kwargs["pixel_values"], + "pixel_attention_mask": mm_kwargs["pixel_attention_mask"], + "position_ids": self.model.get_position_ids( + mm_kwargs["pixel_attention_mask"] + ), + } + ) + + def encoder_cudagraph_forward( + self, + inputs: dict[str, torch.Tensor], + path: str = "default", + ) -> torch.Tensor: + image_features = self.model.image_pixels_to_features( + inputs["pixel_values"], + pixel_attention_mask=inputs["pixel_attention_mask"], + remove_padding=False, + position_ids=inputs["position_ids"], + ) + image_features = self.model.connector(image_features) + return image_features.flatten(0, 1) + + def encoder_eager_forward( + self, + mm_kwargs: dict[str, Any], + path: str = "default", + ) -> torch.Tensor: + image_input = self._parse_and_validate_image_input(**mm_kwargs) + assert image_input is not None and image_input["type"] == "pixel_values" + image_features = self._process_image_pixels(image_input) + image_features = self.model.connector(image_features) + return image_features.flatten(0, 1) + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: image_input = self._parse_and_validate_image_input(**kwargs) if image_input is None: