From 4559c43a9526597c00cbcc4f59979496500268d1 Mon Sep 17 00:00:00 2001 From: Soyaazz <523420504@qq.com> Date: Mon, 29 Jun 2026 12:52:00 +0800 Subject: [PATCH] [MM][CG] Gemma3 Encoder CUDA Graph (#43591) Signed-off-by: JisoLya <523420504@qq.com> Signed-off-by: Soyaazz <523420504@qq.com> Co-authored-by: Isotr0py --- docs/design/cuda_graphs_multimodal.md | 1 + .../generation/test_vit_cudagraph.py | 14 ++ vllm/model_executor/models/gemma3_mm.py | 137 +++++++++++++++++- 3 files changed, 151 insertions(+), 1 deletion(-) diff --git a/docs/design/cuda_graphs_multimodal.md b/docs/design/cuda_graphs_multimodal.md index ceefc19502..7eab425d6e 100644 --- a/docs/design/cuda_graphs_multimodal.md +++ b/docs/design/cuda_graphs_multimodal.md @@ -127,6 +127,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra | Architecture | Models | CG for Image | CG for Video | Dual-Path Graph | | ------------ | ------ | ------------ | ------------ | --------------- | | `DeepseekOCRForCausalLM` | `DeepSeek-OCR` | ✅︎ | ❌︎ | ✅︎ | +| `Gemma3ForConditionalGeneration` | `Gemma3` | ✅︎ | ❌︎ | ❌︎ | | `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ | ❌︎ | | `InternVLChatModel` | `InternVL3.5`, `InternVL3`, `InternVL2.5`, `InternVL2` | ✅︎ | ✅︎ | ❌︎ | | `KimiVLForConditionalGeneration` | `Kimi-VL` | ✅︎ | ❌︎ | ❌︎ | diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index 52b28ca860..954bbdbb9b 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -62,7 +62,21 @@ def step3_vl_chat_template(content: str) -> str: ) +def gemma3_chat_template(content: str) -> str: + return f"user\n{content}\nmodel\n" + + MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { + "gemma3": VitCudagraphTestConfig( + model="google/gemma-3-4b-it", + modalities=["image"], + image_prompt=gemma3_chat_template("What is in this image?"), + compilation_config_overrides={ + "encoder_cudagraph_token_budgets": [512], + }, + dtype="bfloat16", + max_model_len=4096, + ), "llama4": VitCudagraphTestConfig( model="meta-llama/Llama-4-Scout-17B-16E-Instruct", modalities=["image"], diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 6ecadbcd67..9e58438f4c 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -39,6 +39,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import ( MultiModalEmbeddings, + SupportsEncoderCudaGraph, SupportsLoRA, SupportsMultiModal, SupportsPP, @@ -467,7 +468,7 @@ class Gemma3MultiModalProjector(nn.Module): dummy_inputs=Gemma3DummyInputsBuilder, ) class Gemma3ForConditionalGeneration( - nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA + nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsEncoderCudaGraph ): packed_modules_mapping = { "qkv_proj": [ @@ -504,8 +505,12 @@ class Gemma3ForConditionalGeneration( quant_config = vllm_config.quant_config multimodal_config = vllm_config.model_config.multimodal_config self.config = config + self.model_config = vllm_config.model_config self.quant_config = quant_config self.multimodal_config = multimodal_config + self.vit_positions_per_patch = ( + self.config.vision_config.image_size // self.config.vision_config.patch_size + ) ** 2 self.configure_mm_token_handling( vocab_size=config.text_config.vocab_size, @@ -682,3 +687,133 @@ class Gemma3ForConditionalGeneration( """ # The Gemma3 connector maintains a 1:1 token mapping return num_vision_tokens + + def get_encoder_cudagraph_config(self): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphConfig, + ) + + return EncoderCudaGraphConfig( + modalities=["image"], + buffer_keys=["pixel_values"], + out_hidden_size=self.config.text_config.hidden_size, + ) + + def get_encoder_cudagraph_budget_range( + self, + vllm_config: VllmConfig, + ) -> tuple[int, int]: + min_budget = self.config.mm_tokens_per_image + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + self.model_config.max_model_len, + ) + return (min_budget, max_budget) + + def get_encoder_cudagraph_item_specs( + self, + mm_kwargs: dict[str, Any], + ): + from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec + + num_patches = mm_kwargs["num_patches"] + mm_tokens_per_image = self.config.mm_tokens_per_image + + return [ + EncoderItemSpec( + input_size=int(np) * self.vit_positions_per_patch, + output_tokens=int(np) * mm_tokens_per_image, + ) + for np in num_patches + ] + + def select_encoder_cudagraph_items( + self, + mm_kwargs: dict[str, Any], + indices: list[int], + ) -> dict[str, Any]: + pixel_values = mm_kwargs["pixel_values"] + num_patches = mm_kwargs["num_patches"] + + if len(indices) == 0: + return { + "pixel_values": pixel_values[:0], + "num_patches": num_patches[:0], + } + cum_patches = [0] + for p in num_patches: + cum_patches.append(cum_patches[-1] + int(p)) + + selected_pv = torch.cat( + [pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices] + ) + selected_np = num_patches[indices] + + return { + "pixel_values": selected_pv, + "num_patches": selected_np, + } + + def prepare_encoder_cudagraph_capture_inputs( + self, + token_budget: int, + max_batch_size: int, + max_frames_per_batch: int, + device: torch.device, + dtype: torch.dtype, + ): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphCaptureInputs, + ) + + mm_tokens_per_image = self.config.mm_tokens_per_image + num_images = min( + token_budget // mm_tokens_per_image, + max_batch_size, + ) + + image_size = self.config.vision_config.image_size + dummy_pixel_values = torch.randn( + num_images, + 3, + image_size, + image_size, + device=device, + dtype=dtype, + ) + values = {"pixel_values": dummy_pixel_values} + + return EncoderCudaGraphCaptureInputs( + values, + ) + + def prepare_encoder_cudagraph_replay_buffers( + self, + mm_kwargs: dict[str, Any], + max_batch_size: int, + max_frames_per_batch: int, + ): + from vllm.v1.worker.encoder_cudagraph_defs import ( + EncoderCudaGraphReplayBuffers, + ) + + return EncoderCudaGraphReplayBuffers( + values={"pixel_values": mm_kwargs["pixel_values"]}, + ) + + def encoder_cudagraph_forward( + self, + values: dict[str, torch.Tensor], + ) -> torch.Tensor: + pixel_values = values["pixel_values"] + image_features = self.vision_tower(pixel_values) + image_features = self.multi_modal_projector(image_features) + return image_features.flatten(end_dim=1) + + def encoder_eager_forward( + self, + mm_kwargs: dict[str, Any], + ) -> torch.Tensor: + image_input = self._parse_and_validate_image_input(**mm_kwargs) + results = self._process_image_input(image_input) + return torch.cat(results, dim=0)