[MM][CG] Gemma3 Encoder CUDA Graph (#43591)

Signed-off-by: JisoLya <523420504@qq.com>
Signed-off-by: Soyaazz <523420504@qq.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Soyaazz
2026-06-29 04:52:00 +00:00
committed by GitHub
co-authored by Isotr0py
parent 5274c1181d
commit 4559c43a95
3 changed files with 151 additions and 1 deletions
+1
View File
@@ -127,6 +127,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
| Architecture | Models | CG for Image | CG for Video | Dual-Path Graph |
| ------------ | ------ | ------------ | ------------ | --------------- |
| `DeepseekOCRForCausalLM` | `DeepSeek-OCR` | ✅︎ | ❌︎ | ✅︎ |
| `Gemma3ForConditionalGeneration` | `Gemma3` | ✅︎ | ❌︎ | ❌︎ |
| `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ | ❌︎ |
| `InternVLChatModel` | `InternVL3.5`, `InternVL3`, `InternVL2.5`, `InternVL2` | ✅︎ | ✅︎ | ❌︎ |
| `KimiVLForConditionalGeneration` | `Kimi-VL` | ✅︎ | ❌︎ | ❌︎ |
@@ -62,7 +62,21 @@ def step3_vl_chat_template(content: str) -> str:
)
def gemma3_chat_template(content: str) -> str:
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
"gemma3": VitCudagraphTestConfig(
model="google/gemma-3-4b-it",
modalities=["image"],
image_prompt=gemma3_chat_template("<start_of_image>What is in this image?"),
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [512],
},
dtype="bfloat16",
max_model_len=4096,
),
"llama4": VitCudagraphTestConfig(
model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
modalities=["image"],
+136 -1
View File
@@ -39,6 +39,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMultiModal,
SupportsPP,
@@ -467,7 +468,7 @@ class Gemma3MultiModalProjector(nn.Module):
dummy_inputs=Gemma3DummyInputsBuilder,
)
class Gemma3ForConditionalGeneration(
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsEncoderCudaGraph
):
packed_modules_mapping = {
"qkv_proj": [
@@ -504,8 +505,12 @@ class Gemma3ForConditionalGeneration(
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
self.model_config = vllm_config.model_config
self.quant_config = quant_config
self.multimodal_config = multimodal_config
self.vit_positions_per_patch = (
self.config.vision_config.image_size // self.config.vision_config.patch_size
) ** 2
self.configure_mm_token_handling(
vocab_size=config.text_config.vocab_size,
@@ -682,3 +687,133 @@ class Gemma3ForConditionalGeneration(
"""
# The Gemma3 connector maintains a 1:1 token mapping
return num_vision_tokens
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphConfig,
)
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=["pixel_values"],
out_hidden_size=self.config.text_config.hidden_size,
)
def get_encoder_cudagraph_budget_range(
self,
vllm_config: VllmConfig,
) -> tuple[int, int]:
min_budget = self.config.mm_tokens_per_image
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
self.model_config.max_model_len,
)
return (min_budget, max_budget)
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
num_patches = mm_kwargs["num_patches"]
mm_tokens_per_image = self.config.mm_tokens_per_image
return [
EncoderItemSpec(
input_size=int(np) * self.vit_positions_per_patch,
output_tokens=int(np) * mm_tokens_per_image,
)
for np in num_patches
]
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
pixel_values = mm_kwargs["pixel_values"]
num_patches = mm_kwargs["num_patches"]
if len(indices) == 0:
return {
"pixel_values": pixel_values[:0],
"num_patches": num_patches[:0],
}
cum_patches = [0]
for p in num_patches:
cum_patches.append(cum_patches[-1] + int(p))
selected_pv = torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
selected_np = num_patches[indices]
return {
"pixel_values": selected_pv,
"num_patches": selected_np,
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
mm_tokens_per_image = self.config.mm_tokens_per_image
num_images = min(
token_budget // mm_tokens_per_image,
max_batch_size,
)
image_size = self.config.vision_config.image_size
dummy_pixel_values = torch.randn(
num_images,
3,
image_size,
image_size,
device=device,
dtype=dtype,
)
values = {"pixel_values": dummy_pixel_values}
return EncoderCudaGraphCaptureInputs(
values,
)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphReplayBuffers,
)
return EncoderCudaGraphReplayBuffers(
values={"pixel_values": mm_kwargs["pixel_values"]},
)
def encoder_cudagraph_forward(
self,
values: dict[str, torch.Tensor],
) -> torch.Tensor:
pixel_values = values["pixel_values"]
image_features = self.vision_tower(pixel_values)
image_features = self.multi_modal_projector(image_features)
return image_features.flatten(end_dim=1)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
) -> torch.Tensor:
image_input = self._parse_and_validate_image_input(**mm_kwargs)
results = self._process_image_input(image_input)
return torch.cat(results, dim=0)