[MM][CG] Gemma3 Encoder CUDA Graph (#43591)
Signed-off-by: JisoLya <523420504@qq.com> Signed-off-by: Soyaazz <523420504@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -127,6 +127,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra
|
||||
| Architecture | Models | CG for Image | CG for Video | Dual-Path Graph |
|
||||
| ------------ | ------ | ------------ | ------------ | --------------- |
|
||||
| `DeepseekOCRForCausalLM` | `DeepSeek-OCR` | ✅︎ | ❌︎ | ✅︎ |
|
||||
| `Gemma3ForConditionalGeneration` | `Gemma3` | ✅︎ | ❌︎ | ❌︎ |
|
||||
| `Glm4vForConditionalGeneration` | `GLM-4.1V, GLM-4.6V-Flash` | ✅︎ | ✅︎ | ❌︎ |
|
||||
| `InternVLChatModel` | `InternVL3.5`, `InternVL3`, `InternVL2.5`, `InternVL2` | ✅︎ | ✅︎ | ❌︎ |
|
||||
| `KimiVLForConditionalGeneration` | `Kimi-VL` | ✅︎ | ❌︎ | ❌︎ |
|
||||
|
||||
@@ -62,7 +62,21 @@ def step3_vl_chat_template(content: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def gemma3_chat_template(content: str) -> str:
|
||||
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
|
||||
|
||||
|
||||
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"gemma3": VitCudagraphTestConfig(
|
||||
model="google/gemma-3-4b-it",
|
||||
modalities=["image"],
|
||||
image_prompt=gemma3_chat_template("<start_of_image>What is in this image?"),
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [512],
|
||||
},
|
||||
dtype="bfloat16",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"llama4": VitCudagraphTestConfig(
|
||||
model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
modalities=["image"],
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
SupportsEncoderCudaGraph,
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
SupportsPP,
|
||||
@@ -467,7 +468,7 @@ class Gemma3MultiModalProjector(nn.Module):
|
||||
dummy_inputs=Gemma3DummyInputsBuilder,
|
||||
)
|
||||
class Gemma3ForConditionalGeneration(
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
|
||||
nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA, SupportsEncoderCudaGraph
|
||||
):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
@@ -504,8 +505,12 @@ class Gemma3ForConditionalGeneration(
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.quant_config = quant_config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.vit_positions_per_patch = (
|
||||
self.config.vision_config.image_size // self.config.vision_config.patch_size
|
||||
) ** 2
|
||||
|
||||
self.configure_mm_token_handling(
|
||||
vocab_size=config.text_config.vocab_size,
|
||||
@@ -682,3 +687,133 @@ class Gemma3ForConditionalGeneration(
|
||||
"""
|
||||
# The Gemma3 connector maintains a 1:1 token mapping
|
||||
return num_vision_tokens
|
||||
|
||||
def get_encoder_cudagraph_config(self):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphConfig,
|
||||
)
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=["pixel_values"],
|
||||
out_hidden_size=self.config.text_config.hidden_size,
|
||||
)
|
||||
|
||||
def get_encoder_cudagraph_budget_range(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
) -> tuple[int, int]:
|
||||
min_budget = self.config.mm_tokens_per_image
|
||||
max_budget = min(
|
||||
vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
self.model_config.max_model_len,
|
||||
)
|
||||
return (min_budget, max_budget)
|
||||
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
|
||||
|
||||
num_patches = mm_kwargs["num_patches"]
|
||||
mm_tokens_per_image = self.config.mm_tokens_per_image
|
||||
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=int(np) * self.vit_positions_per_patch,
|
||||
output_tokens=int(np) * mm_tokens_per_image,
|
||||
)
|
||||
for np in num_patches
|
||||
]
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
indices: list[int],
|
||||
) -> dict[str, Any]:
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
num_patches = mm_kwargs["num_patches"]
|
||||
|
||||
if len(indices) == 0:
|
||||
return {
|
||||
"pixel_values": pixel_values[:0],
|
||||
"num_patches": num_patches[:0],
|
||||
}
|
||||
cum_patches = [0]
|
||||
for p in num_patches:
|
||||
cum_patches.append(cum_patches[-1] + int(p))
|
||||
|
||||
selected_pv = torch.cat(
|
||||
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
|
||||
)
|
||||
selected_np = num_patches[indices]
|
||||
|
||||
return {
|
||||
"pixel_values": selected_pv,
|
||||
"num_patches": selected_np,
|
||||
}
|
||||
|
||||
def prepare_encoder_cudagraph_capture_inputs(
|
||||
self,
|
||||
token_budget: int,
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
)
|
||||
|
||||
mm_tokens_per_image = self.config.mm_tokens_per_image
|
||||
num_images = min(
|
||||
token_budget // mm_tokens_per_image,
|
||||
max_batch_size,
|
||||
)
|
||||
|
||||
image_size = self.config.vision_config.image_size
|
||||
dummy_pixel_values = torch.randn(
|
||||
num_images,
|
||||
3,
|
||||
image_size,
|
||||
image_size,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
values = {"pixel_values": dummy_pixel_values}
|
||||
|
||||
return EncoderCudaGraphCaptureInputs(
|
||||
values,
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
):
|
||||
from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
)
|
||||
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={"pixel_values": mm_kwargs["pixel_values"]},
|
||||
)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
values: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
pixel_values = values["pixel_values"]
|
||||
image_features = self.vision_tower(pixel_values)
|
||||
image_features = self.multi_modal_projector(image_features)
|
||||
return image_features.flatten(end_dim=1)
|
||||
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> torch.Tensor:
|
||||
image_input = self._parse_and_validate_image_input(**mm_kwargs)
|
||||
results = self._process_image_input(image_input)
|
||||
return torch.cat(results, dim=0)
|
||||
|
||||
Reference in New Issue
Block a user