[MM][CG] Enable encoder CUDA Graph for MiniCPM-V (#42785)

Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Yunzhu Lu
2026-09-10 06:28:48 +00:00
committed by GitHub
co-authored by Isotr0py
parent c3ccc0e957
commit 442d36031c
26 changed files with 843 additions and 103 deletions
@@ -74,6 +74,20 @@ def ernie45_vl_chat_template(content: str) -> str:
)
def minicpmv_25_chat_template(content: str) -> str:
"""Llama3-style chat template used by MiniCPM-V 2.5."""
return (
f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
f"{content}"
f"<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
)
def minicpmv_chat_template(content: str) -> str:
"""ChatML template used by MiniCPM-V 2.6 / 4.0 / 4.5."""
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
"gemma3": VitCudagraphTestConfig(
model="google/gemma-3-4b-it",
@@ -316,6 +330,57 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
needs_video_metadata=True,
marks=[pytest.mark.core_model],
),
"minicpmv_25": VitCudagraphTestConfig(
model="openbmb/MiniCPM-Llama3-V-2_5",
modalities=["image"],
image_prompt=minicpmv_25_chat_template(
"(<image>./</image>)\nWhat is in this image?"
),
# CI runs on 35GB MIG slices: every budget captures one graph per
# patch-grid bucket, so the default budgets through max_model_len
# OOM there. A small budget set covers the test images (each item
# is at most (max_slice_num + 1) * query_num = 640 tokens).
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [64, 1024],
},
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"minicpmv_26": VitCudagraphTestConfig(
model="openbmb/MiniCPM-V-2_6",
image_prompt=minicpmv_chat_template(
"(<image>./</image>)\nWhat is in this image?"
),
video_prompt=minicpmv_chat_template(
"(<video>./</video>)\nDescribe this video in one sentence."
),
max_model_len=2048,
# Fewer frames keep the video item within the 1024 budget.
num_video_frames=4,
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [64, 1024],
"encoder_cudagraph_max_frames_per_batch": 4,
},
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"minicpmv_40": VitCudagraphTestConfig(
model="openbmb/MiniCPM-V-4",
image_prompt=minicpmv_chat_template(
"(<image>./</image>)\nWhat is in this image?"
),
video_prompt=minicpmv_chat_template(
"(<video>./</video>)\nDescribe this video in one sentence."
),
max_model_len=2048,
num_video_frames=4,
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [64, 1024],
"encoder_cudagraph_max_frames_per_batch": 4,
},
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
}
@@ -12,6 +12,7 @@ Test organization:
- TestEncoderCudaGraphVideoReplay — video modality capture, replay
"""
from collections.abc import Hashable
from typing import Any
import pytest
@@ -116,9 +117,15 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
)
mgr.budget_graphs = {"default": {}}
mgr.graph_pool = None
mgr._capture_axes = ()
mgr.graph_hits = 0
mgr.graph_misses = 0
mgr.log_stats_interval = 100
mgr.config = EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=[],
out_hidden_size=32,
)
return mgr
@@ -348,6 +355,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
) -> EncoderCudaGraphCaptureInputs:
per_image_output = token_budget // max_batch_size
grid_config = [
@@ -426,6 +434,7 @@ def _make_manager_for_gpu(
mgr.use_dp = False
mgr.budget_graphs = {"default": {}}
mgr.graph_pool = None
mgr._capture_axes = ()
mgr.graph_hits = 0
mgr.graph_misses = 0
mgr.log_stats_interval = 100
@@ -656,6 +665,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
) -> EncoderCudaGraphCaptureInputs:
per_item_output = token_budget // max_batch_size
frames_per_item = max_frames_per_batch // max_batch_size