[MM][CG] Enable encoder CUDA Graph for MiniCPM-V (#42785)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -74,6 +74,20 @@ def ernie45_vl_chat_template(content: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def minicpmv_25_chat_template(content: str) -> str:
|
||||
"""Llama3-style chat template used by MiniCPM-V 2.5."""
|
||||
return (
|
||||
f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
|
||||
f"{content}"
|
||||
f"<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
|
||||
)
|
||||
|
||||
|
||||
def minicpmv_chat_template(content: str) -> str:
|
||||
"""ChatML template used by MiniCPM-V 2.6 / 4.0 / 4.5."""
|
||||
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
|
||||
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"gemma3": VitCudagraphTestConfig(
|
||||
model="google/gemma-3-4b-it",
|
||||
@@ -316,6 +330,57 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_25": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-Llama3-V-2_5",
|
||||
modalities=["image"],
|
||||
image_prompt=minicpmv_25_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
# CI runs on 35GB MIG slices: every budget captures one graph per
|
||||
# patch-grid bucket, so the default budgets through max_model_len
|
||||
# OOM there. A small budget set covers the test images (each item
|
||||
# is at most (max_slice_num + 1) * query_num = 640 tokens).
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_26": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-V-2_6",
|
||||
image_prompt=minicpmv_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
video_prompt=minicpmv_chat_template(
|
||||
"(<video>./</video>)\nDescribe this video in one sentence."
|
||||
),
|
||||
max_model_len=2048,
|
||||
# Fewer frames keep the video item within the 1024 budget.
|
||||
num_video_frames=4,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
"encoder_cudagraph_max_frames_per_batch": 4,
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_40": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-V-4",
|
||||
image_prompt=minicpmv_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
video_prompt=minicpmv_chat_template(
|
||||
"(<video>./</video>)\nDescribe this video in one sentence."
|
||||
),
|
||||
max_model_len=2048,
|
||||
num_video_frames=4,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
"encoder_cudagraph_max_frames_per_batch": 4,
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ Test organization:
|
||||
- TestEncoderCudaGraphVideoReplay — video modality capture, replay
|
||||
"""
|
||||
|
||||
from collections.abc import Hashable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -116,9 +117,15 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
)
|
||||
mgr.budget_graphs = {"default": {}}
|
||||
mgr.graph_pool = None
|
||||
mgr._capture_axes = ()
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
mgr.log_stats_interval = 100
|
||||
mgr.config = EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=[],
|
||||
out_hidden_size=32,
|
||||
)
|
||||
return mgr
|
||||
|
||||
|
||||
@@ -348,6 +355,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
per_image_output = token_budget // max_batch_size
|
||||
grid_config = [
|
||||
@@ -426,6 +434,7 @@ def _make_manager_for_gpu(
|
||||
mgr.use_dp = False
|
||||
mgr.budget_graphs = {"default": {}}
|
||||
mgr.graph_pool = None
|
||||
mgr._capture_axes = ()
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
mgr.log_stats_interval = 100
|
||||
@@ -656,6 +665,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
per_item_output = token_budget // max_batch_size
|
||||
frames_per_item = max_frames_per_batch // max_batch_size
|
||||
|
||||
Reference in New Issue
Block a user