[MM][CG] Enable encoder CUDA Graph for MiniCPM-V (#42785)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -74,6 +74,20 @@ def ernie45_vl_chat_template(content: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def minicpmv_25_chat_template(content: str) -> str:
|
||||
"""Llama3-style chat template used by MiniCPM-V 2.5."""
|
||||
return (
|
||||
f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
|
||||
f"{content}"
|
||||
f"<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
|
||||
)
|
||||
|
||||
|
||||
def minicpmv_chat_template(content: str) -> str:
|
||||
"""ChatML template used by MiniCPM-V 2.6 / 4.0 / 4.5."""
|
||||
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
|
||||
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"gemma3": VitCudagraphTestConfig(
|
||||
model="google/gemma-3-4b-it",
|
||||
@@ -316,6 +330,57 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_25": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-Llama3-V-2_5",
|
||||
modalities=["image"],
|
||||
image_prompt=minicpmv_25_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
# CI runs on 35GB MIG slices: every budget captures one graph per
|
||||
# patch-grid bucket, so the default budgets through max_model_len
|
||||
# OOM there. A small budget set covers the test images (each item
|
||||
# is at most (max_slice_num + 1) * query_num = 640 tokens).
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_26": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-V-2_6",
|
||||
image_prompt=minicpmv_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
video_prompt=minicpmv_chat_template(
|
||||
"(<video>./</video>)\nDescribe this video in one sentence."
|
||||
),
|
||||
max_model_len=2048,
|
||||
# Fewer frames keep the video item within the 1024 budget.
|
||||
num_video_frames=4,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
"encoder_cudagraph_max_frames_per_batch": 4,
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_40": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-V-4",
|
||||
image_prompt=minicpmv_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
video_prompt=minicpmv_chat_template(
|
||||
"(<video>./</video>)\nDescribe this video in one sentence."
|
||||
),
|
||||
max_model_len=2048,
|
||||
num_video_frames=4,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
"encoder_cudagraph_max_frames_per_batch": 4,
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user