From f25952e59b4a9e21a62d169212f3b072790a409d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?O=C4=9Fuzhan=20KIR?= <86883236+oguzhankir@users.noreply.github.com> Date: Thu, 4 Jun 2026 05:24:25 +0300 Subject: [PATCH] [MM][Perf][CG] Support ViT full CUDA graph for InternVL (#41759) Signed-off-by: oguz Co-authored-by: Isotr0py --- docs/design/cuda_graphs_multimodal.md | 1 + .../multimodal/vision_language_offline.py | 1 + .../generation/test_vit_cudagraph.py | 15 ++ vllm/model_executor/models/internvl.py | 168 +++++++++++++++++- 4 files changed, 183 insertions(+), 2 deletions(-) diff --git a/docs/design/cuda_graphs_multimodal.md b/docs/design/cuda_graphs_multimodal.md index 1fb5c2ba65..5a9edc1ad9 100644 --- a/docs/design/cuda_graphs_multimodal.md +++ b/docs/design/cuda_graphs_multimodal.md @@ -82,6 +82,7 @@ Models opt-in to encoder CUDA Graphs by implementing the [SupportsEncoderCudaGra | Architecture | Models | CG for Image | CG for Video | | ------------ | ------ | ------------ | ------------ | +| `InternVLChatModel` | `InternVL3.5`, `InternVL3`, `InternVL2.5`, `InternVL2` | ✅︎ | ✅︎ | | `Qwen2VLForConditionalGeneration` | `Qwen2-VL` | ✅︎ | ✅︎ | | `Qwen2_5_VLForConditionalGeneration` | `Qwen2.5-VL` | ✅︎ | ✅︎ | | `Qwen3VLForConditionalGeneration` | `Qwen3-VL` | ✅︎ | ✅︎ | diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index b4e34bd643..4d47d9f8b4 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -2554,6 +2554,7 @@ MODELS_NEED_VIDEO_METADATA = [ MODELS_SUPPORT_VIT_CUDA_GRAPH = [ + "internvl_chat", "qwen2_5_vl", "qwen3_vl", "qwen3_vl_moe", diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py index 18630e3559..cbdc5e878a 100644 --- a/tests/models/multimodal/generation/test_vit_cudagraph.py +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -43,6 +43,10 @@ def qwen_vl_chat_template(content: str) -> str: return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n" +def internvl_chat_template(content: str) -> str: + return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n" + + def step3_vl_chat_template(content: str) -> str: return ( "<|begin▁of▁sentence|> You are a helpful assistant.<|BOT|>user\n " @@ -51,6 +55,17 @@ def step3_vl_chat_template(content: str) -> str: MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { + "internvl": VitCudagraphTestConfig( + model="OpenGVLab/InternVL3-1B", + num_video_frames=8, + image_prompt=internvl_chat_template("\nWhat is in this image?"), + video_prompt=internvl_chat_template( + "