[MM][Perf][CG] Support dual-path ViT full CUDA graph for DeepSeek-OCR (#43586)

Signed-off-by: shen-shanshan <467638484@qq.com>
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Shanshan Shen
2026-06-16 04:35:20 -07:00
committed by GitHub
co-authored by Roger Wang Isotr0py
parent b2cfae777d
commit ad32608e24
16 changed files with 809 additions and 69 deletions
@@ -29,6 +29,7 @@ class VitCudagraphTestConfig:
vllm_runner_kwargs: dict = field(default_factory=dict)
compilation_config_overrides: dict = field(default_factory=dict)
marks: list = field(default_factory=list)
skip: bool = False
def params_with_marks(
@@ -75,15 +76,16 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
},
marks=[pytest.mark.core_model],
),
"internvl": VitCudagraphTestConfig(
model="OpenGVLab/InternVL3-1B",
num_video_frames=8,
image_prompt=internvl_chat_template("<image>\nWhat is in this image?"),
video_prompt=internvl_chat_template(
"<video>\nDescribe this video in one sentence."
"qwen2_vl": VitCudagraphTestConfig(
model="Qwen/Qwen2-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
),
video_prompt=qwen_vl_chat_template(
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
),
needs_video_metadata=False,
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"qwen2_5_vl": VitCudagraphTestConfig(
@@ -122,16 +124,15 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
needs_video_metadata=True,
marks=[pytest.mark.core_model],
),
"qwen2_vl": VitCudagraphTestConfig(
model="Qwen/Qwen2-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
),
video_prompt=qwen_vl_chat_template(
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
"internvl": VitCudagraphTestConfig(
model="OpenGVLab/InternVL3-1B",
num_video_frames=8,
image_prompt=internvl_chat_template("<image>\nWhat is in this image?"),
video_prompt=internvl_chat_template(
"<video>\nDescribe this video in one sentence."
),
needs_video_metadata=False,
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"step3_vl": VitCudagraphTestConfig(
@@ -179,6 +180,25 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
),
},
),
"deepseek_ocr": VitCudagraphTestConfig(
model="deepseek-ai/DeepSeek-OCR",
modalities=["image"],
image_prompt="<image>\nWhat is in this image?",
marks=[pytest.mark.core_model],
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [272],
"mode": 0,
"cudagraph_mode": 2,
},
vllm_runner_kwargs={
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="DeepseekOCRForCausalLM",
),
},
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
),
}
@@ -201,6 +221,9 @@ def get_compilation_config(config: VitCudagraphTestConfig):
def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
config = MODEL_CONFIGS[model_id]
if config.skip:
pytest.skip(f"{model_id} is marked to be skipped.")
if "image" not in config.modalities:
pytest.skip(f"{model_id} does not support the image modality.")
@@ -242,6 +265,9 @@ def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
config = MODEL_CONFIGS[model_id]
if config.skip:
pytest.skip(f"{model_id} is marked to be skipped.")
if "video" not in config.modalities:
pytest.skip(f"{model_id} does not support the video modality")
+15 -7
View File
@@ -108,7 +108,7 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
mgr.token_budgets = sorted(budgets)
mgr.max_batch_size = 16
mgr.use_dp = False
mgr.budget_graphs = {}
mgr.budget_graphs = {"default": {}}
mgr.graph_pool = None
mgr.graph_hits = 0
mgr.graph_misses = 0
@@ -341,6 +341,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
) -> EncoderCudaGraphCaptureInputs:
per_image_output = token_budget // max_batch_size
grid_config = [
@@ -365,6 +366,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> EncoderCudaGraphReplayBuffers:
grid_thw = mm_kwargs["image_grid_thw"]
n_out = _count_output_tokens(grid_thw, _SPATIAL_MERGE)
@@ -380,12 +382,14 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
def encoder_cudagraph_forward(
self,
values: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
return self._forward(values["pixel_values"])
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
return self._forward(mm_kwargs["pixel_values"])
@@ -413,7 +417,7 @@ def _make_manager_for_gpu(
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size * 2
)
mgr.use_dp = False
mgr.budget_graphs = {}
mgr.budget_graphs = {"default": {}}
mgr.graph_pool = None
mgr.graph_hits = 0
mgr.graph_misses = 0
@@ -479,15 +483,15 @@ class TestEncoderCudaGraphCaptureReplay:
# --- capture ---
def test_capture_creates_one_graph_per_budget(self):
assert len(self.mgr.budget_graphs) == len(_BUDGETS)
assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS)
assert len(self.mgr.budget_graphs["default"]) == len(_BUDGETS)
assert set(self.mgr.budget_graphs["default"].keys()) == set(_BUDGETS)
def test_capture_uses_supplied_graph_pool(self):
assert self.mgr.graph_pool is self.graph_pool
def test_clear_releases_graphs_and_pool(self):
self.mgr.clear()
assert self.mgr.budget_graphs == {}
assert self.mgr.budget_graphs == {"default": {}}
assert self.mgr.graph_pool is None
# --- output shape ---
@@ -642,6 +646,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
) -> EncoderCudaGraphCaptureInputs:
per_item_output = token_budget // max_batch_size
frames_per_item = max_frames_per_batch // max_batch_size
@@ -678,6 +683,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
) -> EncoderCudaGraphReplayBuffers:
n_out = _count_output_tokens(self._get_grid_thw(mm_kwargs), _SPATIAL_MERGE)
p = next(self.parameters())
@@ -692,12 +698,14 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
def encoder_cudagraph_forward(
self,
values: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
return self._forward(values["pixel_values"])
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
return self._forward(self._get_pixel_values(mm_kwargs))
@@ -763,8 +771,8 @@ class TestEncoderCudaGraphVideoReplay:
# --- capture ---
def test_capture_creates_one_graph_per_budget(self):
assert len(self.mgr.budget_graphs) == len(_BUDGETS)
assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS)
assert len(self.mgr.budget_graphs["default"]) == len(_BUDGETS)
assert set(self.mgr.budget_graphs["default"].keys()) == set(_BUDGETS)
# --- output shape ---