[MM][CG] Enable encoder CUDA Graph for MiniCPM-V (#42785)
Signed-off-by: YunzhuLu <lucia.yunzhu@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -12,6 +12,7 @@ Test organization:
|
||||
- TestEncoderCudaGraphVideoReplay — video modality capture, replay
|
||||
"""
|
||||
|
||||
from collections.abc import Hashable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -116,9 +117,15 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
)
|
||||
mgr.budget_graphs = {"default": {}}
|
||||
mgr.graph_pool = None
|
||||
mgr._capture_axes = ()
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
mgr.log_stats_interval = 100
|
||||
mgr.config = EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
buffer_keys=[],
|
||||
out_hidden_size=32,
|
||||
)
|
||||
return mgr
|
||||
|
||||
|
||||
@@ -348,6 +355,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
per_image_output = token_budget // max_batch_size
|
||||
grid_config = [
|
||||
@@ -426,6 +434,7 @@ def _make_manager_for_gpu(
|
||||
mgr.use_dp = False
|
||||
mgr.budget_graphs = {"default": {}}
|
||||
mgr.graph_pool = None
|
||||
mgr._capture_axes = ()
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
mgr.log_stats_interval = 100
|
||||
@@ -656,6 +665,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
axis_keys: tuple[Hashable, ...] | None = None,
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
per_item_output = token_budget // max_batch_size
|
||||
frames_per_item = max_frames_per_batch // max_batch_size
|
||||
|
||||
Reference in New Issue
Block a user