[MM][Perf][CG] Support dual-path ViT full CUDA graph for DeepSeek-OCR (#43586)
Signed-off-by: shen-shanshan <467638484@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Roger Wang
Isotr0py
parent
b2cfae777d
commit
ad32608e24
@@ -29,6 +29,7 @@ class VitCudagraphTestConfig:
|
||||
vllm_runner_kwargs: dict = field(default_factory=dict)
|
||||
compilation_config_overrides: dict = field(default_factory=dict)
|
||||
marks: list = field(default_factory=list)
|
||||
skip: bool = False
|
||||
|
||||
|
||||
def params_with_marks(
|
||||
@@ -75,15 +76,16 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"internvl": VitCudagraphTestConfig(
|
||||
model="OpenGVLab/InternVL3-1B",
|
||||
num_video_frames=8,
|
||||
image_prompt=internvl_chat_template("<image>\nWhat is in this image?"),
|
||||
video_prompt=internvl_chat_template(
|
||||
"<video>\nDescribe this video in one sentence."
|
||||
"qwen2_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen2-VL-2B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||
),
|
||||
video_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen2_5_vl": VitCudagraphTestConfig(
|
||||
@@ -122,16 +124,15 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen2_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen2-VL-2B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||
),
|
||||
video_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
"internvl": VitCudagraphTestConfig(
|
||||
model="OpenGVLab/InternVL3-1B",
|
||||
num_video_frames=8,
|
||||
image_prompt=internvl_chat_template("<image>\nWhat is in this image?"),
|
||||
video_prompt=internvl_chat_template(
|
||||
"<video>\nDescribe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"step3_vl": VitCudagraphTestConfig(
|
||||
@@ -179,6 +180,25 @@ MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
),
|
||||
},
|
||||
),
|
||||
"deepseek_ocr": VitCudagraphTestConfig(
|
||||
model="deepseek-ai/DeepSeek-OCR",
|
||||
modalities=["image"],
|
||||
image_prompt="<image>\nWhat is in this image?",
|
||||
marks=[pytest.mark.core_model],
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [272],
|
||||
"mode": 0,
|
||||
"cudagraph_mode": 2,
|
||||
},
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="DeepseekOCRForCausalLM",
|
||||
),
|
||||
},
|
||||
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -201,6 +221,9 @@ def get_compilation_config(config: VitCudagraphTestConfig):
|
||||
def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
|
||||
if config.skip:
|
||||
pytest.skip(f"{model_id} is marked to be skipped.")
|
||||
|
||||
if "image" not in config.modalities:
|
||||
pytest.skip(f"{model_id} does not support the image modality.")
|
||||
|
||||
@@ -242,6 +265,9 @@ def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
|
||||
def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
|
||||
if config.skip:
|
||||
pytest.skip(f"{model_id} is marked to be skipped.")
|
||||
|
||||
if "video" not in config.modalities:
|
||||
pytest.skip(f"{model_id} does not support the video modality")
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ def _make_manager_with_budgets(budgets: list[int]) -> EncoderCudaGraphManager:
|
||||
mgr.token_budgets = sorted(budgets)
|
||||
mgr.max_batch_size = 16
|
||||
mgr.use_dp = False
|
||||
mgr.budget_graphs = {}
|
||||
mgr.budget_graphs = {"default": {}}
|
||||
mgr.graph_pool = None
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
@@ -341,6 +341,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
per_image_output = token_budget // max_batch_size
|
||||
grid_config = [
|
||||
@@ -365,6 +366,7 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphReplayBuffers:
|
||||
grid_thw = mm_kwargs["image_grid_thw"]
|
||||
n_out = _count_output_tokens(grid_thw, _SPATIAL_MERGE)
|
||||
@@ -380,12 +382,14 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
values: dict[str, torch.Tensor],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
return self._forward(values["pixel_values"])
|
||||
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
return self._forward(mm_kwargs["pixel_values"])
|
||||
|
||||
@@ -413,7 +417,7 @@ def _make_manager_for_gpu(
|
||||
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size * 2
|
||||
)
|
||||
mgr.use_dp = False
|
||||
mgr.budget_graphs = {}
|
||||
mgr.budget_graphs = {"default": {}}
|
||||
mgr.graph_pool = None
|
||||
mgr.graph_hits = 0
|
||||
mgr.graph_misses = 0
|
||||
@@ -479,15 +483,15 @@ class TestEncoderCudaGraphCaptureReplay:
|
||||
# --- capture ---
|
||||
|
||||
def test_capture_creates_one_graph_per_budget(self):
|
||||
assert len(self.mgr.budget_graphs) == len(_BUDGETS)
|
||||
assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS)
|
||||
assert len(self.mgr.budget_graphs["default"]) == len(_BUDGETS)
|
||||
assert set(self.mgr.budget_graphs["default"].keys()) == set(_BUDGETS)
|
||||
|
||||
def test_capture_uses_supplied_graph_pool(self):
|
||||
assert self.mgr.graph_pool is self.graph_pool
|
||||
|
||||
def test_clear_releases_graphs_and_pool(self):
|
||||
self.mgr.clear()
|
||||
assert self.mgr.budget_graphs == {}
|
||||
assert self.mgr.budget_graphs == {"default": {}}
|
||||
assert self.mgr.graph_pool is None
|
||||
|
||||
# --- output shape ---
|
||||
@@ -642,6 +646,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
max_frames_per_batch: int,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphCaptureInputs:
|
||||
per_item_output = token_budget // max_batch_size
|
||||
frames_per_item = max_frames_per_batch // max_batch_size
|
||||
@@ -678,6 +683,7 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
mm_kwargs: dict[str, Any],
|
||||
max_batch_size: int,
|
||||
max_frames_per_batch: int,
|
||||
path: str = "default",
|
||||
) -> EncoderCudaGraphReplayBuffers:
|
||||
n_out = _count_output_tokens(self._get_grid_thw(mm_kwargs), _SPATIAL_MERGE)
|
||||
p = next(self.parameters())
|
||||
@@ -692,12 +698,14 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
values: dict[str, torch.Tensor],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
return self._forward(values["pixel_values"])
|
||||
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
return self._forward(self._get_pixel_values(mm_kwargs))
|
||||
|
||||
@@ -763,8 +771,8 @@ class TestEncoderCudaGraphVideoReplay:
|
||||
# --- capture ---
|
||||
|
||||
def test_capture_creates_one_graph_per_budget(self):
|
||||
assert len(self.mgr.budget_graphs) == len(_BUDGETS)
|
||||
assert set(self.mgr.budget_graphs.keys()) == set(_BUDGETS)
|
||||
assert len(self.mgr.budget_graphs["default"]) == len(_BUDGETS)
|
||||
assert set(self.mgr.budget_graphs["default"].keys()) == set(_BUDGETS)
|
||||
|
||||
# --- output shape ---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user