[Multimodal] Simplify ViT CUDA graph interfaces (#41234)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -10,7 +10,6 @@ from vllm.multimodal.video import sample_frames_from_video
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, VIDEO_ASSETS
|
||||
from ....utils import create_new_process_for_each_test
|
||||
from ...utils import dummy_hf_overrides
|
||||
from .vlm_utils.builders import sample_frames_with_video_metadata
|
||||
|
||||
@@ -142,7 +141,6 @@ def get_compilation_config(config: VitCudagraphTestConfig):
|
||||
|
||||
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@create_new_process_for_each_test()
|
||||
def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
|
||||
@@ -184,7 +182,6 @@ def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
|
||||
|
||||
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@create_new_process_for_each_test()
|
||||
def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
EncoderItemSpec,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -288,24 +289,18 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
# For tests: min=4, max=128 (small values for fast capture)
|
||||
return (4, 128)
|
||||
|
||||
def get_encoder_cudagraph_num_items(
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> int:
|
||||
return len(mm_kwargs["image_grid_thw"])
|
||||
|
||||
def get_encoder_cudagraph_per_item_output_tokens(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[int]:
|
||||
) -> list[EncoderItemSpec]:
|
||||
m = _SPATIAL_MERGE
|
||||
return [t * (h // m) * (w // m) for t, h, w in mm_kwargs["image_grid_thw"]]
|
||||
|
||||
def get_encoder_cudagraph_per_item_input_sizes(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[int]:
|
||||
return [t * h * w for t, h, w in mm_kwargs["image_grid_thw"]]
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=t * h * w,
|
||||
output_tokens=t * (h // m) * (w // m),
|
||||
)
|
||||
for t, h, w in mm_kwargs["image_grid_thw"]
|
||||
]
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self,
|
||||
@@ -591,19 +586,18 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
# Protocol overrides that depend on modality keys
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_encoder_cudagraph_num_items(self, mm_kwargs: dict[str, Any]) -> int:
|
||||
return len(self._get_grid_thw(mm_kwargs))
|
||||
|
||||
def get_encoder_cudagraph_per_item_output_tokens(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[int]:
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[EncoderItemSpec]:
|
||||
m = _SPATIAL_MERGE
|
||||
return [t * (h // m) * (w // m) for t, h, w in self._get_grid_thw(mm_kwargs)]
|
||||
|
||||
def get_encoder_cudagraph_per_item_input_sizes(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[int]:
|
||||
return [t * h * w for t, h, w in self._get_grid_thw(mm_kwargs)]
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=t * h * w,
|
||||
output_tokens=t * (h // m) * (w // m),
|
||||
)
|
||||
for t, h, w in self._get_grid_thw(mm_kwargs)
|
||||
]
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self, mm_kwargs: dict[str, Any], indices: list[int]
|
||||
@@ -679,11 +673,16 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
return EncoderCudaGraphReplayBuffers(buffers={"dummy_buf": dummy_buf})
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self, mm_kwargs: dict[str, Any], buffers: dict[str, torch.Tensor]
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
buffers: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
return self._forward(self._get_pixel_values(mm_kwargs))
|
||||
|
||||
def encoder_eager_forward(self, mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> torch.Tensor:
|
||||
return self._forward(self._get_pixel_values(mm_kwargs))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user