[Multimodal] Simplify ViT CUDA graph interfaces (#41234)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -26,6 +26,7 @@ from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderCudaGraphCaptureInputs,
|
||||
EncoderCudaGraphConfig,
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
EncoderItemSpec,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -288,24 +289,18 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
# For tests: min=4, max=128 (small values for fast capture)
|
||||
return (4, 128)
|
||||
|
||||
def get_encoder_cudagraph_num_items(
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> int:
|
||||
return len(mm_kwargs["image_grid_thw"])
|
||||
|
||||
def get_encoder_cudagraph_per_item_output_tokens(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[int]:
|
||||
) -> list[EncoderItemSpec]:
|
||||
m = _SPATIAL_MERGE
|
||||
return [t * (h // m) * (w // m) for t, h, w in mm_kwargs["image_grid_thw"]]
|
||||
|
||||
def get_encoder_cudagraph_per_item_input_sizes(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[int]:
|
||||
return [t * h * w for t, h, w in mm_kwargs["image_grid_thw"]]
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=t * h * w,
|
||||
output_tokens=t * (h // m) * (w // m),
|
||||
)
|
||||
for t, h, w in mm_kwargs["image_grid_thw"]
|
||||
]
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self,
|
||||
@@ -591,19 +586,18 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
# Protocol overrides that depend on modality keys
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_encoder_cudagraph_num_items(self, mm_kwargs: dict[str, Any]) -> int:
|
||||
return len(self._get_grid_thw(mm_kwargs))
|
||||
|
||||
def get_encoder_cudagraph_per_item_output_tokens(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[int]:
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[EncoderItemSpec]:
|
||||
m = _SPATIAL_MERGE
|
||||
return [t * (h // m) * (w // m) for t, h, w in self._get_grid_thw(mm_kwargs)]
|
||||
|
||||
def get_encoder_cudagraph_per_item_input_sizes(
|
||||
self, mm_kwargs: dict[str, Any]
|
||||
) -> list[int]:
|
||||
return [t * h * w for t, h, w in self._get_grid_thw(mm_kwargs)]
|
||||
return [
|
||||
EncoderItemSpec(
|
||||
input_size=t * h * w,
|
||||
output_tokens=t * (h // m) * (w // m),
|
||||
)
|
||||
for t, h, w in self._get_grid_thw(mm_kwargs)
|
||||
]
|
||||
|
||||
def select_encoder_cudagraph_items(
|
||||
self, mm_kwargs: dict[str, Any], indices: list[int]
|
||||
@@ -679,11 +673,16 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
return EncoderCudaGraphReplayBuffers(buffers={"dummy_buf": dummy_buf})
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self, mm_kwargs: dict[str, Any], buffers: dict[str, torch.Tensor]
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
buffers: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
return self._forward(self._get_pixel_values(mm_kwargs))
|
||||
|
||||
def encoder_eager_forward(self, mm_kwargs: dict[str, Any]) -> torch.Tensor:
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> torch.Tensor:
|
||||
return self._forward(self._get_pixel_values(mm_kwargs))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user