Adjust design around encoder_cudagraph_forward (#42288)
Signed-off-by: Weida Hong <wdhongtw@google.com>
This commit is contained in:
@@ -87,8 +87,10 @@ class _MockModel(SupportsEncoderCudaGraph):
|
||||
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
input_key_by_modality={"image": "pixel_values"},
|
||||
buffer_keys=["dummy_buf"],
|
||||
buffer_keys=[
|
||||
"pixel_values",
|
||||
"dummy_buf",
|
||||
],
|
||||
out_hidden_size=32,
|
||||
)
|
||||
|
||||
@@ -269,9 +271,6 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
input_key_by_modality={
|
||||
"image": "pixel_values",
|
||||
},
|
||||
buffer_keys=["dummy_buf"],
|
||||
out_hidden_size=_HIDDEN,
|
||||
)
|
||||
@@ -350,11 +349,10 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
n_out = _count_output_tokens(grid_config, _SPATIAL_MERGE)
|
||||
dummy_buf = torch.zeros(n_out, _HIDDEN, device=device, dtype=dtype)
|
||||
return EncoderCudaGraphCaptureInputs(
|
||||
mm_kwargs={
|
||||
values={
|
||||
"pixel_values": dummy_pixel_values,
|
||||
"image_grid_thw": grid_config,
|
||||
"dummy_buf": dummy_buf,
|
||||
},
|
||||
buffers={"dummy_buf": dummy_buf},
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
@@ -367,14 +365,18 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
|
||||
n_out = _count_output_tokens(grid_thw, _SPATIAL_MERGE)
|
||||
p = next(self.parameters())
|
||||
dummy_buf = torch.zeros(n_out, _HIDDEN, device=p.device, dtype=p.dtype)
|
||||
return EncoderCudaGraphReplayBuffers(buffers={"dummy_buf": dummy_buf})
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={
|
||||
"pixel_values": mm_kwargs["pixel_values"],
|
||||
"dummy_buf": dummy_buf,
|
||||
}
|
||||
)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
buffers: dict[str, torch.Tensor],
|
||||
values: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
return self._forward(mm_kwargs["pixel_values"])
|
||||
return self._forward(values["pixel_values"])
|
||||
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
@@ -551,10 +553,6 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image", "video"],
|
||||
input_key_by_modality={
|
||||
"image": "pixel_values",
|
||||
"video": "pixel_values_videos",
|
||||
},
|
||||
buffer_keys=["dummy_buf"],
|
||||
out_hidden_size=_HIDDEN,
|
||||
)
|
||||
@@ -654,11 +652,10 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
n_out = _count_output_tokens(grid_config, _SPATIAL_MERGE)
|
||||
dummy_buf = torch.zeros(n_out, _HIDDEN, device=device, dtype=dtype)
|
||||
return EncoderCudaGraphCaptureInputs(
|
||||
mm_kwargs={
|
||||
values={
|
||||
"pixel_values": dummy_pixel_values,
|
||||
"image_grid_thw": grid_config,
|
||||
"dummy_buf": dummy_buf,
|
||||
},
|
||||
buffers={"dummy_buf": dummy_buf},
|
||||
)
|
||||
|
||||
def prepare_encoder_cudagraph_replay_buffers(
|
||||
@@ -670,14 +667,18 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
|
||||
n_out = _count_output_tokens(self._get_grid_thw(mm_kwargs), _SPATIAL_MERGE)
|
||||
p = next(self.parameters())
|
||||
dummy_buf = torch.zeros(n_out, _HIDDEN, device=p.device, dtype=p.dtype)
|
||||
return EncoderCudaGraphReplayBuffers(buffers={"dummy_buf": dummy_buf})
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
values={
|
||||
"pixel_values": self._get_pixel_values(mm_kwargs),
|
||||
"dummy_buf": dummy_buf,
|
||||
}
|
||||
)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
buffers: dict[str, torch.Tensor],
|
||||
values: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
return self._forward(self._get_pixel_values(mm_kwargs))
|
||||
return self._forward(values["pixel_values"])
|
||||
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
@@ -718,14 +719,6 @@ class TestGetInputModality:
|
||||
}
|
||||
assert model.get_input_modality(mm_kwargs) == "video"
|
||||
|
||||
def test_video_model_config_has_both_modalities(self):
|
||||
model = SimpleMockViTVideoModel()
|
||||
cfg = model.get_encoder_cudagraph_config()
|
||||
assert "image" in cfg.modalities
|
||||
assert "video" in cfg.modalities
|
||||
assert cfg.input_key_by_modality["image"] == "pixel_values"
|
||||
assert cfg.input_key_by_modality["video"] == "pixel_values_videos"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GPU tests — video capture, replay, fallback, and mixed image+video
|
||||
|
||||
Reference in New Issue
Block a user