Adjust design around encoder_cudagraph_forward (#42288)

Signed-off-by: Weida Hong <wdhongtw@google.com>
This commit is contained in:
Weida Hong
2026-05-29 03:02:52 +00:00
committed by GitHub
parent 212deff2ec
commit dfe8ba7c80
8 changed files with 105 additions and 154 deletions
+24 -31
View File
@@ -87,8 +87,10 @@ class _MockModel(SupportsEncoderCudaGraph):
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
return EncoderCudaGraphConfig(
modalities=["image"],
input_key_by_modality={"image": "pixel_values"},
buffer_keys=["dummy_buf"],
buffer_keys=[
"pixel_values",
"dummy_buf",
],
out_hidden_size=32,
)
@@ -269,9 +271,6 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
return EncoderCudaGraphConfig(
modalities=["image"],
input_key_by_modality={
"image": "pixel_values",
},
buffer_keys=["dummy_buf"],
out_hidden_size=_HIDDEN,
)
@@ -350,11 +349,10 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
n_out = _count_output_tokens(grid_config, _SPATIAL_MERGE)
dummy_buf = torch.zeros(n_out, _HIDDEN, device=device, dtype=dtype)
return EncoderCudaGraphCaptureInputs(
mm_kwargs={
values={
"pixel_values": dummy_pixel_values,
"image_grid_thw": grid_config,
"dummy_buf": dummy_buf,
},
buffers={"dummy_buf": dummy_buf},
)
def prepare_encoder_cudagraph_replay_buffers(
@@ -367,14 +365,18 @@ class SimpleMockViTModel(torch.nn.Module, SupportsEncoderCudaGraph):
n_out = _count_output_tokens(grid_thw, _SPATIAL_MERGE)
p = next(self.parameters())
dummy_buf = torch.zeros(n_out, _HIDDEN, device=p.device, dtype=p.dtype)
return EncoderCudaGraphReplayBuffers(buffers={"dummy_buf": dummy_buf})
return EncoderCudaGraphReplayBuffers(
values={
"pixel_values": mm_kwargs["pixel_values"],
"dummy_buf": dummy_buf,
}
)
def encoder_cudagraph_forward(
self,
mm_kwargs: dict[str, Any],
buffers: dict[str, torch.Tensor],
values: dict[str, torch.Tensor],
) -> torch.Tensor:
return self._forward(mm_kwargs["pixel_values"])
return self._forward(values["pixel_values"])
def encoder_eager_forward(
self,
@@ -551,10 +553,6 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
def get_encoder_cudagraph_config(self) -> EncoderCudaGraphConfig:
return EncoderCudaGraphConfig(
modalities=["image", "video"],
input_key_by_modality={
"image": "pixel_values",
"video": "pixel_values_videos",
},
buffer_keys=["dummy_buf"],
out_hidden_size=_HIDDEN,
)
@@ -654,11 +652,10 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
n_out = _count_output_tokens(grid_config, _SPATIAL_MERGE)
dummy_buf = torch.zeros(n_out, _HIDDEN, device=device, dtype=dtype)
return EncoderCudaGraphCaptureInputs(
mm_kwargs={
values={
"pixel_values": dummy_pixel_values,
"image_grid_thw": grid_config,
"dummy_buf": dummy_buf,
},
buffers={"dummy_buf": dummy_buf},
)
def prepare_encoder_cudagraph_replay_buffers(
@@ -670,14 +667,18 @@ class SimpleMockViTVideoModel(SimpleMockViTModel):
n_out = _count_output_tokens(self._get_grid_thw(mm_kwargs), _SPATIAL_MERGE)
p = next(self.parameters())
dummy_buf = torch.zeros(n_out, _HIDDEN, device=p.device, dtype=p.dtype)
return EncoderCudaGraphReplayBuffers(buffers={"dummy_buf": dummy_buf})
return EncoderCudaGraphReplayBuffers(
values={
"pixel_values": self._get_pixel_values(mm_kwargs),
"dummy_buf": dummy_buf,
}
)
def encoder_cudagraph_forward(
self,
mm_kwargs: dict[str, Any],
buffers: dict[str, torch.Tensor],
values: dict[str, torch.Tensor],
) -> torch.Tensor:
return self._forward(self._get_pixel_values(mm_kwargs))
return self._forward(values["pixel_values"])
def encoder_eager_forward(
self,
@@ -718,14 +719,6 @@ class TestGetInputModality:
}
assert model.get_input_modality(mm_kwargs) == "video"
def test_video_model_config_has_both_modalities(self):
model = SimpleMockViTVideoModel()
cfg = model.get_encoder_cudagraph_config()
assert "image" in cfg.modalities
assert "video" in cfg.modalities
assert cfg.input_key_by_modality["image"] == "pixel_values"
assert cfg.input_key_by_modality["video"] == "pixel_values_videos"
# ---------------------------------------------------------------------------
# GPU tests — video capture, replay, fallback, and mixed image+video