Adjust design around encoder_cudagraph_forward (#42288)
Signed-off-by: Weida Hong <wdhongtw@google.com>
This commit is contained in:
@@ -702,8 +702,10 @@ class Step3VLForConditionalGeneration(
|
||||
|
||||
return EncoderCudaGraphConfig(
|
||||
modalities=["image"],
|
||||
input_key_by_modality={"image": "pixel_values"},
|
||||
buffer_keys=["patch_pixel_values"],
|
||||
buffer_keys=[
|
||||
"pixel_values",
|
||||
"patch_pixel_values",
|
||||
],
|
||||
out_hidden_size=self.config.hidden_size,
|
||||
)
|
||||
|
||||
@@ -846,33 +848,27 @@ class Step3VLForConditionalGeneration(
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
# num_patches is NOT in buffers -- the per-item merge is done
|
||||
# num_patches is NOT in values -- the per-item merge is done
|
||||
# CPU-side by finalize_encoder_cudagraph_output using the actual
|
||||
# batch's num_patches from mm_kwargs.
|
||||
mm_kwargs = {
|
||||
values = {
|
||||
"pixel_values": dummy_pixel_values,
|
||||
"patch_pixel_values": dummy_patch_pixel_values,
|
||||
}
|
||||
|
||||
buffers = {
|
||||
"patch_pixel_values": dummy_patch_pixel_values,
|
||||
}
|
||||
|
||||
return EncoderCudaGraphCaptureInputs(
|
||||
mm_kwargs=mm_kwargs,
|
||||
buffers=buffers,
|
||||
values=values,
|
||||
)
|
||||
|
||||
def encoder_cudagraph_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
buffers: dict[str, torch.Tensor],
|
||||
values: dict[str, torch.Tensor],
|
||||
) -> torch.Tensor:
|
||||
# Graph captures only the compute (vision model + conv projector).
|
||||
# Per-item merge happens CPU-side in finalize_encoder_cudagraph_output
|
||||
# using actual num_patches from the batch data.
|
||||
pixel_values = mm_kwargs["pixel_values"]
|
||||
patch_pixel_values = buffers["patch_pixel_values"]
|
||||
pixel_values = values["pixel_values"]
|
||||
patch_pixel_values = values["patch_pixel_values"]
|
||||
|
||||
image_features = self._process_image_features(
|
||||
self._get_vision_model_output(pixel_values)
|
||||
@@ -974,10 +970,11 @@ class Step3VLForConditionalGeneration(
|
||||
EncoderCudaGraphReplayBuffers,
|
||||
)
|
||||
|
||||
# Only patch_pixel_values lives in the buffers dict; num_patches is
|
||||
# Only patch_pixel_values lives in the values dict; num_patches is
|
||||
# processed CPU-side by finalize_encoder_cudagraph_output.
|
||||
return EncoderCudaGraphReplayBuffers(
|
||||
buffers={
|
||||
values={
|
||||
"pixel_values": mm_kwargs["pixel_values"],
|
||||
"patch_pixel_values": mm_kwargs["patch_pixel_values"],
|
||||
},
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user