Adjust design around encoder_cudagraph_forward (#42288)

Signed-off-by: Weida Hong <wdhongtw@google.com>
This commit is contained in:
Weida Hong
2026-05-29 03:02:52 +00:00
committed by GitHub
parent 212deff2ec
commit dfe8ba7c80
8 changed files with 105 additions and 154 deletions
+1 -2
View File
@@ -1623,8 +1623,7 @@ class SupportsEncoderCudaGraph(Protocol):
def encoder_cudagraph_forward(
self,
mm_kwargs: dict[str, Any],
buffers: dict[str, torch.Tensor],
inputs: dict[str, torch.Tensor],
) -> torch.Tensor:
"""Run the encoder forward pass with precomputed buffers.
+15 -18
View File
@@ -1057,7 +1057,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
grid_thw: list[list[int]],
grid_thw: list[list[int]] | None,
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
@@ -1712,11 +1712,8 @@ class Qwen2_5_VLForConditionalGeneration(
)
return EncoderCudaGraphConfig(
modalities=modalities,
input_key_by_modality={
"image": "pixel_values",
"video": "pixel_values_videos",
},
buffer_keys=[
"pixel_values",
"rotary_pos_emb_cos",
"rotary_pos_emb_sin",
"window_index",
@@ -1930,7 +1927,7 @@ class Qwen2_5_VLForConditionalGeneration(
// self.visual.patch_size
)
max_seqlen_window_override = vit_merger_window_size**2 * (spatial_merge_size**2)
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_config,
max_batch_size=max_batch_size,
max_frames_per_batch=max_frames_per_batch,
@@ -1942,14 +1939,12 @@ class Qwen2_5_VLForConditionalGeneration(
# Just use image-modality dummy input_buffer for capturing, since it's also
# compatible for video inputs (has the same shape: [num_patches, C*T*P*P]).
mm_kwargs = {
values = metadata | {
"pixel_values": dummy_pixel_values,
"image_grid_thw": grid_config,
}
return EncoderCudaGraphCaptureInputs(
mm_kwargs=mm_kwargs,
buffers=buffers,
values=values,
)
def prepare_encoder_cudagraph_replay_buffers(
@@ -1968,28 +1963,30 @@ class Qwen2_5_VLForConditionalGeneration(
# bound and can over-pad window attention into many empty FlashAttention
# CTAs.
if modality == "image":
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_batch_size=max_batch_size,
)
elif modality == "video":
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_frames_per_batch=max_frames_per_batch,
)
else:
raise AssertionError("This line should be unreachable.")
return EncoderCudaGraphReplayBuffers(buffers=buffers)
values = metadata | {
"pixel_values": self._get_pixel_values_by_modality(mm_kwargs),
}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
mm_kwargs: dict[str, Any],
buffers: dict[str, torch.Tensor],
values: dict[str, torch.Tensor],
) -> torch.Tensor:
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return self.visual(pixel_values, grid_thw, encoder_metadata=buffers)
pixel_values = values.pop("pixel_values")
metadata = values
return self.visual(pixel_values, None, encoder_metadata=metadata)
def encoder_eager_forward(
self,
+20 -19
View File
@@ -707,7 +707,7 @@ class Qwen2VisionTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
grid_thw: torch.Tensor | list[list[int]],
grid_thw: torch.Tensor | list[list[int]] | None,
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
@@ -715,9 +715,11 @@ class Qwen2VisionTransformer(nn.Module):
x = x.to(device=self.device, dtype=self.dtype)
x = self.patch_embed(x)
grid_thw_list = grid_thw if isinstance(grid_thw, list) else grid_thw.tolist()
if encoder_metadata is None:
assert grid_thw is not None
grid_thw_list = (
grid_thw if isinstance(grid_thw, list) else grid_thw.tolist()
)
encoder_metadata = self.prepare_encoder_metadata(grid_thw_list)
rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
@@ -1460,11 +1462,8 @@ class Qwen2VLForConditionalGeneration(
max_frames = self.get_max_frames_per_video()
return EncoderCudaGraphConfig(
modalities=["image", "video"],
input_key_by_modality={
"image": "pixel_values",
"video": "pixel_values_videos",
},
buffer_keys=[
"pixel_values",
"rotary_pos_emb_cos",
"rotary_pos_emb_sin",
"cu_seqlens",
@@ -1622,7 +1621,7 @@ class Qwen2VLForConditionalGeneration(
)
# max_seqlen.item() gets baked into the CUDA graph at capture time.
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_config,
max_batch_size=max_batch_size,
max_frames_per_batch=max_frames_per_batch,
@@ -1632,14 +1631,12 @@ class Qwen2VLForConditionalGeneration(
# Capture with image-format kwargs; pixel_values shape is compatible with
# both image and video replay paths.
mm_kwargs = {
values = metadata | {
"pixel_values": dummy_pixel_values,
"image_grid_thw": grid_config,
}
return EncoderCudaGraphCaptureInputs(
mm_kwargs=mm_kwargs,
buffers=buffers,
values=values,
)
def prepare_encoder_cudagraph_replay_buffers(
@@ -1652,24 +1649,28 @@ class Qwen2VLForConditionalGeneration(
grid_thw_list = self._get_grid_thw_by_modality(mm_kwargs)
if modality == "image":
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_batch_size=max_batch_size,
)
else:
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_frames_per_batch=max_frames_per_batch,
)
return EncoderCudaGraphReplayBuffers(buffers=buffers)
values = metadata | {
"pixel_values": self._get_pixel_values_by_modality(mm_kwargs),
}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self, mm_kwargs: dict[str, Any], buffers: dict[str, torch.Tensor]
self,
values: dict[str, torch.Tensor],
) -> torch.Tensor:
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return self.visual(pixel_values, grid_thw, encoder_metadata=buffers)
pixel_values = values.pop("pixel_values")
metadata = values
return self.visual(pixel_values, None, encoder_metadata=metadata)
def encoder_eager_forward(
self,
+14 -17
View File
@@ -1784,11 +1784,8 @@ class Qwen3VLForConditionalGeneration(
return EncoderCudaGraphConfig(
modalities=modalities,
input_key_by_modality={
"image": "pixel_values",
"video": "pixel_values_videos",
},
buffer_keys=[
"pixel_values",
"pos_embeds",
"rotary_pos_emb_cos",
"rotary_pos_emb_sin",
@@ -1978,7 +1975,7 @@ class Qwen3VLForConditionalGeneration(
# so the capture value must cover any replay scenario.
# Worst case: 1 item consuming the full budget ->
# seq_len = token_budget * spatial_merge_size^2.
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_config,
max_batch_size=max_batch_size,
max_frames_per_batch=max_frames_per_batch,
@@ -1988,14 +1985,12 @@ class Qwen3VLForConditionalGeneration(
# Just use image-modality dummy input_buffer for capturing, since it's also
# compatible for video inputs (has the same shape: [num_patches, C*T*P*P]).
mm_kwargs = {
values = metadata | {
"pixel_values": dummy_pixel_values,
"image_grid_thw": grid_config,
}
return EncoderCudaGraphCaptureInputs(
mm_kwargs=mm_kwargs,
buffers=buffers,
values=values,
)
def prepare_encoder_cudagraph_replay_buffers(
@@ -2008,28 +2003,30 @@ class Qwen3VLForConditionalGeneration(
grid_thw_list = self._get_grid_thw_by_modality(mm_kwargs)
if modality == "image":
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_batch_size=max_batch_size,
)
elif modality == "video":
buffers = self.visual.prepare_encoder_metadata(
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_frames_per_batch=max_frames_per_batch,
)
else:
raise AssertionError("This line should be unreachable.")
return EncoderCudaGraphReplayBuffers(buffers=buffers)
values = metadata | {
"pixel_values": self._get_pixel_values_by_modality(mm_kwargs),
}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
mm_kwargs: dict[str, Any],
buffers: dict[str, torch.Tensor],
values: dict[str, torch.Tensor],
) -> torch.Tensor:
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return self.visual(pixel_values, grid_thw, encoder_metadata=buffers)
pixel_values = values.pop("pixel_values")
metadata = values
return self.visual(pixel_values, None, encoder_metadata=metadata)
def encoder_eager_forward(
self,
+13 -16
View File
@@ -702,8 +702,10 @@ class Step3VLForConditionalGeneration(
return EncoderCudaGraphConfig(
modalities=["image"],
input_key_by_modality={"image": "pixel_values"},
buffer_keys=["patch_pixel_values"],
buffer_keys=[
"pixel_values",
"patch_pixel_values",
],
out_hidden_size=self.config.hidden_size,
)
@@ -846,33 +848,27 @@ class Step3VLForConditionalGeneration(
device=device,
dtype=dtype,
)
# num_patches is NOT in buffers -- the per-item merge is done
# num_patches is NOT in values -- the per-item merge is done
# CPU-side by finalize_encoder_cudagraph_output using the actual
# batch's num_patches from mm_kwargs.
mm_kwargs = {
values = {
"pixel_values": dummy_pixel_values,
"patch_pixel_values": dummy_patch_pixel_values,
}
buffers = {
"patch_pixel_values": dummy_patch_pixel_values,
}
return EncoderCudaGraphCaptureInputs(
mm_kwargs=mm_kwargs,
buffers=buffers,
values=values,
)
def encoder_cudagraph_forward(
self,
mm_kwargs: dict[str, Any],
buffers: dict[str, torch.Tensor],
values: dict[str, torch.Tensor],
) -> torch.Tensor:
# Graph captures only the compute (vision model + conv projector).
# Per-item merge happens CPU-side in finalize_encoder_cudagraph_output
# using actual num_patches from the batch data.
pixel_values = mm_kwargs["pixel_values"]
patch_pixel_values = buffers["patch_pixel_values"]
pixel_values = values["pixel_values"]
patch_pixel_values = values["patch_pixel_values"]
image_features = self._process_image_features(
self._get_vision_model_output(pixel_values)
@@ -974,10 +970,11 @@ class Step3VLForConditionalGeneration(
EncoderCudaGraphReplayBuffers,
)
# Only patch_pixel_values lives in the buffers dict; num_patches is
# Only patch_pixel_values lives in the values dict; num_patches is
# processed CPU-side by finalize_encoder_cudagraph_output.
return EncoderCudaGraphReplayBuffers(
buffers={
values={
"pixel_values": mm_kwargs["pixel_values"],
"patch_pixel_values": mm_kwargs["patch_pixel_values"],
},
)