[Multimodal] Avoid extra d2d for encoder cudagraph with fused input norm (#56711)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -1976,7 +1976,8 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
for _ in range(max_batch_size)
|
||||
]
|
||||
|
||||
# Create dummy pixel_values
|
||||
# Create dummy pixel_values; uint8 when normalization is fused
|
||||
# on-device. Contents are overwritten before every replay.
|
||||
patch_embed = self.visual.patch_embed
|
||||
in_channels = patch_embed.proj.in_channels
|
||||
patch_size = patch_embed.patch_size
|
||||
@@ -1985,8 +1986,11 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
flattened_patch_size = (
|
||||
in_channels * temporal_patch_size * patch_size * patch_size
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
total_patches, flattened_patch_size, device=device, dtype=dtype
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
flattened_patch_size,
|
||||
device=device,
|
||||
dtype=self.visual.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# Override max_seqlen with a safe upper bound for capture.
|
||||
|
||||
@@ -1616,7 +1616,8 @@ class Qwen2VLForConditionalGeneration(
|
||||
for _ in range(max_batch_size)
|
||||
]
|
||||
|
||||
# Create dummy pixel_values.
|
||||
# Create dummy pixel_values; uint8 when normalization is fused
|
||||
# on-device. Contents are overwritten before every replay.
|
||||
patch_embed = self.visual.patch_embed
|
||||
in_channels = patch_embed.proj.in_channels
|
||||
patch_size = patch_embed.patch_size
|
||||
@@ -1625,8 +1626,11 @@ class Qwen2VLForConditionalGeneration(
|
||||
flattened_patch_size = (
|
||||
in_channels * temporal_patch_size * patch_size * patch_size
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
total_patches, flattened_patch_size, device=device, dtype=dtype
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
flattened_patch_size,
|
||||
device=device,
|
||||
dtype=self.visual.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# max_seqlen.item() gets baked into the CUDA graph at capture time.
|
||||
|
||||
Reference in New Issue
Block a user