[MM] Keep more metadata tensors on CPU (#52827)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-18 19:56:33 -07:00
committed by GitHub
parent aabc1a0a0b
commit deeeae75d0
38 changed files with 148 additions and 83 deletions
@@ -300,11 +300,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# Keep these as batched, as they always have batch size as first dim
if "audio" in modalities:
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
"audio", keep_on_cpu=True
)
if "image" in modalities:
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
# TODO: route to "video" once the video modality is supported
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
@@ -755,6 +761,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
num_image_patches = kwargs.pop("num_image_patches")
# grid_thw fields are registered keep_on_cpu; restore the on-device
# placement that HF get_image_features implementations expect.
for key, value in kwargs.items():
if isinstance(value, torch.Tensor):
kwargs[key] = value.to(pixel_values.device, non_blocking=True)
# The underlying HuggingFace `get_image_features` implementations
# contain model-internal syncs (e.g. Idefics3 filters all-zero
# padding images via boolean-mask indexing, LlavaOnevision