[MM] Keep more metadata tensors on CPU (#52827)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -300,11 +300,17 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
|
||||
# Keep these as batched, as they always have batch size as first dim
|
||||
if "audio" in modalities:
|
||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
|
||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched(
|
||||
"audio", keep_on_cpu=True
|
||||
)
|
||||
if "image" in modalities:
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
# TODO: route to "video" once the video modality is supported
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
@@ -755,6 +761,12 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
num_image_patches = kwargs.pop("num_image_patches")
|
||||
|
||||
# grid_thw fields are registered keep_on_cpu; restore the on-device
|
||||
# placement that HF get_image_features implementations expect.
|
||||
for key, value in kwargs.items():
|
||||
if isinstance(value, torch.Tensor):
|
||||
kwargs[key] = value.to(pixel_values.device, non_blocking=True)
|
||||
|
||||
# The underlying HuggingFace `get_image_features` implementations
|
||||
# contain model-internal syncs (e.g. Idefics3 filters all-zero
|
||||
# padding images via boolean-mask indexing, LlavaOnevision
|
||||
|
||||
Reference in New Issue
Block a user