[MM] Keep more metadata tensors on CPU (#52827)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-08-18 19:56:33 -07:00
committed by GitHub
parent aabc1a0a0b
commit deeeae75d0
38 changed files with 148 additions and 83 deletions
+9 -5
View File
@@ -893,17 +893,21 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
fields: dict[str, MultiModalFieldConfig] = dict(
**_create_qwen2vl_field_factory(merge_size)(hf_inputs),
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
video_start_times=MultiModalFieldConfig.batched("video"),
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
video_start_times=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
audio_features=MultiModalFieldConfig.batched("audio"),
audio_token_lens=MultiModalFieldConfig.batched("audio"),
audio_token_lens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
# video_audio fields: only present when video_audio content was processed
if "video_audio_n_segs" in hf_inputs:
fields["video_audio_n_segs"] = MultiModalFieldConfig.batched("video")
fields["video_audio_n_segs"] = MultiModalFieldConfig.batched(
"video", keep_on_cpu=True
)
# video_audio_seg_lens: list of per-video 1D tensors, batched("video")
if "video_audio_seg_lens" in hf_inputs:
fields["video_audio_seg_lens"] = MultiModalFieldConfig.batched("video")
fields["video_audio_seg_lens"] = MultiModalFieldConfig.batched(
"video", keep_on_cpu=True
)
if "va_audio_features" in hf_inputs:
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
return fields