[MM] Keep more metadata tensors on CPU (#52827)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -893,17 +893,21 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
|
||||
merge_size = self.info.get_hf_config().vision_config.spatial_merge_size
|
||||
fields: dict[str, MultiModalFieldConfig] = dict(
|
||||
**_create_qwen2vl_field_factory(merge_size)(hf_inputs),
|
||||
second_per_grid_ts=MultiModalFieldConfig.batched("video"),
|
||||
video_start_times=MultiModalFieldConfig.batched("video"),
|
||||
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
video_start_times=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
audio_features=MultiModalFieldConfig.batched("audio"),
|
||||
audio_token_lens=MultiModalFieldConfig.batched("audio"),
|
||||
audio_token_lens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
)
|
||||
# video_audio fields: only present when video_audio content was processed
|
||||
if "video_audio_n_segs" in hf_inputs:
|
||||
fields["video_audio_n_segs"] = MultiModalFieldConfig.batched("video")
|
||||
fields["video_audio_n_segs"] = MultiModalFieldConfig.batched(
|
||||
"video", keep_on_cpu=True
|
||||
)
|
||||
# video_audio_seg_lens: list of per-video 1D tensors, batched("video")
|
||||
if "video_audio_seg_lens" in hf_inputs:
|
||||
fields["video_audio_seg_lens"] = MultiModalFieldConfig.batched("video")
|
||||
fields["video_audio_seg_lens"] = MultiModalFieldConfig.batched(
|
||||
"video", keep_on_cpu=True
|
||||
)
|
||||
if "va_audio_features" in hf_inputs:
|
||||
fields["va_audio_features"] = MultiModalFieldConfig.batched("va_audio")
|
||||
return fields
|
||||
|
||||
Reference in New Issue
Block a user