From 279d25f5cbc8d0182672556aa66aae9b09301a23 Mon Sep 17 00:00:00 2001 From: Krishna Chaitanya Date: Mon, 1 Jun 2026 19:57:28 -0700 Subject: [PATCH] [BugFix] Fix TypeError in MiniCPM-O audio feature unpadding (#38053) Signed-off-by: Krishna Chaitanya Balusu Signed-off-by: wjinxu <1299461899@qq.com> Signed-off-by: Kc Balusu Co-authored-by: wjinxu <1299461899@qq.com> Co-authored-by: Cursor Co-authored-by: Cyrus Leung Co-authored-by: Kc Balusu --- vllm/model_executor/models/minicpmo.py | 75 ++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/models/minicpmo.py b/vllm/model_executor/models/minicpmo.py index 9251b14728..4fe8b80f34 100644 --- a/vllm/model_executor/models/minicpmo.py +++ b/vllm/model_executor/models/minicpmo.py @@ -173,9 +173,64 @@ MiniCPMOAudioInputs: TypeAlias = ( def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]): + audio_features = hf_inputs.get("audio_features") + audio_feature_lens = hf_inputs.get("audio_feature_lens") + + # For multi-chunk audio (>30s), audio_features has one item per chunk + # (total_chunks) while audio_feature_lens has one item per audio (N). + # Use flat to group audio_features by audio so both fields + # share the same batch size (N). + audio_features_cfg = MultiModalFieldConfig.batched("audio") + + if audio_features is not None and audio_feature_lens is not None: + num_features = ( + len(audio_features) + if isinstance(audio_features, (list, tuple)) + else audio_features.shape[0] + ) + num_audios = ( + len(audio_feature_lens) + if isinstance(audio_feature_lens, (list, tuple)) + else audio_feature_lens.shape[0] + ) + + if num_features > num_audios: + # Compute the number of chunks belonging to each audio + chunks_per_audio: list[int] = [] + for lens in audio_feature_lens: + if isinstance(lens, torch.Tensor): + chunks_per_audio.append(lens.numel()) + else: + chunks_per_audio.append(1) + + # When audio_feature_lens is padded (e.g. from batched HF + # processor output), numel() over-counts. Fall back to + # counting non-zero entries so the sizes sum to num_features. + if sum(chunks_per_audio) != num_features: + chunks_per_audio = [] + for lens in audio_feature_lens: + if isinstance(lens, torch.Tensor): + n = int((lens != 0).sum()) + chunks_per_audio.append(max(n, 1)) + else: + chunks_per_audio.append(1) + + # Use flat (not flat_from_sizes) because audio_features + # is list[Tensor] with variable-length chunks (post-unpad). + slice_idxs = [0] + for n in chunks_per_audio: + slice_idxs.append(slice_idxs[-1] + n) + audio_features_cfg = MultiModalFieldConfig.flat( + "audio", + [ + slice(slice_idxs[i], slice_idxs[i + 1]) + for i in range(len(chunks_per_audio)) + ], + ) + return dict( **_minicpmv_field_config(hf_inputs), - audio_features=MultiModalFieldConfig.batched("audio"), + audio_features=audio_features_cfg, audio_feature_lens=MultiModalFieldConfig.batched("audio"), audio_embeds=MultiModalFieldConfig.batched("audio"), ) @@ -364,11 +419,23 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing # Avoid padding since we need the output for each audio to be # independent of other audios for the cache to work correctly + # Flatten audio_feature_lens (list of tensors of any + # dimensionality, one per audio, each containing per-chunk + # lengths) into a flat list of integer lengths so there is + # one length per chunk, matching the first dimension of + # audio_features. Using flatten() handles 0-D, 1-D, and + # higher-dimensional tensors uniformly. + flat_feature_lens: list[int] = [] + for lens in audio_inputs["audio_feature_lens"]: + if isinstance(lens, torch.Tensor): + flat_feature_lens.extend(lens.flatten().tolist()) + else: + flat_feature_lens.append(int(lens)) unpadded_audio_features = [ - feat[:, :feature_len] - for feat, feature_len in zip( + feat[:, :length] + for feat, length in zip( audio_inputs["audio_features"], - audio_inputs["audio_feature_lens"], + flat_feature_lens, ) ] audio_inputs["audio_features"] = unpadded_audio_features