[BugFix] Fix TypeError in MiniCPM-O audio feature unpadding (#38053)

Signed-off-by: Krishna Chaitanya Balusu <krishnabkc15@gmail.com>
Signed-off-by: wjinxu <1299461899@qq.com>
Signed-off-by: Kc Balusu <kcbalusu@users.noreply.github.com>
Co-authored-by: wjinxu <1299461899@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Co-authored-by: Kc Balusu <kcbalusu@users.noreply.github.com>
This commit is contained in:
Krishna Chaitanya
2026-06-01 19:57:28 -07:00
committed by GitHub
co-authored by wjinxu Cursor Cyrus Leung Kc Balusu
parent 54d0c36fff
commit 279d25f5cb
+71 -4
View File
@@ -173,9 +173,64 @@ MiniCPMOAudioInputs: TypeAlias = (
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
audio_features = hf_inputs.get("audio_features")
audio_feature_lens = hf_inputs.get("audio_feature_lens")
# For multi-chunk audio (>30s), audio_features has one item per chunk
# (total_chunks) while audio_feature_lens has one item per audio (N).
# Use flat to group audio_features by audio so both fields
# share the same batch size (N).
audio_features_cfg = MultiModalFieldConfig.batched("audio")
if audio_features is not None and audio_feature_lens is not None:
num_features = (
len(audio_features)
if isinstance(audio_features, (list, tuple))
else audio_features.shape[0]
)
num_audios = (
len(audio_feature_lens)
if isinstance(audio_feature_lens, (list, tuple))
else audio_feature_lens.shape[0]
)
if num_features > num_audios:
# Compute the number of chunks belonging to each audio
chunks_per_audio: list[int] = []
for lens in audio_feature_lens:
if isinstance(lens, torch.Tensor):
chunks_per_audio.append(lens.numel())
else:
chunks_per_audio.append(1)
# When audio_feature_lens is padded (e.g. from batched HF
# processor output), numel() over-counts. Fall back to
# counting non-zero entries so the sizes sum to num_features.
if sum(chunks_per_audio) != num_features:
chunks_per_audio = []
for lens in audio_feature_lens:
if isinstance(lens, torch.Tensor):
n = int((lens != 0).sum())
chunks_per_audio.append(max(n, 1))
else:
chunks_per_audio.append(1)
# Use flat (not flat_from_sizes) because audio_features
# is list[Tensor] with variable-length chunks (post-unpad).
slice_idxs = [0]
for n in chunks_per_audio:
slice_idxs.append(slice_idxs[-1] + n)
audio_features_cfg = MultiModalFieldConfig.flat(
"audio",
[
slice(slice_idxs[i], slice_idxs[i + 1])
for i in range(len(chunks_per_audio))
],
)
return dict(
**_minicpmv_field_config(hf_inputs),
audio_features=MultiModalFieldConfig.batched("audio"),
audio_features=audio_features_cfg,
audio_feature_lens=MultiModalFieldConfig.batched("audio"),
audio_embeds=MultiModalFieldConfig.batched("audio"),
)
@@ -364,11 +419,23 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
# Avoid padding since we need the output for each audio to be
# independent of other audios for the cache to work correctly
# Flatten audio_feature_lens (list of tensors of any
# dimensionality, one per audio, each containing per-chunk
# lengths) into a flat list of integer lengths so there is
# one length per chunk, matching the first dimension of
# audio_features. Using flatten() handles 0-D, 1-D, and
# higher-dimensional tensors uniformly.
flat_feature_lens: list[int] = []
for lens in audio_inputs["audio_feature_lens"]:
if isinstance(lens, torch.Tensor):
flat_feature_lens.extend(lens.flatten().tolist())
else:
flat_feature_lens.append(int(lens))
unpadded_audio_features = [
feat[:, :feature_len]
for feat, feature_len in zip(
feat[:, :length]
for feat, length in zip(
audio_inputs["audio_features"],
audio_inputs["audio_feature_lens"],
flat_feature_lens,
)
]
audio_inputs["audio_features"] = unpadded_audio_features