[BugFix] Fix TypeError in MiniCPM-O audio feature unpadding (#38053)
Signed-off-by: Krishna Chaitanya Balusu <krishnabkc15@gmail.com> Signed-off-by: wjinxu <1299461899@qq.com> Signed-off-by: Kc Balusu <kcbalusu@users.noreply.github.com> Co-authored-by: wjinxu <1299461899@qq.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Kc Balusu <kcbalusu@users.noreply.github.com>
This commit is contained in:
co-authored by
wjinxu
Cursor
Cyrus Leung
Kc Balusu
parent
54d0c36fff
commit
279d25f5cb
@@ -173,9 +173,64 @@ MiniCPMOAudioInputs: TypeAlias = (
|
||||
|
||||
|
||||
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
audio_features = hf_inputs.get("audio_features")
|
||||
audio_feature_lens = hf_inputs.get("audio_feature_lens")
|
||||
|
||||
# For multi-chunk audio (>30s), audio_features has one item per chunk
|
||||
# (total_chunks) while audio_feature_lens has one item per audio (N).
|
||||
# Use flat to group audio_features by audio so both fields
|
||||
# share the same batch size (N).
|
||||
audio_features_cfg = MultiModalFieldConfig.batched("audio")
|
||||
|
||||
if audio_features is not None and audio_feature_lens is not None:
|
||||
num_features = (
|
||||
len(audio_features)
|
||||
if isinstance(audio_features, (list, tuple))
|
||||
else audio_features.shape[0]
|
||||
)
|
||||
num_audios = (
|
||||
len(audio_feature_lens)
|
||||
if isinstance(audio_feature_lens, (list, tuple))
|
||||
else audio_feature_lens.shape[0]
|
||||
)
|
||||
|
||||
if num_features > num_audios:
|
||||
# Compute the number of chunks belonging to each audio
|
||||
chunks_per_audio: list[int] = []
|
||||
for lens in audio_feature_lens:
|
||||
if isinstance(lens, torch.Tensor):
|
||||
chunks_per_audio.append(lens.numel())
|
||||
else:
|
||||
chunks_per_audio.append(1)
|
||||
|
||||
# When audio_feature_lens is padded (e.g. from batched HF
|
||||
# processor output), numel() over-counts. Fall back to
|
||||
# counting non-zero entries so the sizes sum to num_features.
|
||||
if sum(chunks_per_audio) != num_features:
|
||||
chunks_per_audio = []
|
||||
for lens in audio_feature_lens:
|
||||
if isinstance(lens, torch.Tensor):
|
||||
n = int((lens != 0).sum())
|
||||
chunks_per_audio.append(max(n, 1))
|
||||
else:
|
||||
chunks_per_audio.append(1)
|
||||
|
||||
# Use flat (not flat_from_sizes) because audio_features
|
||||
# is list[Tensor] with variable-length chunks (post-unpad).
|
||||
slice_idxs = [0]
|
||||
for n in chunks_per_audio:
|
||||
slice_idxs.append(slice_idxs[-1] + n)
|
||||
audio_features_cfg = MultiModalFieldConfig.flat(
|
||||
"audio",
|
||||
[
|
||||
slice(slice_idxs[i], slice_idxs[i + 1])
|
||||
for i in range(len(chunks_per_audio))
|
||||
],
|
||||
)
|
||||
|
||||
return dict(
|
||||
**_minicpmv_field_config(hf_inputs),
|
||||
audio_features=MultiModalFieldConfig.batched("audio"),
|
||||
audio_features=audio_features_cfg,
|
||||
audio_feature_lens=MultiModalFieldConfig.batched("audio"),
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
)
|
||||
@@ -364,11 +419,23 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
|
||||
|
||||
# Avoid padding since we need the output for each audio to be
|
||||
# independent of other audios for the cache to work correctly
|
||||
# Flatten audio_feature_lens (list of tensors of any
|
||||
# dimensionality, one per audio, each containing per-chunk
|
||||
# lengths) into a flat list of integer lengths so there is
|
||||
# one length per chunk, matching the first dimension of
|
||||
# audio_features. Using flatten() handles 0-D, 1-D, and
|
||||
# higher-dimensional tensors uniformly.
|
||||
flat_feature_lens: list[int] = []
|
||||
for lens in audio_inputs["audio_feature_lens"]:
|
||||
if isinstance(lens, torch.Tensor):
|
||||
flat_feature_lens.extend(lens.flatten().tolist())
|
||||
else:
|
||||
flat_feature_lens.append(int(lens))
|
||||
unpadded_audio_features = [
|
||||
feat[:, :feature_len]
|
||||
for feat, feature_len in zip(
|
||||
feat[:, :length]
|
||||
for feat, length in zip(
|
||||
audio_inputs["audio_features"],
|
||||
audio_inputs["audio_feature_lens"],
|
||||
flat_feature_lens,
|
||||
)
|
||||
]
|
||||
audio_inputs["audio_features"] = unpadded_audio_features
|
||||
|
||||
Reference in New Issue
Block a user