[Bugfix] Fix Qwen3-Omni crash on video with no audio track when use_audio_in_video=True (#48420)
Signed-off-by: RyanJHamby <ryanhamby22@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -508,6 +508,20 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
mm_kwargs = dict(
|
||||
**mm_kwargs,
|
||||
)
|
||||
elif (
|
||||
mm_kwargs.get("use_audio_in_video")
|
||||
and "audio" not in mm_data
|
||||
and mm_data.get("videos")
|
||||
):
|
||||
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
|
||||
# and populated mm_data["audio"] itself before delegating here,
|
||||
# in which case this isn't a genuine "no audio" case. Likewise,
|
||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||
# there's nothing to (re-)process this call and the real result
|
||||
# comes from the cache — not a genuine "no audio" case either.
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
|
||||
if mm_data.get("videos") and (
|
||||
@@ -911,7 +925,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
|
||||
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
|
||||
if use_audio_in_video and "video" in mm_counts:
|
||||
assert "audio" in mm_counts
|
||||
if mm_counts.get("audio", 0) < mm_counts["video"]:
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
mm_counts["audio"] -= mm_counts["video"]
|
||||
|
||||
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
|
||||
@@ -1224,6 +1224,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
tok_kwargs = dict(tok_kwargs)
|
||||
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
|
||||
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
|
||||
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
|
||||
# mm_data can be empty on a multimodal-processor-cache hit, where
|
||||
# there's nothing to (re-)process this call and the real result
|
||||
# comes from the cache — not a genuine "no audio" case.
|
||||
raise ValueError(
|
||||
"Video doesn't have audio track with `audio_in_video=True`"
|
||||
)
|
||||
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt=prompt,
|
||||
|
||||
Reference in New Issue
Block a user