[Bugfix] Fix Qwen3-Omni crash on video with no audio track when use_audio_in_video=True (#48420)

Signed-off-by: RyanJHamby <ryanhamby22@gmail.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Ryan Hamby
2026-08-03 16:22:00 -07:00
committed by GitHub
co-authored by Isotr0py
parent c2881ce603
commit 6a9109d865
3 changed files with 57 additions and 1 deletions
@@ -113,3 +113,35 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
f" baseline : {baseline_ids}\n"
f" cache-hit: {second_ids}"
)
@pytest.mark.parametrize("model_id", MODELS)
def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
"""
A video with no audio track, combined with `use_audio_in_video=True`,
must raise a clear `ValueError` naming the mismatch, rather than an
opaque `StopIteration`-derived error from the underlying HF processor.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None)
video_token_id = processor.info.get_hf_config().video_token_id
rng = np.random.RandomState(0)
video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
# No "audio" key at all in this request's data: this is the "video has
# no audio track" case. (The audio limit above is 1, not 0, so that
# `mm_counts["audio"]` is still declared for this request; a 0 limit
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
# trips an unrelated assertion in `_apply_hf_processor_mm_only`.)
mm_data = {"video": [video]}
with pytest.raises(ValueError, match="doesn't have audio track"):
processor(
[video_token_id],
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"use_audio_in_video": True},
)
@@ -508,6 +508,20 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
mm_kwargs = dict(
**mm_kwargs,
)
elif (
mm_kwargs.get("use_audio_in_video")
and "audio" not in mm_data
and mm_data.get("videos")
):
# A subclass (e.g. Qwen3-Omni) may have already popped "audios"
# and populated mm_data["audio"] itself before delegating here,
# in which case this isn't a genuine "no audio" case. Likewise,
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case either.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
if mm_data.get("videos") and (
@@ -911,7 +925,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
if use_audio_in_video and "video" in mm_counts:
assert "audio" in mm_counts
if mm_counts.get("audio", 0) < mm_counts["video"]:
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
mm_counts["audio"] -= mm_counts["video"]
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
@@ -1224,6 +1224,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
tok_kwargs = dict(tok_kwargs)
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
# mm_data can be empty on a multimodal-processor-cache hit, where
# there's nothing to (re-)process this call and the real result
# comes from the cache — not a genuine "no audio" case.
raise ValueError(
"Video doesn't have audio track with `audio_in_video=True`"
)
hf_inputs = super()._call_hf_processor(
prompt=prompt,