From 6a9109d865d8abbfe70777ea52520425f1be6d3f Mon Sep 17 00:00:00 2001 From: Ryan Hamby Date: Mon, 3 Aug 2026 19:22:00 -0400 Subject: [PATCH] [Bugfix] Fix Qwen3-Omni crash on video with no audio track when use_audio_in_video=True (#48420) Signed-off-by: RyanJHamby Signed-off-by: Isotr0py Co-authored-by: Isotr0py --- .../processing/test_audio_in_video.py | 32 +++++++++++++++++++ .../models/qwen2_5_omni_thinker.py | 19 ++++++++++- .../models/qwen3_omni_moe_thinker.py | 7 ++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/models/multimodal/processing/test_audio_in_video.py b/tests/models/multimodal/processing/test_audio_in_video.py index 894b097aba..67235cfc88 100644 --- a/tests/models/multimodal/processing/test_audio_in_video.py +++ b/tests/models/multimodal/processing/test_audio_in_video.py @@ -113,3 +113,35 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non f" baseline : {baseline_ids}\n" f" cache-hit: {second_ids}" ) + + +@pytest.mark.parametrize("model_id", MODELS) +def test_use_audio_in_video_without_audio_track(model_id: str) -> None: + """ + A video with no audio track, combined with `use_audio_in_video=True`, + must raise a clear `ValueError` naming the mismatch, rather than an + opaque `StopIteration`-derived error from the underlying HF processor. + """ + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None) + video_token_id = processor.info.get_hf_config().video_token_id + + rng = np.random.RandomState(0) + video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65) + + # No "audio" key at all in this request's data: this is the "video has + # no audio track" case. (The audio limit above is 1, not 0, so that + # `mm_counts["audio"]` is still declared for this request; a 0 limit + # would make vLLM omit the "audio" key from `mm_counts` entirely, which + # trips an unrelated assertion in `_apply_hf_processor_mm_only`.) + mm_data = {"video": [video]} + + with pytest.raises(ValueError, match="doesn't have audio track"): + processor( + [video_token_id], + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={"use_audio_in_video": True}, + ) diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index a0e9b84f34..470d7f65f0 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -508,6 +508,20 @@ class Qwen2_5OmniThinkerMultiModalProcessor( mm_kwargs = dict( **mm_kwargs, ) + elif ( + mm_kwargs.get("use_audio_in_video") + and "audio" not in mm_data + and mm_data.get("videos") + ): + # A subclass (e.g. Qwen3-Omni) may have already popped "audios" + # and populated mm_data["audio"] itself before delegating here, + # in which case this isn't a genuine "no audio" case. Likewise, + # mm_data can be empty on a multimodal-processor-cache hit, where + # there's nothing to (re-)process this call and the real result + # comes from the cache — not a genuine "no audio" case either. + raise ValueError( + "Video doesn't have audio track with `audio_in_video=True`" + ) merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs) if mm_data.get("videos") and ( @@ -911,7 +925,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor( use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False) if use_audio_in_video and "video" in mm_counts: - assert "audio" in mm_counts + if mm_counts.get("audio", 0) < mm_counts["video"]: + raise ValueError( + "Video doesn't have audio track with `audio_in_video=True`" + ) mm_counts["audio"] -= mm_counts["video"] _, mm_processed_data, _ = self._apply_hf_processor_text_mm( diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 0afc6734af..2b46d4ae55 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1224,6 +1224,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( tok_kwargs = dict(tok_kwargs) mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {}) mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {}) + elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"): + # mm_data can be empty on a multimodal-processor-cache hit, where + # there's nothing to (re-)process this call and the real result + # comes from the cache — not a genuine "no audio" case. + raise ValueError( + "Video doesn't have audio track with `audio_in_video=True`" + ) hf_inputs = super()._call_hf_processor( prompt=prompt,