[Bugfix][Multimodal] Fix Qwen3-Omni use_audio_in_video with mixed image/video inputs (#46213)

Signed-off-by: wendadawen <wendadawen@qq.com>
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
wendadawen
2026-07-17 08:31:16 +00:00
committed by GitHub
co-authored by Tianyu Guo
parent 426d48bfa1
commit 69d4f5ef63
8 changed files with 264 additions and 81 deletions
+26
View File
@@ -78,6 +78,7 @@ def test_draft_lookahead_uses_boundary_feature_when_cached():
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
assert len(mm_embeds) == 2
assert [e.modality for e in mm_embeds] == ["image", "image"]
assert bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 8
@@ -94,6 +95,7 @@ def test_draft_lookahead_tolerates_missing_boundary_feature():
# Only f0 is gathered; f1's boundary position falls back silently.
assert len(mm_embeds) == 1
assert [e.modality for e in mm_embeds] == ["image"]
assert not bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 7
@@ -154,4 +156,28 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead):
# Both requests contribute a feature; with the +1 skew each marks 7 of its
# 8 positions (the skew drops one), otherwise all 8.
assert len(mm_embeds) == 2
assert [e.modality for e in mm_embeds] == ["image", "image"]
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
def test_gather_preserves_mixed_modalities():
"""Modalities must be attached on tensors in gather order."""
video = MultiModalFeatureSpec(
data=None,
modality="video",
identifier="v0",
mm_position=PlaceholderRange(offset=0, length=4),
)
audio = MultiModalFeatureSpec(
data=None,
modality="audio",
identifier="a0",
mm_position=PlaceholderRange(offset=4, length=4),
)
runner = _make_runner([video, audio], cached=[video, audio])
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=0)
assert len(mm_embeds) == 2
assert [e.modality for e in mm_embeds] == ["video", "audio"]
assert int(is_mm_embed.sum()) == 8