[Bugfix][Multimodal] Fix Qwen3-Omni use_audio_in_video with mixed image/video inputs (#46213)

Signed-off-by: wendadawen <wendadawen@qq.com>
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
wendadawen
2026-07-17 08:31:16 +00:00
committed by GitHub
co-authored by Tianyu Guo
parent 426d48bfa1
commit 69d4f5ef63
8 changed files with 264 additions and 81 deletions
@@ -19,6 +19,7 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
check_interleaved_audio_video,
merge_interleaved_embeddings,
)
from vllm.multimodal.utils import set_mm_embedding_modality
# Fake token IDs
AUDIO_TOKEN_ID = 1001
@@ -27,6 +28,10 @@ VIDEO_TOKEN_ID = 1003
TEXT_TOKEN_ID = 0
def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor:
return set_mm_embedding_modality(torch.full(shape, value), modality)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -116,6 +121,24 @@ class TestCheckInterleavedAudioVideo:
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_multi_video_with_boundary_tokens(self):
"""Two interleaved videos separated by boundary tokens → still True.
use_audio_in_video expands each video into a local V/A span bounded by
non-pad tokens. A global density check would fail across those gaps.
"""
# [text][V A V A][text boundary][V A V A]
first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1)
second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0)
boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID])
input_ids = torch.cat([first_ids, boundary, second_ids])
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_batched_non_interleaved_no_false_positive(self):
"""
Regression test for https://github.com/vllm-project/vllm/issues/35394.
@@ -323,13 +346,16 @@ class TestEmbedInputIds:
# mm_embeds come in [video, audio] order (video feature first in
# mm_features when positions are the same for use_audio_in_video)
mm_embeds = [
torch.full((video_n, hidden), video_val),
torch.full((audio_n, hidden), audio_val),
_mm_embed((video_n, hidden), video_val, "video"),
_mm_embed((audio_n, hidden), audio_val, "audio"),
]
model, _ = make_mock_model(hidden)
# Modalities are attached on the embedding tensors (as in encoder gather).
result = model.embed_input_ids(
input_ids, mm_embeds, is_multimodal=is_multimodal
input_ids,
mm_embeds,
is_multimodal=is_multimodal,
)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
@@ -362,8 +388,8 @@ class TestMergeInterleavedEmbeddings:
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
torch.full((num_video, hidden), 30.0),
torch.full((num_audio, hidden), 10.0),
_mm_embed((num_video, hidden), 30.0, "video"),
_mm_embed((num_audio, hidden), 10.0, "audio"),
]
result = merge_interleaved_embeddings(
@@ -372,8 +398,6 @@ class TestMergeInterleavedEmbeddings:
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
video_pos = is_video.nonzero(as_tuple=True)[0]
@@ -381,6 +405,62 @@ class TestMergeInterleavedEmbeddings:
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
def test_image_and_video_mixed(self):
"""Image embeddings must not be misclassified as video."""
hidden = 4
# [text][I I][V A V A]
tokens = (
[TEXT_TOKEN_ID] * 2
+ [IMAGE_TOKEN_ID] * 2
+ [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]
)
input_ids = torch.tensor(tokens)
is_multimodal = (
(input_ids == IMAGE_TOKEN_ID)
| (input_ids == VIDEO_TOKEN_ID)
| (input_ids == AUDIO_TOKEN_ID)
)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID)
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
_mm_embed((2, hidden), 20.0, "image"),
_mm_embed((2, hidden), 30.0, "video"),
_mm_embed((2, hidden), 10.0, "audio"),
]
result = merge_interleaved_embeddings(
inputs_embeds,
mm_embeds,
is_video,
is_audio,
is_multimodal,
)
assert result[is_image.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 20.0)
)
assert result[is_video.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 30.0)
)
assert result[is_audio.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 10.0)
)
def test_missing_modality_raises(self):
hidden = 2
input_ids, is_multimodal = make_interleaved_seq([2], [2])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
with pytest.raises(ValueError, match="Missing modality"):
merge_interleaved_embeddings(
torch.zeros(len(input_ids), hidden),
[torch.zeros(2, hidden), torch.zeros(2, hidden)],
is_video,
is_audio,
is_multimodal,
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])