[Bugfix][Multimodal] Fix Qwen3-Omni use_audio_in_video with mixed image/video inputs (#46213)
Signed-off-by: wendadawen <wendadawen@qq.com> Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
@@ -19,6 +19,7 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
|
||||
check_interleaved_audio_video,
|
||||
merge_interleaved_embeddings,
|
||||
)
|
||||
from vllm.multimodal.utils import set_mm_embedding_modality
|
||||
|
||||
# Fake token IDs
|
||||
AUDIO_TOKEN_ID = 1001
|
||||
@@ -27,6 +28,10 @@ VIDEO_TOKEN_ID = 1003
|
||||
TEXT_TOKEN_ID = 0
|
||||
|
||||
|
||||
def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor:
|
||||
return set_mm_embedding_modality(torch.full(shape, value), modality)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -116,6 +121,24 @@ class TestCheckInterleavedAudioVideo:
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_multi_video_with_boundary_tokens(self):
|
||||
"""Two interleaved videos separated by boundary tokens → still True.
|
||||
|
||||
use_audio_in_video expands each video into a local V/A span bounded by
|
||||
non-pad tokens. A global density check would fail across those gaps.
|
||||
"""
|
||||
# [text][V A V A][text boundary][V A V A]
|
||||
first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1)
|
||||
second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0)
|
||||
boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID])
|
||||
input_ids = torch.cat([first_ids, boundary, second_ids])
|
||||
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
assert check_interleaved_audio_video(
|
||||
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
|
||||
)
|
||||
|
||||
def test_batched_non_interleaved_no_false_positive(self):
|
||||
"""
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/35394.
|
||||
@@ -323,13 +346,16 @@ class TestEmbedInputIds:
|
||||
# mm_embeds come in [video, audio] order (video feature first in
|
||||
# mm_features when positions are the same for use_audio_in_video)
|
||||
mm_embeds = [
|
||||
torch.full((video_n, hidden), video_val),
|
||||
torch.full((audio_n, hidden), audio_val),
|
||||
_mm_embed((video_n, hidden), video_val, "video"),
|
||||
_mm_embed((audio_n, hidden), audio_val, "audio"),
|
||||
]
|
||||
|
||||
model, _ = make_mock_model(hidden)
|
||||
# Modalities are attached on the embedding tensors (as in encoder gather).
|
||||
result = model.embed_input_ids(
|
||||
input_ids, mm_embeds, is_multimodal=is_multimodal
|
||||
input_ids,
|
||||
mm_embeds,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
|
||||
@@ -362,8 +388,8 @@ class TestMergeInterleavedEmbeddings:
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
torch.full((num_video, hidden), 30.0),
|
||||
torch.full((num_audio, hidden), 10.0),
|
||||
_mm_embed((num_video, hidden), 30.0, "video"),
|
||||
_mm_embed((num_audio, hidden), 10.0, "audio"),
|
||||
]
|
||||
|
||||
result = merge_interleaved_embeddings(
|
||||
@@ -372,8 +398,6 @@ class TestMergeInterleavedEmbeddings:
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
num_video,
|
||||
num_audio,
|
||||
)
|
||||
|
||||
video_pos = is_video.nonzero(as_tuple=True)[0]
|
||||
@@ -381,6 +405,62 @@ class TestMergeInterleavedEmbeddings:
|
||||
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
|
||||
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
|
||||
|
||||
def test_image_and_video_mixed(self):
|
||||
"""Image embeddings must not be misclassified as video."""
|
||||
hidden = 4
|
||||
# [text][I I][V A V A]
|
||||
tokens = (
|
||||
[TEXT_TOKEN_ID] * 2
|
||||
+ [IMAGE_TOKEN_ID] * 2
|
||||
+ [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]
|
||||
)
|
||||
input_ids = torch.tensor(tokens)
|
||||
is_multimodal = (
|
||||
(input_ids == IMAGE_TOKEN_ID)
|
||||
| (input_ids == VIDEO_TOKEN_ID)
|
||||
| (input_ids == AUDIO_TOKEN_ID)
|
||||
)
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID)
|
||||
|
||||
inputs_embeds = torch.zeros(len(input_ids), hidden)
|
||||
mm_embeds = [
|
||||
_mm_embed((2, hidden), 20.0, "image"),
|
||||
_mm_embed((2, hidden), 30.0, "video"),
|
||||
_mm_embed((2, hidden), 10.0, "audio"),
|
||||
]
|
||||
result = merge_interleaved_embeddings(
|
||||
inputs_embeds,
|
||||
mm_embeds,
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
assert result[is_image.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 20.0)
|
||||
)
|
||||
assert result[is_video.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 30.0)
|
||||
)
|
||||
assert result[is_audio.nonzero(as_tuple=True)[0]].allclose(
|
||||
torch.full((2, hidden), 10.0)
|
||||
)
|
||||
|
||||
def test_missing_modality_raises(self):
|
||||
hidden = 2
|
||||
input_ids, is_multimodal = make_interleaved_seq([2], [2])
|
||||
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
|
||||
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
|
||||
with pytest.raises(ValueError, match="Missing modality"):
|
||||
merge_interleaved_embeddings(
|
||||
torch.zeros(len(input_ids), hidden),
|
||||
[torch.zeros(2, hidden), torch.zeros(2, hidden)],
|
||||
is_video,
|
||||
is_audio,
|
||||
is_multimodal,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
Reference in New Issue
Block a user