[Bugfix][Multimodal] Fix Qwen3-Omni use_audio_in_video with mixed image/video inputs (#46213)

Signed-off-by: wendadawen <wendadawen@qq.com>
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
wendadawen
2026-07-17 08:31:16 +00:00
committed by GitHub
co-authored by Tianyu Guo
parent 426d48bfa1
commit 69d4f5ef63
8 changed files with 264 additions and 81 deletions
@@ -19,6 +19,7 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
check_interleaved_audio_video,
merge_interleaved_embeddings,
)
from vllm.multimodal.utils import set_mm_embedding_modality
# Fake token IDs
AUDIO_TOKEN_ID = 1001
@@ -27,6 +28,10 @@ VIDEO_TOKEN_ID = 1003
TEXT_TOKEN_ID = 0
def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor:
return set_mm_embedding_modality(torch.full(shape, value), modality)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -116,6 +121,24 @@ class TestCheckInterleavedAudioVideo:
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_multi_video_with_boundary_tokens(self):
"""Two interleaved videos separated by boundary tokens → still True.
use_audio_in_video expands each video into a local V/A span bounded by
non-pad tokens. A global density check would fail across those gaps.
"""
# [text][V A V A][text boundary][V A V A]
first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1)
second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0)
boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID])
input_ids = torch.cat([first_ids, boundary, second_ids])
is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
assert check_interleaved_audio_video(
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
)
def test_batched_non_interleaved_no_false_positive(self):
"""
Regression test for https://github.com/vllm-project/vllm/issues/35394.
@@ -323,13 +346,16 @@ class TestEmbedInputIds:
# mm_embeds come in [video, audio] order (video feature first in
# mm_features when positions are the same for use_audio_in_video)
mm_embeds = [
torch.full((video_n, hidden), video_val),
torch.full((audio_n, hidden), audio_val),
_mm_embed((video_n, hidden), video_val, "video"),
_mm_embed((audio_n, hidden), audio_val, "audio"),
]
model, _ = make_mock_model(hidden)
# Modalities are attached on the embedding tensors (as in encoder gather).
result = model.embed_input_ids(
input_ids, mm_embeds, is_multimodal=is_multimodal
input_ids,
mm_embeds,
is_multimodal=is_multimodal,
)
video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0]
@@ -362,8 +388,8 @@ class TestMergeInterleavedEmbeddings:
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
torch.full((num_video, hidden), 30.0),
torch.full((num_audio, hidden), 10.0),
_mm_embed((num_video, hidden), 30.0, "video"),
_mm_embed((num_audio, hidden), 10.0, "audio"),
]
result = merge_interleaved_embeddings(
@@ -372,8 +398,6 @@ class TestMergeInterleavedEmbeddings:
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
video_pos = is_video.nonzero(as_tuple=True)[0]
@@ -381,6 +405,62 @@ class TestMergeInterleavedEmbeddings:
assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0))
assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0))
def test_image_and_video_mixed(self):
"""Image embeddings must not be misclassified as video."""
hidden = 4
# [text][I I][V A V A]
tokens = (
[TEXT_TOKEN_ID] * 2
+ [IMAGE_TOKEN_ID] * 2
+ [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID]
)
input_ids = torch.tensor(tokens)
is_multimodal = (
(input_ids == IMAGE_TOKEN_ID)
| (input_ids == VIDEO_TOKEN_ID)
| (input_ids == AUDIO_TOKEN_ID)
)
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID)
inputs_embeds = torch.zeros(len(input_ids), hidden)
mm_embeds = [
_mm_embed((2, hidden), 20.0, "image"),
_mm_embed((2, hidden), 30.0, "video"),
_mm_embed((2, hidden), 10.0, "audio"),
]
result = merge_interleaved_embeddings(
inputs_embeds,
mm_embeds,
is_video,
is_audio,
is_multimodal,
)
assert result[is_image.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 20.0)
)
assert result[is_video.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 30.0)
)
assert result[is_audio.nonzero(as_tuple=True)[0]].allclose(
torch.full((2, hidden), 10.0)
)
def test_missing_modality_raises(self):
hidden = 2
input_ids, is_multimodal = make_interleaved_seq([2], [2])
is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID)
is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID)
with pytest.raises(ValueError, match="Missing modality"):
merge_interleaved_embeddings(
torch.zeros(len(input_ids), hidden),
[torch.zeros(2, hidden), torch.zeros(2, hidden)],
is_video,
is_audio,
is_multimodal,
)
if __name__ == "__main__":
pytest.main([__file__, "-v"])
+26
View File
@@ -78,6 +78,7 @@ def test_draft_lookahead_uses_boundary_feature_when_cached():
# f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7.
assert len(mm_embeds) == 2
assert [e.modality for e in mm_embeds] == ["image", "image"]
assert bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 8
@@ -94,6 +95,7 @@ def test_draft_lookahead_tolerates_missing_boundary_feature():
# Only f0 is gathered; f1's boundary position falls back silently.
assert len(mm_embeds) == 1
assert [e.modality for e in mm_embeds] == ["image"]
assert not bool(is_mm_embed[7])
assert int(is_mm_embed.sum()) == 7
@@ -154,4 +156,28 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead):
# Both requests contribute a feature; with the +1 skew each marks 7 of its
# 8 positions (the skew drops one), otherwise all 8.
assert len(mm_embeds) == 2
assert [e.modality for e in mm_embeds] == ["image", "image"]
assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16)
def test_gather_preserves_mixed_modalities():
"""Modalities must be attached on tensors in gather order."""
video = MultiModalFeatureSpec(
data=None,
modality="video",
identifier="v0",
mm_position=PlaceholderRange(offset=0, length=4),
)
audio = MultiModalFeatureSpec(
data=None,
modality="audio",
identifier="a0",
mm_position=PlaceholderRange(offset=4, length=4),
)
runner = _make_runner([video, audio], cached=[video, audio])
mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=0)
assert len(mm_embeds) == 2
assert [e.modality for e in mm_embeds] == ["video", "audio"]
assert int(is_mm_embed.sum()) == 8
@@ -138,42 +138,44 @@ def check_interleaved_audio_video(
num_audio: int,
) -> bool:
"""
Check if video and audio positions are interleaved in the multimodal region.
Check if video and audio positions are interleaved in any per-video span.
Returns True only for the use_audio_in_video=True case, where video and
audio tokens alternate within a single contiguous region with no gaps.
A simple range-overlap check produces false positives when multiple
non-interleaved requests are batched together: audio tokens from request N
fall between video tokens from request N and request N+1, making the
global ranges overlap even though each individual request is non-interleaved.
To distinguish true interleaving from this batching artefact we require
that every position in the combined [first_VA, last_VA] range is occupied
by either a video or an audio token (no text/image gaps).
For use_audio_in_video=True, each video placeholder is expanded into one
local span containing only video/audio pad tokens, bounded by non-pad
tokens such as audio_start/audio_end. Check each contiguous V/A span
independently instead of requiring all V/A tokens in the whole sequence to
form one global dense range; otherwise multi-video requests can be
misclassified as non-interleaved because of the boundary tokens between
videos.
"""
if num_video == 0 or num_audio == 0:
return False
video_pos = is_video.nonzero(as_tuple=True)[0]
audio_pos = is_audio.nonzero(as_tuple=True)[0]
# Quick range-overlap pre-check (necessary but not sufficient).
if not (
video_pos[0].item() < audio_pos[-1].item()
and audio_pos[0].item() < video_pos[-1].item()
):
va_pos = (is_video | is_audio).nonzero(as_tuple=True)[0]
if len(va_pos) == 0:
return False
# Density check: for true use_audio_in_video interleaving every position
# in the combined span is a video or audio token. Batched non-interleaved
# requests have text/image tokens between the per-request V and A blocks.
# combined_start/end encompass all V/A tokens, so num_video + num_audio
# equals the number of V/A tokens in range; compare directly to span size.
combined_start = min(video_pos[0].item(), audio_pos[0].item())
combined_end = max(video_pos[-1].item(), audio_pos[-1].item())
total_in_range = combined_end - combined_start + 1
return (num_video + num_audio) == total_in_range
span_start = 0
for span_end in range(1, len(va_pos) + 1):
is_last = span_end == len(va_pos)
if not is_last and va_pos[span_end].item() == va_pos[span_end - 1].item() + 1:
continue
span = va_pos[span_start:span_end]
span_is_video = is_video[span]
span_is_audio = is_audio[span]
if span_is_video.any() and span_is_audio.any():
video_offsets = span_is_video.nonzero(as_tuple=True)[0]
audio_offsets = span_is_audio.nonzero(as_tuple=True)[0]
if (
video_offsets[0].item() < audio_offsets[-1].item()
and audio_offsets[0].item() < video_offsets[-1].item()
):
return True
span_start = span_end
return False
def merge_interleaved_embeddings(
@@ -182,58 +184,69 @@ def merge_interleaved_embeddings(
is_video: torch.Tensor,
is_audio: torch.Tensor,
is_multimodal: torch.Tensor,
num_video: int,
num_audio: int,
) -> torch.Tensor:
"""
Merge embeddings for interleaved audio-in-video sequences.
When use_audio_in_video=True, video and audio tokens are interleaved in
the token sequence, but embeddings are provided as separate contiguous
tensors (video first, then audio). This function reorders video and audio
embeddings to match sequence position order and scatters them efficiently.
tensors. This function scatters each modality by the ``modality`` attribute
attached to each embedding tensor (set during encoder gather) and also
supports image embeddings in the same interleaved request.
Args:
inputs_embeds: The input embeddings tensor to merge into.
multimodal_embeddings: List of embedding tensors (video, audio, other).
multimodal_embeddings: List of embedding tensors (video, audio, image).
is_video: Boolean mask for video token positions.
is_audio: Boolean mask for audio token positions.
is_multimodal: Boolean mask for all multimodal token positions.
num_video: Total count of video tokens.
num_audio: Total count of audio tokens.
Returns:
The merged inputs_embeds tensor with multimodal embeddings scattered
to their correct positions.
"""
# Categorize embeddings by modality based on token counts.
# Embeddings come grouped by modality but order varies (e.g., image, video, audio
# or video, audio depending on input kwargs order).
video_embeds: list[torch.Tensor] = []
audio_embeds: list[torch.Tensor] = []
other_embeds: list[torch.Tensor] = []
video_remaining = num_video
audio_remaining = num_audio
from vllm.multimodal.utils import get_mm_embedding_modalities
for emb in multimodal_embeddings:
n = emb.shape[0]
if video_remaining > 0 and n <= video_remaining:
video_embeds.append(emb)
video_remaining -= n
elif audio_remaining > 0 and n <= audio_remaining:
audio_embeds.append(emb)
audio_remaining -= n
else:
other_embeds.append(emb)
def _merge_embedding_group(
mask: torch.Tensor,
embeddings: Sequence[torch.Tensor],
modality: str,
) -> None:
num_expected_tokens = mask.sum().item()
num_actual_tokens = sum(embedding.shape[0] for embedding in embeddings)
if num_actual_tokens != num_expected_tokens:
raise ValueError(
f"Attempted to assign {num_actual_tokens} {modality} tokens "
f"to {num_expected_tokens} placeholders"
)
if embeddings:
inputs_embeds[mask] = torch.cat(list(embeddings), dim=0)
# Scatter each modality to its positions
if video_embeds:
inputs_embeds[is_video] = torch.cat(video_embeds, dim=0)
if audio_embeds:
inputs_embeds[is_audio] = torch.cat(audio_embeds, dim=0)
if other_embeds:
other_mask = is_multimodal & ~is_video & ~is_audio
inputs_embeds[other_mask] = torch.cat(other_embeds, dim=0)
# Qwen Omni multimodal placeholders are image/video/audio; after excluding
# video and audio positions, the remaining multimodal positions are images.
is_image = is_multimodal & ~is_video & ~is_audio
embedding_modalities = get_mm_embedding_modalities(multimodal_embeddings)
video_embeds = [
embedding
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
if modality == "video"
]
audio_embeds = [
embedding
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
if modality == "audio"
]
image_embeds = [
embedding
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
if modality == "image"
]
_merge_embedding_group(is_video, video_embeds, "video")
_merge_embedding_group(is_audio, audio_embeds, "audio")
_merge_embedding_group(is_image, image_embeds, "image")
return inputs_embeds
@@ -1497,8 +1510,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
# Default: standard merge (no interleaving), same as parent class
@@ -1833,9 +1833,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
# both the deepstack path and the final embedding merge.
video_token_id = self.config.video_token_id
audio_token_id = self.config.audio_token_id
image_token_id = self.config.image_token_id
input_ids_cpu = input_ids.cpu()
is_video = is_multimodal & (input_ids_cpu == video_token_id)
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
is_image = is_multimodal & (input_ids_cpu == image_token_id)
num_video = is_video.sum().item()
num_audio = is_audio.sum().item()
@@ -1856,9 +1858,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
multimodal_embeddings_multiscale = []
if is_interleaved:
# Use input_ids-based mask for correct vision positions
# when audio and video tokens are interleaved.
is_vision = is_video.clone()
# Use input_ids-based mask for all vision positions when
# audio and video tokens are interleaved.
is_vision = is_video | is_image
else:
is_vision = torch.zeros_like(is_multimodal)
mm_positions = torch.nonzero(is_multimodal, as_tuple=True)[0]
@@ -1917,8 +1919,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
# Default: standard merge (no interleaving), same as parent class.
+38
View File
@@ -346,3 +346,41 @@ def fetch_video(
allowed_local_media_path="/",
)
return media_connector.fetch_video(video_url)
def set_mm_embedding_modality(embed: "torch.Tensor", modality: str) -> "torch.Tensor":
"""Attach modality metadata to a gathered multimodal embedding tensor.
Used by interleaved Omni merge paths that need to group embeddings by
modality without threading a parallel modalities list through
``embed_input_ids``.
"""
embed.modality = modality # type: ignore[attr-defined]
return embed
def copy_mm_embedding_modality(
src: "torch.Tensor", dst: "torch.Tensor"
) -> "torch.Tensor":
"""Copy ``modality`` from ``src`` onto ``dst`` if present."""
modality = getattr(src, "modality", None)
if modality is not None:
dst.modality = modality # type: ignore[attr-defined]
return dst
def get_mm_embedding_modalities(
multimodal_embeddings: Sequence["torch.Tensor"],
) -> list[str]:
"""Collect per-embedding modalities previously set on the tensors."""
modalities: list[str] = []
for i, emb in enumerate(multimodal_embeddings):
modality = getattr(emb, "modality", None)
if modality is None:
raise ValueError(
f"Missing modality on multimodal embedding at index {i}. "
"Encoder gather must set embed.modality before interleaved "
"audio-in-video merge."
)
modalities.append(modality)
return modalities
+8 -1
View File
@@ -5,7 +5,11 @@ import torch
from vllm.model_executor.models.interfaces import SupportsMultiModal, supports_realtime
from vllm.multimodal.inputs import MultiModalKwargsItem
from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
from vllm.multimodal.utils import (
get_mm_features_in_window,
group_and_batch_mm_kwargs,
set_mm_embedding_modality,
)
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs
@@ -136,6 +140,9 @@ class EncoderRunner:
else:
mm_embeds_item = encoder_output[start_idx:end_idx]
# Attach modality for Omni interleaved merge (collected on demand).
set_mm_embedding_modality(mm_embeds_item, mm_feature.modality)
req_start_pos = query_start_loc[i] + start_pos - cur_query_start
is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= (
True if is_embed is None else is_embed
+13 -3
View File
@@ -5,7 +5,10 @@ import torch.nn as nn
from vllm.config import ModelConfig
from vllm.model_executor.models.interfaces import supports_multimodal_pruning
from vllm.multimodal.utils import get_mm_features_in_window
from vllm.multimodal.utils import (
copy_mm_embedding_modality,
get_mm_features_in_window,
)
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.mm.rope import RopeState
@@ -42,7 +45,11 @@ class MultiModalPruner:
speculator reuses the target's already-recomputed positions, hence there is
no position write-back here.
"""
return [mm[:, : self.inputs_embeds_size] for mm in mm_embeds]
stripped: list[torch.Tensor] = []
for mm in mm_embeds:
out = mm[:, : self.inputs_embeds_size]
stripped.append(copy_mm_embedding_modality(mm, out))
return stripped
def recompute(
self,
@@ -82,7 +89,10 @@ class MultiModalPruner:
num_computed_tokens=num_computed,
)
self.rope_state.update_prefill_positions(req_idx, new_positions, delta)
cleaned.extend(req_cleaned)
cleaned.extend(
copy_mm_embedding_modality(src, dst)
for src, dst in zip(req_embeds, req_cleaned)
)
assert pos == len(mm_embeds)
return cleaned
+12 -1
View File
@@ -105,7 +105,12 @@ from vllm.multimodal.inputs import (
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs
from vllm.multimodal.utils import (
copy_mm_embedding_modality,
get_mm_features_in_window,
group_and_batch_mm_kwargs,
set_mm_embedding_modality,
)
from vllm.platforms import current_platform
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingType
@@ -3245,11 +3250,13 @@ class GPUModelRunner(
is_mm_embed[
req_start_pos + start_idx : req_start_pos + end_idx
] |= is_embed
set_mm_embedding_modality(mm_embeds_item, mm_feature.modality)
mm_embeds_req.append(mm_embeds_item)
if self.is_multimodal_pruning_enabled and self.uses_mrope:
assert req_state.mrope_positions is not None
should_sync_mrope_positions = True
old_mm_embeds_req = mm_embeds_req
mm_embeds_req, new_mrope_positions, new_delta = (
self.model.recompute_mrope_positions(
input_ids=req_state.prompt_token_ids,
@@ -3258,6 +3265,10 @@ class GPUModelRunner(
num_computed_tokens=req_state.num_computed_tokens,
)
)
mm_embeds_req = [
copy_mm_embedding_modality(src, dst)
for src, dst in zip(old_mm_embeds_req, mm_embeds_req)
]
req_state.mrope_positions.copy_(new_mrope_positions)
req_state.mrope_position_delta = new_delta