[Bugfix][Multimodal] Fix Qwen3-Omni use_audio_in_video with mixed image/video inputs (#46213)

Signed-off-by: wendadawen <wendadawen@qq.com>
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
Co-authored-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn>
This commit is contained in:
wendadawen
2026-07-17 08:31:16 +00:00
committed by GitHub
co-authored by Tianyu Guo
parent 426d48bfa1
commit 69d4f5ef63
8 changed files with 264 additions and 81 deletions
@@ -138,42 +138,44 @@ def check_interleaved_audio_video(
num_audio: int,
) -> bool:
"""
Check if video and audio positions are interleaved in the multimodal region.
Check if video and audio positions are interleaved in any per-video span.
Returns True only for the use_audio_in_video=True case, where video and
audio tokens alternate within a single contiguous region with no gaps.
A simple range-overlap check produces false positives when multiple
non-interleaved requests are batched together: audio tokens from request N
fall between video tokens from request N and request N+1, making the
global ranges overlap even though each individual request is non-interleaved.
To distinguish true interleaving from this batching artefact we require
that every position in the combined [first_VA, last_VA] range is occupied
by either a video or an audio token (no text/image gaps).
For use_audio_in_video=True, each video placeholder is expanded into one
local span containing only video/audio pad tokens, bounded by non-pad
tokens such as audio_start/audio_end. Check each contiguous V/A span
independently instead of requiring all V/A tokens in the whole sequence to
form one global dense range; otherwise multi-video requests can be
misclassified as non-interleaved because of the boundary tokens between
videos.
"""
if num_video == 0 or num_audio == 0:
return False
video_pos = is_video.nonzero(as_tuple=True)[0]
audio_pos = is_audio.nonzero(as_tuple=True)[0]
# Quick range-overlap pre-check (necessary but not sufficient).
if not (
video_pos[0].item() < audio_pos[-1].item()
and audio_pos[0].item() < video_pos[-1].item()
):
va_pos = (is_video | is_audio).nonzero(as_tuple=True)[0]
if len(va_pos) == 0:
return False
# Density check: for true use_audio_in_video interleaving every position
# in the combined span is a video or audio token. Batched non-interleaved
# requests have text/image tokens between the per-request V and A blocks.
# combined_start/end encompass all V/A tokens, so num_video + num_audio
# equals the number of V/A tokens in range; compare directly to span size.
combined_start = min(video_pos[0].item(), audio_pos[0].item())
combined_end = max(video_pos[-1].item(), audio_pos[-1].item())
total_in_range = combined_end - combined_start + 1
return (num_video + num_audio) == total_in_range
span_start = 0
for span_end in range(1, len(va_pos) + 1):
is_last = span_end == len(va_pos)
if not is_last and va_pos[span_end].item() == va_pos[span_end - 1].item() + 1:
continue
span = va_pos[span_start:span_end]
span_is_video = is_video[span]
span_is_audio = is_audio[span]
if span_is_video.any() and span_is_audio.any():
video_offsets = span_is_video.nonzero(as_tuple=True)[0]
audio_offsets = span_is_audio.nonzero(as_tuple=True)[0]
if (
video_offsets[0].item() < audio_offsets[-1].item()
and audio_offsets[0].item() < video_offsets[-1].item()
):
return True
span_start = span_end
return False
def merge_interleaved_embeddings(
@@ -182,58 +184,69 @@ def merge_interleaved_embeddings(
is_video: torch.Tensor,
is_audio: torch.Tensor,
is_multimodal: torch.Tensor,
num_video: int,
num_audio: int,
) -> torch.Tensor:
"""
Merge embeddings for interleaved audio-in-video sequences.
When use_audio_in_video=True, video and audio tokens are interleaved in
the token sequence, but embeddings are provided as separate contiguous
tensors (video first, then audio). This function reorders video and audio
embeddings to match sequence position order and scatters them efficiently.
tensors. This function scatters each modality by the ``modality`` attribute
attached to each embedding tensor (set during encoder gather) and also
supports image embeddings in the same interleaved request.
Args:
inputs_embeds: The input embeddings tensor to merge into.
multimodal_embeddings: List of embedding tensors (video, audio, other).
multimodal_embeddings: List of embedding tensors (video, audio, image).
is_video: Boolean mask for video token positions.
is_audio: Boolean mask for audio token positions.
is_multimodal: Boolean mask for all multimodal token positions.
num_video: Total count of video tokens.
num_audio: Total count of audio tokens.
Returns:
The merged inputs_embeds tensor with multimodal embeddings scattered
to their correct positions.
"""
# Categorize embeddings by modality based on token counts.
# Embeddings come grouped by modality but order varies (e.g., image, video, audio
# or video, audio depending on input kwargs order).
video_embeds: list[torch.Tensor] = []
audio_embeds: list[torch.Tensor] = []
other_embeds: list[torch.Tensor] = []
video_remaining = num_video
audio_remaining = num_audio
from vllm.multimodal.utils import get_mm_embedding_modalities
for emb in multimodal_embeddings:
n = emb.shape[0]
if video_remaining > 0 and n <= video_remaining:
video_embeds.append(emb)
video_remaining -= n
elif audio_remaining > 0 and n <= audio_remaining:
audio_embeds.append(emb)
audio_remaining -= n
else:
other_embeds.append(emb)
def _merge_embedding_group(
mask: torch.Tensor,
embeddings: Sequence[torch.Tensor],
modality: str,
) -> None:
num_expected_tokens = mask.sum().item()
num_actual_tokens = sum(embedding.shape[0] for embedding in embeddings)
if num_actual_tokens != num_expected_tokens:
raise ValueError(
f"Attempted to assign {num_actual_tokens} {modality} tokens "
f"to {num_expected_tokens} placeholders"
)
if embeddings:
inputs_embeds[mask] = torch.cat(list(embeddings), dim=0)
# Scatter each modality to its positions
if video_embeds:
inputs_embeds[is_video] = torch.cat(video_embeds, dim=0)
if audio_embeds:
inputs_embeds[is_audio] = torch.cat(audio_embeds, dim=0)
if other_embeds:
other_mask = is_multimodal & ~is_video & ~is_audio
inputs_embeds[other_mask] = torch.cat(other_embeds, dim=0)
# Qwen Omni multimodal placeholders are image/video/audio; after excluding
# video and audio positions, the remaining multimodal positions are images.
is_image = is_multimodal & ~is_video & ~is_audio
embedding_modalities = get_mm_embedding_modalities(multimodal_embeddings)
video_embeds = [
embedding
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
if modality == "video"
]
audio_embeds = [
embedding
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
if modality == "audio"
]
image_embeds = [
embedding
for embedding, modality in zip(multimodal_embeddings, embedding_modalities)
if modality == "image"
]
_merge_embedding_group(is_video, video_embeds, "video")
_merge_embedding_group(is_audio, audio_embeds, "audio")
_merge_embedding_group(is_image, image_embeds, "image")
return inputs_embeds
@@ -1497,8 +1510,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
# Default: standard merge (no interleaving), same as parent class
@@ -1833,9 +1833,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
# both the deepstack path and the final embedding merge.
video_token_id = self.config.video_token_id
audio_token_id = self.config.audio_token_id
image_token_id = self.config.image_token_id
input_ids_cpu = input_ids.cpu()
is_video = is_multimodal & (input_ids_cpu == video_token_id)
is_audio = is_multimodal & (input_ids_cpu == audio_token_id)
is_image = is_multimodal & (input_ids_cpu == image_token_id)
num_video = is_video.sum().item()
num_audio = is_audio.sum().item()
@@ -1856,9 +1858,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
multimodal_embeddings_multiscale = []
if is_interleaved:
# Use input_ids-based mask for correct vision positions
# when audio and video tokens are interleaved.
is_vision = is_video.clone()
# Use input_ids-based mask for all vision positions when
# audio and video tokens are interleaved.
is_vision = is_video | is_image
else:
is_vision = torch.zeros_like(is_multimodal)
mm_positions = torch.nonzero(is_multimodal, as_tuple=True)[0]
@@ -1917,8 +1919,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
is_video,
is_audio,
is_multimodal,
num_video,
num_audio,
)
# Default: standard merge (no interleaving), same as parent class.