diff --git a/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py b/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py index 4eb4d03bfe..af12a14a45 100644 --- a/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py +++ b/tests/models/multimodal/processing/test_qwen2_5_omni_embed.py @@ -19,6 +19,7 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import ( check_interleaved_audio_video, merge_interleaved_embeddings, ) +from vllm.multimodal.utils import set_mm_embedding_modality # Fake token IDs AUDIO_TOKEN_ID = 1001 @@ -27,6 +28,10 @@ VIDEO_TOKEN_ID = 1003 TEXT_TOKEN_ID = 0 +def _mm_embed(shape: tuple[int, ...], value: float, modality: str) -> torch.Tensor: + return set_mm_embedding_modality(torch.full(shape, value), modality) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -116,6 +121,24 @@ class TestCheckInterleavedAudioVideo: is_video, is_audio, is_video.sum().item(), is_audio.sum().item() ) + def test_multi_video_with_boundary_tokens(self): + """Two interleaved videos separated by boundary tokens → still True. + + use_audio_in_video expands each video into a local V/A span bounded by + non-pad tokens. A global density check would fail across those gaps. + """ + # [text][V A V A][text boundary][V A V A] + first_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=1) + second_ids, _ = make_interleaved_seq([2, 2], [2, 2], text_prefix=0) + boundary = torch.tensor([TEXT_TOKEN_ID, TEXT_TOKEN_ID]) + input_ids = torch.cat([first_ids, boundary, second_ids]) + is_multimodal = (input_ids == VIDEO_TOKEN_ID) | (input_ids == AUDIO_TOKEN_ID) + is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID) + is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID) + assert check_interleaved_audio_video( + is_video, is_audio, is_video.sum().item(), is_audio.sum().item() + ) + def test_batched_non_interleaved_no_false_positive(self): """ Regression test for https://github.com/vllm-project/vllm/issues/35394. @@ -323,13 +346,16 @@ class TestEmbedInputIds: # mm_embeds come in [video, audio] order (video feature first in # mm_features when positions are the same for use_audio_in_video) mm_embeds = [ - torch.full((video_n, hidden), video_val), - torch.full((audio_n, hidden), audio_val), + _mm_embed((video_n, hidden), video_val, "video"), + _mm_embed((audio_n, hidden), audio_val, "audio"), ] model, _ = make_mock_model(hidden) + # Modalities are attached on the embedding tensors (as in encoder gather). result = model.embed_input_ids( - input_ids, mm_embeds, is_multimodal=is_multimodal + input_ids, + mm_embeds, + is_multimodal=is_multimodal, ) video_pos = (input_ids == VIDEO_TOKEN_ID).nonzero(as_tuple=True)[0] @@ -362,8 +388,8 @@ class TestMergeInterleavedEmbeddings: inputs_embeds = torch.zeros(len(input_ids), hidden) mm_embeds = [ - torch.full((num_video, hidden), 30.0), - torch.full((num_audio, hidden), 10.0), + _mm_embed((num_video, hidden), 30.0, "video"), + _mm_embed((num_audio, hidden), 10.0, "audio"), ] result = merge_interleaved_embeddings( @@ -372,8 +398,6 @@ class TestMergeInterleavedEmbeddings: is_video, is_audio, is_multimodal, - num_video, - num_audio, ) video_pos = is_video.nonzero(as_tuple=True)[0] @@ -381,6 +405,62 @@ class TestMergeInterleavedEmbeddings: assert result[video_pos].allclose(torch.full((num_video, hidden), 30.0)) assert result[audio_pos].allclose(torch.full((num_audio, hidden), 10.0)) + def test_image_and_video_mixed(self): + """Image embeddings must not be misclassified as video.""" + hidden = 4 + # [text][I I][V A V A] + tokens = ( + [TEXT_TOKEN_ID] * 2 + + [IMAGE_TOKEN_ID] * 2 + + [VIDEO_TOKEN_ID, AUDIO_TOKEN_ID, VIDEO_TOKEN_ID, AUDIO_TOKEN_ID] + ) + input_ids = torch.tensor(tokens) + is_multimodal = ( + (input_ids == IMAGE_TOKEN_ID) + | (input_ids == VIDEO_TOKEN_ID) + | (input_ids == AUDIO_TOKEN_ID) + ) + is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID) + is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID) + is_image = is_multimodal & (input_ids == IMAGE_TOKEN_ID) + + inputs_embeds = torch.zeros(len(input_ids), hidden) + mm_embeds = [ + _mm_embed((2, hidden), 20.0, "image"), + _mm_embed((2, hidden), 30.0, "video"), + _mm_embed((2, hidden), 10.0, "audio"), + ] + result = merge_interleaved_embeddings( + inputs_embeds, + mm_embeds, + is_video, + is_audio, + is_multimodal, + ) + assert result[is_image.nonzero(as_tuple=True)[0]].allclose( + torch.full((2, hidden), 20.0) + ) + assert result[is_video.nonzero(as_tuple=True)[0]].allclose( + torch.full((2, hidden), 30.0) + ) + assert result[is_audio.nonzero(as_tuple=True)[0]].allclose( + torch.full((2, hidden), 10.0) + ) + + def test_missing_modality_raises(self): + hidden = 2 + input_ids, is_multimodal = make_interleaved_seq([2], [2]) + is_video = is_multimodal & (input_ids == VIDEO_TOKEN_ID) + is_audio = is_multimodal & (input_ids == AUDIO_TOKEN_ID) + with pytest.raises(ValueError, match="Missing modality"): + merge_interleaved_embeddings( + torch.zeros(len(input_ids), hidden), + [torch.zeros(2, hidden), torch.zeros(2, hidden)], + is_video, + is_audio, + is_multimodal, + ) + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/v1/worker/test_encoder_runner.py b/tests/v1/worker/test_encoder_runner.py index 70c0426640..2cacc20afb 100644 --- a/tests/v1/worker/test_encoder_runner.py +++ b/tests/v1/worker/test_encoder_runner.py @@ -78,6 +78,7 @@ def test_draft_lookahead_uses_boundary_feature_when_cached(): # f0 covers positions 0..6 (+1 skew); f1's first embed covers position 7. assert len(mm_embeds) == 2 + assert [e.modality for e in mm_embeds] == ["image", "image"] assert bool(is_mm_embed[7]) assert int(is_mm_embed.sum()) == 8 @@ -94,6 +95,7 @@ def test_draft_lookahead_tolerates_missing_boundary_feature(): # Only f0 is gathered; f1's boundary position falls back silently. assert len(mm_embeds) == 1 + assert [e.modality for e in mm_embeds] == ["image"] assert not bool(is_mm_embed[7]) assert int(is_mm_embed.sum()) == 7 @@ -154,4 +156,28 @@ def test_multi_request_batch_gathers_per_request(draft_lookahead): # Both requests contribute a feature; with the +1 skew each marks 7 of its # 8 positions (the skew drops one), otherwise all 8. assert len(mm_embeds) == 2 + assert [e.modality for e in mm_embeds] == ["image", "image"] assert int(is_mm_embed.sum()) == (14 if draft_lookahead else 16) + + +def test_gather_preserves_mixed_modalities(): + """Modalities must be attached on tensors in gather order.""" + video = MultiModalFeatureSpec( + data=None, + modality="video", + identifier="v0", + mm_position=PlaceholderRange(offset=0, length=4), + ) + audio = MultiModalFeatureSpec( + data=None, + modality="audio", + identifier="a0", + mm_position=PlaceholderRange(offset=4, length=4), + ) + runner = _make_runner([video, audio], cached=[video, audio]) + + mm_embeds, is_mm_embed = _gather(runner, num_scheduled=8, draft_lookahead=0) + + assert len(mm_embeds) == 2 + assert [e.modality for e in mm_embeds] == ["video", "audio"] + assert int(is_mm_embed.sum()) == 8 diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 9a93a6065a..00382cf0a8 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -138,42 +138,44 @@ def check_interleaved_audio_video( num_audio: int, ) -> bool: """ - Check if video and audio positions are interleaved in the multimodal region. + Check if video and audio positions are interleaved in any per-video span. - Returns True only for the use_audio_in_video=True case, where video and - audio tokens alternate within a single contiguous region with no gaps. - - A simple range-overlap check produces false positives when multiple - non-interleaved requests are batched together: audio tokens from request N - fall between video tokens from request N and request N+1, making the - global ranges overlap even though each individual request is non-interleaved. - - To distinguish true interleaving from this batching artefact we require - that every position in the combined [first_VA, last_VA] range is occupied - by either a video or an audio token (no text/image gaps). + For use_audio_in_video=True, each video placeholder is expanded into one + local span containing only video/audio pad tokens, bounded by non-pad + tokens such as audio_start/audio_end. Check each contiguous V/A span + independently instead of requiring all V/A tokens in the whole sequence to + form one global dense range; otherwise multi-video requests can be + misclassified as non-interleaved because of the boundary tokens between + videos. """ if num_video == 0 or num_audio == 0: return False - video_pos = is_video.nonzero(as_tuple=True)[0] - audio_pos = is_audio.nonzero(as_tuple=True)[0] - - # Quick range-overlap pre-check (necessary but not sufficient). - if not ( - video_pos[0].item() < audio_pos[-1].item() - and audio_pos[0].item() < video_pos[-1].item() - ): + va_pos = (is_video | is_audio).nonzero(as_tuple=True)[0] + if len(va_pos) == 0: return False - # Density check: for true use_audio_in_video interleaving every position - # in the combined span is a video or audio token. Batched non-interleaved - # requests have text/image tokens between the per-request V and A blocks. - # combined_start/end encompass all V/A tokens, so num_video + num_audio - # equals the number of V/A tokens in range; compare directly to span size. - combined_start = min(video_pos[0].item(), audio_pos[0].item()) - combined_end = max(video_pos[-1].item(), audio_pos[-1].item()) - total_in_range = combined_end - combined_start + 1 - return (num_video + num_audio) == total_in_range + span_start = 0 + for span_end in range(1, len(va_pos) + 1): + is_last = span_end == len(va_pos) + if not is_last and va_pos[span_end].item() == va_pos[span_end - 1].item() + 1: + continue + + span = va_pos[span_start:span_end] + span_is_video = is_video[span] + span_is_audio = is_audio[span] + if span_is_video.any() and span_is_audio.any(): + video_offsets = span_is_video.nonzero(as_tuple=True)[0] + audio_offsets = span_is_audio.nonzero(as_tuple=True)[0] + if ( + video_offsets[0].item() < audio_offsets[-1].item() + and audio_offsets[0].item() < video_offsets[-1].item() + ): + return True + + span_start = span_end + + return False def merge_interleaved_embeddings( @@ -182,58 +184,69 @@ def merge_interleaved_embeddings( is_video: torch.Tensor, is_audio: torch.Tensor, is_multimodal: torch.Tensor, - num_video: int, - num_audio: int, ) -> torch.Tensor: """ Merge embeddings for interleaved audio-in-video sequences. When use_audio_in_video=True, video and audio tokens are interleaved in the token sequence, but embeddings are provided as separate contiguous - tensors (video first, then audio). This function reorders video and audio - embeddings to match sequence position order and scatters them efficiently. + tensors. This function scatters each modality by the ``modality`` attribute + attached to each embedding tensor (set during encoder gather) and also + supports image embeddings in the same interleaved request. Args: inputs_embeds: The input embeddings tensor to merge into. - multimodal_embeddings: List of embedding tensors (video, audio, other). + multimodal_embeddings: List of embedding tensors (video, audio, image). is_video: Boolean mask for video token positions. is_audio: Boolean mask for audio token positions. is_multimodal: Boolean mask for all multimodal token positions. - num_video: Total count of video tokens. - num_audio: Total count of audio tokens. Returns: The merged inputs_embeds tensor with multimodal embeddings scattered to their correct positions. """ - # Categorize embeddings by modality based on token counts. - # Embeddings come grouped by modality but order varies (e.g., image, video, audio - # or video, audio depending on input kwargs order). - video_embeds: list[torch.Tensor] = [] - audio_embeds: list[torch.Tensor] = [] - other_embeds: list[torch.Tensor] = [] - video_remaining = num_video - audio_remaining = num_audio + from vllm.multimodal.utils import get_mm_embedding_modalities - for emb in multimodal_embeddings: - n = emb.shape[0] - if video_remaining > 0 and n <= video_remaining: - video_embeds.append(emb) - video_remaining -= n - elif audio_remaining > 0 and n <= audio_remaining: - audio_embeds.append(emb) - audio_remaining -= n - else: - other_embeds.append(emb) + def _merge_embedding_group( + mask: torch.Tensor, + embeddings: Sequence[torch.Tensor], + modality: str, + ) -> None: + num_expected_tokens = mask.sum().item() + num_actual_tokens = sum(embedding.shape[0] for embedding in embeddings) + if num_actual_tokens != num_expected_tokens: + raise ValueError( + f"Attempted to assign {num_actual_tokens} {modality} tokens " + f"to {num_expected_tokens} placeholders" + ) + if embeddings: + inputs_embeds[mask] = torch.cat(list(embeddings), dim=0) - # Scatter each modality to its positions - if video_embeds: - inputs_embeds[is_video] = torch.cat(video_embeds, dim=0) - if audio_embeds: - inputs_embeds[is_audio] = torch.cat(audio_embeds, dim=0) - if other_embeds: - other_mask = is_multimodal & ~is_video & ~is_audio - inputs_embeds[other_mask] = torch.cat(other_embeds, dim=0) + # Qwen Omni multimodal placeholders are image/video/audio; after excluding + # video and audio positions, the remaining multimodal positions are images. + is_image = is_multimodal & ~is_video & ~is_audio + + embedding_modalities = get_mm_embedding_modalities(multimodal_embeddings) + + video_embeds = [ + embedding + for embedding, modality in zip(multimodal_embeddings, embedding_modalities) + if modality == "video" + ] + audio_embeds = [ + embedding + for embedding, modality in zip(multimodal_embeddings, embedding_modalities) + if modality == "audio" + ] + image_embeds = [ + embedding + for embedding, modality in zip(multimodal_embeddings, embedding_modalities) + if modality == "image" + ] + + _merge_embedding_group(is_video, video_embeds, "video") + _merge_embedding_group(is_audio, audio_embeds, "audio") + _merge_embedding_group(is_image, image_embeds, "image") return inputs_embeds @@ -1497,8 +1510,6 @@ class Qwen2_5OmniThinkerForConditionalGeneration( is_video, is_audio, is_multimodal, - num_video, - num_audio, ) # Default: standard merge (no interleaving), same as parent class diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 726329c780..32a622567c 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1833,9 +1833,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( # both the deepstack path and the final embedding merge. video_token_id = self.config.video_token_id audio_token_id = self.config.audio_token_id + image_token_id = self.config.image_token_id input_ids_cpu = input_ids.cpu() is_video = is_multimodal & (input_ids_cpu == video_token_id) is_audio = is_multimodal & (input_ids_cpu == audio_token_id) + is_image = is_multimodal & (input_ids_cpu == image_token_id) num_video = is_video.sum().item() num_audio = is_audio.sum().item() @@ -1856,9 +1858,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( multimodal_embeddings_multiscale = [] if is_interleaved: - # Use input_ids-based mask for correct vision positions - # when audio and video tokens are interleaved. - is_vision = is_video.clone() + # Use input_ids-based mask for all vision positions when + # audio and video tokens are interleaved. + is_vision = is_video | is_image else: is_vision = torch.zeros_like(is_multimodal) mm_positions = torch.nonzero(is_multimodal, as_tuple=True)[0] @@ -1917,8 +1919,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( is_video, is_audio, is_multimodal, - num_video, - num_audio, ) # Default: standard merge (no interleaving), same as parent class. diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index 56c65dc8ea..f34769d838 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -346,3 +346,41 @@ def fetch_video( allowed_local_media_path="/", ) return media_connector.fetch_video(video_url) + + +def set_mm_embedding_modality(embed: "torch.Tensor", modality: str) -> "torch.Tensor": + """Attach modality metadata to a gathered multimodal embedding tensor. + + Used by interleaved Omni merge paths that need to group embeddings by + modality without threading a parallel modalities list through + ``embed_input_ids``. + """ + embed.modality = modality # type: ignore[attr-defined] + return embed + + +def copy_mm_embedding_modality( + src: "torch.Tensor", dst: "torch.Tensor" +) -> "torch.Tensor": + """Copy ``modality`` from ``src`` onto ``dst`` if present.""" + modality = getattr(src, "modality", None) + if modality is not None: + dst.modality = modality # type: ignore[attr-defined] + return dst + + +def get_mm_embedding_modalities( + multimodal_embeddings: Sequence["torch.Tensor"], +) -> list[str]: + """Collect per-embedding modalities previously set on the tensors.""" + modalities: list[str] = [] + for i, emb in enumerate(multimodal_embeddings): + modality = getattr(emb, "modality", None) + if modality is None: + raise ValueError( + f"Missing modality on multimodal embedding at index {i}. " + "Encoder gather must set embed.modality before interleaved " + "audio-in-video merge." + ) + modalities.append(modality) + return modalities diff --git a/vllm/v1/worker/gpu/mm/encoder_runner.py b/vllm/v1/worker/gpu/mm/encoder_runner.py index 48a3af2505..c337ddd4aa 100644 --- a/vllm/v1/worker/gpu/mm/encoder_runner.py +++ b/vllm/v1/worker/gpu/mm/encoder_runner.py @@ -5,7 +5,11 @@ import torch from vllm.model_executor.models.interfaces import SupportsMultiModal, supports_realtime from vllm.multimodal.inputs import MultiModalKwargsItem -from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs +from vllm.multimodal.utils import ( + get_mm_features_in_window, + group_and_batch_mm_kwargs, + set_mm_embedding_modality, +) from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.utils import sanity_check_mm_encoder_outputs @@ -136,6 +140,9 @@ class EncoderRunner: else: mm_embeds_item = encoder_output[start_idx:end_idx] + # Attach modality for Omni interleaved merge (collected on demand). + set_mm_embedding_modality(mm_embeds_item, mm_feature.modality) + req_start_pos = query_start_loc[i] + start_pos - cur_query_start is_mm_embed[req_start_pos + start_idx : req_start_pos + end_idx] |= ( True if is_embed is None else is_embed diff --git a/vllm/v1/worker/gpu/model_states/mm_pruning.py b/vllm/v1/worker/gpu/model_states/mm_pruning.py index 781baa6d15..8bdb85d4a1 100644 --- a/vllm/v1/worker/gpu/model_states/mm_pruning.py +++ b/vllm/v1/worker/gpu/model_states/mm_pruning.py @@ -5,7 +5,10 @@ import torch.nn as nn from vllm.config import ModelConfig from vllm.model_executor.models.interfaces import supports_multimodal_pruning -from vllm.multimodal.utils import get_mm_features_in_window +from vllm.multimodal.utils import ( + copy_mm_embedding_modality, + get_mm_features_in_window, +) from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.rope import RopeState @@ -42,7 +45,11 @@ class MultiModalPruner: speculator reuses the target's already-recomputed positions, hence there is no position write-back here. """ - return [mm[:, : self.inputs_embeds_size] for mm in mm_embeds] + stripped: list[torch.Tensor] = [] + for mm in mm_embeds: + out = mm[:, : self.inputs_embeds_size] + stripped.append(copy_mm_embedding_modality(mm, out)) + return stripped def recompute( self, @@ -82,7 +89,10 @@ class MultiModalPruner: num_computed_tokens=num_computed, ) self.rope_state.update_prefill_positions(req_idx, new_positions, delta) - cleaned.extend(req_cleaned) + cleaned.extend( + copy_mm_embedding_modality(src, dst) + for src, dst in zip(req_embeds, req_cleaned) + ) assert pos == len(mm_embeds) return cleaned diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 5039979b29..79a905adad 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -105,7 +105,12 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItem, PlaceholderRange, ) -from vllm.multimodal.utils import get_mm_features_in_window, group_and_batch_mm_kwargs +from vllm.multimodal.utils import ( + copy_mm_embedding_modality, + get_mm_features_in_window, + group_and_batch_mm_kwargs, + set_mm_embedding_modality, +) from vllm.platforms import current_platform from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingType @@ -3245,11 +3250,13 @@ class GPUModelRunner( is_mm_embed[ req_start_pos + start_idx : req_start_pos + end_idx ] |= is_embed + set_mm_embedding_modality(mm_embeds_item, mm_feature.modality) mm_embeds_req.append(mm_embeds_item) if self.is_multimodal_pruning_enabled and self.uses_mrope: assert req_state.mrope_positions is not None should_sync_mrope_positions = True + old_mm_embeds_req = mm_embeds_req mm_embeds_req, new_mrope_positions, new_delta = ( self.model.recompute_mrope_positions( input_ids=req_state.prompt_token_ids, @@ -3258,6 +3265,10 @@ class GPUModelRunner( num_computed_tokens=req_state.num_computed_tokens, ) ) + mm_embeds_req = [ + copy_mm_embedding_modality(src, dst) + for src, dst in zip(old_mm_embeds_req, mm_embeds_req) + ] req_state.mrope_positions.copy_(new_mrope_positions) req_state.mrope_position_delta = new_delta