diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 00538fd7b3..f08cf5e806 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -734,7 +734,6 @@ class NanoNemotronVLMultiModalProcessor( prompt_ids=prompt_ids, mm_kwargs=mm_info.kwargs, mm_prompt_updates=mm_info.prompt_updates, - is_update_applied=self.renderer_applies_updates, ) mm_placeholder_ranges = { diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 6a210a811b..869b7001c8 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -40,6 +40,7 @@ from vllm.model_executor.models.utils import WeightsMapper from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems from vllm.multimodal.inputs import ( MultiModalFieldConfig, + MultiModalKwargsOptionalItems, NestedTensors, ) from vllm.multimodal.parse import ( @@ -51,6 +52,8 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, + MultiModalPromptUpdates, + PlaceholderFeaturesInfo, ProcessorInputs, PromptReplacement, PromptUpdate, @@ -211,8 +214,26 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): - # The tokens are already inserted by the chat template - renderer_applies_updates = True + # The tokens are already inserted by the chat template, + # so we just double check that they exist + def _maybe_apply_prompt_updates( + self, + mm_items: MultiModalDataItems, + prompt_ids: list[int], + mm_kwargs: MultiModalKwargsOptionalItems, + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + mm_item_counts = mm_items.get_all_counts() + self._validate_mm_kwargs(mm_kwargs, mm_item_counts) + self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + + mm_placeholders = self._find_mm_placeholders( + prompt_ids, + mm_prompt_updates, + ) + self._validate_mm_placeholders(mm_placeholders, mm_item_counts) + + return prompt_ids, mm_placeholders def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 1f335cfc24..58df408a5e 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -575,7 +575,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor( prompt_ids: list[int], mm_kwargs: MultiModalKwargsItems, mm_prompt_updates: MultiModalPromptUpdates, - is_update_applied: bool, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: """ Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`. @@ -605,39 +604,26 @@ class Qwen2_5OmniThinkerMultiModalProcessor( if video_audio_item_num != video_updates_num + audio_updates_num: use_audio_in_video = True - if is_update_applied: - mm_placeholders = self._find_mm_placeholders( + if use_audio_in_video and "audio" in mm_prompt_updates: + # Filter out audio updates - they are embedded in video + filtered_updates = { + k: v for k, v in mm_prompt_updates.items() if k != "audio" + } + prompt_ids, mm_placeholders = self._apply_prompt_updates( + prompt_ids, + filtered_updates, + ) + # Derive audio placeholders from video placeholders + mm_placeholders = self._derive_audio_from_video_placeholders( + mm_placeholders, mm_prompt_updates + ) + else: + prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, mm_prompt_updates, ) - self._validate_mm_placeholders( - mm_placeholders, - mm_item_counts, - ) - else: - if use_audio_in_video and "audio" in mm_prompt_updates: - # Filter out audio updates - they are embedded in video - filtered_updates = { - k: v for k, v in mm_prompt_updates.items() if k != "audio" - } - prompt_ids, mm_placeholders = self._apply_prompt_updates( - prompt_ids, - filtered_updates, - ) - # Derive audio placeholders from video placeholders - mm_placeholders = self._derive_audio_from_video_placeholders( - mm_placeholders, mm_prompt_updates - ) - else: - prompt_ids, mm_placeholders = self._apply_prompt_updates( - prompt_ids, - mm_prompt_updates, - ) - self._validate_mm_placeholders( - mm_placeholders, - mm_item_counts, - ) + self._validate_mm_placeholders(mm_placeholders, mm_item_counts) return prompt_ids, mm_placeholders diff --git a/vllm/model_executor/models/qwen3_asr_realtime.py b/vllm/model_executor/models/qwen3_asr_realtime.py index 6405c101b4..917bfce134 100644 --- a/vllm/model_executor/models/qwen3_asr_realtime.py +++ b/vllm/model_executor/models/qwen3_asr_realtime.py @@ -116,7 +116,6 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor): prompt_ids: list[int], mm_kwargs: MultiModalKwargsOptionalItems, mm_prompt_updates: MultiModalPromptUpdates, - is_update_applied: bool, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: audios = mm_kwargs.get("audio", []) assert len(audios) == 1, ( diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index a3691742ff..0e2aab634c 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1291,7 +1291,6 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( prompt_ids: list[int], mm_kwargs: MultiModalKwargsItems, mm_prompt_updates: MultiModalPromptUpdates, - is_update_applied: bool, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: """ Qwen3-Omni reimplements this function to handle `use_audio_in_video`. @@ -1318,39 +1317,25 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( if video_audio_item_num != video_updates_num + audio_updates_num: use_audio_in_video = True - # normal case with `use_audio_in_video=False` - if is_update_applied: - mm_placeholders = self._find_mm_placeholders( + if use_audio_in_video and "audio" in mm_prompt_updates: + filtered_updates = { + k: v for k, v in mm_prompt_updates.items() if k != "audio" + } + prompt_ids, mm_placeholders = self._apply_prompt_updates( + prompt_ids, + filtered_updates, + ) + # Derive audio placeholders from video placeholders + mm_placeholders = self._derive_audio_from_video_placeholders( + mm_placeholders, mm_prompt_updates + ) + else: + prompt_ids, mm_placeholders = self._apply_prompt_updates( prompt_ids, mm_prompt_updates, ) - self._validate_mm_placeholders( - mm_placeholders, - mm_item_counts, - ) - else: - if use_audio_in_video and "audio" in mm_prompt_updates: - filtered_updates = { - k: v for k, v in mm_prompt_updates.items() if k != "audio" - } - prompt_ids, mm_placeholders = self._apply_prompt_updates( - prompt_ids, - filtered_updates, - ) - # Derive audio placeholders from video placeholders - mm_placeholders = self._derive_audio_from_video_placeholders( - mm_placeholders, mm_prompt_updates - ) - else: - prompt_ids, mm_placeholders = self._apply_prompt_updates( - prompt_ids, - mm_prompt_updates, - ) - self._validate_mm_placeholders( - mm_placeholders, - mm_item_counts, - ) + self._validate_mm_placeholders(mm_placeholders, mm_item_counts) return prompt_ids, mm_placeholders diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 20d166ca2f..e078867af3 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -36,6 +36,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItems, + MultiModalKwargsOptionalItems, NestedTensors, ) from vllm.multimodal.parse import ( @@ -47,6 +48,7 @@ from vllm.multimodal.processing import BaseDummyInputsBuilder from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, BaseProcessingInfo, + MultiModalPromptUpdates, PlaceholderFeaturesInfo, ProcessorInputs, PromptReplacement, @@ -202,8 +204,26 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): - # The tokens are already inserted by the chat template - renderer_applies_updates = True + # The tokens are already inserted by the chat template, + # so we just double check that they exist + def _maybe_apply_prompt_updates( + self, + mm_items: MultiModalDataItems, + prompt_ids: list[int], + mm_kwargs: MultiModalKwargsOptionalItems, + mm_prompt_updates: MultiModalPromptUpdates, + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + mm_item_counts = mm_items.get_all_counts() + self._validate_mm_kwargs(mm_kwargs, mm_item_counts) + self._validate_mm_updates(mm_prompt_updates, mm_item_counts) + + mm_placeholders = self._find_mm_placeholders( + prompt_ids, + mm_prompt_updates, + ) + self._validate_mm_placeholders(mm_placeholders, mm_item_counts) + + return prompt_ids, mm_placeholders def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index b54a4dccba..738185a7ae 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -63,7 +63,6 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor): prompt_ids: list[int], mm_kwargs: MultiModalKwargsOptionalItems, mm_prompt_updates: MultiModalPromptUpdates, - is_update_applied: bool, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: # there are no placeholder audio tokens for streaming # so we need to build the place placeholder positions manually diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 1eda431a8a..dd8a088afa 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -8,7 +8,6 @@ from enum import Enum from functools import lru_cache from typing import ( TYPE_CHECKING, - ClassVar, Generic, NamedTuple, Protocol, @@ -1177,12 +1176,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Not to be confused with `transformers.ProcessorMixin`. """ - renderer_applies_updates: ClassVar[bool] = False - """ - Only set to True if the tokenizer or chat template in the Renderer - (which are run before MM processing) inserts placeholder tokens. - """ - def __init__( self, info: _I, @@ -1780,24 +1773,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): prompt_ids: list[int], mm_kwargs: MultiModalKwargsOptionalItems, mm_prompt_updates: MultiModalPromptUpdates, - is_update_applied: bool, ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: mm_item_counts = mm_items.get_all_counts() self._validate_mm_kwargs(mm_kwargs, mm_item_counts) self._validate_mm_updates(mm_prompt_updates, mm_item_counts) - if is_update_applied: - mm_placeholders = self._find_mm_placeholders( - prompt_ids, - mm_prompt_updates, - ) - self._validate_mm_placeholders(mm_placeholders, mm_item_counts) - else: - prompt_ids, mm_placeholders = self._apply_prompt_updates( - prompt_ids, - mm_prompt_updates, - ) - self._validate_mm_placeholders(mm_placeholders, mm_item_counts) + prompt_ids, mm_placeholders = self._apply_prompt_updates( + prompt_ids, + mm_prompt_updates, + ) + + self._validate_mm_placeholders(mm_placeholders, mm_item_counts) return prompt_ids, mm_placeholders @@ -1829,7 +1815,6 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): prompt_ids=prompt_ids, mm_kwargs=mm_info.kwargs, mm_prompt_updates=mm_info.prompt_updates, - is_update_applied=self.renderer_applies_updates, ) mm_placeholder_ranges = {