From 040700aaa6cae3ef54dad8ba75db1abfb79db449 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 22 Aug 2026 18:27:59 +0800 Subject: [PATCH] [MM] Address comments on #53275 (#53364) Signed-off-by: DarkLight1337 --- vllm/model_executor/models/nano_nemotron_vl.py | 2 +- vllm/model_executor/models/pixtral.py | 2 +- vllm/model_executor/models/voxtral.py | 2 +- vllm/multimodal/processing/processor.py | 15 ++++++++------- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 0e9eda5c82..00538fd7b3 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -734,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor( prompt_ids=prompt_ids, mm_kwargs=mm_info.kwargs, mm_prompt_updates=mm_info.prompt_updates, - is_update_applied=self.hf_processor_applies_updates, + is_update_applied=self.renderer_applies_updates, ) mm_placeholder_ranges = { diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index a24d2484ab..6a210a811b 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -212,7 +212,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]): class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): # The tokens are already inserted by the chat template - hf_processor_applies_updates = True + renderer_applies_updates = True def _get_mm_fields_config( self, diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 07a7f9c325..20d166ca2f 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -203,7 +203,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]): class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): # The tokens are already inserted by the chat template - hf_processor_applies_updates = True + renderer_applies_updates = True def _get_mm_fields_config( self, diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 17bcc2f6dd..1eda431a8a 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1177,9 +1177,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): Not to be confused with `transformers.ProcessorMixin`. """ - hf_processor_applies_updates: ClassVar[bool] = False + renderer_applies_updates: ClassVar[bool] = False """ - Only set to True if the tokenizer or chat template + Only set to True if the tokenizer or chat template in the Renderer (which are run before MM processing) inserts placeholder tokens. """ @@ -1811,12 +1811,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): The main steps are: - 1. Apply HF Processor on prompt text and multi-modal data together, - outputting token IDs and processed tensors. - 2. Find and update sequences in the token IDs with placeholder tokens. + 1. Perform MM-specific post-processing to the token inputs. + 2. Apply HF Processor on multi-modal data together with optional + dummy text, outputting processed multi-modal data tensors. + 3. Find and update sequences in the token IDs with placeholder tokens. The number of placeholder tokens equals the feature size of the multi-modal data outputted by the multi-modal encoder. - 3. Extract information about the placeholder tokens from the + 4. Extract information about the placeholder tokens from the processed token IDs. """ prompt_ids = self._postprocess_prompt(inputs.prompt) @@ -1828,7 +1829,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): prompt_ids=prompt_ids, mm_kwargs=mm_info.kwargs, mm_prompt_updates=mm_info.prompt_updates, - is_update_applied=self.hf_processor_applies_updates, + is_update_applied=self.renderer_applies_updates, ) mm_placeholder_ranges = {