[MM] Address comments on #53275 (#53364)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 10:27:59 +00:00
committed by GitHub
parent 0b19ebcacd
commit 040700aaa6
4 changed files with 11 additions and 10 deletions
@@ -734,7 +734,7 @@ class NanoNemotronVLMultiModalProcessor(
prompt_ids=prompt_ids, prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs, mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates, mm_prompt_updates=mm_info.prompt_updates,
is_update_applied=self.hf_processor_applies_updates, is_update_applied=self.renderer_applies_updates,
) )
mm_placeholder_ranges = { mm_placeholder_ranges = {
+1 -1
View File
@@ -212,7 +212,7 @@ class PixtralDummyInputsBuilder(BaseDummyInputsBuilder[PixtralProcessingInfo]):
class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]): class PixtralMultiModalProcessor(BaseMultiModalProcessor[PixtralProcessingInfo]):
# The tokens are already inserted by the chat template # The tokens are already inserted by the chat template
hf_processor_applies_updates = True renderer_applies_updates = True
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+1 -1
View File
@@ -203,7 +203,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]): class VoxtralMultiModalProcessor(BaseMultiModalProcessor[VoxtralProcessingInfo]):
# The tokens are already inserted by the chat template # The tokens are already inserted by the chat template
hf_processor_applies_updates = True renderer_applies_updates = True
def _get_mm_fields_config( def _get_mm_fields_config(
self, self,
+8 -7
View File
@@ -1177,9 +1177,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
Not to be confused with `transformers.ProcessorMixin`. Not to be confused with `transformers.ProcessorMixin`.
""" """
hf_processor_applies_updates: ClassVar[bool] = False renderer_applies_updates: ClassVar[bool] = False
""" """
Only set to True if the tokenizer or chat template Only set to True if the tokenizer or chat template in the Renderer
(which are run before MM processing) inserts placeholder tokens. (which are run before MM processing) inserts placeholder tokens.
""" """
@@ -1811,12 +1811,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
The main steps are: The main steps are:
1. Apply HF Processor on prompt text and multi-modal data together, 1. Perform MM-specific post-processing to the token inputs.
outputting token IDs and processed tensors. 2. Apply HF Processor on multi-modal data together with optional
2. Find and update sequences in the token IDs with placeholder tokens. dummy text, outputting processed multi-modal data tensors.
3. Find and update sequences in the token IDs with placeholder tokens.
The number of placeholder tokens equals the feature size of the The number of placeholder tokens equals the feature size of the
multi-modal data outputted by the multi-modal encoder. multi-modal data outputted by the multi-modal encoder.
3. Extract information about the placeholder tokens from the 4. Extract information about the placeholder tokens from the
processed token IDs. processed token IDs.
""" """
prompt_ids = self._postprocess_prompt(inputs.prompt) prompt_ids = self._postprocess_prompt(inputs.prompt)
@@ -1828,7 +1829,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
prompt_ids=prompt_ids, prompt_ids=prompt_ids,
mm_kwargs=mm_info.kwargs, mm_kwargs=mm_info.kwargs,
mm_prompt_updates=mm_info.prompt_updates, mm_prompt_updates=mm_info.prompt_updates,
is_update_applied=self.hf_processor_applies_updates, is_update_applied=self.renderer_applies_updates,
) )
mm_placeholder_ranges = { mm_placeholder_ranges = {