[MM] Remove text components from ProcessorInputs (#53093)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-20 20:42:53 -07:00
committed by GitHub
parent 2adf4b9e2a
commit b00f475f09
92 changed files with 220 additions and 806 deletions
@@ -350,24 +350,21 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Run the HF processor and unpad inputs."""
try:
hf_inputs = super()._call_hf_processor(
prompt, mm_data, mm_kwargs, tok_kwargs
)
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
except ValueError:
if any(mm_data.values()):
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
prompt_ids = self.info.get_tokenizer().encode(prompt)
hf_inputs = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(hf_inputs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
return hf_inputs
def _unpad_audios(
@@ -375,7 +372,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
hf_inputs: "BatchFeature",
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> None:
"""Replace the audio fields with each audio processed on its own.
@@ -395,7 +391,7 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
dict(
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
),
dict(**mm_kwargs, **tok_kwargs),
mm_kwargs,
)
for audio in audios
]
@@ -459,13 +455,13 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask which modality owns each token of the expanded prompt, which is the
only marking of the tokens an expansion adds around an item."""
if any(mm_data.values()):
mm_data = {**mm_data, "return_mm_token_type_ids": True}
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
@@ -613,29 +609,26 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
tokenization_kwargs = inputs.tokenization_kwargs
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if not isinstance(prompt, str):
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
prompt = hf_processor.decode(prompt)
tokenization_kwargs = {
**tokenization_kwargs,
"add_special_tokens": False,
}
prompt = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
tokenization_kwargs=tokenization_kwargs,
hf_processor_mm_kwargs={
**hf_processor_mm_kwargs,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (`_call_hf_processor` disables `add_special_tokens`).
"add_special_tokens": False,
},
)
# Use overrides if provided; fallback to data-dependent hashing.
@@ -731,45 +724,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
return updates
def _apply_hf_processor_main(
self,
prompt: str | list[int],
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
*,
enable_hf_prompt_update: bool,
) -> tuple[list[int], "BatchFeature", bool]:
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
in, whatever `enable_hf_prompt_update` asks for.
This differs from `super()` only for a text prompt with
`enable_hf_prompt_update`, where `super()` would keep the expanded token
ids the HF processor returns. Both routes give the same ids, so forcing
this one costs an extra processor call and buys the guarantee that a
request bypassing the cache is identical to a cached one.
A text prompt only arrives with `enable_hf_prompt_update` when there is
no multi-modal processor cache. Requests carrying pre-computed
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
here, so they fail validation shortly afterwards.
"""
if isinstance(prompt, str) and enable_hf_prompt_update:
logger.warning_once(
"Disabling the multi-modal processor cache is extra slow with the "
"Transformers modeling backend: the prompt is still tokenized "
"unexpanded and the expansion spliced in, to keep the token ids "
"identical to what the cache produces, which costs an extra HF "
"processor call per request."
)
return super()._apply_hf_processor_main(
prompt,
mm_items,
hf_processor_mm_kwargs,
tokenization_kwargs,
enable_hf_prompt_update=False,
)
def _get_num_image_patches(
self,
hf_inputs: "BatchFeature",
@@ -824,13 +778,12 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> "BatchFeature":
"""Ask for the replacement each placeholder expands to, and record it as
per-item fields: its token ids, and the tokens or patches behind them."""
if has_mm_data := any(mm_data.values()):
mm_data = {**mm_data, "return_text_replacement_offsets": True}
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
# Drop the inputs the model would reject
hf_inputs.pop("mm_token_type_ids", None)
hf_inputs.pop("token_type_ids", None)