[MM] Remove text components from ProcessorInputs (#53093)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -350,24 +350,21 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Run the HF processor and unpad inputs."""
|
||||
try:
|
||||
hf_inputs = super()._call_hf_processor(
|
||||
prompt, mm_data, mm_kwargs, tok_kwargs
|
||||
)
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt, **tok_kwargs)
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs, tok_kwargs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
|
||||
return hf_inputs
|
||||
|
||||
def _unpad_audios(
|
||||
@@ -375,7 +372,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
hf_inputs: "BatchFeature",
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> None:
|
||||
"""Replace the audio fields with each audio processed on its own.
|
||||
|
||||
@@ -395,7 +391,7 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
dict(
|
||||
text=self.dummy_inputs.get_dummy_text({"audio": 1}), audio=[audio]
|
||||
),
|
||||
dict(**mm_kwargs, **tok_kwargs),
|
||||
mm_kwargs,
|
||||
)
|
||||
for audio in audios
|
||||
]
|
||||
@@ -459,13 +455,13 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask which modality owns each token of the expanded prompt, which is the
|
||||
only marking of the tokens an expansion adds around an item."""
|
||||
if any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
@@ -613,29 +609,26 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
tokenization_kwargs = inputs.tokenization_kwargs
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if not isinstance(prompt, str):
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
prompt = hf_processor.decode(prompt)
|
||||
tokenization_kwargs = {
|
||||
**tokenization_kwargs,
|
||||
"add_special_tokens": False,
|
||||
}
|
||||
prompt = hf_processor.decode(prompt)
|
||||
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
hf_processor_mm_kwargs={
|
||||
**hf_processor_mm_kwargs,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (`_call_hf_processor` disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
},
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
@@ -731,45 +724,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
return updates
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str | list[int],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
tokenization_kwargs: Mapping[str, object],
|
||||
*,
|
||||
enable_hf_prompt_update: bool,
|
||||
) -> tuple[list[int], "BatchFeature", bool]:
|
||||
"""Tokenize the prompt unexpanded, leaving the expansion for vLLM to splice
|
||||
in, whatever `enable_hf_prompt_update` asks for.
|
||||
|
||||
This differs from `super()` only for a text prompt with
|
||||
`enable_hf_prompt_update`, where `super()` would keep the expanded token
|
||||
ids the HF processor returns. Both routes give the same ids, so forcing
|
||||
this one costs an extra processor call and buys the guarantee that a
|
||||
request bypassing the cache is identical to a cached one.
|
||||
|
||||
A text prompt only arrives with `enable_hf_prompt_update` when there is
|
||||
no multi-modal processor cache. Requests carrying pre-computed
|
||||
embeddings bypass the cache too, but `--enable-mm-embeds` is unsupported
|
||||
here, so they fail validation shortly afterwards.
|
||||
"""
|
||||
if isinstance(prompt, str) and enable_hf_prompt_update:
|
||||
logger.warning_once(
|
||||
"Disabling the multi-modal processor cache is extra slow with the "
|
||||
"Transformers modeling backend: the prompt is still tokenized "
|
||||
"unexpanded and the expansion spliced in, to keep the token ids "
|
||||
"identical to what the cache produces, which costs an extra HF "
|
||||
"processor call per request."
|
||||
)
|
||||
return super()._apply_hf_processor_main(
|
||||
prompt,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
tokenization_kwargs,
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
|
||||
def _get_num_image_patches(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -824,13 +778,12 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask for the replacement each placeholder expands to, and record it as
|
||||
per-item fields: its token ids, and the tokens or patches behind them."""
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
hf_inputs.pop("token_type_ids", None)
|
||||
|
||||
Reference in New Issue
Block a user