[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -345,28 +345,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
|
||||
return mm_fields
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Run the HF processor and unpad inputs."""
|
||||
try:
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, mm_kwargs)
|
||||
return hf_inputs
|
||||
|
||||
def _unpad_audios(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -450,19 +428,6 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask which modality owns each token of the expanded prompt, which is the
|
||||
only marking of the tokens an expansion adds around an item."""
|
||||
if any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_mm_token_type_ids": True}
|
||||
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
processors differ from the placeholder written into it.
|
||||
@@ -612,24 +577,55 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
prompt = hf_processor.decode(prompt)
|
||||
prompt_text = hf_processor.decode(prompt)
|
||||
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
prompt_ids, processed_data = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={
|
||||
**hf_processor_mm_kwargs,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (`_call_hf_processor` disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
},
|
||||
hf_processor_mm_kwargs = {
|
||||
**hf_processor_mm_kwargs,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (the HF processor call disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
}
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
# Ask which modality owns each token of the expanded prompt, which
|
||||
# is the only marking of the tokens an expansion adds around an item
|
||||
if any(processor_data.values()):
|
||||
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||
|
||||
try:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **processor_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(processor_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
|
||||
processed_data = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(processed_data)
|
||||
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
@@ -773,17 +769,41 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _call_hf_processor(
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
"""Ask for the replacement each placeholder expands to, and record it as
|
||||
per-item fields: its token ids, and the tokens or patches behind them."""
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
# Ask for the replacement each placeholder expands to, and record it as
|
||||
# per-item fields: its token ids, and the tokens or patches behind them
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
hf_inputs = super()._call_hf_processor(prompt, mm_data, mm_kwargs)
|
||||
|
||||
try:
|
||||
hf_inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
hf_inputs = transformers.BatchFeature(
|
||||
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
|
||||
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
hf_inputs.pop("token_type_ids", None)
|
||||
@@ -802,6 +822,8 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
)
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids", None)
|
||||
return hf_inputs
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
@@ -829,6 +851,9 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids")
|
||||
|
||||
return hf_inputs
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user