[MM] Further cleanup _apply_hf_processor_main (#53610)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -219,18 +219,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
Subclasses add the strategy for locating placeholders in the prompt.
|
||||
"""
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Rename the parser's `audios` key to the `audio` argument HF audio
|
||||
processors take.
|
||||
"""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||
"""Names of the fields the sub-processor for `modality` produces."""
|
||||
# TODO: use else branch only once huggingface/transformers#44394 lands.
|
||||
@@ -353,6 +341,9 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
|
||||
return mm_fields
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _unpad_audios(
|
||||
self,
|
||||
hf_inputs: "BatchFeature",
|
||||
@@ -597,21 +588,31 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
hf_processor_mm_kwargs = {
|
||||
**hf_processor_mm_kwargs,
|
||||
# vLLM needs the untruncated sequence to keep placeholder
|
||||
# tokens aligned. Note that the text inputs are just dummy
|
||||
# text, not the original prompt. The original prompt is
|
||||
# already tokenized by the renderer.
|
||||
"truncation": False,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (the HF processor call disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
**hf_processor_mm_kwargs,
|
||||
}
|
||||
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
# The real prompt is fed to the processor below and the placeholder
|
||||
# ranges are read back out of its output, so the dummy text meant for
|
||||
# media-only callers is discarded here.
|
||||
processor_data.pop("text", None)
|
||||
has_mm_data = any(processor_data.values())
|
||||
|
||||
# Ask which modality owns each token of the expanded prompt, which
|
||||
# is the only marking of the tokens an expansion adds around an item
|
||||
if any(processor_data.values()):
|
||||
if has_mm_data:
|
||||
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||
|
||||
try:
|
||||
@@ -621,7 +622,7 @@ class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(processor_data.values()):
|
||||
if has_mm_data:
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
@@ -792,31 +793,37 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> "BatchFeature":
|
||||
mm_counts = mm_items.get_all_counts()
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
|
||||
valid_mm_items = mm_items.select({k for k, c in mm_counts.items() if c > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
if not hf_data:
|
||||
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
prompt_text = hf_data.pop("text")
|
||||
assert isinstance(prompt_text, str)
|
||||
|
||||
# Ask for the replacement each placeholder expands to, and record it as
|
||||
# per-item fields: its token ids, and the tokens or patches behind them
|
||||
if has_mm_data := any(mm_data.values()):
|
||||
mm_data = {**mm_data, "return_text_replacement_offsets": True}
|
||||
if has_mm_data := any(hf_data.values()):
|
||||
hf_data = {**hf_data, "return_text_replacement_offsets": True}
|
||||
|
||||
try:
|
||||
hf_inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **mm_data),
|
||||
hf_processor_mm_kwargs,
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
dict(text=prompt_text, **hf_data),
|
||||
hf_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if any(mm_data.values()):
|
||||
if any(hf_data.values()):
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
@@ -825,7 +832,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
dict(input_ids=[tokenizer.encode(prompt_text)]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(hf_inputs)
|
||||
self._unpad_audios(hf_inputs, mm_data, hf_processor_mm_kwargs)
|
||||
self._unpad_audios(hf_inputs, hf_data, hf_kwargs)
|
||||
|
||||
# Drop the inputs the model would reject
|
||||
hf_inputs.pop("mm_token_type_ids", None)
|
||||
@@ -845,9 +852,10 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
)
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids", None)
|
||||
return hf_inputs
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, hf_inputs
|
||||
)
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
replacements = defaultdict[str, list[list[int]]](list)
|
||||
@@ -865,7 +873,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
if modality == "image":
|
||||
hf_inputs["num_image_patches"] = self._get_num_image_patches(
|
||||
hf_inputs, mm_data, len(seqs)
|
||||
hf_inputs, hf_data, len(seqs)
|
||||
)
|
||||
elif modality == "audio":
|
||||
counts = []
|
||||
@@ -874,10 +882,11 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
counts.append(int(ids.eq(_get_embed_token_id(ids)).sum()))
|
||||
hf_inputs["num_audio_tokens"] = torch.tensor(counts)
|
||||
|
||||
hf_inputs.update(passthrough_data)
|
||||
hf_inputs.pop("input_ids")
|
||||
|
||||
return hf_inputs
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, hf_inputs
|
||||
)
|
||||
|
||||
|
||||
# From this version on, a processor reporting no offsets is an error rather than a
|
||||
|
||||
Reference in New Issue
Block a user