[MM] Further cleanup _apply_hf_processor_main (#53610)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1008,7 +1008,20 @@ For an item `MultiModalPromptUpdates[k][i]`,
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo)
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(NamedTuple):
|
||||
class HFMultiModalInputs(NamedTuple):
|
||||
hf_data: dict[str, object]
|
||||
"""The data inputs to the HF processor."""
|
||||
|
||||
hf_kwargs: Mapping[str, object]
|
||||
"""The non-data inputs to the HF processor."""
|
||||
|
||||
passthrough_data: dict[str, object]
|
||||
"""The inputs to assign to the output of the HF processor,
|
||||
instead of passing them to the HF processor."""
|
||||
|
||||
|
||||
class MultiModalProcessingResult(NamedTuple):
|
||||
prompt_ids: list[int]
|
||||
kwargs: MultiModalKwargsOptionalItems
|
||||
hashes: MultiModalHashes
|
||||
prompt_updates: MultiModalPromptUpdates
|
||||
@@ -1137,23 +1150,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
|
||||
return find_mm_placeholders(new_token_ids, mm_prompt_updates)
|
||||
|
||||
def _get_hf_mm_data(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Extract processor and passthrough data from multi-modal items."""
|
||||
processor_data = dict[str, object]()
|
||||
passthrough_data = dict[str, object]()
|
||||
|
||||
for items in mm_items.values():
|
||||
processor_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
"""Get the text to pass to the HF processor alongside the multi-modal
|
||||
data.
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
"""Get the text to pass to the HF processor alongside the multi-modal data.
|
||||
|
||||
By default, no text is passed. If the HF processor requires that
|
||||
text and multi-modal items correspond to each other, you should
|
||||
@@ -1162,29 +1160,53 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return None
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""Pre-process the multi-modal data and HF processor keyword arguments
|
||||
before they are passed to the HF processor.
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
"""Extract the multi-modal data and corresponding keyword arguments
|
||||
for HF processing.
|
||||
|
||||
By default, both are returned as-is. If the HF processor expects the
|
||||
multi-modal data under different keys than those provided by the
|
||||
multi-modal items (e.g. `audio` instead of `audios`), or requires
|
||||
additional keyword arguments (e.g. `sampling_rate`), you should
|
||||
override this method.
|
||||
If the HF processor expects the multi-modal data under different keys
|
||||
than those provided by the multi-modal items, or requires additional
|
||||
keyword arguments, you should override this method.
|
||||
"""
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
hf_data = dict[str, object]()
|
||||
passthrough_data = dict[str, object]()
|
||||
|
||||
for items in mm_items.values():
|
||||
if not items:
|
||||
continue
|
||||
|
||||
hf_data.update(items.get_processor_data())
|
||||
passthrough_data.update(items.get_passthrough_data())
|
||||
|
||||
if hf_data:
|
||||
prompt_text = self._get_hf_mm_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
hf_data = dict(text=prompt_text, **hf_data)
|
||||
|
||||
# vLLM needs the untruncated sequence to keep placeholder
|
||||
# tokens aligned. Note that the text inputs are just dummy
|
||||
# text, not the original prompt. The original prompt is
|
||||
# already tokenized by the renderer.
|
||||
hf_kwargs = {"truncation": False, **hf_kwargs}
|
||||
|
||||
# HF processors accept (images, text, videos, audio)
|
||||
# so we need to remap vLLM's "audios" -> HF "audio"
|
||||
if "audios" in hf_data:
|
||||
hf_data["audio"] = hf_data.pop("audios")
|
||||
|
||||
return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data)
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_inputs: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
"""Post-process the output of the HF processor.
|
||||
"""Post-process the combined processor and passthrough data.
|
||||
|
||||
By default, the output is returned as-is. If you need to modify the
|
||||
output of the HF processor before it is converted into multi-modal
|
||||
@@ -1192,41 +1214,51 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
"""
|
||||
return processed_data
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Call the HF processor on the normalized multi-modal inputs."""
|
||||
return self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_kwargs),
|
||||
hf_data,
|
||||
hf_kwargs,
|
||||
)
|
||||
|
||||
def _finalize_hf_mm_data(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
passthrough_data: Mapping[str, object],
|
||||
processed_data: BatchFeature | None = None,
|
||||
) -> BatchFeature:
|
||||
"""Merge passthrough fields and apply model-specific postprocessing."""
|
||||
if processed_data is None:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
processed_data = BatchFeature()
|
||||
|
||||
processed_data.update(passthrough_data)
|
||||
return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
"""Apply the HF processor on the multi-modal data."""
|
||||
valid_mm_items = mm_items.select(
|
||||
{k for k, c in mm_items.get_all_counts().items() if c > 0}
|
||||
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_kwargs
|
||||
)
|
||||
processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
if processor_data:
|
||||
processor_data, hf_processor_mm_kwargs = self._preprocess_hf_mm_data(
|
||||
processor_data, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
prompt_text = self._get_hf_processor_text(mm_items.get_all_counts())
|
||||
if prompt_text is not None:
|
||||
processor_data = dict(text=prompt_text, **processor_data)
|
||||
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
processed_data.update(passthrough_data)
|
||||
if hf_data:
|
||||
processed_data = self._call_hf_processor(hf_data, hf_kwargs)
|
||||
else:
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
processed_data = None
|
||||
|
||||
processed_data = BatchFeature(dict(passthrough_data))
|
||||
|
||||
return self._postprocess_hf_mm_data(
|
||||
processor_data,
|
||||
hf_processor_mm_kwargs,
|
||||
processed_data,
|
||||
return self._finalize_hf_mm_data(
|
||||
hf_data, hf_kwargs, passthrough_data, processed_data
|
||||
)
|
||||
|
||||
def _postprocess_prompt(self, prompt: list[int]) -> list[int]:
|
||||
@@ -1261,6 +1293,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
mm_missing_data = {}
|
||||
for modality, idxs in mm_missing_idxs.items():
|
||||
if not idxs:
|
||||
continue
|
||||
|
||||
missing_modality_data = []
|
||||
for idx in idxs:
|
||||
data = mm_data_items[modality][idx]
|
||||
@@ -1271,6 +1306,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
)
|
||||
else:
|
||||
missing_modality_data.append(data)
|
||||
|
||||
mm_missing_data[modality] = missing_modality_data
|
||||
|
||||
mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False)
|
||||
@@ -1342,11 +1378,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
mm_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=inputs.mm_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
hf_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -1369,26 +1405,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
return MultiModalProcessingResult(
|
||||
prompt_ids=self._postprocess_prompt(inputs.prompt),
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return mm_info
|
||||
|
||||
def _cached_apply_hf_processor(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalProcessingInfo:
|
||||
) -> MultiModalProcessingResult:
|
||||
"""Apply the HF processor on the full prompt text,
|
||||
caching the results and reusing cached results.
|
||||
"""
|
||||
cache = self.cache
|
||||
has_passthrough_data = any(
|
||||
len(items.get_passthrough_data()) > 0
|
||||
for items in inputs.mm_data_items.values()
|
||||
)
|
||||
|
||||
_, passthrough_data = self._get_hf_mm_data(inputs.mm_data_items)
|
||||
if cache is None or passthrough_data:
|
||||
if cache is None or has_passthrough_data:
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
@@ -1410,7 +1448,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
mm_missing_processed_data = self._apply_hf_processor_main(
|
||||
mm_items=mm_missing_data_items,
|
||||
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
hf_kwargs=inputs.hf_processor_mm_kwargs,
|
||||
)
|
||||
|
||||
mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
@@ -1435,14 +1473,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
mm_missing_prompt_updates=mm_missing_prompt_updates,
|
||||
)
|
||||
|
||||
mm_info = MultiModalProcessingInfo(
|
||||
return MultiModalProcessingResult(
|
||||
prompt_ids=self._postprocess_prompt(inputs.prompt),
|
||||
kwargs=mm_kwargs,
|
||||
hashes=mm_hashes,
|
||||
prompt_updates=mm_prompt_updates,
|
||||
)
|
||||
|
||||
return mm_info
|
||||
|
||||
def _apply_token_matches(
|
||||
self,
|
||||
prompt: list[int],
|
||||
@@ -1670,17 +1707,15 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
def _maybe_apply_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
prompt_ids: list[int],
|
||||
mm_kwargs: MultiModalKwargsOptionalItems,
|
||||
mm_prompt_updates: MultiModalPromptUpdates,
|
||||
mm_res: MultiModalProcessingResult,
|
||||
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_prompt_updates, mm_item_counts)
|
||||
self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts)
|
||||
self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts)
|
||||
|
||||
prompt_ids, mm_placeholders = self._apply_prompt_updates(
|
||||
prompt_ids,
|
||||
mm_prompt_updates,
|
||||
mm_res.prompt_ids,
|
||||
mm_res.prompt_updates,
|
||||
)
|
||||
|
||||
self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
|
||||
@@ -1705,15 +1740,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
4. Extract information about the placeholder tokens from the
|
||||
processed token IDs.
|
||||
"""
|
||||
prompt_ids = self._postprocess_prompt(inputs.prompt)
|
||||
mm_info = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
mm_res = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("apply_prompt_updates"):
|
||||
prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
|
||||
mm_items=inputs.mm_data_items,
|
||||
prompt_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_prompt_updates=mm_info.prompt_updates,
|
||||
mm_res=mm_res,
|
||||
)
|
||||
|
||||
mm_placeholder_ranges = {
|
||||
@@ -1723,8 +1755,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_info.kwargs,
|
||||
mm_hashes=mm_info.hashes,
|
||||
mm_kwargs=mm_res.kwargs,
|
||||
mm_hashes=mm_res.hashes,
|
||||
mm_placeholders=mm_placeholder_ranges,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user