[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
@@ -257,40 +258,69 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
|
||||
|
||||
|
||||
class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
# HF Transformers audio processor no longer accepts `audios` key.
|
||||
# We pop `audios` and replace it with `audio` key to suppress
|
||||
# the warning.
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
if "audios" in mm_data:
|
||||
mm_data["audio"] = mm_data.pop("audios")
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
)
|
||||
|
||||
if "input_features" in processed_outputs:
|
||||
# Padding enables audio_tower to run in batched mode
|
||||
processed_outputs["input_features_padded"] = processed_outputs[
|
||||
"input_features"
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
processed_data: BatchFeature,
|
||||
) -> BatchFeature:
|
||||
if "input_features" in processed_data:
|
||||
# The feature extractor pads every clip to the longest one in
|
||||
# the batch, so each clip's frame count depends on the other
|
||||
# clips it was processed with. Trim each clip to the number of
|
||||
# frames it would have if processed alone so that each item's
|
||||
# output (and thus its cache entry) is independent of the other
|
||||
# items in the batch. `batch_audio_features` re-pads the
|
||||
# per-item features so audio_tower can still run batched.
|
||||
num_frames = [
|
||||
self._get_num_audio_frames_alone(len(audio))
|
||||
for audio in mm_data["audio"]
|
||||
]
|
||||
processed_data["input_features_padded"] = [
|
||||
features[:n]
|
||||
for features, n in zip(processed_data.pop("input_features"), num_frames)
|
||||
]
|
||||
processed_data["input_features_mask"] = [
|
||||
mask[:n]
|
||||
for mask, n in zip(processed_data["input_features_mask"], num_frames)
|
||||
]
|
||||
|
||||
# Unpad features here since we need the output of each item to be
|
||||
# independent of other items for the cache to work correctly
|
||||
unpadded_features = [
|
||||
f[mask]
|
||||
for f, mask in zip(
|
||||
processed_outputs["input_features"],
|
||||
processed_outputs["input_features_mask"],
|
||||
)
|
||||
]
|
||||
processed_outputs["input_features"] = unpadded_features
|
||||
return processed_outputs
|
||||
return processed_data
|
||||
|
||||
def _get_num_audio_frames_alone(self, num_samples: int) -> int:
|
||||
"""
|
||||
Get the number of mel frames that the feature extractor produces
|
||||
for a single audio clip of the given length, independent of batch
|
||||
padding.
|
||||
"""
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
call_params = inspect.signature(type(feature_extractor).__call__).parameters
|
||||
max_length = call_params["max_length"].default
|
||||
pad_to_multiple_of = call_params["pad_to_multiple_of"].default
|
||||
|
||||
num_samples = min(num_samples, max_length)
|
||||
remainder = num_samples % pad_to_multiple_of
|
||||
if remainder:
|
||||
num_samples += pad_to_multiple_of - remainder
|
||||
|
||||
frame_size = feature_extractor.frame_length + 1
|
||||
num_frames = (num_samples - frame_size) // feature_extractor.hop_length + 1
|
||||
|
||||
return max(num_frames, 0)
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user