[MM] Further cleanup _apply_hf_processor_main (#53610)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema):
|
||||
"""List of audio embedding sizes for each item in batch."""
|
||||
|
||||
|
||||
class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
|
||||
class GraniteSpeechProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
feature_extractor = self.get_hf_processor().audio_processor
|
||||
|
||||
@@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
### Input Processing & Multimodal utils
|
||||
class GraniteSpeechMultiModalProcessor(
|
||||
BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]
|
||||
BaseMultiModalProcessor[GraniteSpeechProcessingInfo]
|
||||
):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor(
|
||||
)
|
||||
]
|
||||
|
||||
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _preprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
mm_data = dict(mm_data)
|
||||
audios = mm_data.pop("audios", [])
|
||||
|
||||
if audios:
|
||||
# GraniteSpeechFeatureExtractor accepts "audio"
|
||||
mm_data["audio"] = audios
|
||||
|
||||
return mm_data, hf_processor_mm_kwargs
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
@@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor(
|
||||
|
||||
|
||||
class GraniteSpeechDummyInputsBuilder(
|
||||
BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo]
|
||||
BaseDummyInputsBuilder[GraniteSpeechProcessingInfo]
|
||||
):
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
@@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module):
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
GraniteSpeechMultiModalProcessor,
|
||||
info=GraniteSpeechMultiModalProcessingInfo,
|
||||
info=GraniteSpeechProcessingInfo,
|
||||
dummy_inputs=GraniteSpeechDummyInputsBuilder,
|
||||
)
|
||||
class GraniteSpeechForConditionalGeneration(
|
||||
|
||||
Reference in New Issue
Block a user