[MM] Further cleanup _apply_hf_processor_main (#53610)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-16 10:21:00 -07:00
committed by GitHub
parent 6ca2b23e22
commit 0b9e018fb9
74 changed files with 1048 additions and 1323 deletions
+5 -19
View File
@@ -110,7 +110,7 @@ class GraniteSpeechAudioInputs(TensorSchema):
"""List of audio embedding sizes for each item in batch."""
class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
class GraniteSpeechProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
feature_extractor = self.get_hf_processor().audio_processor
@@ -135,7 +135,7 @@ class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
### Input Processing & Multimodal utils
class GraniteSpeechMultiModalProcessor(
BaseMultiModalProcessor[GraniteSpeechMultiModalProcessingInfo]
BaseMultiModalProcessor[GraniteSpeechProcessingInfo]
):
def _get_mm_fields_config(
self,
@@ -181,23 +181,9 @@ class GraniteSpeechMultiModalProcessor(
)
]
def _get_hf_processor_text(self, mm_counts: Mapping[str, int]) -> str:
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _preprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
hf_processor_mm_kwargs: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data)
audios = mm_data.pop("audios", [])
if audios:
# GraniteSpeechFeatureExtractor accepts "audio"
mm_data["audio"] = audios
return mm_data, hf_processor_mm_kwargs
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -216,7 +202,7 @@ class GraniteSpeechMultiModalProcessor(
class GraniteSpeechDummyInputsBuilder(
BaseDummyInputsBuilder[GraniteSpeechMultiModalProcessingInfo]
BaseDummyInputsBuilder[GraniteSpeechProcessingInfo]
):
def get_dummy_mm_data(
self,
@@ -565,7 +551,7 @@ class GraniteSpeechCTCEncoder(nn.Module):
@MULTIMODAL_REGISTRY.register_processor(
GraniteSpeechMultiModalProcessor,
info=GraniteSpeechMultiModalProcessingInfo,
info=GraniteSpeechProcessingInfo,
dummy_inputs=GraniteSpeechDummyInputsBuilder,
)
class GraniteSpeechForConditionalGeneration(