[Refactor] Define MM data parser in processing info instead of processor itself (#33260)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -552,6 +552,29 @@ class FunAudioChatDiscreteEncoder(nn.Module):
|
||||
class FunAudioChatProcessingInfo(BaseProcessingInfo):
|
||||
token_fps: int = 25
|
||||
|
||||
@cached_property
|
||||
def feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
return WhisperFeatureExtractor.from_pretrained(self.model_id)
|
||||
|
||||
@cached_property
|
||||
def speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
return PreTrainedTokenizerFast.from_pretrained(
|
||||
self.model_id, subfolder="speech_tokenizer"
|
||||
)
|
||||
|
||||
def get_feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
return self.feature_extractor
|
||||
|
||||
def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
return self.speech_tokenizer
|
||||
|
||||
def get_data_parser(self):
|
||||
return MultiModalDataParser(
|
||||
target_sr=int(self.feature_extractor.sampling_rate),
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": None}
|
||||
|
||||
@@ -570,22 +593,6 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
|
||||
max_audio_tokens = int(getattr(audio_cfg, "max_source_positions", 1500))
|
||||
return {"audio": max_audio_tokens}
|
||||
|
||||
@cached_property
|
||||
def feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
return WhisperFeatureExtractor.from_pretrained(self.model_id)
|
||||
|
||||
@cached_property
|
||||
def speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
return PreTrainedTokenizerFast.from_pretrained(
|
||||
self.model_id, subfolder="speech_tokenizer"
|
||||
)
|
||||
|
||||
def get_feature_extractor(self) -> WhisperFeatureExtractor:
|
||||
return self.feature_extractor
|
||||
|
||||
def get_speech_tokenizer(self) -> PreTrainedTokenizerFast:
|
||||
return self.speech_tokenizer
|
||||
|
||||
def get_audio_group_size(self) -> int:
|
||||
cfg = self.get_hf_config()
|
||||
audio_cfg = getattr(cfg, "audio_config", None)
|
||||
@@ -635,13 +642,6 @@ class FunAudioChatDummyInputsBuilder(
|
||||
class FunAudioChatMultiModalProcessor(
|
||||
BaseMultiModalProcessor[FunAudioChatProcessingInfo]
|
||||
):
|
||||
def _get_data_parser(self) -> MultiModalDataParser:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
return MultiModalDataParser(
|
||||
target_sr=int(feature_extractor.sampling_rate),
|
||||
target_channels=self.info.get_target_channels(),
|
||||
)
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
|
||||
Reference in New Issue
Block a user