[EPD] Support metadata-only audio inputs (#57887)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
826e300cde
commit
3ee2906f47
@@ -197,6 +197,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
|
||||
return AudioFlamingo3MultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
@@ -237,6 +238,7 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
if chunk_counts is not None:
|
||||
return dict(
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
input_features=MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", chunk_counts, dim=0
|
||||
),
|
||||
@@ -247,6 +249,7 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
)
|
||||
return dict(
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
input_features=MultiModalFieldConfig.batched("audio"),
|
||||
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
||||
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
@@ -354,11 +357,15 @@ def _count_audio_tokens_from_mask(
|
||||
|
||||
|
||||
class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
|
||||
embedding_fields = {
|
||||
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[Any],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data, dict) and "audio_num_tokens" not in data:
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="audio",
|
||||
@@ -442,7 +449,11 @@ class AudioFlamingo3MultiModalProcessor(
|
||||
chunk_counts = out_mm_data.get("chunk_counts")
|
||||
|
||||
def get_replacement_audioflamingo3(item_idx: int):
|
||||
if feature_attention_mask is not None:
|
||||
if "audio_num_tokens" in out_mm_data:
|
||||
counts = out_mm_data["audio_num_tokens"]
|
||||
assert isinstance(counts, torch.Tensor)
|
||||
num_features = int(counts[item_idx])
|
||||
elif feature_attention_mask is not None:
|
||||
num_features = _count_audio_tokens_from_mask(
|
||||
feature_attention_mask,
|
||||
chunk_counts,
|
||||
|
||||
@@ -282,7 +282,7 @@ def create_qwen2_5_omni_thinker_field_factory(
|
||||
def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
audio_feature_lengths = hf_inputs.get(
|
||||
"audio_feature_lengths", torch.empty((0,))
|
||||
)
|
||||
).flatten()
|
||||
|
||||
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
|
||||
image_pixel_grid_sizes = image_grid_thw.prod(-1)
|
||||
@@ -332,6 +332,14 @@ def create_qwen2_5_omni_thinker_field_factory(
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
embedding_fields = {
|
||||
**Qwen2VLMultiModalDataParser.embedding_fields,
|
||||
"audio": {
|
||||
"input_audio_features": "values",
|
||||
"audio_feature_lengths": "metadata",
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, spatial_merge_size: int, *args, **kwargs):
|
||||
self._spatial_merge_size = spatial_merge_size
|
||||
super().__init__(self._spatial_merge_size, *args, **kwargs)
|
||||
@@ -341,10 +349,15 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any]:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("audio")
|
||||
data = dict(data)
|
||||
if "audio_feature_lengths" in data:
|
||||
data["audio_feature_lengths"] = data["audio_feature_lengths"].flatten()
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="audio",
|
||||
required_fields={"input_audio_features", "audio_feature_lengths"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=create_qwen2_5_omni_thinker_field_factory(
|
||||
self._spatial_merge_size
|
||||
),
|
||||
|
||||
@@ -128,17 +128,22 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
|
||||
def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
return dict(
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
input_features=MultiModalFieldConfig.batched("audio"),
|
||||
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
|
||||
)
|
||||
|
||||
|
||||
class Qwen2AudioMultiModalDataParser(MultiModalDataParser):
|
||||
embedding_fields = {
|
||||
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[AudioItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
if isinstance(data, dict) and "audio_num_tokens" not in data:
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="audio",
|
||||
@@ -169,6 +174,7 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
@@ -279,7 +285,9 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
|
||||
audio_output_lengths = audio_output_lens.tolist()
|
||||
|
||||
def get_replacement_qwen2_audio(item_idx: int):
|
||||
if audio_output_lengths:
|
||||
if "audio_num_tokens" in out_mm_data:
|
||||
num_features = int(out_mm_data["audio_num_tokens"][item_idx])
|
||||
elif audio_output_lengths:
|
||||
num_features = audio_output_lengths[item_idx]
|
||||
else:
|
||||
audio_embeds = out_mm_data["audio_embeds"][item_idx]
|
||||
|
||||
@@ -100,6 +100,7 @@ from .qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniAudioFeatureInputs,
|
||||
Qwen2_5OmniConditionalGenerationMixin,
|
||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||
Qwen2_5OmniThinkerMultiModalDataParser,
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
check_interleaved_audio_video,
|
||||
merge_interleaved_embeddings,
|
||||
@@ -1152,6 +1153,13 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
|
||||
)
|
||||
|
||||
|
||||
class Qwen3OmniMoeThinkerMultiModalDataParser(Qwen2_5OmniThinkerMultiModalDataParser):
|
||||
# Keep the existing raw-media fallback for vision inputs.
|
||||
embedding_fields = {
|
||||
"audio": Qwen2_5OmniThinkerMultiModalDataParser.embedding_fields["audio"],
|
||||
}
|
||||
|
||||
|
||||
class Qwen3OmniMoeThinkerProcessingInfo(
|
||||
Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo
|
||||
):
|
||||
@@ -1178,6 +1186,15 @@ class Qwen3OmniMoeThinkerProcessingInfo(
|
||||
assert isinstance(feature_extractor, WhisperFeatureExtractor)
|
||||
return feature_extractor
|
||||
|
||||
def get_data_parser(self):
|
||||
return Qwen3OmniMoeThinkerMultiModalDataParser(
|
||||
spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size,
|
||||
target_sr=self.get_feature_extractor().sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": None, "image": None, "video": None}
|
||||
|
||||
|
||||
@@ -114,6 +114,12 @@ UltravoxAudioInputs: TypeAlias = (
|
||||
)
|
||||
|
||||
|
||||
class UltravoxMultiModalDataParser(MultiModalDataParser):
|
||||
embedding_fields = {
|
||||
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
|
||||
}
|
||||
|
||||
|
||||
class UltravoxProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
|
||||
config = self.ctx.model_config.hf_config
|
||||
@@ -145,10 +151,11 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
|
||||
return MultiModalDataParser(
|
||||
return UltravoxMultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
@@ -247,6 +254,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
# num_chunks can convert audio_chunked to audio batch dimension
|
||||
audio_num_chunks=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
audio_embeds=MultiModalFieldConfig.batched("audio"),
|
||||
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
@@ -273,6 +281,8 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
|
||||
)
|
||||
|
||||
def get_replacement_ultravox(item_idx: int):
|
||||
if "audio_num_tokens" in out_mm_data:
|
||||
return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx])
|
||||
start = chunks_start_idx[item_idx]
|
||||
end = chunks_start_idx[item_idx + 1]
|
||||
audio_token_len = out_mm_data["audio_token_len"][start:end].sum()
|
||||
|
||||
Reference in New Issue
Block a user