[EPD] Support metadata-only audio inputs (#57887)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Tianyu Guo
2026-09-23 09:03:50 +08:00
committed by GitHub
co-authored by Cyrus Leung
parent 826e300cde
commit 3ee2906f47
14 changed files with 387 additions and 27 deletions
+13 -2
View File
@@ -197,6 +197,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
return AudioFlamingo3MultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
@@ -237,6 +238,7 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
if chunk_counts is not None:
return dict(
audio_embeds=MultiModalFieldConfig.batched("audio"),
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
input_features=MultiModalFieldConfig.flat_from_sizes(
"audio", chunk_counts, dim=0
),
@@ -247,6 +249,7 @@ def _audioflamingo3_field_config(hf_inputs: Mapping[str, torch.Tensor]):
)
return dict(
audio_embeds=MultiModalFieldConfig.batched("audio"),
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
input_features=MultiModalFieldConfig.batched("audio"),
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
chunk_counts=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
@@ -354,11 +357,15 @@ def _count_audio_tokens_from_mask(
class AudioFlamingo3MultiModalDataParser(MultiModalDataParser):
embedding_fields = {
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
}
def _parse_audio_data(
self,
data: dict[str, torch.Tensor] | ModalityData[Any],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
if isinstance(data, dict) and "audio_num_tokens" not in data:
return DictEmbeddingItems(
data,
modality="audio",
@@ -442,7 +449,11 @@ class AudioFlamingo3MultiModalProcessor(
chunk_counts = out_mm_data.get("chunk_counts")
def get_replacement_audioflamingo3(item_idx: int):
if feature_attention_mask is not None:
if "audio_num_tokens" in out_mm_data:
counts = out_mm_data["audio_num_tokens"]
assert isinstance(counts, torch.Tensor)
num_features = int(counts[item_idx])
elif feature_attention_mask is not None:
num_features = _count_audio_tokens_from_mask(
feature_attention_mask,
chunk_counts,
@@ -282,7 +282,7 @@ def create_qwen2_5_omni_thinker_field_factory(
def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
audio_feature_lengths = hf_inputs.get(
"audio_feature_lengths", torch.empty((0,))
)
).flatten()
image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
image_pixel_grid_sizes = image_grid_thw.prod(-1)
@@ -332,6 +332,14 @@ def create_qwen2_5_omni_thinker_field_factory(
class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
embedding_fields = {
**Qwen2VLMultiModalDataParser.embedding_fields,
"audio": {
"input_audio_features": "values",
"audio_feature_lengths": "metadata",
},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(self._spatial_merge_size, *args, **kwargs)
@@ -341,10 +349,15 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any]:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("audio")
data = dict(data)
if "audio_feature_lengths" in data:
data["audio_feature_lengths"] = data["audio_feature_lengths"].flatten()
return DictEmbeddingItems(
data,
modality="audio",
required_fields={"input_audio_features", "audio_feature_lengths"},
required_fields=required,
optional_fields=optional,
fields_factory=create_qwen2_5_omni_thinker_field_factory(
self._spatial_merge_size
),
+10 -2
View File
@@ -128,17 +128,22 @@ def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
def _qwen2audio_field_config(hf_inputs: Mapping[str, torch.Tensor]):
return dict(
audio_embeds=MultiModalFieldConfig.batched("audio"),
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
input_features=MultiModalFieldConfig.batched("audio"),
feature_attention_mask=MultiModalFieldConfig.batched("audio"),
)
class Qwen2AudioMultiModalDataParser(MultiModalDataParser):
embedding_fields = {
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
}
def _parse_audio_data(
self,
data: dict[str, torch.Tensor] | ModalityData[AudioItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
if isinstance(data, dict) and "audio_num_tokens" not in data:
return DictEmbeddingItems(
data,
modality="audio",
@@ -169,6 +174,7 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_target_channels(self) -> int:
@@ -279,7 +285,9 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
audio_output_lengths = audio_output_lens.tolist()
def get_replacement_qwen2_audio(item_idx: int):
if audio_output_lengths:
if "audio_num_tokens" in out_mm_data:
num_features = int(out_mm_data["audio_num_tokens"][item_idx])
elif audio_output_lengths:
num_features = audio_output_lengths[item_idx]
else:
audio_embeds = out_mm_data["audio_embeds"][item_idx]
@@ -100,6 +100,7 @@ from .qwen2_5_omni_thinker import (
Qwen2_5OmniAudioFeatureInputs,
Qwen2_5OmniConditionalGenerationMixin,
Qwen2_5OmniThinkerDummyInputsBuilder,
Qwen2_5OmniThinkerMultiModalDataParser,
Qwen2_5OmniThinkerMultiModalProcessor,
check_interleaved_audio_video,
merge_interleaved_embeddings,
@@ -1152,6 +1153,13 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
)
class Qwen3OmniMoeThinkerMultiModalDataParser(Qwen2_5OmniThinkerMultiModalDataParser):
# Keep the existing raw-media fallback for vision inputs.
embedding_fields = {
"audio": Qwen2_5OmniThinkerMultiModalDataParser.embedding_fields["audio"],
}
class Qwen3OmniMoeThinkerProcessingInfo(
Qwen2AudioProcessingInfo, Qwen2_5_VLProcessingInfo
):
@@ -1178,6 +1186,15 @@ class Qwen3OmniMoeThinkerProcessingInfo(
assert isinstance(feature_extractor, WhisperFeatureExtractor)
return feature_extractor
def get_data_parser(self):
return Qwen3OmniMoeThinkerMultiModalDataParser(
spatial_merge_size=self.get_hf_config().vision_config.spatial_merge_size,
target_sr=self.get_feature_extractor().sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None, "image": None, "video": None}
+11 -1
View File
@@ -114,6 +114,12 @@ UltravoxAudioInputs: TypeAlias = (
)
class UltravoxMultiModalDataParser(MultiModalDataParser):
embedding_fields = {
"audio": {"audio_embeds": "values", "audio_num_tokens": "metadata"},
}
class UltravoxProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:
config = self.ctx.model_config.hf_config
@@ -145,10 +151,11 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
feature_extractor = self.get_feature_extractor()
return MultiModalDataParser(
return UltravoxMultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_target_channels(self) -> int:
@@ -247,6 +254,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
# num_chunks can convert audio_chunked to audio batch dimension
audio_num_chunks=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
audio_embeds=MultiModalFieldConfig.batched("audio"),
audio_num_tokens=MultiModalFieldConfig.batched("audio", keep_on_cpu=True),
)
def _get_prompt_updates(
@@ -273,6 +281,8 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor[UltravoxProcessingInfo
)
def get_replacement_ultravox(item_idx: int):
if "audio_num_tokens" in out_mm_data:
return [replacement_id] * int(out_mm_data["audio_num_tokens"][item_idx])
start = chunks_start_idx[item_idx]
end = chunks_start_idx[item_idx + 1]
audio_token_len = out_mm_data["audio_token_len"][start:end].sum()