[EPD] Support metadata-only audio inputs (#57887)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Tianyu Guo
2026-09-23 09:03:50 +08:00
committed by GitHub
co-authored by Cyrus Leung
parent 826e300cde
commit 3ee2906f47
14 changed files with 387 additions and 27 deletions
+43 -1
View File
@@ -664,9 +664,46 @@ class MultiModalDataParser:
assert_never(video)
def _parse_audio_embedding_data(
self, data: dict[str, torch.Tensor]
) -> DictEmbeddingItems:
counts = data.get("audio_num_tokens")
if (
not isinstance(counts, torch.Tensor)
or counts.dtype not in (torch.int32, torch.int64)
or counts.ndim not in (1, 2)
or (counts.ndim == 2 and counts.shape[1] != 1)
or not torch.all(counts > 0)
):
raise ValueError(
"audio_num_tokens must contain one positive integer per audio"
)
counts = counts.flatten()
if "audio_embeds" in data:
embeds = data["audio_embeds"]
if len(embeds) != len(counts) or any(
len(embedding) != count for embedding, count in zip(embeds, counts)
):
raise ValueError("audio_num_tokens does not match audio_embeds")
required, optional = self.embedding_field_sets("audio")
return DictEmbeddingItems(
{**data, "audio_num_tokens": counts},
modality="audio",
required_fields=required,
optional_fields=optional,
fields_factory=lambda _: {
"audio_embeds": MultiModalFieldConfig.batched("audio"),
"audio_num_tokens": MultiModalFieldConfig.batched(
"audio", keep_on_cpu=True
),
},
)
def _parse_audio_data(
self,
data: ModalityData[AudioItem],
data: dict[str, torch.Tensor] | ModalityData[AudioItem],
) -> ModalityDataItems[Any, Any] | None:
if data is None:
return None
@@ -674,6 +711,11 @@ class MultiModalDataParser:
if self.is_embeddings(data):
return AudioEmbeddingItems(data, self.expected_hidden_size)
if isinstance(data, dict) and "audio_num_tokens" in self.embedding_fields.get(
"audio", {}
):
return self._parse_audio_embedding_data(data)
data_items: list[AudioItem | None]
if (
(is_list_of(data, float) and len(data) > 0)