[ASR] Add Long Audio benchmark and correctness test (#44587)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-12 04:11:16 +00:00
committed by GitHub
parent b927004c44
commit 226ba9fc9e
5 changed files with 530 additions and 46 deletions
+83 -14
View File
@@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset):
Dataset class for processing a ASR dataset for transcription.
Tested on the following set:
+----------------+----------------------------------------+--------------------------+-----------------------------+
| Dataset | Domain | Speaking Style | hf-subset |
+----------------+----------------------------------------+--------------------------+-----------------------------+
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
| | | | release3-speaker-adaptation |
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
| AMI | Meetings | Spontaneous | ihm, sdm |
+----------------+----------------------------------------+--------------------------+-----------------------------+
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
| Dataset | Domain | Speaking Style | hf-subset |
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
| | | | release3-speaker-adaptation |
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
| Earnings22-Cleaned-AA | Long form earnings calls | Prepared remarks, Q&A | test |
| Earnings22-Tiny-Filtered | Earnings calls | Prepared remarks, Q&A | validation |
| AMI | Meetings | Spontaneous | ihm, sdm |
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
""" # noqa: E501
EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA"
EARNINGS22_TINY_FILTERED_DATASET = (
"D4nt3/esb-datasets-earnings22-validation-tiny-filtered"
)
SUPPORTED_DATASET_PATHS = {
"openslr/librispeech_asr",
"facebook/voxpopuli",
@@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset):
"edinburghcstr/ami",
"speechcolab/gigaspeech",
"kensho/spgispeech",
EARNINGS22_CLEANED_DATASET,
EARNINGS22_TINY_FILTERED_DATASET,
}
DEFAULT_OUTPUT_LEN = 1024
IS_MULTIMODAL = True
def load_data(self) -> None:
if self.hf_name == self.EARNINGS22_CLEANED_DATASET:
# This subset stores repo-local MP3 paths instead of a HF `Audio`
# column, so eagerly materialize it back into the common schema.
self.data = load_dataset(
self.dataset_path,
name=self.dataset_subset,
split=self.dataset_split,
streaming=False,
trust_remote_code=self.trust_remote_code,
)
if not getattr(self, "disable_shuffle", False):
self.data = self.data.shuffle(seed=self.random_seed)
self._materialize_local_audio_column()
return
if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET:
super().load_data()
self._disable_audio_decode()
return
super().load_data()
def _disable_audio_decode(self) -> None:
from datasets import Audio
self.data = self.data.cast_column("audio", Audio(decode=False))
def _materialize_local_audio_column(self) -> None:
local_path_root = Path(
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
)
self.data = self.data.map(
lambda item: {
"audio": str(local_path_root / item["url"]),
"text": item["transcript"],
}
)
self._disable_audio_decode()
def sample(
self,
tokenizer: TokenizerLike,
@@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset):
if len(sampled_requests) >= num_requests:
break
audio = item["audio"]
y, sr = audio["array"], audio["sampling_rate"]
duration_s = get_audio_duration(y=y, sr=sr)
if (
isinstance(audio, dict)
and "array" in audio
and "sampling_rate" in audio
):
y, sr = audio["array"], audio["sampling_rate"]
duration_s = get_audio_duration(y=y, sr=sr)
mm_content = {"audio": (y, sr)}
elif isinstance(audio, str):
duration_s = sf.info(audio).duration
mm_content = {"audio_path": audio}
elif isinstance(audio, dict) and audio.get("path"):
duration_s = sf.info(audio["path"]).duration
mm_content = {"audio_path": audio["path"]}
elif isinstance(audio, dict) and audio.get("bytes") is not None:
with BytesIO(audio["bytes"]) as audio_buffer:
y, sr = sf.read(audio_buffer, dtype="float32")
duration_s = get_audio_duration(y=y, sr=sr)
mm_content = {"audio": (y, sr)}
else:
raise ValueError(
"ASR samples must provide decoded audio arrays, "
"embedded audio bytes, or a local audio path."
)
if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:
skipped += 1
continue
durations.append(duration_s)
mm_content = {"audio": (y, sr)}
sampled_requests.append(
SampleRequest(
prompt=prompt,