[ASR] Add Long Audio benchmark and correctness test (#44587)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
@@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset):
|
||||
Dataset class for processing a ASR dataset for transcription.
|
||||
Tested on the following set:
|
||||
|
||||
+----------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| Dataset | Domain | Speaking Style | hf-subset |
|
||||
+----------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
|
||||
| | | | release3-speaker-adaptation |
|
||||
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
|
||||
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
|
||||
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
|
||||
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
|
||||
| AMI | Meetings | Spontaneous | ihm, sdm |
|
||||
+----------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| Dataset | Domain | Speaking Style | hf-subset |
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
|
||||
| | | | release3-speaker-adaptation |
|
||||
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
|
||||
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
|
||||
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
|
||||
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
|
||||
| Earnings22-Cleaned-AA | Long form earnings calls | Prepared remarks, Q&A | test |
|
||||
| Earnings22-Tiny-Filtered | Earnings calls | Prepared remarks, Q&A | validation |
|
||||
| AMI | Meetings | Spontaneous | ihm, sdm |
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA"
|
||||
EARNINGS22_TINY_FILTERED_DATASET = (
|
||||
"D4nt3/esb-datasets-earnings22-validation-tiny-filtered"
|
||||
)
|
||||
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"openslr/librispeech_asr",
|
||||
"facebook/voxpopuli",
|
||||
@@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset):
|
||||
"edinburghcstr/ami",
|
||||
"speechcolab/gigaspeech",
|
||||
"kensho/spgispeech",
|
||||
EARNINGS22_CLEANED_DATASET,
|
||||
EARNINGS22_TINY_FILTERED_DATASET,
|
||||
}
|
||||
|
||||
DEFAULT_OUTPUT_LEN = 1024
|
||||
IS_MULTIMODAL = True
|
||||
|
||||
def load_data(self) -> None:
|
||||
if self.hf_name == self.EARNINGS22_CLEANED_DATASET:
|
||||
# This subset stores repo-local MP3 paths instead of a HF `Audio`
|
||||
# column, so eagerly materialize it back into the common schema.
|
||||
self.data = load_dataset(
|
||||
self.dataset_path,
|
||||
name=self.dataset_subset,
|
||||
split=self.dataset_split,
|
||||
streaming=False,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
)
|
||||
if not getattr(self, "disable_shuffle", False):
|
||||
self.data = self.data.shuffle(seed=self.random_seed)
|
||||
self._materialize_local_audio_column()
|
||||
return
|
||||
if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET:
|
||||
super().load_data()
|
||||
self._disable_audio_decode()
|
||||
return
|
||||
|
||||
super().load_data()
|
||||
|
||||
def _disable_audio_decode(self) -> None:
|
||||
from datasets import Audio
|
||||
|
||||
self.data = self.data.cast_column("audio", Audio(decode=False))
|
||||
|
||||
def _materialize_local_audio_column(self) -> None:
|
||||
local_path_root = Path(
|
||||
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
|
||||
)
|
||||
self.data = self.data.map(
|
||||
lambda item: {
|
||||
"audio": str(local_path_root / item["url"]),
|
||||
"text": item["transcript"],
|
||||
}
|
||||
)
|
||||
self._disable_audio_decode()
|
||||
|
||||
def sample(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
audio = item["audio"]
|
||||
y, sr = audio["array"], audio["sampling_rate"]
|
||||
duration_s = get_audio_duration(y=y, sr=sr)
|
||||
if (
|
||||
isinstance(audio, dict)
|
||||
and "array" in audio
|
||||
and "sampling_rate" in audio
|
||||
):
|
||||
y, sr = audio["array"], audio["sampling_rate"]
|
||||
duration_s = get_audio_duration(y=y, sr=sr)
|
||||
mm_content = {"audio": (y, sr)}
|
||||
elif isinstance(audio, str):
|
||||
duration_s = sf.info(audio).duration
|
||||
mm_content = {"audio_path": audio}
|
||||
elif isinstance(audio, dict) and audio.get("path"):
|
||||
duration_s = sf.info(audio["path"]).duration
|
||||
mm_content = {"audio_path": audio["path"]}
|
||||
elif isinstance(audio, dict) and audio.get("bytes") is not None:
|
||||
with BytesIO(audio["bytes"]) as audio_buffer:
|
||||
y, sr = sf.read(audio_buffer, dtype="float32")
|
||||
duration_s = get_audio_duration(y=y, sr=sr)
|
||||
mm_content = {"audio": (y, sr)}
|
||||
else:
|
||||
raise ValueError(
|
||||
"ASR samples must provide decoded audio arrays, "
|
||||
"embedded audio bytes, or a local audio path."
|
||||
)
|
||||
if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
durations.append(duration_s)
|
||||
mm_content = {"audio": (y, sr)}
|
||||
sampled_requests.append(
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
|
||||
Reference in New Issue
Block a user