[ASR] Add Long Audio benchmark and correctness test (#44587)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-12 04:11:16 +00:00
committed by GitHub
parent b927004c44
commit 226ba9fc9e
5 changed files with 530 additions and 46 deletions
+83 -14
View File
@@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset):
Dataset class for processing a ASR dataset for transcription.
Tested on the following set:
+----------------+----------------------------------------+--------------------------+-----------------------------+
| Dataset | Domain | Speaking Style | hf-subset |
+----------------+----------------------------------------+--------------------------+-----------------------------+
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
| | | | release3-speaker-adaptation |
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
| AMI | Meetings | Spontaneous | ihm, sdm |
+----------------+----------------------------------------+--------------------------+-----------------------------+
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
| Dataset | Domain | Speaking Style | hf-subset |
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
| | | | release3-speaker-adaptation |
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
| Earnings22-Cleaned-AA | Long form earnings calls | Prepared remarks, Q&A | test |
| Earnings22-Tiny-Filtered | Earnings calls | Prepared remarks, Q&A | validation |
| AMI | Meetings | Spontaneous | ihm, sdm |
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
""" # noqa: E501
EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA"
EARNINGS22_TINY_FILTERED_DATASET = (
"D4nt3/esb-datasets-earnings22-validation-tiny-filtered"
)
SUPPORTED_DATASET_PATHS = {
"openslr/librispeech_asr",
"facebook/voxpopuli",
@@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset):
"edinburghcstr/ami",
"speechcolab/gigaspeech",
"kensho/spgispeech",
EARNINGS22_CLEANED_DATASET,
EARNINGS22_TINY_FILTERED_DATASET,
}
DEFAULT_OUTPUT_LEN = 1024
IS_MULTIMODAL = True
def load_data(self) -> None:
if self.hf_name == self.EARNINGS22_CLEANED_DATASET:
# This subset stores repo-local MP3 paths instead of a HF `Audio`
# column, so eagerly materialize it back into the common schema.
self.data = load_dataset(
self.dataset_path,
name=self.dataset_subset,
split=self.dataset_split,
streaming=False,
trust_remote_code=self.trust_remote_code,
)
if not getattr(self, "disable_shuffle", False):
self.data = self.data.shuffle(seed=self.random_seed)
self._materialize_local_audio_column()
return
if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET:
super().load_data()
self._disable_audio_decode()
return
super().load_data()
def _disable_audio_decode(self) -> None:
from datasets import Audio
self.data = self.data.cast_column("audio", Audio(decode=False))
def _materialize_local_audio_column(self) -> None:
local_path_root = Path(
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
)
self.data = self.data.map(
lambda item: {
"audio": str(local_path_root / item["url"]),
"text": item["transcript"],
}
)
self._disable_audio_decode()
def sample(
self,
tokenizer: TokenizerLike,
@@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset):
if len(sampled_requests) >= num_requests:
break
audio = item["audio"]
y, sr = audio["array"], audio["sampling_rate"]
duration_s = get_audio_duration(y=y, sr=sr)
if (
isinstance(audio, dict)
and "array" in audio
and "sampling_rate" in audio
):
y, sr = audio["array"], audio["sampling_rate"]
duration_s = get_audio_duration(y=y, sr=sr)
mm_content = {"audio": (y, sr)}
elif isinstance(audio, str):
duration_s = sf.info(audio).duration
mm_content = {"audio_path": audio}
elif isinstance(audio, dict) and audio.get("path"):
duration_s = sf.info(audio["path"]).duration
mm_content = {"audio_path": audio["path"]}
elif isinstance(audio, dict) and audio.get("bytes") is not None:
with BytesIO(audio["bytes"]) as audio_buffer:
y, sr = sf.read(audio_buffer, dtype="float32")
duration_s = get_audio_duration(y=y, sr=sr)
mm_content = {"audio": (y, sr)}
else:
raise ValueError(
"ASR samples must provide decoded audio arrays, "
"embedded audio bytes, or a local audio path."
)
if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:
skipped += 1
continue
durations.append(duration_s)
mm_content = {"audio": (y, sr)}
sampled_requests.append(
SampleRequest(
prompt=prompt,
+44 -11
View File
@@ -445,7 +445,6 @@ async def async_request_openai_audio(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Audio API", {"transcriptions", "translations"})
content = [{"type": "text", "text": request_func_input.prompt}]
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
@@ -469,19 +468,26 @@ async def async_request_openai_audio(
buffer.seek(0)
return buffer
mm_audio = request_func_input.multi_modal_content
if not isinstance(mm_audio, dict) or "audio" not in mm_audio:
raise TypeError("multi_modal_content must be a dict containing 'audio'")
with to_bytes(*mm_audio["audio"]) as f:
async def send_audio_file(
audio_file: io.BytesIO | Any,
*,
input_audio_duration: float,
filename: str | None = None,
content_type: str | None = None,
) -> RequestFuncOutput:
form = aiohttp.FormData()
form.add_field("file", f, content_type="audio/wav")
add_field_kwargs: dict[str, str] = {}
if filename is not None:
add_field_kwargs["filename"] = filename
if content_type is not None:
add_field_kwargs["content_type"] = content_type
form.add_field("file", audio_file, **add_field_kwargs)
for key, value in payload.items():
form.add_field(key, str(value))
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output.input_audio_duration = soundfile.info(f).duration
f.seek(0)
output.input_audio_duration = input_audio_duration
generated_text = ""
ttft = 0.0
@@ -541,9 +547,36 @@ async def async_request_openai_audio(
exc_info = sys.exc_info()
output.error = "".join(traceback.format_exception(*exc_info))
if pbar:
pbar.update(1)
return output
if pbar:
pbar.update(1)
return output
mm_audio = request_func_input.multi_modal_content
if not isinstance(mm_audio, dict):
raise TypeError(
"multi_modal_content must be a dict containing 'audio' or 'audio_path'"
)
if "audio" in mm_audio:
with to_bytes(*mm_audio["audio"]) as f:
input_audio_duration = soundfile.info(f).duration
f.seek(0)
return await send_audio_file(
f,
input_audio_duration=input_audio_duration,
filename="audio.wav",
content_type="audio/wav",
)
if "audio_path" in mm_audio:
audio_path = mm_audio["audio_path"]
with open(audio_path, "rb") as f:
return await send_audio_file(
f,
input_audio_duration=soundfile.info(audio_path).duration,
filename=os.path.basename(audio_path),
)
raise TypeError(
"multi_modal_content must be a dict containing 'audio' or 'audio_path'"
)
async def _run_pooling_request(