[ASR] Add Long Audio benchmark and correctness test (#44587)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-12 04:11:16 +00:00
committed by GitHub
parent b927004c44
commit 226ba9fc9e
5 changed files with 530 additions and 46 deletions
@@ -16,10 +16,11 @@ from statistics import mean, median
import pytest
import soundfile
import torch
from datasets import load_dataset
from datasets import Audio, load_dataset
from evaluate import load
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
from vllm.benchmarks.datasets.datasets import ASRDataset
from vllm.multimodal.audio import get_audio_duration
from vllm.tokenizers import get_tokenizer
@@ -38,6 +39,20 @@ def to_bytes(y, sr):
return buffer
def load_audio_sample(audio):
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
if "array" in audio and "sampling_rate" in audio:
return audio["array"], audio["sampling_rate"]
if audio.get("path"):
return soundfile.read(audio["path"], dtype="float32")
if audio.get("bytes") is not None:
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
raise ValueError("Audio sample did not contain array, path, or bytes data")
# not all models have a normalizer so use the one from whisper as a standard option
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
normalizer_tokenizer = get_tokenizer(
@@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer(
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
async def transcribe_audio(client, tokenizer, y, sr):
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
# Send loaded audio directly instead of loading from disk,
# don't account for that time though
with to_bytes(y, sr) as f:
@@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr):
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
# NOTE there's no streaming in transcriptions, can't measure ttft
@@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr):
return latency, num_output_tokens, transcription.text
async def bound_transcribe(sem, client, tokenizer, audio, reference):
async def bound_transcribe(
sem, client, tokenizer, audio, sr, reference, extra_body=None
):
# Use semaphore to limit concurrent requests.
async with sem:
result = await transcribe_audio(client, tokenizer, *audio)
result = await transcribe_audio(
client, tokenizer, audio, sr, extra_body=extra_body
)
# Normalize *english* output/reference for evaluation.
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_dataset(model, client, data, concurrent_request):
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
@@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request):
)
# Warmup call as the first `load_audio` server-side is quite slow.
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
_ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
audio, sr = load_audio_sample(data[0]["audio"])
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
audio, sr = load_audio_sample(sample["audio"])
task = asyncio.create_task(
bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"])
bound_transcribe(
sem, client, tokenizer, audio, sr, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
@@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time):
def add_duration(sample):
y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
y, sr = load_audio_sample(sample["audio"])
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
return sample
def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset
dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
if "duration_ms" not in dataset[0]:
# compute duration to filter
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
asr_dataset_kwargs = {
"dataset_path": dataset_repo,
"dataset_split": split,
"disable_shuffle": True,
"no_stream": True,
}
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
if key in hf_kwargs:
asr_dataset_kwargs[key] = hf_kwargs[key]
return ASRDataset(**asr_dataset_kwargs).data
return load_dataset(dataset_repo, split=split, **hf_kwargs)
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset.
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
dataset = dataset.cast_column("audio", Audio(decode=False))
if "duration_ms" not in dataset.column_names:
# Compute duration to filter.
dataset = dataset.map(add_duration)
# Whisper max supported duration
# Whisper max supported duration.
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
return dataset
@@ -145,11 +184,16 @@ def run_evaluation(
max_concurrent_reqs: int,
n_examples: int = -1,
print_metrics: bool = True,
extra_body=None,
):
if n_examples > 0:
dataset = dataset.select(range(n_examples))
start = time.perf_counter()
results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs))
results = asyncio.run(
process_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
@@ -164,6 +208,106 @@ def run_evaluation(
return wer_score
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
LONGFORM_DATASET_SPLIT = "test"
LONGFORM_NUM_SAMPLES = 6
def load_longform_dataset():
dataset = load_asr_dataset_rows(
LONGFORM_DATASET_REPO,
split=LONGFORM_DATASET_SPLIT,
)
assert len(dataset) >= LONGFORM_NUM_SAMPLES
return dataset.select(range(LONGFORM_NUM_SAMPLES))
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
with open(audio_path, "rb") as f:
start_time = time.perf_counter()
transcription = await client.audio.transcriptions.create(
file=f,
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
latency = end_time - start_time
num_output_tokens = len(
tokenizer(transcription.text, add_special_tokens=False).input_ids
)
return latency, num_output_tokens, transcription.text
async def bound_transcribe_path(
sem, client, tokenizer, audio_path, reference, extra_body=None
):
async with sem:
result = await transcribe_audio_path(
client, tokenizer, audio_path, extra_body=extra_body
)
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_longform_dataset(
model, client, data, concurrent_request, extra_body=None
):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
tokenizer = get_tokenizer(
model,
tokenizer_mode=model_info.tokenizer_mode,
trust_remote_code=model_info.trust_remote_code,
)
warmup_path = data[0]["audio"]["path"]
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio_path = sample["audio"]["path"]
task = asyncio.create_task(
bound_transcribe_path(
sem, client, tokenizer, audio_path, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
def run_longform_evaluation(
model: str,
client,
dataset,
max_concurrent_reqs: int,
print_metrics: bool = True,
extra_body=None,
):
start = time.perf_counter()
results = asyncio.run(
process_longform_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
if print_metrics:
print_performance_metrics(results, total_time)
predictions = [res[2] for res in results]
references = [res[3] for res in results]
wer = load("wer")
wer_score = 100 * wer.compute(references=references, predictions=predictions)
print("WER:", wer_score)
return wer_score
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
# NOTE: Expected WER measured with equivalent hf.transformers args:
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
@@ -184,7 +328,6 @@ def test_wer_correctness(
):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
# TODO refactor to use `ASRDataset`
server_args = [
"--enforce-eager",
f"--tokenizer_mode={model_info.tokenizer_mode}",
@@ -197,7 +340,7 @@ def test_wer_correctness(
model_name,
server_args,
) as remote_server:
dataset = load_hf_dataset(dataset_repo)
dataset = load_shortform_eval_dataset(dataset_repo)
if not max_concurrent_request:
# No max concurrency
@@ -216,3 +359,42 @@ def test_wer_correctness(
if expected_wer:
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
# checks for long audio transcription correctness and RMS split.
@pytest.mark.parametrize(
"model_config",
[("openai/whisper-large-v3", 9.5)],
)
def test_long_audio_wer_correctness(model_config):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
server_args = [
f"--tokenizer_mode={model_info.tokenizer_mode}",
]
if model_info.trust_remote_code:
server_args.append("--trust-remote-code")
# 1800 seconds is 30 minutes
env_dict = {
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
}
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=env_dict,
) as remote_server:
dataset = load_longform_dataset()
client = remote_server.get_async_client()
wer = run_longform_evaluation(
model=model_name,
client=client,
dataset=dataset,
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
)
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)