[ASR] Add Long Audio benchmark and correctness test (#44587)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-12 04:11:16 +00:00
committed by GitHub
parent b927004c44
commit 226ba9fc9e
5 changed files with 530 additions and 46 deletions
+200
View File
@@ -0,0 +1,200 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from pathlib import Path
from typing import Protocol, cast
import numpy as np
import pytest
import soundfile as sf
import vllm.benchmarks.datasets.datasets as datasets_module
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
pytestmark = pytest.mark.skip_global_cleanup
class _ReadableBinary(Protocol):
def read(self, size: int = -1) -> bytes: ...
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
self.name_or_path = name_or_path
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
num_samples = int(duration_s * sample_rate)
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
class _FakeFormData:
def __init__(self) -> None:
self.fields: list[tuple[str, object, dict[str, str]]] = []
def add_field(self, name: str, value: object, **kwargs: str) -> None:
self.fields.append((name, value, kwargs))
class _FakeContent:
async def iter_any(self):
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
yield b"data: [DONE]\n\n"
class _FakeResponse:
def __init__(self) -> None:
self.status = 200
self.reason = "OK"
self.content = _FakeContent()
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
class _FakeSession:
def __init__(self) -> None:
self.uploaded_bytes: bytes | None = None
self.upload_filename: str | None = None
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
del url, headers
self.fields = list(data.fields)
_, file_obj, file_kwargs = self.fields[0]
file_obj = cast(_ReadableBinary, file_obj)
self.uploaded_bytes = file_obj.read()
self.upload_filename = file_kwargs.get("filename")
return _FakeResponse()
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": str(audio_path),
"bytes": None,
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
assert (
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
)
def test_asr_dataset_sample_handles_embedded_audio_bytes(tmp_path: Path) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": None,
"bytes": audio_path.read_bytes(),
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, dict)
audio, sample_rate = samples[0].multi_modal_data["audio"]
assert sample_rate == 16_000
assert isinstance(audio, np.ndarray)
assert audio.size > 0
def test_async_request_openai_audio_handles_local_audio_paths(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.25)
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={"audio_path": str(audio_path)},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == audio_path.name
assert session.uploaded_bytes == audio_path.read_bytes()
assert output.success is True
assert output.generated_text == "hello"
assert output.output_tokens == 1
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
def test_async_request_openai_audio_handles_decoded_audio_arrays(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == "audio.wav"
assert session.uploaded_bytes is not None
assert output.success is True
assert output.generated_text == "hello"
@@ -16,10 +16,11 @@ from statistics import mean, median
import pytest
import soundfile
import torch
from datasets import load_dataset
from datasets import Audio, load_dataset
from evaluate import load
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
from vllm.benchmarks.datasets.datasets import ASRDataset
from vllm.multimodal.audio import get_audio_duration
from vllm.tokenizers import get_tokenizer
@@ -38,6 +39,20 @@ def to_bytes(y, sr):
return buffer
def load_audio_sample(audio):
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
if "array" in audio and "sampling_rate" in audio:
return audio["array"], audio["sampling_rate"]
if audio.get("path"):
return soundfile.read(audio["path"], dtype="float32")
if audio.get("bytes") is not None:
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
raise ValueError("Audio sample did not contain array, path, or bytes data")
# not all models have a normalizer so use the one from whisper as a standard option
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
normalizer_tokenizer = get_tokenizer(
@@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer(
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
async def transcribe_audio(client, tokenizer, y, sr):
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
# Send loaded audio directly instead of loading from disk,
# don't account for that time though
with to_bytes(y, sr) as f:
@@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr):
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
# NOTE there's no streaming in transcriptions, can't measure ttft
@@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr):
return latency, num_output_tokens, transcription.text
async def bound_transcribe(sem, client, tokenizer, audio, reference):
async def bound_transcribe(
sem, client, tokenizer, audio, sr, reference, extra_body=None
):
# Use semaphore to limit concurrent requests.
async with sem:
result = await transcribe_audio(client, tokenizer, *audio)
result = await transcribe_audio(
client, tokenizer, audio, sr, extra_body=extra_body
)
# Normalize *english* output/reference for evaluation.
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_dataset(model, client, data, concurrent_request):
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
@@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request):
)
# Warmup call as the first `load_audio` server-side is quite slow.
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
_ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
audio, sr = load_audio_sample(data[0]["audio"])
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
audio, sr = load_audio_sample(sample["audio"])
task = asyncio.create_task(
bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"])
bound_transcribe(
sem, client, tokenizer, audio, sr, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
@@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time):
def add_duration(sample):
y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
y, sr = load_audio_sample(sample["audio"])
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
return sample
def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset
dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
if "duration_ms" not in dataset[0]:
# compute duration to filter
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
asr_dataset_kwargs = {
"dataset_path": dataset_repo,
"dataset_split": split,
"disable_shuffle": True,
"no_stream": True,
}
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
if key in hf_kwargs:
asr_dataset_kwargs[key] = hf_kwargs[key]
return ASRDataset(**asr_dataset_kwargs).data
return load_dataset(dataset_repo, split=split, **hf_kwargs)
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset.
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
dataset = dataset.cast_column("audio", Audio(decode=False))
if "duration_ms" not in dataset.column_names:
# Compute duration to filter.
dataset = dataset.map(add_duration)
# Whisper max supported duration
# Whisper max supported duration.
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
return dataset
@@ -145,11 +184,16 @@ def run_evaluation(
max_concurrent_reqs: int,
n_examples: int = -1,
print_metrics: bool = True,
extra_body=None,
):
if n_examples > 0:
dataset = dataset.select(range(n_examples))
start = time.perf_counter()
results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs))
results = asyncio.run(
process_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
@@ -164,6 +208,106 @@ def run_evaluation(
return wer_score
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
LONGFORM_DATASET_SPLIT = "test"
LONGFORM_NUM_SAMPLES = 6
def load_longform_dataset():
dataset = load_asr_dataset_rows(
LONGFORM_DATASET_REPO,
split=LONGFORM_DATASET_SPLIT,
)
assert len(dataset) >= LONGFORM_NUM_SAMPLES
return dataset.select(range(LONGFORM_NUM_SAMPLES))
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
with open(audio_path, "rb") as f:
start_time = time.perf_counter()
transcription = await client.audio.transcriptions.create(
file=f,
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
latency = end_time - start_time
num_output_tokens = len(
tokenizer(transcription.text, add_special_tokens=False).input_ids
)
return latency, num_output_tokens, transcription.text
async def bound_transcribe_path(
sem, client, tokenizer, audio_path, reference, extra_body=None
):
async with sem:
result = await transcribe_audio_path(
client, tokenizer, audio_path, extra_body=extra_body
)
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_longform_dataset(
model, client, data, concurrent_request, extra_body=None
):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
tokenizer = get_tokenizer(
model,
tokenizer_mode=model_info.tokenizer_mode,
trust_remote_code=model_info.trust_remote_code,
)
warmup_path = data[0]["audio"]["path"]
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio_path = sample["audio"]["path"]
task = asyncio.create_task(
bound_transcribe_path(
sem, client, tokenizer, audio_path, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
def run_longform_evaluation(
model: str,
client,
dataset,
max_concurrent_reqs: int,
print_metrics: bool = True,
extra_body=None,
):
start = time.perf_counter()
results = asyncio.run(
process_longform_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
if print_metrics:
print_performance_metrics(results, total_time)
predictions = [res[2] for res in results]
references = [res[3] for res in results]
wer = load("wer")
wer_score = 100 * wer.compute(references=references, predictions=predictions)
print("WER:", wer_score)
return wer_score
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
# NOTE: Expected WER measured with equivalent hf.transformers args:
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
@@ -184,7 +328,6 @@ def test_wer_correctness(
):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
# TODO refactor to use `ASRDataset`
server_args = [
"--enforce-eager",
f"--tokenizer_mode={model_info.tokenizer_mode}",
@@ -197,7 +340,7 @@ def test_wer_correctness(
model_name,
server_args,
) as remote_server:
dataset = load_hf_dataset(dataset_repo)
dataset = load_shortform_eval_dataset(dataset_repo)
if not max_concurrent_request:
# No max concurrency
@@ -216,3 +359,42 @@ def test_wer_correctness(
if expected_wer:
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
# checks for long audio transcription correctness and RMS split.
@pytest.mark.parametrize(
"model_config",
[("openai/whisper-large-v3", 9.5)],
)
def test_long_audio_wer_correctness(model_config):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
server_args = [
f"--tokenizer_mode={model_info.tokenizer_mode}",
]
if model_info.trust_remote_code:
server_args.append("--trust-remote-code")
# 1800 seconds is 30 minutes
env_dict = {
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
}
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=env_dict,
) as remote_server:
dataset = load_longform_dataset()
client = remote_server.get_async_client()
wer = run_longform_evaluation(
model=model_name,
client=client,
dataset=dataset,
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
)
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)