[ASR] Add Long Audio benchmark and correctness test (#44587)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
+201
-19
@@ -16,10 +16,11 @@ from statistics import mean, median
|
||||
import pytest
|
||||
import soundfile
|
||||
import torch
|
||||
from datasets import load_dataset
|
||||
from datasets import Audio, load_dataset
|
||||
from evaluate import load
|
||||
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
|
||||
|
||||
from vllm.benchmarks.datasets.datasets import ASRDataset
|
||||
from vllm.multimodal.audio import get_audio_duration
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
@@ -38,6 +39,20 @@ def to_bytes(y, sr):
|
||||
return buffer
|
||||
|
||||
|
||||
def load_audio_sample(audio):
|
||||
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
|
||||
if "array" in audio and "sampling_rate" in audio:
|
||||
return audio["array"], audio["sampling_rate"]
|
||||
|
||||
if audio.get("path"):
|
||||
return soundfile.read(audio["path"], dtype="float32")
|
||||
|
||||
if audio.get("bytes") is not None:
|
||||
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
|
||||
|
||||
raise ValueError("Audio sample did not contain array, path, or bytes data")
|
||||
|
||||
|
||||
# not all models have a normalizer so use the one from whisper as a standard option
|
||||
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
|
||||
normalizer_tokenizer = get_tokenizer(
|
||||
@@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer(
|
||||
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
|
||||
|
||||
|
||||
async def transcribe_audio(client, tokenizer, y, sr):
|
||||
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
|
||||
# Send loaded audio directly instead of loading from disk,
|
||||
# don't account for that time though
|
||||
with to_bytes(y, sr) as f:
|
||||
@@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr):
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
# NOTE there's no streaming in transcriptions, can't measure ttft
|
||||
@@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr):
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe(sem, client, tokenizer, audio, reference):
|
||||
async def bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, reference, extra_body=None
|
||||
):
|
||||
# Use semaphore to limit concurrent requests.
|
||||
async with sem:
|
||||
result = await transcribe_audio(client, tokenizer, *audio)
|
||||
result = await transcribe_audio(
|
||||
client, tokenizer, audio, sr, extra_body=extra_body
|
||||
)
|
||||
# Normalize *english* output/reference for evaluation.
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_dataset(model, client, data, concurrent_request):
|
||||
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
@@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request):
|
||||
)
|
||||
|
||||
# Warmup call as the first `load_audio` server-side is quite slow.
|
||||
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
|
||||
_ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
|
||||
audio, sr = load_audio_sample(data[0]["audio"])
|
||||
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
|
||||
audio, sr = load_audio_sample(sample["audio"])
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"])
|
||||
bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
@@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time):
|
||||
|
||||
|
||||
def add_duration(sample):
|
||||
y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
|
||||
y, sr = load_audio_sample(sample["audio"])
|
||||
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
|
||||
return sample
|
||||
|
||||
|
||||
def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
## Load and filter the dataset
|
||||
dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
|
||||
if "duration_ms" not in dataset[0]:
|
||||
# compute duration to filter
|
||||
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
|
||||
asr_dataset_kwargs = {
|
||||
"dataset_path": dataset_repo,
|
||||
"dataset_split": split,
|
||||
"disable_shuffle": True,
|
||||
"no_stream": True,
|
||||
}
|
||||
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
|
||||
if key in hf_kwargs:
|
||||
asr_dataset_kwargs[key] = hf_kwargs[key]
|
||||
return ASRDataset(**asr_dataset_kwargs).data
|
||||
|
||||
return load_dataset(dataset_repo, split=split, **hf_kwargs)
|
||||
|
||||
|
||||
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
## Load and filter the dataset.
|
||||
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
|
||||
dataset = dataset.cast_column("audio", Audio(decode=False))
|
||||
if "duration_ms" not in dataset.column_names:
|
||||
# Compute duration to filter.
|
||||
dataset = dataset.map(add_duration)
|
||||
|
||||
# Whisper max supported duration
|
||||
# Whisper max supported duration.
|
||||
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
|
||||
return dataset
|
||||
|
||||
@@ -145,11 +184,16 @@ def run_evaluation(
|
||||
max_concurrent_reqs: int,
|
||||
n_examples: int = -1,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
if n_examples > 0:
|
||||
dataset = dataset.select(range(n_examples))
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs))
|
||||
results = asyncio.run(
|
||||
process_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
@@ -164,6 +208,106 @@ def run_evaluation(
|
||||
return wer_score
|
||||
|
||||
|
||||
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
|
||||
LONGFORM_DATASET_SPLIT = "test"
|
||||
LONGFORM_NUM_SAMPLES = 6
|
||||
|
||||
|
||||
def load_longform_dataset():
|
||||
dataset = load_asr_dataset_rows(
|
||||
LONGFORM_DATASET_REPO,
|
||||
split=LONGFORM_DATASET_SPLIT,
|
||||
)
|
||||
assert len(dataset) >= LONGFORM_NUM_SAMPLES
|
||||
return dataset.select(range(LONGFORM_NUM_SAMPLES))
|
||||
|
||||
|
||||
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
|
||||
with open(audio_path, "rb") as f:
|
||||
start_time = time.perf_counter()
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
|
||||
latency = end_time - start_time
|
||||
num_output_tokens = len(
|
||||
tokenizer(transcription.text, add_special_tokens=False).input_ids
|
||||
)
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, reference, extra_body=None
|
||||
):
|
||||
async with sem:
|
||||
result = await transcribe_audio_path(
|
||||
client, tokenizer, audio_path, extra_body=extra_body
|
||||
)
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_longform_dataset(
|
||||
model, client, data, concurrent_request, extra_body=None
|
||||
):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
)
|
||||
|
||||
warmup_path = data[0]["audio"]["path"]
|
||||
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio_path = sample["audio"]["path"]
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
def run_longform_evaluation(
|
||||
model: str,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_reqs: int,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(
|
||||
process_longform_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
if print_metrics:
|
||||
print_performance_metrics(results, total_time)
|
||||
|
||||
predictions = [res[2] for res in results]
|
||||
references = [res[3] for res in results]
|
||||
wer = load("wer")
|
||||
wer_score = 100 * wer.compute(references=references, predictions=predictions)
|
||||
print("WER:", wer_score)
|
||||
return wer_score
|
||||
|
||||
|
||||
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
|
||||
# NOTE: Expected WER measured with equivalent hf.transformers args:
|
||||
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
|
||||
@@ -184,7 +328,6 @@ def test_wer_correctness(
|
||||
):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
# TODO refactor to use `ASRDataset`
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
@@ -197,7 +340,7 @@ def test_wer_correctness(
|
||||
model_name,
|
||||
server_args,
|
||||
) as remote_server:
|
||||
dataset = load_hf_dataset(dataset_repo)
|
||||
dataset = load_shortform_eval_dataset(dataset_repo)
|
||||
|
||||
if not max_concurrent_request:
|
||||
# No max concurrency
|
||||
@@ -216,3 +359,42 @@ def test_wer_correctness(
|
||||
|
||||
if expected_wer:
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
|
||||
|
||||
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
|
||||
# checks for long audio transcription correctness and RMS split.
|
||||
@pytest.mark.parametrize(
|
||||
"model_config",
|
||||
[("openai/whisper-large-v3", 9.5)],
|
||||
)
|
||||
def test_long_audio_wer_correctness(model_config):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
server_args = [
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
]
|
||||
|
||||
if model_info.trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
|
||||
# 1800 seconds is 30 minutes
|
||||
env_dict = {
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
dataset = load_longform_dataset()
|
||||
client = remote_server.get_async_client()
|
||||
wer = run_longform_evaluation(
|
||||
model=model_name,
|
||||
client=client,
|
||||
dataset=dataset,
|
||||
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
|
||||
)
|
||||
|
||||
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
|
||||
Reference in New Issue
Block a user