[ASR] Add Long Audio benchmark and correctness test (#44587)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
@@ -37,7 +37,7 @@ th {
|
||||
| HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` |
|
||||
| HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` |
|
||||
| HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |
|
||||
| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` |
|
||||
| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered` |
|
||||
| Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |
|
||||
| SPEED-Bench | ✅ | ✅ | `curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \| python3 -` |
|
||||
| Custom | ✅ | ✅ | Local file: `data.jsonl` |
|
||||
@@ -532,7 +532,7 @@ vllm bench serve \
|
||||
--blazedit-max-distance 0.99
|
||||
```
|
||||
|
||||
`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`
|
||||
`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered`
|
||||
|
||||
```bash
|
||||
vllm bench serve \
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Protocol, cast
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import soundfile as sf
|
||||
|
||||
import vllm.benchmarks.datasets.datasets as datasets_module
|
||||
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
|
||||
class _ReadableBinary(Protocol):
|
||||
def read(self, size: int = -1) -> bytes: ...
|
||||
|
||||
|
||||
class _TokenizedPrompt:
|
||||
def __init__(self, prompt: str) -> None:
|
||||
self.input_ids = prompt.split()
|
||||
|
||||
|
||||
class _Tokenizer:
|
||||
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
|
||||
self.name_or_path = name_or_path
|
||||
|
||||
def __call__(self, prompt: str) -> _TokenizedPrompt:
|
||||
return _TokenizedPrompt(prompt)
|
||||
|
||||
|
||||
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
|
||||
num_samples = int(duration_s * sample_rate)
|
||||
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
|
||||
|
||||
|
||||
class _FakeFormData:
|
||||
def __init__(self) -> None:
|
||||
self.fields: list[tuple[str, object, dict[str, str]]] = []
|
||||
|
||||
def add_field(self, name: str, value: object, **kwargs: str) -> None:
|
||||
self.fields.append((name, value, kwargs))
|
||||
|
||||
|
||||
class _FakeContent:
|
||||
async def iter_any(self):
|
||||
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
|
||||
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
|
||||
yield b"data: [DONE]\n\n"
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self) -> None:
|
||||
self.status = 200
|
||||
self.reason = "OK"
|
||||
self.content = _FakeContent()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self) -> None:
|
||||
self.uploaded_bytes: bytes | None = None
|
||||
self.upload_filename: str | None = None
|
||||
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
|
||||
|
||||
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
|
||||
del url, headers
|
||||
self.fields = list(data.fields)
|
||||
_, file_obj, file_kwargs = self.fields[0]
|
||||
file_obj = cast(_ReadableBinary, file_obj)
|
||||
self.uploaded_bytes = file_obj.read()
|
||||
self.upload_filename = file_kwargs.get("filename")
|
||||
return _FakeResponse()
|
||||
|
||||
|
||||
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {
|
||||
"path": str(audio_path),
|
||||
"bytes": None,
|
||||
},
|
||||
"text": "quarterly earnings call",
|
||||
}
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
|
||||
assert (
|
||||
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
|
||||
)
|
||||
|
||||
|
||||
def test_asr_dataset_sample_handles_embedded_audio_bytes(tmp_path: Path) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.1)
|
||||
|
||||
dataset = object.__new__(datasets_module.ASRDataset)
|
||||
dataset.data = [
|
||||
{
|
||||
"audio": {
|
||||
"path": None,
|
||||
"bytes": audio_path.read_bytes(),
|
||||
},
|
||||
"text": "quarterly earnings call",
|
||||
}
|
||||
]
|
||||
|
||||
samples = dataset.sample(
|
||||
tokenizer=_Tokenizer(),
|
||||
num_requests=1,
|
||||
output_len=32,
|
||||
asr_min_audio_len_sec=0.0,
|
||||
asr_max_audio_len_sec=1.0,
|
||||
)
|
||||
|
||||
assert len(samples) == 1
|
||||
assert isinstance(samples[0].multi_modal_data, dict)
|
||||
audio, sample_rate = samples[0].multi_modal_data["audio"]
|
||||
assert sample_rate == 16_000
|
||||
assert isinstance(audio, np.ndarray)
|
||||
assert audio.size > 0
|
||||
|
||||
|
||||
def test_async_request_openai_audio_handles_local_audio_paths(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
audio_path = tmp_path / "earnings.wav"
|
||||
_write_wav(audio_path, duration_s=0.25)
|
||||
|
||||
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
|
||||
session = _FakeSession()
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/v1/audio/transcriptions",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
multi_modal_content={"audio_path": str(audio_path)},
|
||||
)
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_openai_audio(request_input, session)
|
||||
)
|
||||
|
||||
assert session.upload_filename == audio_path.name
|
||||
assert session.uploaded_bytes == audio_path.read_bytes()
|
||||
assert output.success is True
|
||||
assert output.generated_text == "hello"
|
||||
assert output.output_tokens == 1
|
||||
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
|
||||
|
||||
|
||||
def test_async_request_openai_audio_handles_decoded_audio_arrays(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
|
||||
session = _FakeSession()
|
||||
request_input = RequestFuncInput(
|
||||
prompt="",
|
||||
api_url="http://localhost:8000/v1/audio/transcriptions",
|
||||
prompt_len=1,
|
||||
output_len=32,
|
||||
model="openai/whisper-large-v3",
|
||||
multi_modal_content={
|
||||
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
|
||||
},
|
||||
)
|
||||
|
||||
output = asyncio.run(
|
||||
request_func_module.async_request_openai_audio(request_input, session)
|
||||
)
|
||||
|
||||
assert session.upload_filename == "audio.wav"
|
||||
assert session.uploaded_bytes is not None
|
||||
assert output.success is True
|
||||
assert output.generated_text == "hello"
|
||||
+201
-19
@@ -16,10 +16,11 @@ from statistics import mean, median
|
||||
import pytest
|
||||
import soundfile
|
||||
import torch
|
||||
from datasets import load_dataset
|
||||
from datasets import Audio, load_dataset
|
||||
from evaluate import load
|
||||
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
|
||||
|
||||
from vllm.benchmarks.datasets.datasets import ASRDataset
|
||||
from vllm.multimodal.audio import get_audio_duration
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
@@ -38,6 +39,20 @@ def to_bytes(y, sr):
|
||||
return buffer
|
||||
|
||||
|
||||
def load_audio_sample(audio):
|
||||
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
|
||||
if "array" in audio and "sampling_rate" in audio:
|
||||
return audio["array"], audio["sampling_rate"]
|
||||
|
||||
if audio.get("path"):
|
||||
return soundfile.read(audio["path"], dtype="float32")
|
||||
|
||||
if audio.get("bytes") is not None:
|
||||
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
|
||||
|
||||
raise ValueError("Audio sample did not contain array, path, or bytes data")
|
||||
|
||||
|
||||
# not all models have a normalizer so use the one from whisper as a standard option
|
||||
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
|
||||
normalizer_tokenizer = get_tokenizer(
|
||||
@@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer(
|
||||
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
|
||||
|
||||
|
||||
async def transcribe_audio(client, tokenizer, y, sr):
|
||||
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
|
||||
# Send loaded audio directly instead of loading from disk,
|
||||
# don't account for that time though
|
||||
with to_bytes(y, sr) as f:
|
||||
@@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr):
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
# NOTE there's no streaming in transcriptions, can't measure ttft
|
||||
@@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr):
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe(sem, client, tokenizer, audio, reference):
|
||||
async def bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, reference, extra_body=None
|
||||
):
|
||||
# Use semaphore to limit concurrent requests.
|
||||
async with sem:
|
||||
result = await transcribe_audio(client, tokenizer, *audio)
|
||||
result = await transcribe_audio(
|
||||
client, tokenizer, audio, sr, extra_body=extra_body
|
||||
)
|
||||
# Normalize *english* output/reference for evaluation.
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_dataset(model, client, data, concurrent_request):
|
||||
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
@@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request):
|
||||
)
|
||||
|
||||
# Warmup call as the first `load_audio` server-side is quite slow.
|
||||
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
|
||||
_ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
|
||||
audio, sr = load_audio_sample(data[0]["audio"])
|
||||
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
|
||||
audio, sr = load_audio_sample(sample["audio"])
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"])
|
||||
bound_transcribe(
|
||||
sem, client, tokenizer, audio, sr, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
@@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time):
|
||||
|
||||
|
||||
def add_duration(sample):
|
||||
y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
|
||||
y, sr = load_audio_sample(sample["audio"])
|
||||
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
|
||||
return sample
|
||||
|
||||
|
||||
def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
## Load and filter the dataset
|
||||
dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
|
||||
if "duration_ms" not in dataset[0]:
|
||||
# compute duration to filter
|
||||
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
|
||||
asr_dataset_kwargs = {
|
||||
"dataset_path": dataset_repo,
|
||||
"dataset_split": split,
|
||||
"disable_shuffle": True,
|
||||
"no_stream": True,
|
||||
}
|
||||
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
|
||||
if key in hf_kwargs:
|
||||
asr_dataset_kwargs[key] = hf_kwargs[key]
|
||||
return ASRDataset(**asr_dataset_kwargs).data
|
||||
|
||||
return load_dataset(dataset_repo, split=split, **hf_kwargs)
|
||||
|
||||
|
||||
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
|
||||
## Load and filter the dataset.
|
||||
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
|
||||
dataset = dataset.cast_column("audio", Audio(decode=False))
|
||||
if "duration_ms" not in dataset.column_names:
|
||||
# Compute duration to filter.
|
||||
dataset = dataset.map(add_duration)
|
||||
|
||||
# Whisper max supported duration
|
||||
# Whisper max supported duration.
|
||||
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
|
||||
return dataset
|
||||
|
||||
@@ -145,11 +184,16 @@ def run_evaluation(
|
||||
max_concurrent_reqs: int,
|
||||
n_examples: int = -1,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
if n_examples > 0:
|
||||
dataset = dataset.select(range(n_examples))
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs))
|
||||
results = asyncio.run(
|
||||
process_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
@@ -164,6 +208,106 @@ def run_evaluation(
|
||||
return wer_score
|
||||
|
||||
|
||||
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
|
||||
LONGFORM_DATASET_SPLIT = "test"
|
||||
LONGFORM_NUM_SAMPLES = 6
|
||||
|
||||
|
||||
def load_longform_dataset():
|
||||
dataset = load_asr_dataset_rows(
|
||||
LONGFORM_DATASET_REPO,
|
||||
split=LONGFORM_DATASET_SPLIT,
|
||||
)
|
||||
assert len(dataset) >= LONGFORM_NUM_SAMPLES
|
||||
return dataset.select(range(LONGFORM_NUM_SAMPLES))
|
||||
|
||||
|
||||
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
|
||||
with open(audio_path, "rb") as f:
|
||||
start_time = time.perf_counter()
|
||||
transcription = await client.audio.transcriptions.create(
|
||||
file=f,
|
||||
model=tokenizer.name_or_path,
|
||||
language="en",
|
||||
temperature=0.0,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
end_time = time.perf_counter()
|
||||
|
||||
latency = end_time - start_time
|
||||
num_output_tokens = len(
|
||||
tokenizer(transcription.text, add_special_tokens=False).input_ids
|
||||
)
|
||||
return latency, num_output_tokens, transcription.text
|
||||
|
||||
|
||||
async def bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, reference, extra_body=None
|
||||
):
|
||||
async with sem:
|
||||
result = await transcribe_audio_path(
|
||||
client, tokenizer, audio_path, extra_body=extra_body
|
||||
)
|
||||
out = normalizer(result[2])
|
||||
ref = normalizer(reference)
|
||||
return result[:2] + (out, ref)
|
||||
|
||||
|
||||
async def process_longform_dataset(
|
||||
model, client, data, concurrent_request, extra_body=None
|
||||
):
|
||||
sem = asyncio.Semaphore(concurrent_request)
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
tokenizer = get_tokenizer(
|
||||
model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
)
|
||||
|
||||
warmup_path = data[0]["audio"]["path"]
|
||||
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
|
||||
|
||||
tasks: list[asyncio.Task] = []
|
||||
for sample in data:
|
||||
audio_path = sample["audio"]["path"]
|
||||
task = asyncio.create_task(
|
||||
bound_transcribe_path(
|
||||
sem, client, tokenizer, audio_path, sample["text"], extra_body
|
||||
)
|
||||
)
|
||||
tasks.append(task)
|
||||
return await asyncio.gather(*tasks)
|
||||
|
||||
|
||||
def run_longform_evaluation(
|
||||
model: str,
|
||||
client,
|
||||
dataset,
|
||||
max_concurrent_reqs: int,
|
||||
print_metrics: bool = True,
|
||||
extra_body=None,
|
||||
):
|
||||
start = time.perf_counter()
|
||||
results = asyncio.run(
|
||||
process_longform_dataset(
|
||||
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
|
||||
)
|
||||
)
|
||||
end = time.perf_counter()
|
||||
total_time = end - start
|
||||
print(f"Total Test Time: {total_time:.4f} seconds")
|
||||
if print_metrics:
|
||||
print_performance_metrics(results, total_time)
|
||||
|
||||
predictions = [res[2] for res in results]
|
||||
references = [res[3] for res in results]
|
||||
wer = load("wer")
|
||||
wer_score = 100 * wer.compute(references=references, predictions=predictions)
|
||||
print("WER:", wer_score)
|
||||
return wer_score
|
||||
|
||||
|
||||
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
|
||||
# NOTE: Expected WER measured with equivalent hf.transformers args:
|
||||
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
|
||||
@@ -184,7 +328,6 @@ def test_wer_correctness(
|
||||
):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
# TODO refactor to use `ASRDataset`
|
||||
server_args = [
|
||||
"--enforce-eager",
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
@@ -197,7 +340,7 @@ def test_wer_correctness(
|
||||
model_name,
|
||||
server_args,
|
||||
) as remote_server:
|
||||
dataset = load_hf_dataset(dataset_repo)
|
||||
dataset = load_shortform_eval_dataset(dataset_repo)
|
||||
|
||||
if not max_concurrent_request:
|
||||
# No max concurrency
|
||||
@@ -216,3 +359,42 @@ def test_wer_correctness(
|
||||
|
||||
if expected_wer:
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
|
||||
|
||||
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
|
||||
# checks for long audio transcription correctness and RMS split.
|
||||
@pytest.mark.parametrize(
|
||||
"model_config",
|
||||
[("openai/whisper-large-v3", 9.5)],
|
||||
)
|
||||
def test_long_audio_wer_correctness(model_config):
|
||||
model_name, expected_wer = model_config
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
|
||||
server_args = [
|
||||
f"--tokenizer_mode={model_info.tokenizer_mode}",
|
||||
]
|
||||
|
||||
if model_info.trust_remote_code:
|
||||
server_args.append("--trust-remote-code")
|
||||
|
||||
# 1800 seconds is 30 minutes
|
||||
env_dict = {
|
||||
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
|
||||
}
|
||||
|
||||
with RemoteOpenAIServer(
|
||||
model_name,
|
||||
server_args,
|
||||
env_dict=env_dict,
|
||||
) as remote_server:
|
||||
dataset = load_longform_dataset()
|
||||
client = remote_server.get_async_client()
|
||||
wer = run_longform_evaluation(
|
||||
model=model_name,
|
||||
client=client,
|
||||
dataset=dataset,
|
||||
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
|
||||
)
|
||||
|
||||
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
|
||||
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
|
||||
|
||||
@@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset):
|
||||
Dataset class for processing a ASR dataset for transcription.
|
||||
Tested on the following set:
|
||||
|
||||
+----------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| Dataset | Domain | Speaking Style | hf-subset |
|
||||
+----------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
|
||||
| | | | release3-speaker-adaptation |
|
||||
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
|
||||
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
|
||||
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
|
||||
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
|
||||
| AMI | Meetings | Spontaneous | ihm, sdm |
|
||||
+----------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| Dataset | Domain | Speaking Style | hf-subset |
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
|
||||
| | | | release3-speaker-adaptation |
|
||||
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
|
||||
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
|
||||
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
|
||||
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
|
||||
| Earnings22-Cleaned-AA | Long form earnings calls | Prepared remarks, Q&A | test |
|
||||
| Earnings22-Tiny-Filtered | Earnings calls | Prepared remarks, Q&A | validation |
|
||||
| AMI | Meetings | Spontaneous | ihm, sdm |
|
||||
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
|
||||
|
||||
""" # noqa: E501
|
||||
|
||||
EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA"
|
||||
EARNINGS22_TINY_FILTERED_DATASET = (
|
||||
"D4nt3/esb-datasets-earnings22-validation-tiny-filtered"
|
||||
)
|
||||
|
||||
SUPPORTED_DATASET_PATHS = {
|
||||
"openslr/librispeech_asr",
|
||||
"facebook/voxpopuli",
|
||||
@@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset):
|
||||
"edinburghcstr/ami",
|
||||
"speechcolab/gigaspeech",
|
||||
"kensho/spgispeech",
|
||||
EARNINGS22_CLEANED_DATASET,
|
||||
EARNINGS22_TINY_FILTERED_DATASET,
|
||||
}
|
||||
|
||||
DEFAULT_OUTPUT_LEN = 1024
|
||||
IS_MULTIMODAL = True
|
||||
|
||||
def load_data(self) -> None:
|
||||
if self.hf_name == self.EARNINGS22_CLEANED_DATASET:
|
||||
# This subset stores repo-local MP3 paths instead of a HF `Audio`
|
||||
# column, so eagerly materialize it back into the common schema.
|
||||
self.data = load_dataset(
|
||||
self.dataset_path,
|
||||
name=self.dataset_subset,
|
||||
split=self.dataset_split,
|
||||
streaming=False,
|
||||
trust_remote_code=self.trust_remote_code,
|
||||
)
|
||||
if not getattr(self, "disable_shuffle", False):
|
||||
self.data = self.data.shuffle(seed=self.random_seed)
|
||||
self._materialize_local_audio_column()
|
||||
return
|
||||
if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET:
|
||||
super().load_data()
|
||||
self._disable_audio_decode()
|
||||
return
|
||||
|
||||
super().load_data()
|
||||
|
||||
def _disable_audio_decode(self) -> None:
|
||||
from datasets import Audio
|
||||
|
||||
self.data = self.data.cast_column("audio", Audio(decode=False))
|
||||
|
||||
def _materialize_local_audio_column(self) -> None:
|
||||
local_path_root = Path(
|
||||
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
|
||||
)
|
||||
self.data = self.data.map(
|
||||
lambda item: {
|
||||
"audio": str(local_path_root / item["url"]),
|
||||
"text": item["transcript"],
|
||||
}
|
||||
)
|
||||
self._disable_audio_decode()
|
||||
|
||||
def sample(
|
||||
self,
|
||||
tokenizer: TokenizerLike,
|
||||
@@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset):
|
||||
if len(sampled_requests) >= num_requests:
|
||||
break
|
||||
audio = item["audio"]
|
||||
y, sr = audio["array"], audio["sampling_rate"]
|
||||
duration_s = get_audio_duration(y=y, sr=sr)
|
||||
if (
|
||||
isinstance(audio, dict)
|
||||
and "array" in audio
|
||||
and "sampling_rate" in audio
|
||||
):
|
||||
y, sr = audio["array"], audio["sampling_rate"]
|
||||
duration_s = get_audio_duration(y=y, sr=sr)
|
||||
mm_content = {"audio": (y, sr)}
|
||||
elif isinstance(audio, str):
|
||||
duration_s = sf.info(audio).duration
|
||||
mm_content = {"audio_path": audio}
|
||||
elif isinstance(audio, dict) and audio.get("path"):
|
||||
duration_s = sf.info(audio["path"]).duration
|
||||
mm_content = {"audio_path": audio["path"]}
|
||||
elif isinstance(audio, dict) and audio.get("bytes") is not None:
|
||||
with BytesIO(audio["bytes"]) as audio_buffer:
|
||||
y, sr = sf.read(audio_buffer, dtype="float32")
|
||||
duration_s = get_audio_duration(y=y, sr=sr)
|
||||
mm_content = {"audio": (y, sr)}
|
||||
else:
|
||||
raise ValueError(
|
||||
"ASR samples must provide decoded audio arrays, "
|
||||
"embedded audio bytes, or a local audio path."
|
||||
)
|
||||
if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
durations.append(duration_s)
|
||||
mm_content = {"audio": (y, sr)}
|
||||
sampled_requests.append(
|
||||
SampleRequest(
|
||||
prompt=prompt,
|
||||
|
||||
@@ -445,7 +445,6 @@ async def async_request_openai_audio(
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Audio API", {"transcriptions", "translations"})
|
||||
|
||||
content = [{"type": "text", "text": request_func_input.prompt}]
|
||||
payload = {
|
||||
"model": request_func_input.model_name
|
||||
if request_func_input.model_name
|
||||
@@ -469,19 +468,26 @@ async def async_request_openai_audio(
|
||||
buffer.seek(0)
|
||||
return buffer
|
||||
|
||||
mm_audio = request_func_input.multi_modal_content
|
||||
if not isinstance(mm_audio, dict) or "audio" not in mm_audio:
|
||||
raise TypeError("multi_modal_content must be a dict containing 'audio'")
|
||||
with to_bytes(*mm_audio["audio"]) as f:
|
||||
async def send_audio_file(
|
||||
audio_file: io.BytesIO | Any,
|
||||
*,
|
||||
input_audio_duration: float,
|
||||
filename: str | None = None,
|
||||
content_type: str | None = None,
|
||||
) -> RequestFuncOutput:
|
||||
form = aiohttp.FormData()
|
||||
form.add_field("file", f, content_type="audio/wav")
|
||||
add_field_kwargs: dict[str, str] = {}
|
||||
if filename is not None:
|
||||
add_field_kwargs["filename"] = filename
|
||||
if content_type is not None:
|
||||
add_field_kwargs["content_type"] = content_type
|
||||
form.add_field("file", audio_file, **add_field_kwargs)
|
||||
for key, value in payload.items():
|
||||
form.add_field(key, str(value))
|
||||
|
||||
output = RequestFuncOutput()
|
||||
output.prompt_len = request_func_input.prompt_len
|
||||
output.input_audio_duration = soundfile.info(f).duration
|
||||
f.seek(0)
|
||||
output.input_audio_duration = input_audio_duration
|
||||
|
||||
generated_text = ""
|
||||
ttft = 0.0
|
||||
@@ -541,9 +547,36 @@ async def async_request_openai_audio(
|
||||
exc_info = sys.exc_info()
|
||||
output.error = "".join(traceback.format_exception(*exc_info))
|
||||
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
if pbar:
|
||||
pbar.update(1)
|
||||
return output
|
||||
|
||||
mm_audio = request_func_input.multi_modal_content
|
||||
if not isinstance(mm_audio, dict):
|
||||
raise TypeError(
|
||||
"multi_modal_content must be a dict containing 'audio' or 'audio_path'"
|
||||
)
|
||||
if "audio" in mm_audio:
|
||||
with to_bytes(*mm_audio["audio"]) as f:
|
||||
input_audio_duration = soundfile.info(f).duration
|
||||
f.seek(0)
|
||||
return await send_audio_file(
|
||||
f,
|
||||
input_audio_duration=input_audio_duration,
|
||||
filename="audio.wav",
|
||||
content_type="audio/wav",
|
||||
)
|
||||
if "audio_path" in mm_audio:
|
||||
audio_path = mm_audio["audio_path"]
|
||||
with open(audio_path, "rb") as f:
|
||||
return await send_audio_file(
|
||||
f,
|
||||
input_audio_duration=soundfile.info(audio_path).duration,
|
||||
filename=os.path.basename(audio_path),
|
||||
)
|
||||
raise TypeError(
|
||||
"multi_modal_content must be a dict containing 'audio' or 'audio_path'"
|
||||
)
|
||||
|
||||
|
||||
async def _run_pooling_request(
|
||||
|
||||
Reference in New Issue
Block a user