From 226ba9fc9e285556e7269e4efe102530a4d9fedb Mon Sep 17 00:00:00 2001 From: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Date: Fri, 12 Jun 2026 00:11:16 -0400 Subject: [PATCH] [ASR] Add Long Audio benchmark and correctness test (#44587) Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> --- docs/benchmarking/cli.md | 4 +- tests/benchmarks/test_audio_dataset.py | 200 ++++++++++++++++ .../test_transcription_api_correctness.py | 220 ++++++++++++++++-- vllm/benchmarks/datasets/datasets.py | 97 ++++++-- vllm/benchmarks/lib/endpoint_request_func.py | 55 ++++- 5 files changed, 530 insertions(+), 46 deletions(-) create mode 100644 tests/benchmarks/test_audio_dataset.py diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index 3d8fda95a3..22406f2eaa 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -37,7 +37,7 @@ th { | HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` | | HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` | | HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` | -| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` | +| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered` | | Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` | | SPEED-Bench | ✅ | ✅ | `curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \| python3 -` | | Custom | ✅ | ✅ | Local file: `data.jsonl` | @@ -532,7 +532,7 @@ vllm bench serve \ --blazedit-max-distance 0.99 ``` -`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` +`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered` ```bash vllm bench serve \ diff --git a/tests/benchmarks/test_audio_dataset.py b/tests/benchmarks/test_audio_dataset.py new file mode 100644 index 0000000000..5957011c48 --- /dev/null +++ b/tests/benchmarks/test_audio_dataset.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import asyncio +from pathlib import Path +from typing import Protocol, cast + +import numpy as np +import pytest +import soundfile as sf + +import vllm.benchmarks.datasets.datasets as datasets_module +import vllm.benchmarks.lib.endpoint_request_func as request_func_module +from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput + +pytestmark = pytest.mark.skip_global_cleanup + + +class _ReadableBinary(Protocol): + def read(self, size: int = -1) -> bytes: ... + + +class _TokenizedPrompt: + def __init__(self, prompt: str) -> None: + self.input_ids = prompt.split() + + +class _Tokenizer: + def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None: + self.name_or_path = name_or_path + + def __call__(self, prompt: str) -> _TokenizedPrompt: + return _TokenizedPrompt(prompt) + + +def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None: + num_samples = int(duration_s * sample_rate) + sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate) + + +class _FakeFormData: + def __init__(self) -> None: + self.fields: list[tuple[str, object, dict[str, str]]] = [] + + def add_field(self, name: str, value: object, **kwargs: str) -> None: + self.fields.append((name, value, kwargs)) + + +class _FakeContent: + async def iter_any(self): + yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n' + yield b'data: {"usage":{"completion_tokens":1}}\n\n' + yield b"data: [DONE]\n\n" + + +class _FakeResponse: + def __init__(self) -> None: + self.status = 200 + self.reason = "OK" + self.content = _FakeContent() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return False + + +class _FakeSession: + def __init__(self) -> None: + self.uploaded_bytes: bytes | None = None + self.upload_filename: str | None = None + self.fields: list[tuple[str, object, dict[str, str]]] | None = None + + def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]): + del url, headers + self.fields = list(data.fields) + _, file_obj, file_kwargs = self.fields[0] + file_obj = cast(_ReadableBinary, file_obj) + self.uploaded_bytes = file_obj.read() + self.upload_filename = file_kwargs.get("filename") + return _FakeResponse() + + +def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None: + audio_path = tmp_path / "earnings.wav" + _write_wav(audio_path, duration_s=0.1) + + dataset = object.__new__(datasets_module.ASRDataset) + dataset.data = [ + { + "audio": { + "path": str(audio_path), + "bytes": None, + }, + "text": "quarterly earnings call", + } + ] + + samples = dataset.sample( + tokenizer=_Tokenizer(), + num_requests=1, + output_len=32, + asr_min_audio_len_sec=0.0, + asr_max_audio_len_sec=1.0, + ) + + assert len(samples) == 1 + assert samples[0].multi_modal_data == {"audio_path": str(audio_path)} + assert ( + samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>" + ) + + +def test_asr_dataset_sample_handles_embedded_audio_bytes(tmp_path: Path) -> None: + audio_path = tmp_path / "earnings.wav" + _write_wav(audio_path, duration_s=0.1) + + dataset = object.__new__(datasets_module.ASRDataset) + dataset.data = [ + { + "audio": { + "path": None, + "bytes": audio_path.read_bytes(), + }, + "text": "quarterly earnings call", + } + ] + + samples = dataset.sample( + tokenizer=_Tokenizer(), + num_requests=1, + output_len=32, + asr_min_audio_len_sec=0.0, + asr_max_audio_len_sec=1.0, + ) + + assert len(samples) == 1 + assert isinstance(samples[0].multi_modal_data, dict) + audio, sample_rate = samples[0].multi_modal_data["audio"] + assert sample_rate == 16_000 + assert isinstance(audio, np.ndarray) + assert audio.size > 0 + + +def test_async_request_openai_audio_handles_local_audio_paths( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + audio_path = tmp_path / "earnings.wav" + _write_wav(audio_path, duration_s=0.25) + + monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData) + session = _FakeSession() + request_input = RequestFuncInput( + prompt="", + api_url="http://localhost:8000/v1/audio/transcriptions", + prompt_len=1, + output_len=32, + model="openai/whisper-large-v3", + multi_modal_content={"audio_path": str(audio_path)}, + ) + + output = asyncio.run( + request_func_module.async_request_openai_audio(request_input, session) + ) + + assert session.upload_filename == audio_path.name + assert session.uploaded_bytes == audio_path.read_bytes() + assert output.success is True + assert output.generated_text == "hello" + assert output.output_tokens == 1 + assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2) + + +def test_async_request_openai_audio_handles_decoded_audio_arrays( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData) + session = _FakeSession() + request_input = RequestFuncInput( + prompt="", + api_url="http://localhost:8000/v1/audio/transcriptions", + prompt_len=1, + output_len=32, + model="openai/whisper-large-v3", + multi_modal_content={ + "audio": (np.zeros(1_600, dtype=np.float32), 16_000), + }, + ) + + output = asyncio.run( + request_func_module.async_request_openai_audio(request_input, session) + ) + + assert session.upload_filename == "audio.wav" + assert session.uploaded_bytes is not None + assert output.success is True + assert output.generated_text == "hello" diff --git a/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py index fedbd74795..af61ebc526 100644 --- a/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py +++ b/tests/entrypoints/speech_to_text/correctness/test_transcription_api_correctness.py @@ -16,10 +16,11 @@ from statistics import mean, median import pytest import soundfile import torch -from datasets import load_dataset +from datasets import Audio, load_dataset from evaluate import load from transformers.models.whisper.english_normalizer import EnglishTextNormalizer +from vllm.benchmarks.datasets.datasets import ASRDataset from vllm.multimodal.audio import get_audio_duration from vllm.tokenizers import get_tokenizer @@ -38,6 +39,20 @@ def to_bytes(y, sr): return buffer +def load_audio_sample(audio): + # Avoid torchcodec in CI by decoding dataset audio with soundfile. + if "array" in audio and "sampling_rate" in audio: + return audio["array"], audio["sampling_rate"] + + if audio.get("path"): + return soundfile.read(audio["path"], dtype="float32") + + if audio.get("bytes") is not None: + return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32") + + raise ValueError("Audio sample did not contain array, path, or bytes data") + + # not all models have a normalizer so use the one from whisper as a standard option normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3") normalizer_tokenizer = get_tokenizer( @@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer( normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer) -async def transcribe_audio(client, tokenizer, y, sr): +async def transcribe_audio(client, tokenizer, y, sr, extra_body=None): # Send loaded audio directly instead of loading from disk, # don't account for that time though with to_bytes(y, sr) as f: @@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr): model=tokenizer.name_or_path, language="en", temperature=0.0, + extra_body=extra_body, ) end_time = time.perf_counter() # NOTE there's no streaming in transcriptions, can't measure ttft @@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr): return latency, num_output_tokens, transcription.text -async def bound_transcribe(sem, client, tokenizer, audio, reference): +async def bound_transcribe( + sem, client, tokenizer, audio, sr, reference, extra_body=None +): # Use semaphore to limit concurrent requests. async with sem: - result = await transcribe_audio(client, tokenizer, *audio) + result = await transcribe_audio( + client, tokenizer, audio, sr, extra_body=extra_body + ) # Normalize *english* output/reference for evaluation. out = normalizer(result[2]) ref = normalizer(reference) return result[:2] + (out, ref) -async def process_dataset(model, client, data, concurrent_request): +async def process_dataset(model, client, data, concurrent_request, extra_body=None): sem = asyncio.Semaphore(concurrent_request) model_info = HF_EXAMPLE_MODELS.find_hf_info(model) @@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request): ) # Warmup call as the first `load_audio` server-side is quite slow. - audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"] - _ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "") + audio, sr = load_audio_sample(data[0]["audio"]) + _ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body) tasks: list[asyncio.Task] = [] for sample in data: - audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"] + audio, sr = load_audio_sample(sample["audio"]) task = asyncio.create_task( - bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"]) + bound_transcribe( + sem, client, tokenizer, audio, sr, sample["text"], extra_body + ) ) tasks.append(task) return await asyncio.gather(*tasks) @@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time): def add_duration(sample): - y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"] + y, sr = load_audio_sample(sample["audio"]) sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000 return sample -def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs): - ## Load and filter the dataset - dataset = load_dataset(dataset_repo, split=split, **hf_kwargs) - if "duration_ms" not in dataset[0]: - # compute duration to filter +def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs): + if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS: + asr_dataset_kwargs = { + "dataset_path": dataset_repo, + "dataset_split": split, + "disable_shuffle": True, + "no_stream": True, + } + for key in ("dataset_subset", "hf_name", "trust_remote_code"): + if key in hf_kwargs: + asr_dataset_kwargs[key] = hf_kwargs[key] + return ASRDataset(**asr_dataset_kwargs).data + + return load_dataset(dataset_repo, split=split, **hf_kwargs) + + +def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs): + ## Load and filter the dataset. + dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs) + dataset = dataset.cast_column("audio", Audio(decode=False)) + if "duration_ms" not in dataset.column_names: + # Compute duration to filter. dataset = dataset.map(add_duration) - # Whisper max supported duration + # Whisper max supported duration. dataset = dataset.filter(lambda example: example["duration_ms"] < 30000) return dataset @@ -145,11 +184,16 @@ def run_evaluation( max_concurrent_reqs: int, n_examples: int = -1, print_metrics: bool = True, + extra_body=None, ): if n_examples > 0: dataset = dataset.select(range(n_examples)) start = time.perf_counter() - results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs)) + results = asyncio.run( + process_dataset( + model, client, dataset, max_concurrent_reqs, extra_body=extra_body + ) + ) end = time.perf_counter() total_time = end - start print(f"Total Test Time: {total_time:.4f} seconds") @@ -164,6 +208,106 @@ def run_evaluation( return wer_score +LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET +LONGFORM_DATASET_SPLIT = "test" +LONGFORM_NUM_SAMPLES = 6 + + +def load_longform_dataset(): + dataset = load_asr_dataset_rows( + LONGFORM_DATASET_REPO, + split=LONGFORM_DATASET_SPLIT, + ) + assert len(dataset) >= LONGFORM_NUM_SAMPLES + return dataset.select(range(LONGFORM_NUM_SAMPLES)) + + +async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None): + with open(audio_path, "rb") as f: + start_time = time.perf_counter() + transcription = await client.audio.transcriptions.create( + file=f, + model=tokenizer.name_or_path, + language="en", + temperature=0.0, + extra_body=extra_body, + ) + end_time = time.perf_counter() + + latency = end_time - start_time + num_output_tokens = len( + tokenizer(transcription.text, add_special_tokens=False).input_ids + ) + return latency, num_output_tokens, transcription.text + + +async def bound_transcribe_path( + sem, client, tokenizer, audio_path, reference, extra_body=None +): + async with sem: + result = await transcribe_audio_path( + client, tokenizer, audio_path, extra_body=extra_body + ) + out = normalizer(result[2]) + ref = normalizer(reference) + return result[:2] + (out, ref) + + +async def process_longform_dataset( + model, client, data, concurrent_request, extra_body=None +): + sem = asyncio.Semaphore(concurrent_request) + + model_info = HF_EXAMPLE_MODELS.find_hf_info(model) + tokenizer = get_tokenizer( + model, + tokenizer_mode=model_info.tokenizer_mode, + trust_remote_code=model_info.trust_remote_code, + ) + + warmup_path = data[0]["audio"]["path"] + _ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body) + + tasks: list[asyncio.Task] = [] + for sample in data: + audio_path = sample["audio"]["path"] + task = asyncio.create_task( + bound_transcribe_path( + sem, client, tokenizer, audio_path, sample["text"], extra_body + ) + ) + tasks.append(task) + return await asyncio.gather(*tasks) + + +def run_longform_evaluation( + model: str, + client, + dataset, + max_concurrent_reqs: int, + print_metrics: bool = True, + extra_body=None, +): + start = time.perf_counter() + results = asyncio.run( + process_longform_dataset( + model, client, dataset, max_concurrent_reqs, extra_body=extra_body + ) + ) + end = time.perf_counter() + total_time = end - start + print(f"Total Test Time: {total_time:.4f} seconds") + if print_metrics: + print_performance_metrics(results, total_time) + + predictions = [res[2] for res in results] + references = [res[3] for res in results] + wer = load("wer") + wer_score = 100 * wer.compute(references=references, predictions=predictions) + print("WER:", wer_score) + return wer_score + + # alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo".. # NOTE: Expected WER measured with equivalent hf.transformers args: # whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered. @@ -184,7 +328,6 @@ def test_wer_correctness( ): model_name, expected_wer = model_config model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name) - # TODO refactor to use `ASRDataset` server_args = [ "--enforce-eager", f"--tokenizer_mode={model_info.tokenizer_mode}", @@ -197,7 +340,7 @@ def test_wer_correctness( model_name, server_args, ) as remote_server: - dataset = load_hf_dataset(dataset_repo) + dataset = load_shortform_eval_dataset(dataset_repo) if not max_concurrent_request: # No max concurrency @@ -216,3 +359,42 @@ def test_wer_correctness( if expected_wer: torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2) + + +# 14-22mins of 6 audio samples of total ~115 mins and just 37MB. +# checks for long audio transcription correctness and RMS split. +@pytest.mark.parametrize( + "model_config", + [("openai/whisper-large-v3", 9.5)], +) +def test_long_audio_wer_correctness(model_config): + model_name, expected_wer = model_config + model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name) + server_args = [ + f"--tokenizer_mode={model_info.tokenizer_mode}", + ] + + if model_info.trust_remote_code: + server_args.append("--trust-remote-code") + + # 1800 seconds is 30 minutes + env_dict = { + "VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800", + } + + with RemoteOpenAIServer( + model_name, + server_args, + env_dict=env_dict, + ) as remote_server: + dataset = load_longform_dataset() + client = remote_server.get_async_client() + wer = run_longform_evaluation( + model=model_name, + client=client, + dataset=dataset, + max_concurrent_reqs=LONGFORM_NUM_SAMPLES, + ) + + print(f"Expected WER: {expected_wer}, Actual WER: {wer}") + torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2) diff --git a/vllm/benchmarks/datasets/datasets.py b/vllm/benchmarks/datasets/datasets.py index abdcedd12b..25ceadc41a 100644 --- a/vllm/benchmarks/datasets/datasets.py +++ b/vllm/benchmarks/datasets/datasets.py @@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset): Dataset class for processing a ASR dataset for transcription. Tested on the following set: - +----------------+----------------------------------------+--------------------------+-----------------------------+ - | Dataset | Domain | Speaking Style | hf-subset | - +----------------+----------------------------------------+--------------------------+-----------------------------+ - | TED-LIUM | TED talks | Oratory | release1, release2, release3| - | | | | release3-speaker-adaptation | - | VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... | - | LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" | - | GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test | - | SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test | - | AMI | Meetings | Spontaneous | ihm, sdm | - +----------------+----------------------------------------+--------------------------+-----------------------------+ + +---------------------------+----------------------------------------+--------------------------+-----------------------------+ + | Dataset | Domain | Speaking Style | hf-subset | + +---------------------------+----------------------------------------+--------------------------+-----------------------------+ + | TED-LIUM | TED talks | Oratory | release1, release2, release3| + | | | | release3-speaker-adaptation | + | VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... | + | LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" | + | GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test | + | SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test | + | Earnings22-Cleaned-AA | Long form earnings calls | Prepared remarks, Q&A | test | + | Earnings22-Tiny-Filtered | Earnings calls | Prepared remarks, Q&A | validation | + | AMI | Meetings | Spontaneous | ihm, sdm | + +---------------------------+----------------------------------------+--------------------------+-----------------------------+ """ # noqa: E501 + EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA" + EARNINGS22_TINY_FILTERED_DATASET = ( + "D4nt3/esb-datasets-earnings22-validation-tiny-filtered" + ) + SUPPORTED_DATASET_PATHS = { "openslr/librispeech_asr", "facebook/voxpopuli", @@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset): "edinburghcstr/ami", "speechcolab/gigaspeech", "kensho/spgispeech", + EARNINGS22_CLEANED_DATASET, + EARNINGS22_TINY_FILTERED_DATASET, } DEFAULT_OUTPUT_LEN = 1024 IS_MULTIMODAL = True + def load_data(self) -> None: + if self.hf_name == self.EARNINGS22_CLEANED_DATASET: + # This subset stores repo-local MP3 paths instead of a HF `Audio` + # column, so eagerly materialize it back into the common schema. + self.data = load_dataset( + self.dataset_path, + name=self.dataset_subset, + split=self.dataset_split, + streaming=False, + trust_remote_code=self.trust_remote_code, + ) + if not getattr(self, "disable_shuffle", False): + self.data = self.data.shuffle(seed=self.random_seed) + self._materialize_local_audio_column() + return + if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET: + super().load_data() + self._disable_audio_decode() + return + + super().load_data() + + def _disable_audio_decode(self) -> None: + from datasets import Audio + + self.data = self.data.cast_column("audio", Audio(decode=False)) + + def _materialize_local_audio_column(self) -> None: + local_path_root = Path( + hf_api().snapshot_download(self.hf_name, repo_type="dataset") + ) + self.data = self.data.map( + lambda item: { + "audio": str(local_path_root / item["url"]), + "text": item["transcript"], + } + ) + self._disable_audio_decode() + def sample( self, tokenizer: TokenizerLike, @@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset): if len(sampled_requests) >= num_requests: break audio = item["audio"] - y, sr = audio["array"], audio["sampling_rate"] - duration_s = get_audio_duration(y=y, sr=sr) + if ( + isinstance(audio, dict) + and "array" in audio + and "sampling_rate" in audio + ): + y, sr = audio["array"], audio["sampling_rate"] + duration_s = get_audio_duration(y=y, sr=sr) + mm_content = {"audio": (y, sr)} + elif isinstance(audio, str): + duration_s = sf.info(audio).duration + mm_content = {"audio_path": audio} + elif isinstance(audio, dict) and audio.get("path"): + duration_s = sf.info(audio["path"]).duration + mm_content = {"audio_path": audio["path"]} + elif isinstance(audio, dict) and audio.get("bytes") is not None: + with BytesIO(audio["bytes"]) as audio_buffer: + y, sr = sf.read(audio_buffer, dtype="float32") + duration_s = get_audio_duration(y=y, sr=sr) + mm_content = {"audio": (y, sr)} + else: + raise ValueError( + "ASR samples must provide decoded audio arrays, " + "embedded audio bytes, or a local audio path." + ) if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec: skipped += 1 continue durations.append(duration_s) - mm_content = {"audio": (y, sr)} sampled_requests.append( SampleRequest( prompt=prompt, diff --git a/vllm/benchmarks/lib/endpoint_request_func.py b/vllm/benchmarks/lib/endpoint_request_func.py index d282033ba1..db58f422b8 100644 --- a/vllm/benchmarks/lib/endpoint_request_func.py +++ b/vllm/benchmarks/lib/endpoint_request_func.py @@ -445,7 +445,6 @@ async def async_request_openai_audio( api_url = request_func_input.api_url _validate_api_url(api_url, "OpenAI Audio API", {"transcriptions", "translations"}) - content = [{"type": "text", "text": request_func_input.prompt}] payload = { "model": request_func_input.model_name if request_func_input.model_name @@ -469,19 +468,26 @@ async def async_request_openai_audio( buffer.seek(0) return buffer - mm_audio = request_func_input.multi_modal_content - if not isinstance(mm_audio, dict) or "audio" not in mm_audio: - raise TypeError("multi_modal_content must be a dict containing 'audio'") - with to_bytes(*mm_audio["audio"]) as f: + async def send_audio_file( + audio_file: io.BytesIO | Any, + *, + input_audio_duration: float, + filename: str | None = None, + content_type: str | None = None, + ) -> RequestFuncOutput: form = aiohttp.FormData() - form.add_field("file", f, content_type="audio/wav") + add_field_kwargs: dict[str, str] = {} + if filename is not None: + add_field_kwargs["filename"] = filename + if content_type is not None: + add_field_kwargs["content_type"] = content_type + form.add_field("file", audio_file, **add_field_kwargs) for key, value in payload.items(): form.add_field(key, str(value)) output = RequestFuncOutput() output.prompt_len = request_func_input.prompt_len - output.input_audio_duration = soundfile.info(f).duration - f.seek(0) + output.input_audio_duration = input_audio_duration generated_text = "" ttft = 0.0 @@ -541,9 +547,36 @@ async def async_request_openai_audio( exc_info = sys.exc_info() output.error = "".join(traceback.format_exception(*exc_info)) - if pbar: - pbar.update(1) - return output + if pbar: + pbar.update(1) + return output + + mm_audio = request_func_input.multi_modal_content + if not isinstance(mm_audio, dict): + raise TypeError( + "multi_modal_content must be a dict containing 'audio' or 'audio_path'" + ) + if "audio" in mm_audio: + with to_bytes(*mm_audio["audio"]) as f: + input_audio_duration = soundfile.info(f).duration + f.seek(0) + return await send_audio_file( + f, + input_audio_duration=input_audio_duration, + filename="audio.wav", + content_type="audio/wav", + ) + if "audio_path" in mm_audio: + audio_path = mm_audio["audio_path"] + with open(audio_path, "rb") as f: + return await send_audio_file( + f, + input_audio_duration=soundfile.info(audio_path).duration, + filename=os.path.basename(audio_path), + ) + raise TypeError( + "multi_modal_content must be a dict containing 'audio' or 'audio_path'" + ) async def _run_pooling_request(