[ASR] Add Long Audio benchmark and correctness test (#44587)

Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com>
This commit is contained in:
Ekagra Ranjan
2026-06-12 04:11:16 +00:00
committed by GitHub
parent b927004c44
commit 226ba9fc9e
5 changed files with 530 additions and 46 deletions
+2 -2
View File
@@ -37,7 +37,7 @@ th {
| HuggingFace-HumanEval | ✅ | ✅ | `openai/openai_humaneval` |
| HuggingFace-GSM8K | ✅ | ✅ | `openai/gsm8k` |
| HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |
| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech` |
| HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered` |
| Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |
| SPEED-Bench | ✅ | ✅ | `curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \| python3 -` |
| Custom | ✅ | ✅ | Local file: `data.jsonl` |
@@ -532,7 +532,7 @@ vllm bench serve \
--blazedit-max-distance 0.99
```
`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`
`openslr/librispeech_asr`, `facebook/voxpopuli`, `LIUM/tedlium`, `edinburghcstr/ami`, `speechcolab/gigaspeech`, `kensho/spgispeech`, `ArtificialAnalysis/Earnings22-Cleaned-AA`, `D4nt3/esb-datasets-earnings22-validation-tiny-filtered`
```bash
vllm bench serve \
+200
View File
@@ -0,0 +1,200 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from pathlib import Path
from typing import Protocol, cast
import numpy as np
import pytest
import soundfile as sf
import vllm.benchmarks.datasets.datasets as datasets_module
import vllm.benchmarks.lib.endpoint_request_func as request_func_module
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
pytestmark = pytest.mark.skip_global_cleanup
class _ReadableBinary(Protocol):
def read(self, size: int = -1) -> bytes: ...
class _TokenizedPrompt:
def __init__(self, prompt: str) -> None:
self.input_ids = prompt.split()
class _Tokenizer:
def __init__(self, name_or_path: str = "openai/whisper-large-v3") -> None:
self.name_or_path = name_or_path
def __call__(self, prompt: str) -> _TokenizedPrompt:
return _TokenizedPrompt(prompt)
def _write_wav(path: Path, duration_s: float = 0.1, sample_rate: int = 16_000) -> None:
num_samples = int(duration_s * sample_rate)
sf.write(path, np.zeros(num_samples, dtype=np.float32), sample_rate)
class _FakeFormData:
def __init__(self) -> None:
self.fields: list[tuple[str, object, dict[str, str]]] = []
def add_field(self, name: str, value: object, **kwargs: str) -> None:
self.fields.append((name, value, kwargs))
class _FakeContent:
async def iter_any(self):
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
yield b'data: {"usage":{"completion_tokens":1}}\n\n'
yield b"data: [DONE]\n\n"
class _FakeResponse:
def __init__(self) -> None:
self.status = 200
self.reason = "OK"
self.content = _FakeContent()
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return False
class _FakeSession:
def __init__(self) -> None:
self.uploaded_bytes: bytes | None = None
self.upload_filename: str | None = None
self.fields: list[tuple[str, object, dict[str, str]]] | None = None
def post(self, *, url: str, data: _FakeFormData, headers: dict[str, str]):
del url, headers
self.fields = list(data.fields)
_, file_obj, file_kwargs = self.fields[0]
file_obj = cast(_ReadableBinary, file_obj)
self.uploaded_bytes = file_obj.read()
self.upload_filename = file_kwargs.get("filename")
return _FakeResponse()
def test_asr_dataset_sample_handles_local_audio_paths(tmp_path: Path) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": str(audio_path),
"bytes": None,
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert samples[0].multi_modal_data == {"audio_path": str(audio_path)}
assert (
samples[0].prompt == "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
)
def test_asr_dataset_sample_handles_embedded_audio_bytes(tmp_path: Path) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.1)
dataset = object.__new__(datasets_module.ASRDataset)
dataset.data = [
{
"audio": {
"path": None,
"bytes": audio_path.read_bytes(),
},
"text": "quarterly earnings call",
}
]
samples = dataset.sample(
tokenizer=_Tokenizer(),
num_requests=1,
output_len=32,
asr_min_audio_len_sec=0.0,
asr_max_audio_len_sec=1.0,
)
assert len(samples) == 1
assert isinstance(samples[0].multi_modal_data, dict)
audio, sample_rate = samples[0].multi_modal_data["audio"]
assert sample_rate == 16_000
assert isinstance(audio, np.ndarray)
assert audio.size > 0
def test_async_request_openai_audio_handles_local_audio_paths(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
audio_path = tmp_path / "earnings.wav"
_write_wav(audio_path, duration_s=0.25)
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={"audio_path": str(audio_path)},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == audio_path.name
assert session.uploaded_bytes == audio_path.read_bytes()
assert output.success is True
assert output.generated_text == "hello"
assert output.output_tokens == 1
assert output.input_audio_duration == pytest.approx(0.25, abs=1e-2)
def test_async_request_openai_audio_handles_decoded_audio_arrays(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(request_func_module.aiohttp, "FormData", _FakeFormData)
session = _FakeSession()
request_input = RequestFuncInput(
prompt="",
api_url="http://localhost:8000/v1/audio/transcriptions",
prompt_len=1,
output_len=32,
model="openai/whisper-large-v3",
multi_modal_content={
"audio": (np.zeros(1_600, dtype=np.float32), 16_000),
},
)
output = asyncio.run(
request_func_module.async_request_openai_audio(request_input, session)
)
assert session.upload_filename == "audio.wav"
assert session.uploaded_bytes is not None
assert output.success is True
assert output.generated_text == "hello"
@@ -16,10 +16,11 @@ from statistics import mean, median
import pytest
import soundfile
import torch
from datasets import load_dataset
from datasets import Audio, load_dataset
from evaluate import load
from transformers.models.whisper.english_normalizer import EnglishTextNormalizer
from vllm.benchmarks.datasets.datasets import ASRDataset
from vllm.multimodal.audio import get_audio_duration
from vllm.tokenizers import get_tokenizer
@@ -38,6 +39,20 @@ def to_bytes(y, sr):
return buffer
def load_audio_sample(audio):
# Avoid torchcodec in CI by decoding dataset audio with soundfile.
if "array" in audio and "sampling_rate" in audio:
return audio["array"], audio["sampling_rate"]
if audio.get("path"):
return soundfile.read(audio["path"], dtype="float32")
if audio.get("bytes") is not None:
return soundfile.read(io.BytesIO(audio["bytes"]), dtype="float32")
raise ValueError("Audio sample did not contain array, path, or bytes data")
# not all models have a normalizer so use the one from whisper as a standard option
normalizer_model_info = HF_EXAMPLE_MODELS.find_hf_info("openai/whisper-large-v3")
normalizer_tokenizer = get_tokenizer(
@@ -48,7 +63,7 @@ normalizer_tokenizer = get_tokenizer(
normalizer = EnglishTextNormalizer(normalizer_tokenizer.english_spelling_normalizer)
async def transcribe_audio(client, tokenizer, y, sr):
async def transcribe_audio(client, tokenizer, y, sr, extra_body=None):
# Send loaded audio directly instead of loading from disk,
# don't account for that time though
with to_bytes(y, sr) as f:
@@ -58,6 +73,7 @@ async def transcribe_audio(client, tokenizer, y, sr):
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
# NOTE there's no streaming in transcriptions, can't measure ttft
@@ -68,17 +84,21 @@ async def transcribe_audio(client, tokenizer, y, sr):
return latency, num_output_tokens, transcription.text
async def bound_transcribe(sem, client, tokenizer, audio, reference):
async def bound_transcribe(
sem, client, tokenizer, audio, sr, reference, extra_body=None
):
# Use semaphore to limit concurrent requests.
async with sem:
result = await transcribe_audio(client, tokenizer, *audio)
result = await transcribe_audio(
client, tokenizer, audio, sr, extra_body=extra_body
)
# Normalize *english* output/reference for evaluation.
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_dataset(model, client, data, concurrent_request):
async def process_dataset(model, client, data, concurrent_request, extra_body=None):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
@@ -89,14 +109,16 @@ async def process_dataset(model, client, data, concurrent_request):
)
# Warmup call as the first `load_audio` server-side is quite slow.
audio, sr = data[0]["audio"]["array"], data[0]["audio"]["sampling_rate"]
_ = await bound_transcribe(sem, client, tokenizer, (audio, sr), "")
audio, sr = load_audio_sample(data[0]["audio"])
_ = await bound_transcribe(sem, client, tokenizer, audio, sr, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
audio, sr = load_audio_sample(sample["audio"])
task = asyncio.create_task(
bound_transcribe(sem, client, tokenizer, (audio, sr), sample["text"])
bound_transcribe(
sem, client, tokenizer, audio, sr, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
@@ -121,19 +143,36 @@ def print_performance_metrics(results, total_time):
def add_duration(sample):
y, sr = sample["audio"]["array"], sample["audio"]["sampling_rate"]
y, sr = load_audio_sample(sample["audio"])
sample["duration_ms"] = get_audio_duration(y=y, sr=sr) * 1000
return sample
def load_hf_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset
dataset = load_dataset(dataset_repo, split=split, **hf_kwargs)
if "duration_ms" not in dataset[0]:
# compute duration to filter
def load_asr_dataset_rows(dataset_repo: str, split="validation", **hf_kwargs):
if dataset_repo in ASRDataset.SUPPORTED_DATASET_PATHS:
asr_dataset_kwargs = {
"dataset_path": dataset_repo,
"dataset_split": split,
"disable_shuffle": True,
"no_stream": True,
}
for key in ("dataset_subset", "hf_name", "trust_remote_code"):
if key in hf_kwargs:
asr_dataset_kwargs[key] = hf_kwargs[key]
return ASRDataset(**asr_dataset_kwargs).data
return load_dataset(dataset_repo, split=split, **hf_kwargs)
def load_shortform_eval_dataset(dataset_repo: str, split="validation", **hf_kwargs):
## Load and filter the dataset.
dataset = load_asr_dataset_rows(dataset_repo, split=split, **hf_kwargs)
dataset = dataset.cast_column("audio", Audio(decode=False))
if "duration_ms" not in dataset.column_names:
# Compute duration to filter.
dataset = dataset.map(add_duration)
# Whisper max supported duration
# Whisper max supported duration.
dataset = dataset.filter(lambda example: example["duration_ms"] < 30000)
return dataset
@@ -145,11 +184,16 @@ def run_evaluation(
max_concurrent_reqs: int,
n_examples: int = -1,
print_metrics: bool = True,
extra_body=None,
):
if n_examples > 0:
dataset = dataset.select(range(n_examples))
start = time.perf_counter()
results = asyncio.run(process_dataset(model, client, dataset, max_concurrent_reqs))
results = asyncio.run(
process_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
@@ -164,6 +208,106 @@ def run_evaluation(
return wer_score
LONGFORM_DATASET_REPO = ASRDataset.EARNINGS22_CLEANED_DATASET
LONGFORM_DATASET_SPLIT = "test"
LONGFORM_NUM_SAMPLES = 6
def load_longform_dataset():
dataset = load_asr_dataset_rows(
LONGFORM_DATASET_REPO,
split=LONGFORM_DATASET_SPLIT,
)
assert len(dataset) >= LONGFORM_NUM_SAMPLES
return dataset.select(range(LONGFORM_NUM_SAMPLES))
async def transcribe_audio_path(client, tokenizer, audio_path: str, extra_body=None):
with open(audio_path, "rb") as f:
start_time = time.perf_counter()
transcription = await client.audio.transcriptions.create(
file=f,
model=tokenizer.name_or_path,
language="en",
temperature=0.0,
extra_body=extra_body,
)
end_time = time.perf_counter()
latency = end_time - start_time
num_output_tokens = len(
tokenizer(transcription.text, add_special_tokens=False).input_ids
)
return latency, num_output_tokens, transcription.text
async def bound_transcribe_path(
sem, client, tokenizer, audio_path, reference, extra_body=None
):
async with sem:
result = await transcribe_audio_path(
client, tokenizer, audio_path, extra_body=extra_body
)
out = normalizer(result[2])
ref = normalizer(reference)
return result[:2] + (out, ref)
async def process_longform_dataset(
model, client, data, concurrent_request, extra_body=None
):
sem = asyncio.Semaphore(concurrent_request)
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
tokenizer = get_tokenizer(
model,
tokenizer_mode=model_info.tokenizer_mode,
trust_remote_code=model_info.trust_remote_code,
)
warmup_path = data[0]["audio"]["path"]
_ = await bound_transcribe_path(sem, client, tokenizer, warmup_path, "", extra_body)
tasks: list[asyncio.Task] = []
for sample in data:
audio_path = sample["audio"]["path"]
task = asyncio.create_task(
bound_transcribe_path(
sem, client, tokenizer, audio_path, sample["text"], extra_body
)
)
tasks.append(task)
return await asyncio.gather(*tasks)
def run_longform_evaluation(
model: str,
client,
dataset,
max_concurrent_reqs: int,
print_metrics: bool = True,
extra_body=None,
):
start = time.perf_counter()
results = asyncio.run(
process_longform_dataset(
model, client, dataset, max_concurrent_reqs, extra_body=extra_body
)
)
end = time.perf_counter()
total_time = end - start
print(f"Total Test Time: {total_time:.4f} seconds")
if print_metrics:
print_performance_metrics(results, total_time)
predictions = [res[2] for res in results]
references = [res[3] for res in results]
wer = load("wer")
wer_score = 100 * wer.compute(references=references, predictions=predictions)
print("WER:", wer_score)
return wer_score
# alternatives "openai/whisper-large-v2", "openai/whisper-large-v3-turbo"..
# NOTE: Expected WER measured with equivalent hf.transformers args:
# whisper-large-v3 + esb-datasets-earnings22-validation-tiny-filtered.
@@ -184,7 +328,6 @@ def test_wer_correctness(
):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
# TODO refactor to use `ASRDataset`
server_args = [
"--enforce-eager",
f"--tokenizer_mode={model_info.tokenizer_mode}",
@@ -197,7 +340,7 @@ def test_wer_correctness(
model_name,
server_args,
) as remote_server:
dataset = load_hf_dataset(dataset_repo)
dataset = load_shortform_eval_dataset(dataset_repo)
if not max_concurrent_request:
# No max concurrency
@@ -216,3 +359,42 @@ def test_wer_correctness(
if expected_wer:
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
# 14-22mins of 6 audio samples of total ~115 mins and just 37MB.
# checks for long audio transcription correctness and RMS split.
@pytest.mark.parametrize(
"model_config",
[("openai/whisper-large-v3", 9.5)],
)
def test_long_audio_wer_correctness(model_config):
model_name, expected_wer = model_config
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_name)
server_args = [
f"--tokenizer_mode={model_info.tokenizer_mode}",
]
if model_info.trust_remote_code:
server_args.append("--trust-remote-code")
# 1800 seconds is 30 minutes
env_dict = {
"VLLM_MAX_AUDIO_DECODE_DURATION_S": "1800",
}
with RemoteOpenAIServer(
model_name,
server_args,
env_dict=env_dict,
) as remote_server:
dataset = load_longform_dataset()
client = remote_server.get_async_client()
wer = run_longform_evaluation(
model=model_name,
client=client,
dataset=dataset,
max_concurrent_reqs=LONGFORM_NUM_SAMPLES,
)
print(f"Expected WER: {expected_wer}, Actual WER: {wer}")
torch.testing.assert_close(wer, expected_wer, atol=1e-1, rtol=1e-2)
+83 -14
View File
@@ -4001,20 +4001,27 @@ class ASRDataset(HuggingFaceDataset):
Dataset class for processing a ASR dataset for transcription.
Tested on the following set:
+----------------+----------------------------------------+--------------------------+-----------------------------+
| Dataset | Domain | Speaking Style | hf-subset |
+----------------+----------------------------------------+--------------------------+-----------------------------+
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
| | | | release3-speaker-adaptation |
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
| AMI | Meetings | Spontaneous | ihm, sdm |
+----------------+----------------------------------------+--------------------------+-----------------------------+
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
| Dataset | Domain | Speaking Style | hf-subset |
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
| TED-LIUM | TED talks | Oratory | release1, release2, release3|
| | | | release3-speaker-adaptation |
| VoxPopuli | European Parliament | Oratory | en, de, it, fr, ... |
| LibriSpeech | Audiobook | Narrated | "LIUM/tedlium" |
| GigaSpeech | Audiobook, podcast, YouTube | Narrated, spontaneous | xs, s, m, l, xl, dev, test |
| SPGISpeech | Financial meetings | Oratory, spontaneous | S, M, L, dev, test |
| Earnings22-Cleaned-AA | Long form earnings calls | Prepared remarks, Q&A | test |
| Earnings22-Tiny-Filtered | Earnings calls | Prepared remarks, Q&A | validation |
| AMI | Meetings | Spontaneous | ihm, sdm |
+---------------------------+----------------------------------------+--------------------------+-----------------------------+
""" # noqa: E501
EARNINGS22_CLEANED_DATASET = "ArtificialAnalysis/Earnings22-Cleaned-AA"
EARNINGS22_TINY_FILTERED_DATASET = (
"D4nt3/esb-datasets-earnings22-validation-tiny-filtered"
)
SUPPORTED_DATASET_PATHS = {
"openslr/librispeech_asr",
"facebook/voxpopuli",
@@ -4022,11 +4029,52 @@ class ASRDataset(HuggingFaceDataset):
"edinburghcstr/ami",
"speechcolab/gigaspeech",
"kensho/spgispeech",
EARNINGS22_CLEANED_DATASET,
EARNINGS22_TINY_FILTERED_DATASET,
}
DEFAULT_OUTPUT_LEN = 1024
IS_MULTIMODAL = True
def load_data(self) -> None:
if self.hf_name == self.EARNINGS22_CLEANED_DATASET:
# This subset stores repo-local MP3 paths instead of a HF `Audio`
# column, so eagerly materialize it back into the common schema.
self.data = load_dataset(
self.dataset_path,
name=self.dataset_subset,
split=self.dataset_split,
streaming=False,
trust_remote_code=self.trust_remote_code,
)
if not getattr(self, "disable_shuffle", False):
self.data = self.data.shuffle(seed=self.random_seed)
self._materialize_local_audio_column()
return
if self.hf_name == self.EARNINGS22_TINY_FILTERED_DATASET:
super().load_data()
self._disable_audio_decode()
return
super().load_data()
def _disable_audio_decode(self) -> None:
from datasets import Audio
self.data = self.data.cast_column("audio", Audio(decode=False))
def _materialize_local_audio_column(self) -> None:
local_path_root = Path(
hf_api().snapshot_download(self.hf_name, repo_type="dataset")
)
self.data = self.data.map(
lambda item: {
"audio": str(local_path_root / item["url"]),
"text": item["transcript"],
}
)
self._disable_audio_decode()
def sample(
self,
tokenizer: TokenizerLike,
@@ -4052,14 +4100,35 @@ class ASRDataset(HuggingFaceDataset):
if len(sampled_requests) >= num_requests:
break
audio = item["audio"]
y, sr = audio["array"], audio["sampling_rate"]
duration_s = get_audio_duration(y=y, sr=sr)
if (
isinstance(audio, dict)
and "array" in audio
and "sampling_rate" in audio
):
y, sr = audio["array"], audio["sampling_rate"]
duration_s = get_audio_duration(y=y, sr=sr)
mm_content = {"audio": (y, sr)}
elif isinstance(audio, str):
duration_s = sf.info(audio).duration
mm_content = {"audio_path": audio}
elif isinstance(audio, dict) and audio.get("path"):
duration_s = sf.info(audio["path"]).duration
mm_content = {"audio_path": audio["path"]}
elif isinstance(audio, dict) and audio.get("bytes") is not None:
with BytesIO(audio["bytes"]) as audio_buffer:
y, sr = sf.read(audio_buffer, dtype="float32")
duration_s = get_audio_duration(y=y, sr=sr)
mm_content = {"audio": (y, sr)}
else:
raise ValueError(
"ASR samples must provide decoded audio arrays, "
"embedded audio bytes, or a local audio path."
)
if duration_s < asr_min_audio_len_sec or duration_s > asr_max_audio_len_sec:
skipped += 1
continue
durations.append(duration_s)
mm_content = {"audio": (y, sr)}
sampled_requests.append(
SampleRequest(
prompt=prompt,
+44 -11
View File
@@ -445,7 +445,6 @@ async def async_request_openai_audio(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Audio API", {"transcriptions", "translations"})
content = [{"type": "text", "text": request_func_input.prompt}]
payload = {
"model": request_func_input.model_name
if request_func_input.model_name
@@ -469,19 +468,26 @@ async def async_request_openai_audio(
buffer.seek(0)
return buffer
mm_audio = request_func_input.multi_modal_content
if not isinstance(mm_audio, dict) or "audio" not in mm_audio:
raise TypeError("multi_modal_content must be a dict containing 'audio'")
with to_bytes(*mm_audio["audio"]) as f:
async def send_audio_file(
audio_file: io.BytesIO | Any,
*,
input_audio_duration: float,
filename: str | None = None,
content_type: str | None = None,
) -> RequestFuncOutput:
form = aiohttp.FormData()
form.add_field("file", f, content_type="audio/wav")
add_field_kwargs: dict[str, str] = {}
if filename is not None:
add_field_kwargs["filename"] = filename
if content_type is not None:
add_field_kwargs["content_type"] = content_type
form.add_field("file", audio_file, **add_field_kwargs)
for key, value in payload.items():
form.add_field(key, str(value))
output = RequestFuncOutput()
output.prompt_len = request_func_input.prompt_len
output.input_audio_duration = soundfile.info(f).duration
f.seek(0)
output.input_audio_duration = input_audio_duration
generated_text = ""
ttft = 0.0
@@ -541,9 +547,36 @@ async def async_request_openai_audio(
exc_info = sys.exc_info()
output.error = "".join(traceback.format_exception(*exc_info))
if pbar:
pbar.update(1)
return output
if pbar:
pbar.update(1)
return output
mm_audio = request_func_input.multi_modal_content
if not isinstance(mm_audio, dict):
raise TypeError(
"multi_modal_content must be a dict containing 'audio' or 'audio_path'"
)
if "audio" in mm_audio:
with to_bytes(*mm_audio["audio"]) as f:
input_audio_duration = soundfile.info(f).duration
f.seek(0)
return await send_audio_file(
f,
input_audio_duration=input_audio_duration,
filename="audio.wav",
content_type="audio/wav",
)
if "audio_path" in mm_audio:
audio_path = mm_audio["audio_path"]
with open(audio_path, "rb") as f:
return await send_audio_file(
f,
input_audio_duration=soundfile.info(audio_path).duration,
filename=os.path.basename(audio_path),
)
raise TypeError(
"multi_modal_content must be a dict containing 'audio' or 'audio_path'"
)
async def _run_pooling_request(