Implement custom dataset class for ASR benchmarking (#41576)

Signed-off-by: Yasmin Moslem <48152713+ymoslem@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Yasmin Moslem
2026-05-12 12:17:58 +08:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent 630492da30
commit 28ee78af54
3 changed files with 216 additions and 12 deletions
+6 -2
View File
@@ -9,8 +9,9 @@ from vllm.benchmarks.datasets.datasets import (
BlazeditDataset,
BurstGPTDataset,
ConversationDataset,
CustomAudioDataset,
CustomDataset,
CustomMMDataset,
CustomImageDataset,
HuggingFaceDataset,
InstructCoderDataset,
MLPerfDataset,
@@ -36,6 +37,7 @@ from vllm.benchmarks.datasets.datasets import (
is_valid_sequence,
lora_path_on_disk,
lora_tokenizer_cache,
process_audio,
process_image,
process_video,
zeta_prompt,
@@ -51,7 +53,8 @@ __all__ = [
"BurstGPTDataset",
"ConversationDataset",
"CustomDataset",
"CustomMMDataset",
"CustomAudioDataset",
"CustomImageDataset",
"HuggingFaceDataset",
"InstructCoderDataset",
"MLPerfDataset",
@@ -77,6 +80,7 @@ __all__ = [
"is_valid_sequence",
"lora_path_on_disk",
"lora_tokenizer_cache",
"process_audio",
"process_image",
"process_video",
"RangeRatio",
+150 -4
View File
@@ -60,6 +60,11 @@ try:
except ImportError:
pd = PlaceholderModule("pandas")
try:
import soundfile as sf
except ImportError:
sf = PlaceholderModule("soundfile")
logger = logging.getLogger(__name__)
@@ -441,6 +446,27 @@ def process_video(video: Any) -> Mapping[str, Any]:
)
def process_audio(audio: Any) -> tuple:
"""
Process a single audio input and return a (array, sample_rate) tuple.
Supports:
1. String: treated as a file path, loaded with soundfile.
2. Dict with 'array' and 'sampling_rate' keys: HuggingFace audio format.
3. Tuple (array, sr): passed through directly.
"""
if isinstance(audio, str):
return sf.read(audio)
if isinstance(audio, dict) and "array" in audio and "sampling_rate" in audio:
return audio["array"], audio["sampling_rate"]
if isinstance(audio, tuple) and len(audio) == 2:
return audio
raise ValueError(
f"Invalid audio input {audio}. Must be a file path string, "
"a dict with 'array' and 'sampling_rate', or a (array, sr) tuple."
)
def gen_prompt_decode_to_target_len(
tokenizer: TokenizerLike,
token_sequence: list[int],
@@ -1399,6 +1425,8 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
"random-rerank",
"hf",
"custom",
"custom_audio",
"custom_image",
"custom_mm",
"prefix_repetition",
"spec_bench",
@@ -1816,8 +1844,28 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
no_oversample=args.no_oversample,
)
elif args.dataset_name == "custom_mm":
dataset = CustomMMDataset(
elif args.dataset_name in ("custom_image", "custom_mm"):
if args.dataset_name == "custom_mm":
logger.warning(
"Dataset name 'custom_mm' is deprecated and will be removed in v0.24. "
"Use '--dataset-name custom_image' instead."
)
dataset = CustomImageDataset(
dataset_path=args.dataset_path,
disable_shuffle=args.disable_shuffle,
random_seed=args.seed,
)
input_requests = dataset.sample(
num_requests=args.num_prompts,
tokenizer=tokenizer,
output_len=args.custom_output_len,
enable_multimodal_chat=args.enable_multimodal_chat,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
elif args.dataset_name == "custom_audio":
dataset = CustomAudioDataset(
dataset_path=args.dataset_path,
disable_shuffle=args.disable_shuffle,
random_seed=args.seed,
@@ -2249,9 +2297,9 @@ class CustomDataset(BenchmarkDataset):
return sampled_requests
class CustomMMDataset(CustomDataset):
class CustomImageDataset(CustomDataset):
"""
Implements the Custom MultiModal dataset. Loads data from a JSONL file and generates
Implements the Custom image dataset. Loads data from a JSONL file and generates
sample requests based on conversation turns. E.g.,
```
{
@@ -2328,6 +2376,104 @@ class CustomMMDataset(CustomDataset):
return sampled_requests
class CustomAudioDataset(CustomDataset):
"""
Custom dataset for audio benchmarking. Loads data from a JSONL file. E.g.,
{"prompt": "Transcribe the audio.", "audio": "/path/to/audio.wav"}
Supports both:
- Dedicated ASR models (e.g. Whisper) via openai-audio & /v1/audio/transcriptions
- Chat-based audio models (e.g. Qwen2-Audio) via openai-chat & /v1/chat/completions
"""
IS_MULTIMODAL = True
def sample(
self,
tokenizer: TokenizerLike,
num_requests: int,
output_len: int | None = None,
request_id_prefix: str = "",
no_oversample: bool = False,
skip_chat_template: bool = False,
enable_multimodal_chat: bool = False,
**kwargs,
) -> list[SampleRequest]:
self.num_available_samples = len(self.data)
if num_requests <= 0:
num_requests = self.num_available_samples
sampled_requests = []
for i, item in enumerate(self.data):
if len(sampled_requests) >= num_requests:
break
prompt = item.get("prompt", "")
if tokenizer is None:
prompt_len = 1
new_output_len = output_len if output_len not in (None, -1) else 256
mm_content = None
else:
use_chat_template = (
not skip_chat_template
and hasattr(tokenizer, "chat_template")
and tokenizer.chat_template is not None
)
if enable_multimodal_chat:
# Chat-based audio models (e.g., Qwen2-Audio):
# encode audio as base64; serve.py assembles the chat message
# as: {"role": "user", "content": [
# {"type": "text", "text": prompt},
# {"type": "input_audio", "input_audio": {...}}
# ]}
y, sr = process_audio(item["audio"])
buf = io.BytesIO()
sf.write(buf, y, sr, format="WAV")
audio_base64 = base64.b64encode(buf.getvalue()).decode("utf-8")
mm_content = {
"type": "input_audio",
"input_audio": {
"data": audio_base64,
"format": "wav",
},
}
# prompt stays as plain string; serve.py handles wrapping
else:
# Whisper-style models: load audio array locally
y, sr = process_audio(item["audio"])
mm_content = {"audio": (y, sr)}
if use_chat_template:
# ASR models with a chat template but not multimodal chat
prompt = tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
add_generation_prompt=True,
tokenize=False,
)
# else: plain prompt for Whisper-style models
prompt_len = (
len(tokenizer(prompt).input_ids) if isinstance(prompt, str) else 1
)
new_output_len = output_len
if output_len is None or output_len == -1:
if "output_tokens" not in item:
raise ValueError(
"If no output length is provided the "
"custom dataset must contain an 'output_tokens' field."
)
new_output_len = int(item["output_tokens"])
sampled_requests.append(
SampleRequest(
prompt=prompt,
prompt_len=prompt_len,
expected_output_len=new_output_len,
multi_modal_data=mm_content,
request_id=request_id_prefix + str(i),
)
)
self.maybe_oversample_requests(
sampled_requests, num_requests, request_id_prefix, no_oversample
)
return sampled_requests
# -----------------------------------------------------------------------------
# Spec Bench Dataset Implementation
# -----------------------------------------------------------------------------