feat[vLLM × v5]: Add audio support for the Transformers backend (#39330)

Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harshal Janjani
2026-07-25 04:20:58 -07:00
committed by GitHub
co-authored by Harry Mellor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ca0defa343
commit b9b6306ebe
8 changed files with 682 additions and 152 deletions
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from transformers import AutoModelForSeq2SeqLM
from vllm.assets.audio import AudioAsset
from vllm.envs import disable_envs_cache
from vllm.lora.request import LoRARequest
from vllm.multimodal.audio import AudioResampler
from ....conftest import HfRunner, VllmRunner
from ...utils import check_logprobs_close
AUDIO_ASSET = AudioAsset("mary_had_lamb")
AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
"<|start_of_role|>system<|end_of_role|>"
"You are a helpful AI assistant<|end_of_text|>\n"
"<|start_of_role|>user<|end_of_role|>"
"<|audio|>can you transcribe the speech into a written format?"
"<|end_of_text|>\n"
"<|start_of_role|>assistant<|end_of_role|>"
),
"audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
},
"nvidia/audio-flamingo-3-hf": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\n"
"<sound>Transcribe the input speech.<|im_end|>\n"
"<|im_start|>assistant\n"
),
"vllm_runner_kwargs": {
"gpu_memory_utilization": 0.85,
},
},
"microsoft/VibeVoice-ASR-HF": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant that transcribes audio input "
"into text output in JSON format.<|im_end|>\n"
"<|im_start|>user\n"
"<|object_ref_start|><|box_start|><|object_ref_end|>\n"
"This is a 1.0 seconds audio, please transcribe it with "
"these keys: Start time, End time, Speaker ID, Content"
"<|im_end|>\n"
"<|im_start|>assistant\n"
),
"sampling_rate": 24000,
"vllm_runner_kwargs": {
"max_num_batched_tokens": 2048,
"gpu_memory_utilization": 0.85,
},
},
"zai-org/GLM-ASR-Nano-2512": {
"prompt": (
"<|user|>\n"
"<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
"Please transcribe this audio into text"
"<|assistant|>\n"
),
},
}
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
def test_transformers_audio_generation(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
monkeypatch,
model_id: str,
):
"""Single-process workaround for V1 fork safety deadlock issue
(vllm-project/vllm/issues/17676). Running multiple audio models together
under pytest can cause (possibly flaky) hangs, so they are grouped under
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
deadlock and allows worker processes to terminate cleanly, and release
GPU memory between test runs until the issue is fixed."""
# TODO: Remove monkeypatch once
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
disable_envs_cache()
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
settings = AUDIO_MODEL_SETTINGS[model_id]
audio_lora_path = settings.get("audio_lora_path")
audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
target_sr = settings.get("sampling_rate", orig_sr)
if orig_sr != target_sr:
audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr)
audio = (audio, target_sr)
with vllm_runner(
model_id,
model_impl="transformers",
dtype="bfloat16",
max_model_len=2048,
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
enable_lora=audio_lora_path is not None,
max_lora_rank=64,
**settings.get("vllm_runner_kwargs", {}),
) as vllm_model:
lora_request = (
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
)
vllm_outputs = vllm_model.generate_greedy_logprobs(
[settings["prompt"]],
128,
num_logprobs=10,
audios=[audio],
lora_request=lora_request,
)
with hf_runner(
model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM
) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
[settings["prompt"]], 128, num_logprobs=10, audios=[audio]
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@@ -452,6 +452,14 @@ def test_processing_correctness(
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
# TODO: Remove when transformers 5.15.0 is released, which contains
# https://github.com/huggingface/transformers/pull/47483.
if model_id == "microsoft/VibeVoice-ASR-HF":
pytest.skip(
"VibeVoice ASR requires audio as a positional argument and hence "
"cannot pass the processing correctness test as is. Its generation "
"is covered by test_transformers_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
AUDIO_MODEL_SETTINGS = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
"<|start_of_role|>system<|end_of_role|>"
"You are a helpful AI assistant<|end_of_text|>\n"
"<|start_of_role|>user<|end_of_role|>"
"<|audio|>can you transcribe the speech into a written format?"
"<|end_of_text|>\n"
"<|start_of_role|>assistant<|end_of_role|>"
),
},
"nvidia/audio-flamingo-3-hf": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\n"
"<sound>Transcribe the input speech.<|im_end|>\n"
"<|im_start|>assistant\n"
),
},
"mistralai/Voxtral-Mini-3B-2507": {
"prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
},
"microsoft/VibeVoice-ASR-HF": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant that transcribes audio input "
"into text output in JSON format.<|im_end|>\n"
"<|im_start|>user\n"
"<|object_ref_start|><|box_start|><|object_ref_end|>\n"
"This is a 1.0 seconds audio, please transcribe it with "
"these keys: Start time, End time, Speaker ID, Content"
"<|im_end|>\n"
"<|im_start|>assistant\n"
),
},
"zai-org/GLM-ASR-Nano-2512": {
"prompt": (
"<|user|>\n"
"<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
"Please transcribe this audio into text"
"<|assistant|>\n"
),
},
}
@pytest.mark.parametrize(
"model_id",
[
"ibm-granite/granite-speech-3.3-2b",
"nvidia/audio-flamingo-3-hf",
pytest.param(
"mistralai/Voxtral-Mini-3B-2507",
marks=pytest.mark.xfail(
reason="MistralCommonBackend.encode does not produce the audio "
"placeholder token (ID 24) from raw text. apply_chat_template "
"yields token IDs with placeholders, but MultiModalProcessor."
"apply() decodes the prompt back to text and re-tokenizes, at "
"which point the placeholders are lost. Fix belongs in "
"mistral_common or in the Voxtral-specific path.",
strict=False,
),
),
"microsoft/VibeVoice-ASR-HF",
"zai-org/GLM-ASR-Nano-2512",
],
)
def test_audio_multimodal_processor(model_id):
settings = AUDIO_MODEL_SETTINGS[model_id]
model_config = ModelConfig(
model=model_id,
model_impl="transformers",
)
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
audio = np.zeros(16000, dtype=np.float32)
mm_data = {"audio": (audio, 16000)}
result = mm_processor(
prompt=settings["prompt"],
mm_items=mm_processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
assert "prompt_token_ids" in result
assert len(result["prompt_token_ids"]) > 0
mm_placeholders = result.get("mm_placeholders", {})
assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}"
assert len(mm_placeholders["audio"]) == 1
placeholder = mm_placeholders["audio"][0]
assert placeholder.length > 0
assert placeholder.offset >= 0
audio_items = result.get("mm_kwargs", {}).get("audio", [])
assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
item_keys = list(audio_items[0].keys())
has_features = "input_features" in item_keys or "input_values" in item_keys
assert has_features, (
f"No audio features (input_features/input_values) in {item_keys} for {model_id}"
)
def test_audio_multiple_inputs():
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers backend."""
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
audio_token = mm_processor.info.get_hf_processor().audio_token
# One token per audio; the processor expands each to its placeholder run.
prompt = (
"<|start_of_role|>user<|end_of_role|>"
f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
@@ -54,3 +54,26 @@ def test_multimodal_processor(model_id):
str_processed_inputs["prompt_token_ids"]
== ids_processed_inputs["prompt_token_ids"]
)
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers backend."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
"<|im_start|>user <image>\n and <image>\n"
"What do these images show?<|im_end|><|im_start|>assistant\n"
)
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
+4
View File
@@ -1393,6 +1393,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
trust_remote_code=True,
),
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
"microsoft/VibeVoice-ASR-HF",
min_transformers_version="5.13.0",
),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
tokenizer_mode="mistral",