diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 3ee942c175..566a3e1f95 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -19,7 +19,7 @@ vLLM also supports model implementations that are available in Transformers. We
Currently, the Transformers modeling backend works for the following:
-- Modalities: embedding models, language models and vision-language models*
+- Modalities: embedding models, language models, vision-language models* and audio-language models
- Architectures: encoder-only, decoder-only, mixture-of-experts
- Attention types: full attention and/or sliding attention
@@ -606,7 +606,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor
| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
| ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
-| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
+| `Emu3ForConditionalGeneration` | Emu3 | T + I+ | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
+| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A+ | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ |
^ You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
E Pre-computed embeddings can be inputted for this modality.
diff --git a/tests/models/multimodal/generation/test_transformers_audio.py b/tests/models/multimodal/generation/test_transformers_audio.py
new file mode 100644
index 0000000000..9195579877
--- /dev/null
+++ b/tests/models/multimodal/generation/test_transformers_audio.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Any
+
+import pytest
+from transformers import AutoModelForSeq2SeqLM
+
+from vllm.assets.audio import AudioAsset
+from vllm.envs import disable_envs_cache
+from vllm.lora.request import LoRARequest
+from vllm.multimodal.audio import AudioResampler
+
+from ....conftest import HfRunner, VllmRunner
+from ...utils import check_logprobs_close
+
+AUDIO_ASSET = AudioAsset("mary_had_lamb")
+
+AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
+ "ibm-granite/granite-speech-3.3-2b": {
+ "prompt": (
+ "<|start_of_role|>system<|end_of_role|>"
+ "You are a helpful AI assistant<|end_of_text|>\n"
+ "<|start_of_role|>user<|end_of_role|>"
+ "<|audio|>can you transcribe the speech into a written format?"
+ "<|end_of_text|>\n"
+ "<|start_of_role|>assistant<|end_of_role|>"
+ ),
+ "audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
+ },
+ "nvidia/audio-flamingo-3-hf": {
+ "prompt": (
+ "<|im_start|>system\n"
+ "You are a helpful assistant.<|im_end|>\n"
+ "<|im_start|>user\n"
+ "Transcribe the input speech.<|im_end|>\n"
+ "<|im_start|>assistant\n"
+ ),
+ "vllm_runner_kwargs": {
+ "gpu_memory_utilization": 0.85,
+ },
+ },
+ "microsoft/VibeVoice-ASR-HF": {
+ "prompt": (
+ "<|im_start|>system\n"
+ "You are a helpful assistant that transcribes audio input "
+ "into text output in JSON format.<|im_end|>\n"
+ "<|im_start|>user\n"
+ "<|object_ref_start|><|box_start|><|object_ref_end|>\n"
+ "This is a 1.0 seconds audio, please transcribe it with "
+ "these keys: Start time, End time, Speaker ID, Content"
+ "<|im_end|>\n"
+ "<|im_start|>assistant\n"
+ ),
+ "sampling_rate": 24000,
+ "vllm_runner_kwargs": {
+ "max_num_batched_tokens": 2048,
+ "gpu_memory_utilization": 0.85,
+ },
+ },
+ "zai-org/GLM-ASR-Nano-2512": {
+ "prompt": (
+ "<|user|>\n"
+ "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
+ "Please transcribe this audio into text"
+ "<|assistant|>\n"
+ ),
+ },
+}
+
+
+@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
+def test_transformers_audio_generation(
+ hf_runner: type[HfRunner],
+ vllm_runner: type[VllmRunner],
+ monkeypatch,
+ model_id: str,
+):
+ """Single-process workaround for V1 fork safety deadlock issue
+ (vllm-project/vllm/issues/17676). Running multiple audio models together
+ under pytest can cause (possibly flaky) hangs, so they are grouped under
+ the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
+ deadlock and allows worker processes to terminate cleanly, and release
+ GPU memory between test runs until the issue is fixed."""
+ # TODO: Remove monkeypatch once
+ # https://github.com/vllm-project/vllm/issues/17676 is fixed.
+ disable_envs_cache()
+ monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+ settings = AUDIO_MODEL_SETTINGS[model_id]
+ audio_lora_path = settings.get("audio_lora_path")
+
+ audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
+ target_sr = settings.get("sampling_rate", orig_sr)
+ if orig_sr != target_sr:
+ audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr)
+ audio = (audio, target_sr)
+
+ with vllm_runner(
+ model_id,
+ model_impl="transformers",
+ dtype="bfloat16",
+ max_model_len=2048,
+ enforce_eager=True,
+ limit_mm_per_prompt={"audio": 1},
+ enable_lora=audio_lora_path is not None,
+ max_lora_rank=64,
+ **settings.get("vllm_runner_kwargs", {}),
+ ) as vllm_model:
+ lora_request = (
+ LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
+ )
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ [settings["prompt"]],
+ 128,
+ num_logprobs=10,
+ audios=[audio],
+ lora_request=lora_request,
+ )
+
+ with hf_runner(
+ model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM
+ ) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ [settings["prompt"]], 128, num_logprobs=10, audios=[audio]
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py
index 1ef39cfaa7..9b586be479 100644
--- a/tests/models/multimodal/processing/test_common.py
+++ b/tests/models/multimodal/processing/test_common.py
@@ -452,6 +452,14 @@ def test_processing_correctness(
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
+ # TODO: Remove when transformers 5.15.0 is released, which contains
+ # https://github.com/huggingface/transformers/pull/47483.
+ if model_id == "microsoft/VibeVoice-ASR-HF":
+ pytest.skip(
+ "VibeVoice ASR requires audio as a positional argument and hence "
+ "cannot pass the processing correctness test as is. Its generation "
+ "is covered by test_transformers_audio.py."
+ )
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py
new file mode 100644
index 0000000000..3ff3e04379
--- /dev/null
+++ b/tests/models/multimodal/processing/test_transformers_audio.py
@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import numpy as np
+import pytest
+
+from vllm.config import ModelConfig
+from vllm.multimodal import MULTIMODAL_REGISTRY
+
+AUDIO_MODEL_SETTINGS = {
+ "ibm-granite/granite-speech-3.3-2b": {
+ "prompt": (
+ "<|start_of_role|>system<|end_of_role|>"
+ "You are a helpful AI assistant<|end_of_text|>\n"
+ "<|start_of_role|>user<|end_of_role|>"
+ "<|audio|>can you transcribe the speech into a written format?"
+ "<|end_of_text|>\n"
+ "<|start_of_role|>assistant<|end_of_role|>"
+ ),
+ },
+ "nvidia/audio-flamingo-3-hf": {
+ "prompt": (
+ "<|im_start|>system\n"
+ "You are a helpful assistant.<|im_end|>\n"
+ "<|im_start|>user\n"
+ "Transcribe the input speech.<|im_end|>\n"
+ "<|im_start|>assistant\n"
+ ),
+ },
+ "mistralai/Voxtral-Mini-3B-2507": {
+ "prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
+ },
+ "microsoft/VibeVoice-ASR-HF": {
+ "prompt": (
+ "<|im_start|>system\n"
+ "You are a helpful assistant that transcribes audio input "
+ "into text output in JSON format.<|im_end|>\n"
+ "<|im_start|>user\n"
+ "<|object_ref_start|><|box_start|><|object_ref_end|>\n"
+ "This is a 1.0 seconds audio, please transcribe it with "
+ "these keys: Start time, End time, Speaker ID, Content"
+ "<|im_end|>\n"
+ "<|im_start|>assistant\n"
+ ),
+ },
+ "zai-org/GLM-ASR-Nano-2512": {
+ "prompt": (
+ "<|user|>\n"
+ "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
+ "Please transcribe this audio into text"
+ "<|assistant|>\n"
+ ),
+ },
+}
+
+
+@pytest.mark.parametrize(
+ "model_id",
+ [
+ "ibm-granite/granite-speech-3.3-2b",
+ "nvidia/audio-flamingo-3-hf",
+ pytest.param(
+ "mistralai/Voxtral-Mini-3B-2507",
+ marks=pytest.mark.xfail(
+ reason="MistralCommonBackend.encode does not produce the audio "
+ "placeholder token (ID 24) from raw text. apply_chat_template "
+ "yields token IDs with placeholders, but MultiModalProcessor."
+ "apply() decodes the prompt back to text and re-tokenizes, at "
+ "which point the placeholders are lost. Fix belongs in "
+ "mistral_common or in the Voxtral-specific path.",
+ strict=False,
+ ),
+ ),
+ "microsoft/VibeVoice-ASR-HF",
+ "zai-org/GLM-ASR-Nano-2512",
+ ],
+)
+def test_audio_multimodal_processor(model_id):
+ settings = AUDIO_MODEL_SETTINGS[model_id]
+
+ model_config = ModelConfig(
+ model=model_id,
+ model_impl="transformers",
+ )
+
+ mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+
+ audio = np.zeros(16000, dtype=np.float32)
+ mm_data = {"audio": (audio, 16000)}
+
+ result = mm_processor(
+ prompt=settings["prompt"],
+ mm_items=mm_processor.info.parse_mm_data(mm_data),
+ hf_processor_mm_kwargs={},
+ )
+
+ assert "prompt_token_ids" in result
+ assert len(result["prompt_token_ids"]) > 0
+
+ mm_placeholders = result.get("mm_placeholders", {})
+ assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}"
+ assert len(mm_placeholders["audio"]) == 1
+
+ placeholder = mm_placeholders["audio"][0]
+ assert placeholder.length > 0
+ assert placeholder.offset >= 0
+
+ audio_items = result.get("mm_kwargs", {}).get("audio", [])
+ assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
+ item_keys = list(audio_items[0].keys())
+ has_features = "input_features" in item_keys or "input_values" in item_keys
+ assert has_features, (
+ f"No audio features (input_features/input_values) in {item_keys} for {model_id}"
+ )
+
+
+def test_audio_multiple_inputs():
+ """Multiple audios per prompt are each detected as a separate placeholder
+ and multi-modal item by the Transformers backend."""
+ model_id = "ibm-granite/granite-speech-3.3-2b"
+ model_config = ModelConfig(model=model_id, model_impl="transformers")
+ mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+
+ audio_token = mm_processor.info.get_hf_processor().audio_token
+ # One token per audio; the processor expands each to its placeholder run.
+ prompt = (
+ "<|start_of_role|>user<|end_of_role|>"
+ f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
+ )
+ audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
+
+ result = mm_processor(
+ prompt=prompt,
+ mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
+ hf_processor_mm_kwargs={},
+ )
+
+ assert len(result["mm_placeholders"]["audio"]) == 2
+ assert len(result["mm_kwargs"]["audio"]) == 2
diff --git a/tests/models/multimodal/processing/test_transformers.py b/tests/models/multimodal/processing/test_transformers_image.py
similarity index 63%
rename from tests/models/multimodal/processing/test_transformers.py
rename to tests/models/multimodal/processing/test_transformers_image.py
index a556b8f10a..2c31bcc634 100644
--- a/tests/models/multimodal/processing/test_transformers.py
+++ b/tests/models/multimodal/processing/test_transformers_image.py
@@ -54,3 +54,26 @@ def test_multimodal_processor(model_id):
str_processed_inputs["prompt_token_ids"]
== ids_processed_inputs["prompt_token_ids"]
)
+
+
+def test_image_multiple_inputs():
+ """Multiple images per prompt are each detected as a separate placeholder
+ and multi-modal item by the Transformers backend."""
+ model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
+ model_config = ModelConfig(model=model_id, model_impl="transformers")
+ mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
+
+ image = ImageAsset("cherry_blossom").pil_image
+ prompt = (
+ "<|im_start|>user \n and \n"
+ "What do these images show?<|im_end|><|im_start|>assistant\n"
+ )
+
+ result = mm_processor(
+ prompt=prompt,
+ mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
+ hf_processor_mm_kwargs={},
+ )
+
+ assert len(result["mm_placeholders"]["image"]) == 2
+ assert len(result["mm_kwargs"]["image"]) == 2
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 2d8ecfb560..f55daa5103 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1393,6 +1393,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
trust_remote_code=True,
),
+ "VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
+ "microsoft/VibeVoice-ASR-HF",
+ min_transformers_version="5.13.0",
+ ),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
tokenizer_mode="mistral",
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 229cf4fe87..c130a7f4a2 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -664,6 +664,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
"transformers",
"TransformersMultiModalForCausalLM",
),
+ "VibeVoiceAsrForConditionalGeneration": (
+ "transformers",
+ "TransformersMultiModalForCausalLM",
+ ),
}
_TRANSFORMERS_BACKEND_MODELS = {
diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py
index ae203d6bdb..5c6b0e286a 100644
--- a/vllm/model_executor/models/transformers/multimodal.py
+++ b/vllm/model_executor/models/transformers/multimodal.py
@@ -17,7 +17,8 @@
"""Transformers modeling backend mixin for multi-modal models."""
from collections.abc import Mapping
-from typing import TYPE_CHECKING
+from contextlib import nullcontext
+from typing import TYPE_CHECKING, Any
import torch
from transformers import AutoModel
@@ -27,6 +28,7 @@ from vllm.config.utils import getattr_iter
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal
+from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -36,6 +38,7 @@ from vllm.multimodal.inputs import (
from vllm.multimodal.parse import (
ImageProcessorItems,
MultiModalDataItems,
+ MultiModalDataParser,
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
@@ -59,11 +62,90 @@ _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
class MultiModalProcessingInfo(BaseProcessingInfo):
+ def _get_audio_processor(self) -> Any:
+ # TODO: drop feature_extractor branch once huggingface/transformers#44394 lands.
+ return getattr_iter(
+ self.get_hf_processor(), ("audio_processor", "feature_extractor")
+ )
+
+ def _is_audio_model(self) -> bool:
+ return self._get_audio_processor() is not None
+
+ def _is_image_model(self) -> bool:
+ return hasattr(self.get_hf_processor(), "image_processor")
+
+ def _is_video_model(self) -> bool:
+ return hasattr(self.get_hf_processor(), "video_processor")
+
+ def _get_audio_token_id(self) -> int:
+ processor = self.get_hf_processor()
+ if hasattr(processor, "audio_token_id"):
+ return processor.audio_token_id
+ config = self.get_hf_config()
+ val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
+ if val is not None:
+ return val
+ if hasattr(processor, "audio_token"):
+ tokenizer = self.get_tokenizer()
+ vocab = tokenizer.get_vocab()
+ if processor.audio_token in vocab:
+ return vocab[processor.audio_token]
+ raise ValueError("Cannot find audio_token_id on processor or model config")
+
+ def _get_audio_sampling_rate(self) -> float:
+ sub = self._get_audio_processor()
+ if sub is not None and hasattr(sub, "sampling_rate"):
+ return sub.sampling_rate
+ return 16000.0
+
+ def get_data_parser(self) -> MultiModalDataParser:
+ if self._is_audio_model():
+ return MultiModalDataParser(
+ target_sr=self._get_audio_sampling_rate(),
+ expected_hidden_size=self._get_expected_hidden_size(),
+ )
+ return super().get_data_parser()
+
def get_supported_mm_limits(self):
- return {"image": None}
+ limits = {}
+ if self._is_audio_model():
+ limits["audio"] = None
+ if self._is_image_model():
+ limits["image"] = None
+ if not limits:
+ raise ValueError(
+ f"Unable to detect a supported modality on "
+ f"{type(self.get_hf_processor()).__name__}. "
+ "Checked `_is_audio_model` and `_is_image_model`."
+ )
+ return limits
def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
- return {"image": self.get_max_image_tokens()}
+ result = {}
+ if self._is_audio_model():
+ result["audio"] = self.get_max_audio_tokens()
+ if self._is_image_model():
+ result["image"] = self.get_max_image_tokens()
+ if not result:
+ raise ValueError(
+ f"Unable to detect a supported modality on "
+ f"{type(self.get_hf_processor()).__name__}. "
+ "Checked `_is_audio_model` and `_is_image_model`."
+ )
+ return result
+
+ def get_max_audio_tokens(self) -> int:
+ config = self.get_hf_config()
+ audio_config_names = ("audio_config", "encoder_config")
+ names = ("max_source_positions", "max_position_embeddings", "max_pos_emb")
+ audio_config = getattr_iter(config, audio_config_names, default=config)
+ val = getattr_iter(audio_config, names)
+ if val is not None:
+ return int(val)
+ raise ValueError(
+ f"Unable to get max input length from {type(audio_config).__name__}. "
+ f"The following attribute names were checked: {names}."
+ )
def get_max_image_tokens(self) -> int:
width, height = self.get_max_image_size()
@@ -82,14 +164,19 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_images = mm_counts.get("image", 0)
-
- processor = self.info.get_hf_processor()
- if "gemma3" in processor.__class__.__name__.lower():
- image_token = processor.boi_token
- else:
- image_token = getattr(processor, "image_token", "")
- return image_token * num_images
+ text = ""
+ if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
+ processor = self.info.get_hf_processor()
+ audio_token = getattr(processor, "audio_token", "")
+ text += audio_token * num_audios
+ if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
+ processor = self.info.get_hf_processor()
+ if "gemma3" in processor.__class__.__name__.lower():
+ image_token = processor.boi_token
+ else:
+ image_token = getattr(processor, "image_token", "")
+ text += image_token * num_images
+ return text
def get_dummy_mm_data(
self,
@@ -97,20 +184,28 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
mm_counts: Mapping[str, int],
mm_options: Mapping[str, "BaseDummyOptions"],
) -> MultiModalDataDict:
- num_images = mm_counts.get("image", 0)
-
- target_width, target_height = self.info.get_max_image_size()
-
- image_overrides = mm_options.get("image")
-
- return {
- "image": self._get_dummy_images(
+ data: MultiModalDataDict = {}
+ if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
+ sampling_rate = self.info._get_audio_sampling_rate()
+ sub = self.info._get_audio_processor()
+ chunk_length = getattr(sub, "chunk_length", None) if sub else None
+ if chunk_length is None:
+ chunk_length = 30
+ audio_len = int(chunk_length * sampling_rate)
+ data["audio"] = self._get_dummy_audios(
+ length=audio_len,
+ num_audios=num_audios,
+ overrides=mm_options.get("audio"),
+ )
+ if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
+ target_width, target_height = self.info.get_max_image_size()
+ data["image"] = self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
- overrides=image_overrides,
- ),
- }
+ overrides=mm_options.get("image"),
+ )
+ return data
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
@@ -142,21 +237,39 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
) -> Mapping[str, MultiModalFieldConfig]:
# HF Processors always return a mask but vLLM doesn't need it
hf_inputs.pop("attention_mask", None)
- num_image_patches = hf_inputs.get("num_image_patches")
- mm_fields = {
- key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
- for key in hf_inputs
- }
- mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
- "image", num_image_patches
- )
- # Keep these as batched, as they always have batch size as first dim
- mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
- mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
- mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
- "image", keep_on_cpu=True
- )
+ mm_fields: dict[str, MultiModalFieldConfig] = {}
+ if self.info._is_audio_model():
+ num_audio_tokens = hf_inputs.get("num_audio_tokens")
+ mm_fields.update(
+ {
+ key: MultiModalFieldConfig.flat_from_sizes(
+ "audio", num_audio_tokens
+ )
+ for key in hf_inputs
+ }
+ )
+ mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
+ if self.info._is_image_model():
+ num_image_patches = hf_inputs.get("num_image_patches")
+ mm_fields.update(
+ {
+ key: MultiModalFieldConfig.flat_from_sizes(
+ "image", num_image_patches
+ )
+ for key in hf_inputs
+ }
+ )
+ mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
+ "image", num_image_patches
+ )
+
+ # Keep these as batched, as they always have batch size as first dim
+ mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
+ mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
+ mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
+ "image", keep_on_cpu=True
+ )
return mm_fields
def _get_hf_mm_data(
@@ -164,13 +277,93 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""
- In contrast to the base class, this method always adds
- `return_mm_token_type_ids` to the processor data
+ In contrast to the base class, this method requests
+ `return_mm_token_type_ids` and remaps the `audios` key to `audio` for
+ audio models.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
+ if self.info._is_audio_model() and "audios" in processor_data:
+ processor_data["audio"] = processor_data.pop("audios")
processor_data["return_mm_token_type_ids"] = True
return processor_data, passthrough_data
+ def _apply_audio(
+ self,
+ prompt_ids: list[int],
+ processed_data: "BatchFeature",
+ ) -> dict[str, list[PlaceholderRange]]:
+ audio_token_id = self.info._get_audio_token_id()
+ prompt_tensor = torch.tensor(prompt_ids)
+ is_audio = prompt_tensor == audio_token_id
+
+ if not is_audio.any():
+ return {}
+
+ padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
+ transitions = padded.int().diff()
+ starts = torch.where(transitions == 1)[0]
+ ends = torch.where(transitions == -1)[0]
+ lengths = ends - starts
+
+ ranges = [
+ PlaceholderRange(
+ offset=s.item(),
+ length=ln.item(),
+ is_embed=torch.ones(ln.item(), dtype=torch.bool),
+ )
+ for s, ln in zip(starts, lengths)
+ ]
+ processed_data["num_audio_tokens"] = lengths
+ return {"audio": ranges}
+
+ def _apply_vision(
+ self,
+ prompt_ids: list[int],
+ processed_data: "BatchFeature",
+ mm_items: MultiModalDataItems,
+ hf_processor_mm_kwargs: Mapping[str, object],
+ mm_token_type_ids: torch.Tensor | None,
+ ) -> dict[str, list[PlaceholderRange]]:
+ if mm_token_type_ids is None:
+ return {}
+
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
+
+ # We can infer vLLM style placeholder from token type ids, if we split
+ # it for each input `mm_data`.
+ mm_positions = torch.where(mm_token_type_ids == 1)[1]
+ images = mm_items.get_items("image", ImageProcessorItems)
+ image_sizes = []
+ for item_idx in range(len(images)):
+ image_size = images.get_image_size(item_idx)
+ image_sizes.append((image_size.height, image_size.width))
+
+ mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
+ image_sizes=image_sizes,
+ **self.info.ctx.get_merged_mm_kwargs({}),
+ )
+
+ mm_placeholders: dict[str, list[PlaceholderRange]] = {}
+ split_sizes = mm_tokens_per_modality["num_image_tokens"]
+ if split_sizes:
+ chunked_mm_positions = torch.split(mm_positions, split_sizes)
+ mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
+ chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
+ ranges = [
+ PlaceholderRange(
+ offset=positions[0].item(),
+ length=positions.shape[0],
+ is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
+ )
+ for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
+ ]
+ mm_placeholders = {"image": ranges}
+
+ processed_data["num_image_patches"] = torch.tensor(
+ mm_tokens_per_modality["num_image_patches"]
+ )
+ return mm_placeholders
+
def apply(
self,
inputs: ProcessorInputs,
@@ -198,8 +391,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
- # work for Transformers. We have a lot of logic tied to
- # `mm_tokens_per_modality` below
+ # work for Transformers. The vision path has logic tied to
+ # `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
@@ -207,52 +400,33 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
tokenization_kwargs=tokenization_kwargs,
)
+ # Use overrides if provided; fallback to data-dependent hashing.
+ with timing_ctx.record("get_mm_hashes"):
+ mm_hashes = inputs.get_mm_hashes(self.info.model_id)
+
# For gemma3 we check `token_type_ids` as the key
mm_token_type_ids = processed_data.pop("token_type_ids", None)
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
- # We can infer vLLM style placeholder from token type ids, if we split
- # it for each input `mm_data`.
- mm_positions = torch.where(mm_token_type_ids == 1)[1]
- images = mm_items.get_items("image", ImageProcessorItems)
- image_sizes = []
- for item_idx in range(len(images)):
- image_size = images.get_image_size(item_idx)
- image_sizes.append((image_size.height, image_size.width))
-
- mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
- image_sizes=image_sizes,
- **self.info.ctx.get_merged_mm_kwargs({}),
- )
-
- mm_placeholders = {}
- split_sizes = mm_tokens_per_modality["num_image_tokens"]
- if split_sizes:
- chunked_mm_positions = torch.split(mm_positions, split_sizes)
- mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
- chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
- ranges = [
- PlaceholderRange(
- offset=positions[0].item(),
- length=positions.shape[0],
- is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
+ mm_placeholders: dict[str, list[PlaceholderRange]] = {}
+ if self.info._is_audio_model():
+ mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
+ if self.info._is_image_model():
+ mm_placeholders.update(
+ self._apply_vision(
+ prompt_ids,
+ processed_data,
+ mm_items,
+ hf_processor_mm_kwargs,
+ mm_token_type_ids,
)
- for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
- ]
- mm_placeholders = {"image": ranges}
+ )
- processed_data["num_image_patches"] = torch.tensor(
- mm_tokens_per_modality["num_image_patches"]
- )
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
processed_data,
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
)
- # Use overrides if provided; fallback to data-dependent hashing.
- with timing_ctx.record("get_mm_hashes"):
- mm_hashes = inputs.get_mm_hashes(self.info.model_id)
-
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_kwargs,
@@ -364,7 +538,79 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
return LanguageModel(self)
- def embed_multimodal(self, **kwargs):
+ def get_mm_mapping(self) -> MultiModelKeys:
+ """
+ Get the module prefix in multimodal models
+ """
+ for name in ("language_model", "text_model"):
+ if getattr(self.model, name, None) is not None:
+ return MultiModelKeys.from_string_field(language_model=f"model.{name}")
+ raise ValueError(
+ "Could not locate the language model submodule for LoRA support"
+ )
+
+ def _split_embeddings(
+ self, embeddings: torch.Tensor, split_sizes: list[int]
+ ) -> list[torch.Tensor]:
+ total_expected = sum(split_sizes)
+
+ # Flatten to 2D: [total_tokens, hidden_dim]
+ if embeddings.ndim == 3:
+ embeddings = embeddings.view(-1, embeddings.shape[-1])
+
+ total_tokens = embeddings.shape[0]
+ if total_tokens == total_expected:
+ # Direct match: split_sizes are actual token counts
+ token_split_sizes = split_sizes
+ elif total_expected > 0 and total_tokens % total_expected == 0:
+ # Uniform expansion: each item expands to N tokens
+ tokens_per_item = total_tokens // total_expected
+ token_split_sizes = [s * tokens_per_item for s in split_sizes]
+ elif total_expected > 0:
+ # Mismatch (profiling with dummy data) - pad/truncate
+ if total_tokens == 0:
+ raise ValueError(
+ "Encoder returned empty embeddings. "
+ f"Expected {total_expected} tokens from "
+ f"split_sizes={split_sizes}"
+ )
+ if total_tokens < total_expected:
+ repeat_factor = (total_expected + total_tokens - 1) // total_tokens
+ embeddings = embeddings.repeat(repeat_factor, 1)
+ embeddings = embeddings[:total_expected]
+ token_split_sizes = split_sizes
+ else:
+ return []
+
+ return list(torch.split(embeddings, token_split_sizes, dim=0))
+
+ def _embed_audio(self, **kwargs) -> list[torch.Tensor] | None:
+ self.check_version("5.13.0", "audio models support")
+ input_features: torch.Tensor | None = kwargs.pop("input_features", None)
+ if input_features is None:
+ input_features = kwargs.pop("input_values", None)
+ if input_features is None:
+ return None
+
+ num_audio_tokens = kwargs.pop("num_audio_tokens")
+ kwargs.pop("token_type_ids", None)
+ kwargs.pop("mm_token_type_ids", None)
+
+ context = nullcontext()
+ if current_platform.is_rocm():
+ context = torch.nn.attention.sdpa_kernel(
+ backends=[torch.nn.attention.SDPBackend.MATH]
+ )
+ with context:
+ audio_output = self.model.get_audio_features(
+ input_features, return_dict=True, **kwargs
+ )
+ audio_embeddings = audio_output.pooler_output
+
+ split_sizes = num_audio_tokens.flatten().tolist()
+ return self._split_embeddings(audio_embeddings, split_sizes)
+
+ def _embed_vision(self, **kwargs) -> list[torch.Tensor] | torch.Tensor | None:
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
# Model might use `image_patches` instead of `pixel_values`
@@ -379,84 +625,57 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
num_image_patches = kwargs.pop("num_image_patches")
- if pixel_values is not None:
+ context = nullcontext()
+ if current_platform.is_rocm():
# ROCm: Force math SDP backend for vision encoder to avoid accuracy issues
# with flash_sdp and mem_efficient_sdp
- if current_platform.is_rocm():
- # TODO: [ROCm] Fix accuracy issues with flash backend
- logger.debug(
- "ROCm platform detected. Forcing math SDP backend "
- "for vision encoder. Currently ROCm platform has "
- "accuracy issues with `flash_sdp` and"
- "`mem_efficient_sdp` backends. See issue: "
- "https://github.com/vllm-project/vllm/issues/30167"
- )
- with torch.nn.attention.sdpa_kernel(
- backends=[torch.nn.attention.SDPBackend.MATH]
- ):
- vision_embeddings = self.model.get_image_features(
- pixel_values, **kwargs
- )
- else:
- vision_embeddings = self.model.get_image_features(
- pixel_values, **kwargs
- )
-
- # Transformers `v5`, `self.get_image_features` returns a tuple
- # containing the features and optionally attentions/hidden_states
- # After v5 is settled, we can enable qwen3-vl with several outputs
- # from `self.get_image_features`
- if isinstance(vision_embeddings, tuple):
- vision_embeddings = vision_embeddings[0]
- elif isinstance(vision_embeddings, dict):
- vision_embeddings = vision_embeddings.pooler_output
-
- if isinstance(vision_embeddings, torch.Tensor):
- split_sizes = num_image_patches.flatten().tolist()
- total_patches = sum(split_sizes)
-
- # Flatten to 2D: [total_tokens, hidden_dim]
- if vision_embeddings.ndim == 3:
- vision_embeddings = vision_embeddings.view(
- -1, vision_embeddings.shape[-1]
- )
-
- total_tokens = vision_embeddings.shape[0]
- if total_tokens == total_patches:
- # Direct match: num_image_patches are actual token counts
- # (e.g., Qwen2.5-VL style)
- token_split_sizes = split_sizes
- elif total_patches > 0 and total_tokens % total_patches == 0:
- # Uniform expansion: each patch expands to N tokens
- # (e.g., Idefics3 style)
- tokens_per_patch = total_tokens // total_patches
- token_split_sizes = [s * tokens_per_patch for s in split_sizes]
- elif total_patches > 0:
- # Mismatch (profiling with dummy data) - pad/truncate
- if total_tokens == 0:
- raise ValueError(
- "Vision encoder returned empty embeddings. "
- f"Expected {total_patches} patches from "
- f"num_image_patches={split_sizes}"
- )
- if total_tokens < total_patches:
- repeat_factor = (
- total_patches + total_tokens - 1
- ) // total_tokens
- vision_embeddings = vision_embeddings.repeat(repeat_factor, 1)
- vision_embeddings = vision_embeddings[:total_patches]
- token_split_sizes = split_sizes
- else:
- return []
-
- return list(torch.split(vision_embeddings, token_split_sizes, dim=0))
-
- return vision_embeddings
- else:
+ # TODO: [ROCm] Fix accuracy issues with flash backend
logger.debug(
- "No pixel values or image embeddings provided for multimodal embedding."
+ "ROCm platform detected. Forcing math SDP backend "
+ "for vision encoder. Currently ROCm platform has "
+ "accuracy issues with `flash_sdp` and"
+ "`mem_efficient_sdp` backends. See issue: "
+ "https://github.com/vllm-project/vllm/issues/30167"
)
- return None
+ context = torch.nn.attention.sdpa_kernel(
+ backends=[torch.nn.attention.SDPBackend.MATH]
+ )
+ with context:
+ vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
+
+ # Transformers `v5`, `self.get_image_features` returns a tuple
+ # containing the features and optionally attentions/hidden_states
+ # After v5 is settled, we can enable qwen3-vl with several outputs
+ # from `self.get_image_features`
+ if isinstance(vision_embeddings, tuple):
+ vision_embeddings = vision_embeddings[0]
+ elif isinstance(vision_embeddings, dict):
+ vision_embeddings = vision_embeddings.pooler_output
+
+ if isinstance(vision_embeddings, torch.Tensor):
+ split_sizes = num_image_patches.flatten().tolist()
+ return self._split_embeddings(vision_embeddings, split_sizes)
+
+ return vision_embeddings
+
+ def embed_multimodal(self, **kwargs):
+ embeddings: tuple[torch.Tensor, ...] = ()
+ if "input_features" in kwargs or "input_values" in kwargs:
+ audio_embeddings = self._embed_audio(**kwargs)
+ if audio_embeddings is not None:
+ embeddings += tuple(audio_embeddings)
+ if (
+ "pixel_values" in kwargs
+ or "image_embeds" in kwargs
+ or "image_patches" in kwargs
+ ):
+ vision_embeddings = self._embed_vision(**kwargs)
+ if vision_embeddings is not None:
+ if isinstance(vision_embeddings, torch.Tensor):
+ embeddings += (vision_embeddings,)
+ else:
+ embeddings += tuple(vision_embeddings)
+ return embeddings
def get_mrope_input_positions(
self,