From b9b6306ebe0f5fa5046553667192e56bb3934af0 Mon Sep 17 00:00:00 2001 From: Harshal Janjani Date: Sat, 25 Jul 2026 15:20:58 +0400 Subject: [PATCH] =?UTF-8?q?feat[vLLM=20=C3=97=20v5]:=20Add=20audio=20suppo?= =?UTF-8?q?rt=20for=20the=20Transformers=20backend=20(#39330)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Harshal Janjani Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docs/models/supported_models.md | 5 +- .../generation/test_transformers_audio.py | 133 +++++ .../multimodal/processing/test_common.py | 8 + .../processing/test_transformers_audio.py | 138 +++++ ...sformers.py => test_transformers_image.py} | 23 + tests/models/registry.py | 4 + vllm/model_executor/models/registry.py | 4 + .../models/transformers/multimodal.py | 519 +++++++++++++----- 8 files changed, 682 insertions(+), 152 deletions(-) create mode 100644 tests/models/multimodal/generation/test_transformers_audio.py create mode 100644 tests/models/multimodal/processing/test_transformers_audio.py rename tests/models/multimodal/processing/{test_transformers.py => test_transformers_image.py} (63%) diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 3ee942c175..566a3e1f95 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -19,7 +19,7 @@ vLLM also supports model implementations that are available in Transformers. We Currently, the Transformers modeling backend works for the following: -- Modalities: embedding models, language models and vision-language models* +- Modalities: embedding models, language models, vision-language models* and audio-language models - Architectures: encoder-only, decoder-only, mixture-of-experts - Attention types: full attention and/or sliding attention @@ -606,7 +606,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | | ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- | -| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ | +| `Emu3ForConditionalGeneration` | Emu3 | T + I+ | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ | +| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A+ | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ | ^ You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
E Pre-computed embeddings can be inputted for this modality.
diff --git a/tests/models/multimodal/generation/test_transformers_audio.py b/tests/models/multimodal/generation/test_transformers_audio.py new file mode 100644 index 0000000000..9195579877 --- /dev/null +++ b/tests/models/multimodal/generation/test_transformers_audio.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import pytest +from transformers import AutoModelForSeq2SeqLM + +from vllm.assets.audio import AudioAsset +from vllm.envs import disable_envs_cache +from vllm.lora.request import LoRARequest +from vllm.multimodal.audio import AudioResampler + +from ....conftest import HfRunner, VllmRunner +from ...utils import check_logprobs_close + +AUDIO_ASSET = AudioAsset("mary_had_lamb") + +AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = { + "ibm-granite/granite-speech-3.3-2b": { + "prompt": ( + "<|start_of_role|>system<|end_of_role|>" + "You are a helpful AI assistant<|end_of_text|>\n" + "<|start_of_role|>user<|end_of_role|>" + "<|audio|>can you transcribe the speech into a written format?" + "<|end_of_text|>\n" + "<|start_of_role|>assistant<|end_of_role|>" + ), + "audio_lora_path": "ibm-granite/granite-speech-3.3-2b", + }, + "nvidia/audio-flamingo-3-hf": { + "prompt": ( + "<|im_start|>system\n" + "You are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + "Transcribe the input speech.<|im_end|>\n" + "<|im_start|>assistant\n" + ), + "vllm_runner_kwargs": { + "gpu_memory_utilization": 0.85, + }, + }, + "microsoft/VibeVoice-ASR-HF": { + "prompt": ( + "<|im_start|>system\n" + "You are a helpful assistant that transcribes audio input " + "into text output in JSON format.<|im_end|>\n" + "<|im_start|>user\n" + "<|object_ref_start|><|box_start|><|object_ref_end|>\n" + "This is a 1.0 seconds audio, please transcribe it with " + "these keys: Start time, End time, Speaker ID, Content" + "<|im_end|>\n" + "<|im_start|>assistant\n" + ), + "sampling_rate": 24000, + "vllm_runner_kwargs": { + "max_num_batched_tokens": 2048, + "gpu_memory_utilization": 0.85, + }, + }, + "zai-org/GLM-ASR-Nano-2512": { + "prompt": ( + "<|user|>\n" + "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n" + "Please transcribe this audio into text" + "<|assistant|>\n" + ), + }, +} + + +@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS)) +def test_transformers_audio_generation( + hf_runner: type[HfRunner], + vllm_runner: type[VllmRunner], + monkeypatch, + model_id: str, +): + """Single-process workaround for V1 fork safety deadlock issue + (vllm-project/vllm/issues/17676). Running multiple audio models together + under pytest can cause (possibly flaky) hangs, so they are grouped under + the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the + deadlock and allows worker processes to terminate cleanly, and release + GPU memory between test runs until the issue is fixed.""" + # TODO: Remove monkeypatch once + # https://github.com/vllm-project/vllm/issues/17676 is fixed. + disable_envs_cache() + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + + settings = AUDIO_MODEL_SETTINGS[model_id] + audio_lora_path = settings.get("audio_lora_path") + + audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate + target_sr = settings.get("sampling_rate", orig_sr) + if orig_sr != target_sr: + audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr) + audio = (audio, target_sr) + + with vllm_runner( + model_id, + model_impl="transformers", + dtype="bfloat16", + max_model_len=2048, + enforce_eager=True, + limit_mm_per_prompt={"audio": 1}, + enable_lora=audio_lora_path is not None, + max_lora_rank=64, + **settings.get("vllm_runner_kwargs", {}), + ) as vllm_model: + lora_request = ( + LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None + ) + vllm_outputs = vllm_model.generate_greedy_logprobs( + [settings["prompt"]], + 128, + num_logprobs=10, + audios=[audio], + lora_request=lora_request, + ) + + with hf_runner( + model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM + ) as hf_model: + hf_outputs = hf_model.generate_greedy_logprobs_limit( + [settings["prompt"]], 128, num_logprobs=10, audios=[audio] + ) + + check_logprobs_close( + outputs_0_lst=hf_outputs, + outputs_1_lst=vllm_outputs, + name_0="hf", + name_1="vllm", + ) diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 1ef39cfaa7..9b586be479 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -452,6 +452,14 @@ def test_processing_correctness( "audio placeholders from processed audio lengths. Its vLLM " "processor paths are covered by test_moss_audio.py." ) + # TODO: Remove when transformers 5.15.0 is released, which contains + # https://github.com/huggingface/transformers/pull/47483. + if model_id == "microsoft/VibeVoice-ASR-HF": + pytest.skip( + "VibeVoice ASR requires audio as a positional argument and hence " + "cannot pass the processing correctness test as is. Its generation " + "is covered by test_transformers_audio.py." + ) if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct": pytest.skip( "LLaVA-OneVision-2 video processing routes frames through custom " diff --git a/tests/models/multimodal/processing/test_transformers_audio.py b/tests/models/multimodal/processing/test_transformers_audio.py new file mode 100644 index 0000000000..3ff3e04379 --- /dev/null +++ b/tests/models/multimodal/processing/test_transformers_audio.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import numpy as np +import pytest + +from vllm.config import ModelConfig +from vllm.multimodal import MULTIMODAL_REGISTRY + +AUDIO_MODEL_SETTINGS = { + "ibm-granite/granite-speech-3.3-2b": { + "prompt": ( + "<|start_of_role|>system<|end_of_role|>" + "You are a helpful AI assistant<|end_of_text|>\n" + "<|start_of_role|>user<|end_of_role|>" + "<|audio|>can you transcribe the speech into a written format?" + "<|end_of_text|>\n" + "<|start_of_role|>assistant<|end_of_role|>" + ), + }, + "nvidia/audio-flamingo-3-hf": { + "prompt": ( + "<|im_start|>system\n" + "You are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + "Transcribe the input speech.<|im_end|>\n" + "<|im_start|>assistant\n" + ), + }, + "mistralai/Voxtral-Mini-3B-2507": { + "prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"), + }, + "microsoft/VibeVoice-ASR-HF": { + "prompt": ( + "<|im_start|>system\n" + "You are a helpful assistant that transcribes audio input " + "into text output in JSON format.<|im_end|>\n" + "<|im_start|>user\n" + "<|object_ref_start|><|box_start|><|object_ref_end|>\n" + "This is a 1.0 seconds audio, please transcribe it with " + "these keys: Start time, End time, Speaker ID, Content" + "<|im_end|>\n" + "<|im_start|>assistant\n" + ), + }, + "zai-org/GLM-ASR-Nano-2512": { + "prompt": ( + "<|user|>\n" + "<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n" + "Please transcribe this audio into text" + "<|assistant|>\n" + ), + }, +} + + +@pytest.mark.parametrize( + "model_id", + [ + "ibm-granite/granite-speech-3.3-2b", + "nvidia/audio-flamingo-3-hf", + pytest.param( + "mistralai/Voxtral-Mini-3B-2507", + marks=pytest.mark.xfail( + reason="MistralCommonBackend.encode does not produce the audio " + "placeholder token (ID 24) from raw text. apply_chat_template " + "yields token IDs with placeholders, but MultiModalProcessor." + "apply() decodes the prompt back to text and re-tokenizes, at " + "which point the placeholders are lost. Fix belongs in " + "mistral_common or in the Voxtral-specific path.", + strict=False, + ), + ), + "microsoft/VibeVoice-ASR-HF", + "zai-org/GLM-ASR-Nano-2512", + ], +) +def test_audio_multimodal_processor(model_id): + settings = AUDIO_MODEL_SETTINGS[model_id] + + model_config = ModelConfig( + model=model_id, + model_impl="transformers", + ) + + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + + audio = np.zeros(16000, dtype=np.float32) + mm_data = {"audio": (audio, 16000)} + + result = mm_processor( + prompt=settings["prompt"], + mm_items=mm_processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={}, + ) + + assert "prompt_token_ids" in result + assert len(result["prompt_token_ids"]) > 0 + + mm_placeholders = result.get("mm_placeholders", {}) + assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}" + assert len(mm_placeholders["audio"]) == 1 + + placeholder = mm_placeholders["audio"][0] + assert placeholder.length > 0 + assert placeholder.offset >= 0 + + audio_items = result.get("mm_kwargs", {}).get("audio", []) + assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}" + item_keys = list(audio_items[0].keys()) + has_features = "input_features" in item_keys or "input_values" in item_keys + assert has_features, ( + f"No audio features (input_features/input_values) in {item_keys} for {model_id}" + ) + + +def test_audio_multiple_inputs(): + """Multiple audios per prompt are each detected as a separate placeholder + and multi-modal item by the Transformers backend.""" + model_id = "ibm-granite/granite-speech-3.3-2b" + model_config = ModelConfig(model=model_id, model_impl="transformers") + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + + audio_token = mm_processor.info.get_hf_processor().audio_token + # One token per audio; the processor expands each to its placeholder run. + prompt = ( + "<|start_of_role|>user<|end_of_role|>" + f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n" + ) + audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)] + + result = mm_processor( + prompt=prompt, + mm_items=mm_processor.info.parse_mm_data({"audio": audios}), + hf_processor_mm_kwargs={}, + ) + + assert len(result["mm_placeholders"]["audio"]) == 2 + assert len(result["mm_kwargs"]["audio"]) == 2 diff --git a/tests/models/multimodal/processing/test_transformers.py b/tests/models/multimodal/processing/test_transformers_image.py similarity index 63% rename from tests/models/multimodal/processing/test_transformers.py rename to tests/models/multimodal/processing/test_transformers_image.py index a556b8f10a..2c31bcc634 100644 --- a/tests/models/multimodal/processing/test_transformers.py +++ b/tests/models/multimodal/processing/test_transformers_image.py @@ -54,3 +54,26 @@ def test_multimodal_processor(model_id): str_processed_inputs["prompt_token_ids"] == ids_processed_inputs["prompt_token_ids"] ) + + +def test_image_multiple_inputs(): + """Multiple images per prompt are each detected as a separate placeholder + and multi-modal item by the Transformers backend.""" + model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf" + model_config = ModelConfig(model=model_id, model_impl="transformers") + mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config) + + image = ImageAsset("cherry_blossom").pil_image + prompt = ( + "<|im_start|>user \n and \n" + "What do these images show?<|im_end|><|im_start|>assistant\n" + ) + + result = mm_processor( + prompt=prompt, + mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}), + hf_processor_mm_kwargs={}, + ) + + assert len(result["mm_placeholders"]["image"]) == 2 + assert len(result["mm_kwargs"]["image"]) == 2 diff --git a/tests/models/registry.py b/tests/models/registry.py index 2d8ecfb560..f55daa5103 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1393,6 +1393,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { "fixie-ai/ultravox-v0_5-llama-3_2-1b", trust_remote_code=True, ), + "VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo( + "microsoft/VibeVoice-ASR-HF", + min_transformers_version="5.13.0", + ), "VoxtralForConditionalGeneration": _HfExamplesInfo( "mistralai/Voxtral-Mini-3B-2507", tokenizer_mode="mistral", diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 229cf4fe87..c130a7f4a2 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -664,6 +664,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = { "transformers", "TransformersMultiModalForCausalLM", ), + "VibeVoiceAsrForConditionalGeneration": ( + "transformers", + "TransformersMultiModalForCausalLM", + ), } _TRANSFORMERS_BACKEND_MODELS = { diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index ae203d6bdb..5c6b0e286a 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -17,7 +17,8 @@ """Transformers modeling backend mixin for multi-modal models.""" from collections.abc import Mapping -from typing import TYPE_CHECKING +from contextlib import nullcontext +from typing import TYPE_CHECKING, Any import torch from transformers import AutoModel @@ -27,6 +28,7 @@ from vllm.config.utils import getattr_iter from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input from vllm.logger import init_logger from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal +from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MultiModalKwargsItems from vllm.multimodal.inputs import ( MultiModalFeatureSpec, @@ -36,6 +38,7 @@ from vllm.multimodal.inputs import ( from vllm.multimodal.parse import ( ImageProcessorItems, MultiModalDataItems, + MultiModalDataParser, ) from vllm.multimodal.processing import ( BaseDummyInputsBuilder, @@ -59,11 +62,90 @@ _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3} class MultiModalProcessingInfo(BaseProcessingInfo): + def _get_audio_processor(self) -> Any: + # TODO: drop feature_extractor branch once huggingface/transformers#44394 lands. + return getattr_iter( + self.get_hf_processor(), ("audio_processor", "feature_extractor") + ) + + def _is_audio_model(self) -> bool: + return self._get_audio_processor() is not None + + def _is_image_model(self) -> bool: + return hasattr(self.get_hf_processor(), "image_processor") + + def _is_video_model(self) -> bool: + return hasattr(self.get_hf_processor(), "video_processor") + + def _get_audio_token_id(self) -> int: + processor = self.get_hf_processor() + if hasattr(processor, "audio_token_id"): + return processor.audio_token_id + config = self.get_hf_config() + val = getattr_iter(config, ("audio_token_id", "audio_token_index")) + if val is not None: + return val + if hasattr(processor, "audio_token"): + tokenizer = self.get_tokenizer() + vocab = tokenizer.get_vocab() + if processor.audio_token in vocab: + return vocab[processor.audio_token] + raise ValueError("Cannot find audio_token_id on processor or model config") + + def _get_audio_sampling_rate(self) -> float: + sub = self._get_audio_processor() + if sub is not None and hasattr(sub, "sampling_rate"): + return sub.sampling_rate + return 16000.0 + + def get_data_parser(self) -> MultiModalDataParser: + if self._is_audio_model(): + return MultiModalDataParser( + target_sr=self._get_audio_sampling_rate(), + expected_hidden_size=self._get_expected_hidden_size(), + ) + return super().get_data_parser() + def get_supported_mm_limits(self): - return {"image": None} + limits = {} + if self._is_audio_model(): + limits["audio"] = None + if self._is_image_model(): + limits["image"] = None + if not limits: + raise ValueError( + f"Unable to detect a supported modality on " + f"{type(self.get_hf_processor()).__name__}. " + "Checked `_is_audio_model` and `_is_image_model`." + ) + return limits def get_mm_max_tokens_per_item(self, seq_len, mm_counts): - return {"image": self.get_max_image_tokens()} + result = {} + if self._is_audio_model(): + result["audio"] = self.get_max_audio_tokens() + if self._is_image_model(): + result["image"] = self.get_max_image_tokens() + if not result: + raise ValueError( + f"Unable to detect a supported modality on " + f"{type(self.get_hf_processor()).__name__}. " + "Checked `_is_audio_model` and `_is_image_model`." + ) + return result + + def get_max_audio_tokens(self) -> int: + config = self.get_hf_config() + audio_config_names = ("audio_config", "encoder_config") + names = ("max_source_positions", "max_position_embeddings", "max_pos_emb") + audio_config = getattr_iter(config, audio_config_names, default=config) + val = getattr_iter(audio_config, names) + if val is not None: + return int(val) + raise ValueError( + f"Unable to get max input length from {type(audio_config).__name__}. " + f"The following attribute names were checked: {names}." + ) def get_max_image_tokens(self) -> int: width, height = self.get_max_image_size() @@ -82,14 +164,19 @@ class MultiModalProcessingInfo(BaseProcessingInfo): class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_images = mm_counts.get("image", 0) - - processor = self.info.get_hf_processor() - if "gemma3" in processor.__class__.__name__.lower(): - image_token = processor.boi_token - else: - image_token = getattr(processor, "image_token", "") - return image_token * num_images + text = "" + if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)): + processor = self.info.get_hf_processor() + audio_token = getattr(processor, "audio_token", "") + text += audio_token * num_audios + if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)): + processor = self.info.get_hf_processor() + if "gemma3" in processor.__class__.__name__.lower(): + image_token = processor.boi_token + else: + image_token = getattr(processor, "image_token", "") + text += image_token * num_images + return text def get_dummy_mm_data( self, @@ -97,20 +184,28 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn mm_counts: Mapping[str, int], mm_options: Mapping[str, "BaseDummyOptions"], ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - - target_width, target_height = self.info.get_max_image_size() - - image_overrides = mm_options.get("image") - - return { - "image": self._get_dummy_images( + data: MultiModalDataDict = {} + if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)): + sampling_rate = self.info._get_audio_sampling_rate() + sub = self.info._get_audio_processor() + chunk_length = getattr(sub, "chunk_length", None) if sub else None + if chunk_length is None: + chunk_length = 30 + audio_len = int(chunk_length * sampling_rate) + data["audio"] = self._get_dummy_audios( + length=audio_len, + num_audios=num_audios, + overrides=mm_options.get("audio"), + ) + if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)): + target_width, target_height = self.info.get_max_image_size() + data["image"] = self._get_dummy_images( width=target_width, height=target_height, num_images=num_images, - overrides=image_overrides, - ), - } + overrides=mm_options.get("image"), + ) + return data class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): @@ -142,21 +237,39 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): ) -> Mapping[str, MultiModalFieldConfig]: # HF Processors always return a mask but vLLM doesn't need it hf_inputs.pop("attention_mask", None) - num_image_patches = hf_inputs.get("num_image_patches") - mm_fields = { - key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches) - for key in hf_inputs - } - mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes( - "image", num_image_patches - ) - # Keep these as batched, as they always have batch size as first dim - mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image") - mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image") - mm_fields["num_image_patches"] = MultiModalFieldConfig.batched( - "image", keep_on_cpu=True - ) + mm_fields: dict[str, MultiModalFieldConfig] = {} + if self.info._is_audio_model(): + num_audio_tokens = hf_inputs.get("num_audio_tokens") + mm_fields.update( + { + key: MultiModalFieldConfig.flat_from_sizes( + "audio", num_audio_tokens + ) + for key in hf_inputs + } + ) + mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio") + if self.info._is_image_model(): + num_image_patches = hf_inputs.get("num_image_patches") + mm_fields.update( + { + key: MultiModalFieldConfig.flat_from_sizes( + "image", num_image_patches + ) + for key in hf_inputs + } + ) + mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes( + "image", num_image_patches + ) + + # Keep these as batched, as they always have batch size as first dim + mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image") + mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image") + mm_fields["num_image_patches"] = MultiModalFieldConfig.batched( + "image", keep_on_cpu=True + ) return mm_fields def _get_hf_mm_data( @@ -164,13 +277,93 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): mm_items: MultiModalDataItems, ) -> tuple[Mapping[str, object], Mapping[str, object]]: """ - In contrast to the base class, this method always adds - `return_mm_token_type_ids` to the processor data + In contrast to the base class, this method requests + `return_mm_token_type_ids` and remaps the `audios` key to `audio` for + audio models. """ processor_data, passthrough_data = super()._get_hf_mm_data(mm_items) + if self.info._is_audio_model() and "audios" in processor_data: + processor_data["audio"] = processor_data.pop("audios") processor_data["return_mm_token_type_ids"] = True return processor_data, passthrough_data + def _apply_audio( + self, + prompt_ids: list[int], + processed_data: "BatchFeature", + ) -> dict[str, list[PlaceholderRange]]: + audio_token_id = self.info._get_audio_token_id() + prompt_tensor = torch.tensor(prompt_ids) + is_audio = prompt_tensor == audio_token_id + + if not is_audio.any(): + return {} + + padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])]) + transitions = padded.int().diff() + starts = torch.where(transitions == 1)[0] + ends = torch.where(transitions == -1)[0] + lengths = ends - starts + + ranges = [ + PlaceholderRange( + offset=s.item(), + length=ln.item(), + is_embed=torch.ones(ln.item(), dtype=torch.bool), + ) + for s, ln in zip(starts, lengths) + ] + processed_data["num_audio_tokens"] = lengths + return {"audio": ranges} + + def _apply_vision( + self, + prompt_ids: list[int], + processed_data: "BatchFeature", + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + mm_token_type_ids: torch.Tensor | None, + ) -> dict[str, list[PlaceholderRange]]: + if mm_token_type_ids is None: + return {} + + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) + + # We can infer vLLM style placeholder from token type ids, if we split + # it for each input `mm_data`. + mm_positions = torch.where(mm_token_type_ids == 1)[1] + images = mm_items.get_items("image", ImageProcessorItems) + image_sizes = [] + for item_idx in range(len(images)): + image_size = images.get_image_size(item_idx) + image_sizes.append((image_size.height, image_size.width)) + + mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens( + image_sizes=image_sizes, + **self.info.ctx.get_merged_mm_kwargs({}), + ) + + mm_placeholders: dict[str, list[PlaceholderRange]] = {} + split_sizes = mm_tokens_per_modality["num_image_tokens"] + if split_sizes: + chunked_mm_positions = torch.split(mm_positions, split_sizes) + mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()] + chunked_mm_tokens = torch.split(mm_tokens, split_sizes) + ranges = [ + PlaceholderRange( + offset=positions[0].item(), + length=positions.shape[0], + is_embed=(mm_tokens == hf_processor.image_token_id).bool(), + ) + for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens) + ] + mm_placeholders = {"image": ranges} + + processed_data["num_image_patches"] = torch.tensor( + mm_tokens_per_modality["num_image_patches"] + ) + return mm_placeholders + def apply( self, inputs: ProcessorInputs, @@ -198,8 +391,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): # Bypass cached processor and always apply to the full set of mm inputs # NOTE: we can't just set caching=False because base class method # transforms outputs to `MultiModalKwargs` which is not going to - # work for Transformers. We have a lot of logic tied to - # `mm_tokens_per_modality` below + # work for Transformers. The vision path has logic tied to + # `mm_tokens_per_modality` in _apply_vision() prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm( prompt_text=prompt, mm_items=mm_items, @@ -207,52 +400,33 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): tokenization_kwargs=tokenization_kwargs, ) + # Use overrides if provided; fallback to data-dependent hashing. + with timing_ctx.record("get_mm_hashes"): + mm_hashes = inputs.get_mm_hashes(self.info.model_id) + # For gemma3 we check `token_type_ids` as the key mm_token_type_ids = processed_data.pop("token_type_ids", None) mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids) - # We can infer vLLM style placeholder from token type ids, if we split - # it for each input `mm_data`. - mm_positions = torch.where(mm_token_type_ids == 1)[1] - images = mm_items.get_items("image", ImageProcessorItems) - image_sizes = [] - for item_idx in range(len(images)): - image_size = images.get_image_size(item_idx) - image_sizes.append((image_size.height, image_size.width)) - - mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens( - image_sizes=image_sizes, - **self.info.ctx.get_merged_mm_kwargs({}), - ) - - mm_placeholders = {} - split_sizes = mm_tokens_per_modality["num_image_tokens"] - if split_sizes: - chunked_mm_positions = torch.split(mm_positions, split_sizes) - mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()] - chunked_mm_tokens = torch.split(mm_tokens, split_sizes) - ranges = [ - PlaceholderRange( - offset=positions[0].item(), - length=positions.shape[0], - is_embed=(mm_tokens == hf_processor.image_token_id).bool(), + mm_placeholders: dict[str, list[PlaceholderRange]] = {} + if self.info._is_audio_model(): + mm_placeholders.update(self._apply_audio(prompt_ids, processed_data)) + if self.info._is_image_model(): + mm_placeholders.update( + self._apply_vision( + prompt_ids, + processed_data, + mm_items, + hf_processor_mm_kwargs, + mm_token_type_ids, ) - for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens) - ] - mm_placeholders = {"image": ranges} + ) - processed_data["num_image_patches"] = torch.tensor( - mm_tokens_per_modality["num_image_patches"] - ) mm_kwargs = MultiModalKwargsItems.from_hf_inputs( processed_data, self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs), ) - # Use overrides if provided; fallback to data-dependent hashing. - with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes(self.info.model_id) - return mm_input( prompt_token_ids=prompt_ids, mm_kwargs=mm_kwargs, @@ -364,7 +538,79 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): return LanguageModel(self) - def embed_multimodal(self, **kwargs): + def get_mm_mapping(self) -> MultiModelKeys: + """ + Get the module prefix in multimodal models + """ + for name in ("language_model", "text_model"): + if getattr(self.model, name, None) is not None: + return MultiModelKeys.from_string_field(language_model=f"model.{name}") + raise ValueError( + "Could not locate the language model submodule for LoRA support" + ) + + def _split_embeddings( + self, embeddings: torch.Tensor, split_sizes: list[int] + ) -> list[torch.Tensor]: + total_expected = sum(split_sizes) + + # Flatten to 2D: [total_tokens, hidden_dim] + if embeddings.ndim == 3: + embeddings = embeddings.view(-1, embeddings.shape[-1]) + + total_tokens = embeddings.shape[0] + if total_tokens == total_expected: + # Direct match: split_sizes are actual token counts + token_split_sizes = split_sizes + elif total_expected > 0 and total_tokens % total_expected == 0: + # Uniform expansion: each item expands to N tokens + tokens_per_item = total_tokens // total_expected + token_split_sizes = [s * tokens_per_item for s in split_sizes] + elif total_expected > 0: + # Mismatch (profiling with dummy data) - pad/truncate + if total_tokens == 0: + raise ValueError( + "Encoder returned empty embeddings. " + f"Expected {total_expected} tokens from " + f"split_sizes={split_sizes}" + ) + if total_tokens < total_expected: + repeat_factor = (total_expected + total_tokens - 1) // total_tokens + embeddings = embeddings.repeat(repeat_factor, 1) + embeddings = embeddings[:total_expected] + token_split_sizes = split_sizes + else: + return [] + + return list(torch.split(embeddings, token_split_sizes, dim=0)) + + def _embed_audio(self, **kwargs) -> list[torch.Tensor] | None: + self.check_version("5.13.0", "audio models support") + input_features: torch.Tensor | None = kwargs.pop("input_features", None) + if input_features is None: + input_features = kwargs.pop("input_values", None) + if input_features is None: + return None + + num_audio_tokens = kwargs.pop("num_audio_tokens") + kwargs.pop("token_type_ids", None) + kwargs.pop("mm_token_type_ids", None) + + context = nullcontext() + if current_platform.is_rocm(): + context = torch.nn.attention.sdpa_kernel( + backends=[torch.nn.attention.SDPBackend.MATH] + ) + with context: + audio_output = self.model.get_audio_features( + input_features, return_dict=True, **kwargs + ) + audio_embeddings = audio_output.pooler_output + + split_sizes = num_audio_tokens.flatten().tolist() + return self._split_embeddings(audio_embeddings, split_sizes) + + def _embed_vision(self, **kwargs) -> list[torch.Tensor] | torch.Tensor | None: pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None) image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None) # Model might use `image_patches` instead of `pixel_values` @@ -379,84 +625,57 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE): num_image_patches = kwargs.pop("num_image_patches") - if pixel_values is not None: + context = nullcontext() + if current_platform.is_rocm(): # ROCm: Force math SDP backend for vision encoder to avoid accuracy issues # with flash_sdp and mem_efficient_sdp - if current_platform.is_rocm(): - # TODO: [ROCm] Fix accuracy issues with flash backend - logger.debug( - "ROCm platform detected. Forcing math SDP backend " - "for vision encoder. Currently ROCm platform has " - "accuracy issues with `flash_sdp` and" - "`mem_efficient_sdp` backends. See issue: " - "https://github.com/vllm-project/vllm/issues/30167" - ) - with torch.nn.attention.sdpa_kernel( - backends=[torch.nn.attention.SDPBackend.MATH] - ): - vision_embeddings = self.model.get_image_features( - pixel_values, **kwargs - ) - else: - vision_embeddings = self.model.get_image_features( - pixel_values, **kwargs - ) - - # Transformers `v5`, `self.get_image_features` returns a tuple - # containing the features and optionally attentions/hidden_states - # After v5 is settled, we can enable qwen3-vl with several outputs - # from `self.get_image_features` - if isinstance(vision_embeddings, tuple): - vision_embeddings = vision_embeddings[0] - elif isinstance(vision_embeddings, dict): - vision_embeddings = vision_embeddings.pooler_output - - if isinstance(vision_embeddings, torch.Tensor): - split_sizes = num_image_patches.flatten().tolist() - total_patches = sum(split_sizes) - - # Flatten to 2D: [total_tokens, hidden_dim] - if vision_embeddings.ndim == 3: - vision_embeddings = vision_embeddings.view( - -1, vision_embeddings.shape[-1] - ) - - total_tokens = vision_embeddings.shape[0] - if total_tokens == total_patches: - # Direct match: num_image_patches are actual token counts - # (e.g., Qwen2.5-VL style) - token_split_sizes = split_sizes - elif total_patches > 0 and total_tokens % total_patches == 0: - # Uniform expansion: each patch expands to N tokens - # (e.g., Idefics3 style) - tokens_per_patch = total_tokens // total_patches - token_split_sizes = [s * tokens_per_patch for s in split_sizes] - elif total_patches > 0: - # Mismatch (profiling with dummy data) - pad/truncate - if total_tokens == 0: - raise ValueError( - "Vision encoder returned empty embeddings. " - f"Expected {total_patches} patches from " - f"num_image_patches={split_sizes}" - ) - if total_tokens < total_patches: - repeat_factor = ( - total_patches + total_tokens - 1 - ) // total_tokens - vision_embeddings = vision_embeddings.repeat(repeat_factor, 1) - vision_embeddings = vision_embeddings[:total_patches] - token_split_sizes = split_sizes - else: - return [] - - return list(torch.split(vision_embeddings, token_split_sizes, dim=0)) - - return vision_embeddings - else: + # TODO: [ROCm] Fix accuracy issues with flash backend logger.debug( - "No pixel values or image embeddings provided for multimodal embedding." + "ROCm platform detected. Forcing math SDP backend " + "for vision encoder. Currently ROCm platform has " + "accuracy issues with `flash_sdp` and" + "`mem_efficient_sdp` backends. See issue: " + "https://github.com/vllm-project/vllm/issues/30167" ) - return None + context = torch.nn.attention.sdpa_kernel( + backends=[torch.nn.attention.SDPBackend.MATH] + ) + with context: + vision_embeddings = self.model.get_image_features(pixel_values, **kwargs) + + # Transformers `v5`, `self.get_image_features` returns a tuple + # containing the features and optionally attentions/hidden_states + # After v5 is settled, we can enable qwen3-vl with several outputs + # from `self.get_image_features` + if isinstance(vision_embeddings, tuple): + vision_embeddings = vision_embeddings[0] + elif isinstance(vision_embeddings, dict): + vision_embeddings = vision_embeddings.pooler_output + + if isinstance(vision_embeddings, torch.Tensor): + split_sizes = num_image_patches.flatten().tolist() + return self._split_embeddings(vision_embeddings, split_sizes) + + return vision_embeddings + + def embed_multimodal(self, **kwargs): + embeddings: tuple[torch.Tensor, ...] = () + if "input_features" in kwargs or "input_values" in kwargs: + audio_embeddings = self._embed_audio(**kwargs) + if audio_embeddings is not None: + embeddings += tuple(audio_embeddings) + if ( + "pixel_values" in kwargs + or "image_embeds" in kwargs + or "image_patches" in kwargs + ): + vision_embeddings = self._embed_vision(**kwargs) + if vision_embeddings is not None: + if isinstance(vision_embeddings, torch.Tensor): + embeddings += (vision_embeddings,) + else: + embeddings += tuple(vision_embeddings) + return embeddings def get_mrope_input_positions( self,