feat[vLLM × v5]: Add audio support for the Transformers backend (#39330)

Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harshal Janjani
2026-07-25 04:20:58 -07:00
committed by GitHub
co-authored by Harry Mellor mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent ca0defa343
commit b9b6306ebe
8 changed files with 682 additions and 152 deletions
+3 -2
View File
@@ -19,7 +19,7 @@ vLLM also supports model implementations that are available in Transformers. We
Currently, the Transformers modeling backend works for the following:
- Modalities: embedding models, language models and vision-language models*
- Modalities: embedding models, language models, vision-language models* and audio-language models
- Architectures: encoder-only, decoder-only, mixture-of-experts
- Attention types: full attention and/or sliding attention
@@ -606,7 +606,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor
| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
| ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
| `Emu3ForConditionalGeneration` | Emu3 | T + I<sup>+</sup> | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A<sup>+</sup> | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ |
<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.</br>
<sup>E</sup> Pre-computed embeddings can be inputted for this modality.</br>
@@ -0,0 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from transformers import AutoModelForSeq2SeqLM
from vllm.assets.audio import AudioAsset
from vllm.envs import disable_envs_cache
from vllm.lora.request import LoRARequest
from vllm.multimodal.audio import AudioResampler
from ....conftest import HfRunner, VllmRunner
from ...utils import check_logprobs_close
AUDIO_ASSET = AudioAsset("mary_had_lamb")
AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
"<|start_of_role|>system<|end_of_role|>"
"You are a helpful AI assistant<|end_of_text|>\n"
"<|start_of_role|>user<|end_of_role|>"
"<|audio|>can you transcribe the speech into a written format?"
"<|end_of_text|>\n"
"<|start_of_role|>assistant<|end_of_role|>"
),
"audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
},
"nvidia/audio-flamingo-3-hf": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\n"
"<sound>Transcribe the input speech.<|im_end|>\n"
"<|im_start|>assistant\n"
),
"vllm_runner_kwargs": {
"gpu_memory_utilization": 0.85,
},
},
"microsoft/VibeVoice-ASR-HF": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant that transcribes audio input "
"into text output in JSON format.<|im_end|>\n"
"<|im_start|>user\n"
"<|object_ref_start|><|box_start|><|object_ref_end|>\n"
"This is a 1.0 seconds audio, please transcribe it with "
"these keys: Start time, End time, Speaker ID, Content"
"<|im_end|>\n"
"<|im_start|>assistant\n"
),
"sampling_rate": 24000,
"vllm_runner_kwargs": {
"max_num_batched_tokens": 2048,
"gpu_memory_utilization": 0.85,
},
},
"zai-org/GLM-ASR-Nano-2512": {
"prompt": (
"<|user|>\n"
"<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
"Please transcribe this audio into text"
"<|assistant|>\n"
),
},
}
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
def test_transformers_audio_generation(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
monkeypatch,
model_id: str,
):
"""Single-process workaround for V1 fork safety deadlock issue
(vllm-project/vllm/issues/17676). Running multiple audio models together
under pytest can cause (possibly flaky) hangs, so they are grouped under
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
deadlock and allows worker processes to terminate cleanly, and release
GPU memory between test runs until the issue is fixed."""
# TODO: Remove monkeypatch once
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
disable_envs_cache()
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
settings = AUDIO_MODEL_SETTINGS[model_id]
audio_lora_path = settings.get("audio_lora_path")
audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
target_sr = settings.get("sampling_rate", orig_sr)
if orig_sr != target_sr:
audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr)
audio = (audio, target_sr)
with vllm_runner(
model_id,
model_impl="transformers",
dtype="bfloat16",
max_model_len=2048,
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
enable_lora=audio_lora_path is not None,
max_lora_rank=64,
**settings.get("vllm_runner_kwargs", {}),
) as vllm_model:
lora_request = (
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
)
vllm_outputs = vllm_model.generate_greedy_logprobs(
[settings["prompt"]],
128,
num_logprobs=10,
audios=[audio],
lora_request=lora_request,
)
with hf_runner(
model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM
) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
[settings["prompt"]], 128, num_logprobs=10, audios=[audio]
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@@ -452,6 +452,14 @@ def test_processing_correctness(
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
# TODO: Remove when transformers 5.15.0 is released, which contains
# https://github.com/huggingface/transformers/pull/47483.
if model_id == "microsoft/VibeVoice-ASR-HF":
pytest.skip(
"VibeVoice ASR requires audio as a positional argument and hence "
"cannot pass the processing correctness test as is. Its generation "
"is covered by test_transformers_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
AUDIO_MODEL_SETTINGS = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
"<|start_of_role|>system<|end_of_role|>"
"You are a helpful AI assistant<|end_of_text|>\n"
"<|start_of_role|>user<|end_of_role|>"
"<|audio|>can you transcribe the speech into a written format?"
"<|end_of_text|>\n"
"<|start_of_role|>assistant<|end_of_role|>"
),
},
"nvidia/audio-flamingo-3-hf": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant.<|im_end|>\n"
"<|im_start|>user\n"
"<sound>Transcribe the input speech.<|im_end|>\n"
"<|im_start|>assistant\n"
),
},
"mistralai/Voxtral-Mini-3B-2507": {
"prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
},
"microsoft/VibeVoice-ASR-HF": {
"prompt": (
"<|im_start|>system\n"
"You are a helpful assistant that transcribes audio input "
"into text output in JSON format.<|im_end|>\n"
"<|im_start|>user\n"
"<|object_ref_start|><|box_start|><|object_ref_end|>\n"
"This is a 1.0 seconds audio, please transcribe it with "
"these keys: Start time, End time, Speaker ID, Content"
"<|im_end|>\n"
"<|im_start|>assistant\n"
),
},
"zai-org/GLM-ASR-Nano-2512": {
"prompt": (
"<|user|>\n"
"<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
"Please transcribe this audio into text"
"<|assistant|>\n"
),
},
}
@pytest.mark.parametrize(
"model_id",
[
"ibm-granite/granite-speech-3.3-2b",
"nvidia/audio-flamingo-3-hf",
pytest.param(
"mistralai/Voxtral-Mini-3B-2507",
marks=pytest.mark.xfail(
reason="MistralCommonBackend.encode does not produce the audio "
"placeholder token (ID 24) from raw text. apply_chat_template "
"yields token IDs with placeholders, but MultiModalProcessor."
"apply() decodes the prompt back to text and re-tokenizes, at "
"which point the placeholders are lost. Fix belongs in "
"mistral_common or in the Voxtral-specific path.",
strict=False,
),
),
"microsoft/VibeVoice-ASR-HF",
"zai-org/GLM-ASR-Nano-2512",
],
)
def test_audio_multimodal_processor(model_id):
settings = AUDIO_MODEL_SETTINGS[model_id]
model_config = ModelConfig(
model=model_id,
model_impl="transformers",
)
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
audio = np.zeros(16000, dtype=np.float32)
mm_data = {"audio": (audio, 16000)}
result = mm_processor(
prompt=settings["prompt"],
mm_items=mm_processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
assert "prompt_token_ids" in result
assert len(result["prompt_token_ids"]) > 0
mm_placeholders = result.get("mm_placeholders", {})
assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}"
assert len(mm_placeholders["audio"]) == 1
placeholder = mm_placeholders["audio"][0]
assert placeholder.length > 0
assert placeholder.offset >= 0
audio_items = result.get("mm_kwargs", {}).get("audio", [])
assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
item_keys = list(audio_items[0].keys())
has_features = "input_features" in item_keys or "input_values" in item_keys
assert has_features, (
f"No audio features (input_features/input_values) in {item_keys} for {model_id}"
)
def test_audio_multiple_inputs():
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers backend."""
model_id = "ibm-granite/granite-speech-3.3-2b"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
audio_token = mm_processor.info.get_hf_processor().audio_token
# One token per audio; the processor expands each to its placeholder run.
prompt = (
"<|start_of_role|>user<|end_of_role|>"
f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
@@ -54,3 +54,26 @@ def test_multimodal_processor(model_id):
str_processed_inputs["prompt_token_ids"]
== ids_processed_inputs["prompt_token_ids"]
)
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers backend."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
model_config = ModelConfig(model=model_id, model_impl="transformers")
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
"<|im_start|>user <image>\n and <image>\n"
"What do these images show?<|im_end|><|im_start|>assistant\n"
)
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
+4
View File
@@ -1393,6 +1393,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
trust_remote_code=True,
),
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
"microsoft/VibeVoice-ASR-HF",
min_transformers_version="5.13.0",
),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
tokenizer_mode="mistral",
+4
View File
@@ -664,6 +664,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
"transformers",
"TransformersMultiModalForCausalLM",
),
"VibeVoiceAsrForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
}
_TRANSFORMERS_BACKEND_MODELS = {
@@ -17,7 +17,8 @@
"""Transformers modeling backend mixin for multi-modal models."""
from collections.abc import Mapping
from typing import TYPE_CHECKING
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any
import torch
from transformers import AutoModel
@@ -27,6 +28,7 @@ from vllm.config.utils import getattr_iter
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -36,6 +38,7 @@ from vllm.multimodal.inputs import (
from vllm.multimodal.parse import (
ImageProcessorItems,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
@@ -59,11 +62,90 @@ _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
class MultiModalProcessingInfo(BaseProcessingInfo):
def _get_audio_processor(self) -> Any:
# TODO: drop feature_extractor branch once huggingface/transformers#44394 lands.
return getattr_iter(
self.get_hf_processor(), ("audio_processor", "feature_extractor")
)
def _is_audio_model(self) -> bool:
return self._get_audio_processor() is not None
def _is_image_model(self) -> bool:
return hasattr(self.get_hf_processor(), "image_processor")
def _is_video_model(self) -> bool:
return hasattr(self.get_hf_processor(), "video_processor")
def _get_audio_token_id(self) -> int:
processor = self.get_hf_processor()
if hasattr(processor, "audio_token_id"):
return processor.audio_token_id
config = self.get_hf_config()
val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
if val is not None:
return val
if hasattr(processor, "audio_token"):
tokenizer = self.get_tokenizer()
vocab = tokenizer.get_vocab()
if processor.audio_token in vocab:
return vocab[processor.audio_token]
raise ValueError("Cannot find audio_token_id on processor or model config")
def _get_audio_sampling_rate(self) -> float:
sub = self._get_audio_processor()
if sub is not None and hasattr(sub, "sampling_rate"):
return sub.sampling_rate
return 16000.0
def get_data_parser(self) -> MultiModalDataParser:
if self._is_audio_model():
return MultiModalDataParser(
target_sr=self._get_audio_sampling_rate(),
expected_hidden_size=self._get_expected_hidden_size(),
)
return super().get_data_parser()
def get_supported_mm_limits(self):
return {"image": None}
limits = {}
if self._is_audio_model():
limits["audio"] = None
if self._is_image_model():
limits["image"] = None
if not limits:
raise ValueError(
f"Unable to detect a supported modality on "
f"{type(self.get_hf_processor()).__name__}. "
"Checked `_is_audio_model` and `_is_image_model`."
)
return limits
def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
return {"image": self.get_max_image_tokens()}
result = {}
if self._is_audio_model():
result["audio"] = self.get_max_audio_tokens()
if self._is_image_model():
result["image"] = self.get_max_image_tokens()
if not result:
raise ValueError(
f"Unable to detect a supported modality on "
f"{type(self.get_hf_processor()).__name__}. "
"Checked `_is_audio_model` and `_is_image_model`."
)
return result
def get_max_audio_tokens(self) -> int:
config = self.get_hf_config()
audio_config_names = ("audio_config", "encoder_config")
names = ("max_source_positions", "max_position_embeddings", "max_pos_emb")
audio_config = getattr_iter(config, audio_config_names, default=config)
val = getattr_iter(audio_config, names)
if val is not None:
return int(val)
raise ValueError(
f"Unable to get max input length from {type(audio_config).__name__}. "
f"The following attribute names were checked: {names}."
)
def get_max_image_tokens(self) -> int:
width, height = self.get_max_image_size()
@@ -82,14 +164,19 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
processor = self.info.get_hf_processor()
if "gemma3" in processor.__class__.__name__.lower():
image_token = processor.boi_token
else:
image_token = getattr(processor, "image_token", "")
return image_token * num_images
text = ""
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
processor = self.info.get_hf_processor()
audio_token = getattr(processor, "audio_token", "")
text += audio_token * num_audios
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
processor = self.info.get_hf_processor()
if "gemma3" in processor.__class__.__name__.lower():
image_token = processor.boi_token
else:
image_token = getattr(processor, "image_token", "")
text += image_token * num_images
return text
def get_dummy_mm_data(
self,
@@ -97,20 +184,28 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
mm_counts: Mapping[str, int],
mm_options: Mapping[str, "BaseDummyOptions"],
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
target_width, target_height = self.info.get_max_image_size()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
data: MultiModalDataDict = {}
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
sampling_rate = self.info._get_audio_sampling_rate()
sub = self.info._get_audio_processor()
chunk_length = getattr(sub, "chunk_length", None) if sub else None
if chunk_length is None:
chunk_length = 30
audio_len = int(chunk_length * sampling_rate)
data["audio"] = self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=mm_options.get("audio"),
)
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
target_width, target_height = self.info.get_max_image_size()
data["image"] = self._get_dummy_images(
width=target_width,
height=target_height,
num_images=num_images,
overrides=image_overrides,
),
}
overrides=mm_options.get("image"),
)
return data
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
@@ -142,21 +237,39 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
) -> Mapping[str, MultiModalFieldConfig]:
# HF Processors always return a mask but vLLM doesn't need it
hf_inputs.pop("attention_mask", None)
num_image_patches = hf_inputs.get("num_image_patches")
mm_fields = {
key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
for key in hf_inputs
}
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
"image", num_image_patches
)
# Keep these as batched, as they always have batch size as first dim
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
mm_fields: dict[str, MultiModalFieldConfig] = {}
if self.info._is_audio_model():
num_audio_tokens = hf_inputs.get("num_audio_tokens")
mm_fields.update(
{
key: MultiModalFieldConfig.flat_from_sizes(
"audio", num_audio_tokens
)
for key in hf_inputs
}
)
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
if self.info._is_image_model():
num_image_patches = hf_inputs.get("num_image_patches")
mm_fields.update(
{
key: MultiModalFieldConfig.flat_from_sizes(
"image", num_image_patches
)
for key in hf_inputs
}
)
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
"image", num_image_patches
)
# Keep these as batched, as they always have batch size as first dim
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
"image", keep_on_cpu=True
)
return mm_fields
def _get_hf_mm_data(
@@ -164,13 +277,93 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
mm_items: MultiModalDataItems,
) -> tuple[Mapping[str, object], Mapping[str, object]]:
"""
In contrast to the base class, this method always adds
`return_mm_token_type_ids` to the processor data
In contrast to the base class, this method requests
`return_mm_token_type_ids` and remaps the `audios` key to `audio` for
audio models.
"""
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
if self.info._is_audio_model() and "audios" in processor_data:
processor_data["audio"] = processor_data.pop("audios")
processor_data["return_mm_token_type_ids"] = True
return processor_data, passthrough_data
def _apply_audio(
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
) -> dict[str, list[PlaceholderRange]]:
audio_token_id = self.info._get_audio_token_id()
prompt_tensor = torch.tensor(prompt_ids)
is_audio = prompt_tensor == audio_token_id
if not is_audio.any():
return {}
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
transitions = padded.int().diff()
starts = torch.where(transitions == 1)[0]
ends = torch.where(transitions == -1)[0]
lengths = ends - starts
ranges = [
PlaceholderRange(
offset=s.item(),
length=ln.item(),
is_embed=torch.ones(ln.item(), dtype=torch.bool),
)
for s, ln in zip(starts, lengths)
]
processed_data["num_audio_tokens"] = lengths
return {"audio": ranges}
def _apply_vision(
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
if mm_token_type_ids is None:
return {}
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
# We can infer vLLM style placeholder from token type ids, if we split
# it for each input `mm_data`.
mm_positions = torch.where(mm_token_type_ids == 1)[1]
images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = []
for item_idx in range(len(images)):
image_size = images.get_image_size(item_idx)
image_sizes.append((image_size.height, image_size.width))
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
image_sizes=image_sizes,
**self.info.ctx.get_merged_mm_kwargs({}),
)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
split_sizes = mm_tokens_per_modality["num_image_tokens"]
if split_sizes:
chunked_mm_positions = torch.split(mm_positions, split_sizes)
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
ranges = [
PlaceholderRange(
offset=positions[0].item(),
length=positions.shape[0],
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
)
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
]
mm_placeholders = {"image": ranges}
processed_data["num_image_patches"] = torch.tensor(
mm_tokens_per_modality["num_image_patches"]
)
return mm_placeholders
def apply(
self,
inputs: ProcessorInputs,
@@ -198,8 +391,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. We have a lot of logic tied to
# `mm_tokens_per_modality` below
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
prompt_text=prompt,
mm_items=mm_items,
@@ -207,52 +400,33 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
tokenization_kwargs=tokenization_kwargs,
)
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
# For gemma3 we check `token_type_ids` as the key
mm_token_type_ids = processed_data.pop("token_type_ids", None)
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
# We can infer vLLM style placeholder from token type ids, if we split
# it for each input `mm_data`.
mm_positions = torch.where(mm_token_type_ids == 1)[1]
images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = []
for item_idx in range(len(images)):
image_size = images.get_image_size(item_idx)
image_sizes.append((image_size.height, image_size.width))
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
image_sizes=image_sizes,
**self.info.ctx.get_merged_mm_kwargs({}),
)
mm_placeholders = {}
split_sizes = mm_tokens_per_modality["num_image_tokens"]
if split_sizes:
chunked_mm_positions = torch.split(mm_positions, split_sizes)
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
ranges = [
PlaceholderRange(
offset=positions[0].item(),
length=positions.shape[0],
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
if self.info._is_audio_model():
mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
if self.info._is_image_model():
mm_placeholders.update(
self._apply_vision(
prompt_ids,
processed_data,
mm_items,
hf_processor_mm_kwargs,
mm_token_type_ids,
)
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
]
mm_placeholders = {"image": ranges}
)
processed_data["num_image_patches"] = torch.tensor(
mm_tokens_per_modality["num_image_patches"]
)
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
processed_data,
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
)
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_kwargs,
@@ -364,7 +538,79 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
return LanguageModel(self)
def embed_multimodal(self, **kwargs):
def get_mm_mapping(self) -> MultiModelKeys:
"""
Get the module prefix in multimodal models
"""
for name in ("language_model", "text_model"):
if getattr(self.model, name, None) is not None:
return MultiModelKeys.from_string_field(language_model=f"model.{name}")
raise ValueError(
"Could not locate the language model submodule for LoRA support"
)
def _split_embeddings(
self, embeddings: torch.Tensor, split_sizes: list[int]
) -> list[torch.Tensor]:
total_expected = sum(split_sizes)
# Flatten to 2D: [total_tokens, hidden_dim]
if embeddings.ndim == 3:
embeddings = embeddings.view(-1, embeddings.shape[-1])
total_tokens = embeddings.shape[0]
if total_tokens == total_expected:
# Direct match: split_sizes are actual token counts
token_split_sizes = split_sizes
elif total_expected > 0 and total_tokens % total_expected == 0:
# Uniform expansion: each item expands to N tokens
tokens_per_item = total_tokens // total_expected
token_split_sizes = [s * tokens_per_item for s in split_sizes]
elif total_expected > 0:
# Mismatch (profiling with dummy data) - pad/truncate
if total_tokens == 0:
raise ValueError(
"Encoder returned empty embeddings. "
f"Expected {total_expected} tokens from "
f"split_sizes={split_sizes}"
)
if total_tokens < total_expected:
repeat_factor = (total_expected + total_tokens - 1) // total_tokens
embeddings = embeddings.repeat(repeat_factor, 1)
embeddings = embeddings[:total_expected]
token_split_sizes = split_sizes
else:
return []
return list(torch.split(embeddings, token_split_sizes, dim=0))
def _embed_audio(self, **kwargs) -> list[torch.Tensor] | None:
self.check_version("5.13.0", "audio models support")
input_features: torch.Tensor | None = kwargs.pop("input_features", None)
if input_features is None:
input_features = kwargs.pop("input_values", None)
if input_features is None:
return None
num_audio_tokens = kwargs.pop("num_audio_tokens")
kwargs.pop("token_type_ids", None)
kwargs.pop("mm_token_type_ids", None)
context = nullcontext()
if current_platform.is_rocm():
context = torch.nn.attention.sdpa_kernel(
backends=[torch.nn.attention.SDPBackend.MATH]
)
with context:
audio_output = self.model.get_audio_features(
input_features, return_dict=True, **kwargs
)
audio_embeddings = audio_output.pooler_output
split_sizes = num_audio_tokens.flatten().tolist()
return self._split_embeddings(audio_embeddings, split_sizes)
def _embed_vision(self, **kwargs) -> list[torch.Tensor] | torch.Tensor | None:
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
# Model might use `image_patches` instead of `pixel_values`
@@ -379,84 +625,57 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
num_image_patches = kwargs.pop("num_image_patches")
if pixel_values is not None:
context = nullcontext()
if current_platform.is_rocm():
# ROCm: Force math SDP backend for vision encoder to avoid accuracy issues
# with flash_sdp and mem_efficient_sdp
if current_platform.is_rocm():
# TODO: [ROCm] Fix accuracy issues with flash backend
logger.debug(
"ROCm platform detected. Forcing math SDP backend "
"for vision encoder. Currently ROCm platform has "
"accuracy issues with `flash_sdp` and"
"`mem_efficient_sdp` backends. See issue: "
"https://github.com/vllm-project/vllm/issues/30167"
)
with torch.nn.attention.sdpa_kernel(
backends=[torch.nn.attention.SDPBackend.MATH]
):
vision_embeddings = self.model.get_image_features(
pixel_values, **kwargs
)
else:
vision_embeddings = self.model.get_image_features(
pixel_values, **kwargs
)
# Transformers `v5`, `self.get_image_features` returns a tuple
# containing the features and optionally attentions/hidden_states
# After v5 is settled, we can enable qwen3-vl with several outputs
# from `self.get_image_features`
if isinstance(vision_embeddings, tuple):
vision_embeddings = vision_embeddings[0]
elif isinstance(vision_embeddings, dict):
vision_embeddings = vision_embeddings.pooler_output
if isinstance(vision_embeddings, torch.Tensor):
split_sizes = num_image_patches.flatten().tolist()
total_patches = sum(split_sizes)
# Flatten to 2D: [total_tokens, hidden_dim]
if vision_embeddings.ndim == 3:
vision_embeddings = vision_embeddings.view(
-1, vision_embeddings.shape[-1]
)
total_tokens = vision_embeddings.shape[0]
if total_tokens == total_patches:
# Direct match: num_image_patches are actual token counts
# (e.g., Qwen2.5-VL style)
token_split_sizes = split_sizes
elif total_patches > 0 and total_tokens % total_patches == 0:
# Uniform expansion: each patch expands to N tokens
# (e.g., Idefics3 style)
tokens_per_patch = total_tokens // total_patches
token_split_sizes = [s * tokens_per_patch for s in split_sizes]
elif total_patches > 0:
# Mismatch (profiling with dummy data) - pad/truncate
if total_tokens == 0:
raise ValueError(
"Vision encoder returned empty embeddings. "
f"Expected {total_patches} patches from "
f"num_image_patches={split_sizes}"
)
if total_tokens < total_patches:
repeat_factor = (
total_patches + total_tokens - 1
) // total_tokens
vision_embeddings = vision_embeddings.repeat(repeat_factor, 1)
vision_embeddings = vision_embeddings[:total_patches]
token_split_sizes = split_sizes
else:
return []
return list(torch.split(vision_embeddings, token_split_sizes, dim=0))
return vision_embeddings
else:
# TODO: [ROCm] Fix accuracy issues with flash backend
logger.debug(
"No pixel values or image embeddings provided for multimodal embedding."
"ROCm platform detected. Forcing math SDP backend "
"for vision encoder. Currently ROCm platform has "
"accuracy issues with `flash_sdp` and"
"`mem_efficient_sdp` backends. See issue: "
"https://github.com/vllm-project/vllm/issues/30167"
)
return None
context = torch.nn.attention.sdpa_kernel(
backends=[torch.nn.attention.SDPBackend.MATH]
)
with context:
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
# Transformers `v5`, `self.get_image_features` returns a tuple
# containing the features and optionally attentions/hidden_states
# After v5 is settled, we can enable qwen3-vl with several outputs
# from `self.get_image_features`
if isinstance(vision_embeddings, tuple):
vision_embeddings = vision_embeddings[0]
elif isinstance(vision_embeddings, dict):
vision_embeddings = vision_embeddings.pooler_output
if isinstance(vision_embeddings, torch.Tensor):
split_sizes = num_image_patches.flatten().tolist()
return self._split_embeddings(vision_embeddings, split_sizes)
return vision_embeddings
def embed_multimodal(self, **kwargs):
embeddings: tuple[torch.Tensor, ...] = ()
if "input_features" in kwargs or "input_values" in kwargs:
audio_embeddings = self._embed_audio(**kwargs)
if audio_embeddings is not None:
embeddings += tuple(audio_embeddings)
if (
"pixel_values" in kwargs
or "image_embeds" in kwargs
or "image_patches" in kwargs
):
vision_embeddings = self._embed_vision(**kwargs)
if vision_embeddings is not None:
if isinstance(vision_embeddings, torch.Tensor):
embeddings += (vision_embeddings,)
else:
embeddings += tuple(vision_embeddings)
return embeddings
def get_mrope_input_positions(
self,