feat[vLLM × v5]: Add audio support for the Transformers backend (#39330)
Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ca0defa343
commit
b9b6306ebe
@@ -19,7 +19,7 @@ vLLM also supports model implementations that are available in Transformers. We
|
||||
|
||||
Currently, the Transformers modeling backend works for the following:
|
||||
|
||||
- Modalities: embedding models, language models and vision-language models*
|
||||
- Modalities: embedding models, language models, vision-language models* and audio-language models
|
||||
- Architectures: encoder-only, decoder-only, mixture-of-experts
|
||||
- Attention types: full attention and/or sliding attention
|
||||
|
||||
@@ -606,7 +606,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor
|
||||
|
||||
| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
|
||||
| ------------ | ------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
|
||||
| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
|
||||
| `Emu3ForConditionalGeneration` | Emu3 | T + I<sup>+</sup> | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
|
||||
| `VibeVoiceAsrForConditionalGeneration` | VibeVoice-ASR | T + A<sup>+</sup> | `microsoft/VibeVoice-ASR-HF` | ✅︎ | ✅︎ |
|
||||
|
||||
<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.</br>
|
||||
<sup>E</sup> Pre-computed embeddings can be inputted for this modality.</br>
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from transformers import AutoModelForSeq2SeqLM
|
||||
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.envs import disable_envs_cache
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
|
||||
from ....conftest import HfRunner, VllmRunner
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
AUDIO_ASSET = AudioAsset("mary_had_lamb")
|
||||
|
||||
AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
"<|start_of_role|>system<|end_of_role|>"
|
||||
"You are a helpful AI assistant<|end_of_text|>\n"
|
||||
"<|start_of_role|>user<|end_of_role|>"
|
||||
"<|audio|>can you transcribe the speech into a written format?"
|
||||
"<|end_of_text|>\n"
|
||||
"<|start_of_role|>assistant<|end_of_role|>"
|
||||
),
|
||||
"audio_lora_path": "ibm-granite/granite-speech-3.3-2b",
|
||||
},
|
||||
"nvidia/audio-flamingo-3-hf": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\n"
|
||||
"You are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<sound>Transcribe the input speech.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
"vllm_runner_kwargs": {
|
||||
"gpu_memory_utilization": 0.85,
|
||||
},
|
||||
},
|
||||
"microsoft/VibeVoice-ASR-HF": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\n"
|
||||
"You are a helpful assistant that transcribes audio input "
|
||||
"into text output in JSON format.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|object_ref_start|><|box_start|><|object_ref_end|>\n"
|
||||
"This is a 1.0 seconds audio, please transcribe it with "
|
||||
"these keys: Start time, End time, Speaker ID, Content"
|
||||
"<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
"sampling_rate": 24000,
|
||||
"vllm_runner_kwargs": {
|
||||
"max_num_batched_tokens": 2048,
|
||||
"gpu_memory_utilization": 0.85,
|
||||
},
|
||||
},
|
||||
"zai-org/GLM-ASR-Nano-2512": {
|
||||
"prompt": (
|
||||
"<|user|>\n"
|
||||
"<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
|
||||
"Please transcribe this audio into text"
|
||||
"<|assistant|>\n"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
|
||||
def test_transformers_audio_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
monkeypatch,
|
||||
model_id: str,
|
||||
):
|
||||
"""Single-process workaround for V1 fork safety deadlock issue
|
||||
(vllm-project/vllm/issues/17676). Running multiple audio models together
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
|
||||
deadlock and allows worker processes to terminate cleanly, and release
|
||||
GPU memory between test runs until the issue is fixed."""
|
||||
# TODO: Remove monkeypatch once
|
||||
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
|
||||
disable_envs_cache()
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
audio_lora_path = settings.get("audio_lora_path")
|
||||
|
||||
audio, orig_sr = AUDIO_ASSET.audio_and_sample_rate
|
||||
target_sr = settings.get("sampling_rate", orig_sr)
|
||||
if orig_sr != target_sr:
|
||||
audio = AudioResampler(target_sr=target_sr).resample(audio, orig_sr=orig_sr)
|
||||
audio = (audio, target_sr)
|
||||
|
||||
with vllm_runner(
|
||||
model_id,
|
||||
model_impl="transformers",
|
||||
dtype="bfloat16",
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
enable_lora=audio_lora_path is not None,
|
||||
max_lora_rank=64,
|
||||
**settings.get("vllm_runner_kwargs", {}),
|
||||
) as vllm_model:
|
||||
lora_request = (
|
||||
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
|
||||
)
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[settings["prompt"]],
|
||||
128,
|
||||
num_logprobs=10,
|
||||
audios=[audio],
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model_id, dtype="bfloat16", auto_cls=AutoModelForSeq2SeqLM
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
[settings["prompt"]], 128, num_logprobs=10, audios=[audio]
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
@@ -452,6 +452,14 @@ def test_processing_correctness(
|
||||
"audio placeholders from processed audio lengths. Its vLLM "
|
||||
"processor paths are covered by test_moss_audio.py."
|
||||
)
|
||||
# TODO: Remove when transformers 5.15.0 is released, which contains
|
||||
# https://github.com/huggingface/transformers/pull/47483.
|
||||
if model_id == "microsoft/VibeVoice-ASR-HF":
|
||||
pytest.skip(
|
||||
"VibeVoice ASR requires audio as a positional argument and hence "
|
||||
"cannot pass the processing correctness test as is. Its generation "
|
||||
"is covered by test_transformers_audio.py."
|
||||
)
|
||||
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
|
||||
pytest.skip(
|
||||
"LLaVA-OneVision-2 video processing routes frames through custom "
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
AUDIO_MODEL_SETTINGS = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
"<|start_of_role|>system<|end_of_role|>"
|
||||
"You are a helpful AI assistant<|end_of_text|>\n"
|
||||
"<|start_of_role|>user<|end_of_role|>"
|
||||
"<|audio|>can you transcribe the speech into a written format?"
|
||||
"<|end_of_text|>\n"
|
||||
"<|start_of_role|>assistant<|end_of_role|>"
|
||||
),
|
||||
},
|
||||
"nvidia/audio-flamingo-3-hf": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\n"
|
||||
"You are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<sound>Transcribe the input speech.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"mistralai/Voxtral-Mini-3B-2507": {
|
||||
"prompt": ("[INST][AUDIO]What can you tell me about this audio?[/INST]"),
|
||||
},
|
||||
"microsoft/VibeVoice-ASR-HF": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\n"
|
||||
"You are a helpful assistant that transcribes audio input "
|
||||
"into text output in JSON format.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|object_ref_start|><|box_start|><|object_ref_end|>\n"
|
||||
"This is a 1.0 seconds audio, please transcribe it with "
|
||||
"these keys: Start time, End time, Speaker ID, Content"
|
||||
"<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"zai-org/GLM-ASR-Nano-2512": {
|
||||
"prompt": (
|
||||
"<|user|>\n"
|
||||
"<|begin_of_audio|><|pad|><|end_of_audio|><|user|>\n"
|
||||
"Please transcribe this audio into text"
|
||||
"<|assistant|>\n"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
"ibm-granite/granite-speech-3.3-2b",
|
||||
"nvidia/audio-flamingo-3-hf",
|
||||
pytest.param(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="MistralCommonBackend.encode does not produce the audio "
|
||||
"placeholder token (ID 24) from raw text. apply_chat_template "
|
||||
"yields token IDs with placeholders, but MultiModalProcessor."
|
||||
"apply() decodes the prompt back to text and re-tokenizes, at "
|
||||
"which point the placeholders are lost. Fix belongs in "
|
||||
"mistral_common or in the Voxtral-specific path.",
|
||||
strict=False,
|
||||
),
|
||||
),
|
||||
"microsoft/VibeVoice-ASR-HF",
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
],
|
||||
)
|
||||
def test_audio_multimodal_processor(model_id):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_id,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
mm_data = {"audio": (audio, 16000)}
|
||||
|
||||
result = mm_processor(
|
||||
prompt=settings["prompt"],
|
||||
mm_items=mm_processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert "prompt_token_ids" in result
|
||||
assert len(result["prompt_token_ids"]) > 0
|
||||
|
||||
mm_placeholders = result.get("mm_placeholders", {})
|
||||
assert "audio" in mm_placeholders, f"No audio placeholders found for {model_id}"
|
||||
assert len(mm_placeholders["audio"]) == 1
|
||||
|
||||
placeholder = mm_placeholders["audio"][0]
|
||||
assert placeholder.length > 0
|
||||
assert placeholder.offset >= 0
|
||||
|
||||
audio_items = result.get("mm_kwargs", {}).get("audio", [])
|
||||
assert len(audio_items) == 1, f"Expected 1 audio item, got {len(audio_items)}"
|
||||
item_keys = list(audio_items[0].keys())
|
||||
has_features = "input_features" in item_keys or "input_values" in item_keys
|
||||
assert has_features, (
|
||||
f"No audio features (input_features/input_values) in {item_keys} for {model_id}"
|
||||
)
|
||||
|
||||
|
||||
def test_audio_multiple_inputs():
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
prompt = (
|
||||
"<|start_of_role|>user<|end_of_role|>"
|
||||
f"{audio_token} and {audio_token} transcribe<|end_of_text|>\n"
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
+23
@@ -54,3 +54,26 @@ def test_multimodal_processor(model_id):
|
||||
str_processed_inputs["prompt_token_ids"]
|
||||
== ids_processed_inputs["prompt_token_ids"]
|
||||
)
|
||||
|
||||
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers backend."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
"<|im_start|>user <image>\n and <image>\n"
|
||||
"What do these images show?<|im_end|><|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
@@ -1393,6 +1393,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
|
||||
trust_remote_code=True,
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"microsoft/VibeVoice-ASR-HF",
|
||||
min_transformers_version="5.13.0",
|
||||
),
|
||||
"VoxtralForConditionalGeneration": _HfExamplesInfo(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
tokenizer_mode="mistral",
|
||||
|
||||
@@ -664,6 +664,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
"""Transformers modeling backend mixin for multi-modal models."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
@@ -27,6 +28,7 @@ from vllm.config.utils import getattr_iter
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -36,6 +38,7 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.multimodal.parse import (
|
||||
ImageProcessorItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
@@ -59,11 +62,90 @@ _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
def _get_audio_processor(self) -> Any:
|
||||
# TODO: drop feature_extractor branch once huggingface/transformers#44394 lands.
|
||||
return getattr_iter(
|
||||
self.get_hf_processor(), ("audio_processor", "feature_extractor")
|
||||
)
|
||||
|
||||
def _is_audio_model(self) -> bool:
|
||||
return self._get_audio_processor() is not None
|
||||
|
||||
def _is_image_model(self) -> bool:
|
||||
return hasattr(self.get_hf_processor(), "image_processor")
|
||||
|
||||
def _is_video_model(self) -> bool:
|
||||
return hasattr(self.get_hf_processor(), "video_processor")
|
||||
|
||||
def _get_audio_token_id(self) -> int:
|
||||
processor = self.get_hf_processor()
|
||||
if hasattr(processor, "audio_token_id"):
|
||||
return processor.audio_token_id
|
||||
config = self.get_hf_config()
|
||||
val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
|
||||
if val is not None:
|
||||
return val
|
||||
if hasattr(processor, "audio_token"):
|
||||
tokenizer = self.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
if processor.audio_token in vocab:
|
||||
return vocab[processor.audio_token]
|
||||
raise ValueError("Cannot find audio_token_id on processor or model config")
|
||||
|
||||
def _get_audio_sampling_rate(self) -> float:
|
||||
sub = self._get_audio_processor()
|
||||
if sub is not None and hasattr(sub, "sampling_rate"):
|
||||
return sub.sampling_rate
|
||||
return 16000.0
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
if self._is_audio_model():
|
||||
return MultiModalDataParser(
|
||||
target_sr=self._get_audio_sampling_rate(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
return super().get_data_parser()
|
||||
|
||||
def get_supported_mm_limits(self):
|
||||
return {"image": None}
|
||||
limits = {}
|
||||
if self._is_audio_model():
|
||||
limits["audio"] = None
|
||||
if self._is_image_model():
|
||||
limits["image"] = None
|
||||
if not limits:
|
||||
raise ValueError(
|
||||
f"Unable to detect a supported modality on "
|
||||
f"{type(self.get_hf_processor()).__name__}. "
|
||||
"Checked `_is_audio_model` and `_is_image_model`."
|
||||
)
|
||||
return limits
|
||||
|
||||
def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
|
||||
return {"image": self.get_max_image_tokens()}
|
||||
result = {}
|
||||
if self._is_audio_model():
|
||||
result["audio"] = self.get_max_audio_tokens()
|
||||
if self._is_image_model():
|
||||
result["image"] = self.get_max_image_tokens()
|
||||
if not result:
|
||||
raise ValueError(
|
||||
f"Unable to detect a supported modality on "
|
||||
f"{type(self.get_hf_processor()).__name__}. "
|
||||
"Checked `_is_audio_model` and `_is_image_model`."
|
||||
)
|
||||
return result
|
||||
|
||||
def get_max_audio_tokens(self) -> int:
|
||||
config = self.get_hf_config()
|
||||
audio_config_names = ("audio_config", "encoder_config")
|
||||
names = ("max_source_positions", "max_position_embeddings", "max_pos_emb")
|
||||
audio_config = getattr_iter(config, audio_config_names, default=config)
|
||||
val = getattr_iter(audio_config, names)
|
||||
if val is not None:
|
||||
return int(val)
|
||||
raise ValueError(
|
||||
f"Unable to get max input length from {type(audio_config).__name__}. "
|
||||
f"The following attribute names were checked: {names}."
|
||||
)
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
width, height = self.get_max_image_size()
|
||||
@@ -82,14 +164,19 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
|
||||
processor = self.info.get_hf_processor()
|
||||
if "gemma3" in processor.__class__.__name__.lower():
|
||||
image_token = processor.boi_token
|
||||
else:
|
||||
image_token = getattr(processor, "image_token", "")
|
||||
return image_token * num_images
|
||||
text = ""
|
||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||
processor = self.info.get_hf_processor()
|
||||
audio_token = getattr(processor, "audio_token", "")
|
||||
text += audio_token * num_audios
|
||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||
processor = self.info.get_hf_processor()
|
||||
if "gemma3" in processor.__class__.__name__.lower():
|
||||
image_token = processor.boi_token
|
||||
else:
|
||||
image_token = getattr(processor, "image_token", "")
|
||||
text += image_token * num_images
|
||||
return text
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
@@ -97,20 +184,28 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, "BaseDummyOptions"],
|
||||
) -> MultiModalDataDict:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
|
||||
target_width, target_height = self.info.get_max_image_size()
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
data: MultiModalDataDict = {}
|
||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||
sampling_rate = self.info._get_audio_sampling_rate()
|
||||
sub = self.info._get_audio_processor()
|
||||
chunk_length = getattr(sub, "chunk_length", None) if sub else None
|
||||
if chunk_length is None:
|
||||
chunk_length = 30
|
||||
audio_len = int(chunk_length * sampling_rate)
|
||||
data["audio"] = self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=mm_options.get("audio"),
|
||||
)
|
||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||
target_width, target_height = self.info.get_max_image_size()
|
||||
data["image"] = self._get_dummy_images(
|
||||
width=target_width,
|
||||
height=target_height,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides,
|
||||
),
|
||||
}
|
||||
overrides=mm_options.get("image"),
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
@@ -142,21 +237,39 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
# HF Processors always return a mask but vLLM doesn't need it
|
||||
hf_inputs.pop("attention_mask", None)
|
||||
num_image_patches = hf_inputs.get("num_image_patches")
|
||||
mm_fields = {
|
||||
key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
|
||||
for key in hf_inputs
|
||||
}
|
||||
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_image_patches
|
||||
)
|
||||
|
||||
# Keep these as batched, as they always have batch size as first dim
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
mm_fields: dict[str, MultiModalFieldConfig] = {}
|
||||
if self.info._is_audio_model():
|
||||
num_audio_tokens = hf_inputs.get("num_audio_tokens")
|
||||
mm_fields.update(
|
||||
{
|
||||
key: MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", num_audio_tokens
|
||||
)
|
||||
for key in hf_inputs
|
||||
}
|
||||
)
|
||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
|
||||
if self.info._is_image_model():
|
||||
num_image_patches = hf_inputs.get("num_image_patches")
|
||||
mm_fields.update(
|
||||
{
|
||||
key: MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_image_patches
|
||||
)
|
||||
for key in hf_inputs
|
||||
}
|
||||
)
|
||||
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_image_patches
|
||||
)
|
||||
|
||||
# Keep these as batched, as they always have batch size as first dim
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
return mm_fields
|
||||
|
||||
def _get_hf_mm_data(
|
||||
@@ -164,13 +277,93 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""
|
||||
In contrast to the base class, this method always adds
|
||||
`return_mm_token_type_ids` to the processor data
|
||||
In contrast to the base class, this method requests
|
||||
`return_mm_token_type_ids` and remaps the `audios` key to `audio` for
|
||||
audio models.
|
||||
"""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
processor_data["return_mm_token_type_ids"] = True
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _apply_audio(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
audio_token_id = self.info._get_audio_token_id()
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = prompt_tensor == audio_token_id
|
||||
|
||||
if not is_audio.any():
|
||||
return {}
|
||||
|
||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||
transitions = padded.int().diff()
|
||||
starts = torch.where(transitions == 1)[0]
|
||||
ends = torch.where(transitions == -1)[0]
|
||||
lengths = ends - starts
|
||||
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=s.item(),
|
||||
length=ln.item(),
|
||||
is_embed=torch.ones(ln.item(), dtype=torch.bool),
|
||||
)
|
||||
for s, ln in zip(starts, lengths)
|
||||
]
|
||||
processed_data["num_audio_tokens"] = lengths
|
||||
return {"audio": ranges}
|
||||
|
||||
def _apply_vision(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
if mm_token_type_ids is None:
|
||||
return {}
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
# We can infer vLLM style placeholder from token type ids, if we split
|
||||
# it for each input `mm_data`.
|
||||
mm_positions = torch.where(mm_token_type_ids == 1)[1]
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = []
|
||||
for item_idx in range(len(images)):
|
||||
image_size = images.get_image_size(item_idx)
|
||||
image_sizes.append((image_size.height, image_size.width))
|
||||
|
||||
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if split_sizes:
|
||||
chunked_mm_positions = torch.split(mm_positions, split_sizes)
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
|
||||
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
|
||||
)
|
||||
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
|
||||
processed_data["num_image_patches"] = torch.tensor(
|
||||
mm_tokens_per_modality["num_image_patches"]
|
||||
)
|
||||
return mm_placeholders
|
||||
|
||||
def apply(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
@@ -198,8 +391,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. We have a lot of logic tied to
|
||||
# `mm_tokens_per_modality` below
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
@@ -207,52 +400,33 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
|
||||
# For gemma3 we check `token_type_ids` as the key
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||
|
||||
# We can infer vLLM style placeholder from token type ids, if we split
|
||||
# it for each input `mm_data`.
|
||||
mm_positions = torch.where(mm_token_type_ids == 1)[1]
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = []
|
||||
for item_idx in range(len(images)):
|
||||
image_size = images.get_image_size(item_idx)
|
||||
image_sizes.append((image_size.height, image_size.width))
|
||||
|
||||
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
mm_placeholders = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if split_sizes:
|
||||
chunked_mm_positions = torch.split(mm_positions, split_sizes)
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
|
||||
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
if self.info._is_audio_model():
|
||||
mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
|
||||
if self.info._is_image_model():
|
||||
mm_placeholders.update(
|
||||
self._apply_vision(
|
||||
prompt_ids,
|
||||
processed_data,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_token_type_ids,
|
||||
)
|
||||
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
)
|
||||
|
||||
processed_data["num_image_patches"] = torch.tensor(
|
||||
mm_tokens_per_modality["num_image_patches"]
|
||||
)
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
processed_data,
|
||||
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
@@ -364,7 +538,79 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
return LanguageModel(self)
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
for name in ("language_model", "text_model"):
|
||||
if getattr(self.model, name, None) is not None:
|
||||
return MultiModelKeys.from_string_field(language_model=f"model.{name}")
|
||||
raise ValueError(
|
||||
"Could not locate the language model submodule for LoRA support"
|
||||
)
|
||||
|
||||
def _split_embeddings(
|
||||
self, embeddings: torch.Tensor, split_sizes: list[int]
|
||||
) -> list[torch.Tensor]:
|
||||
total_expected = sum(split_sizes)
|
||||
|
||||
# Flatten to 2D: [total_tokens, hidden_dim]
|
||||
if embeddings.ndim == 3:
|
||||
embeddings = embeddings.view(-1, embeddings.shape[-1])
|
||||
|
||||
total_tokens = embeddings.shape[0]
|
||||
if total_tokens == total_expected:
|
||||
# Direct match: split_sizes are actual token counts
|
||||
token_split_sizes = split_sizes
|
||||
elif total_expected > 0 and total_tokens % total_expected == 0:
|
||||
# Uniform expansion: each item expands to N tokens
|
||||
tokens_per_item = total_tokens // total_expected
|
||||
token_split_sizes = [s * tokens_per_item for s in split_sizes]
|
||||
elif total_expected > 0:
|
||||
# Mismatch (profiling with dummy data) - pad/truncate
|
||||
if total_tokens == 0:
|
||||
raise ValueError(
|
||||
"Encoder returned empty embeddings. "
|
||||
f"Expected {total_expected} tokens from "
|
||||
f"split_sizes={split_sizes}"
|
||||
)
|
||||
if total_tokens < total_expected:
|
||||
repeat_factor = (total_expected + total_tokens - 1) // total_tokens
|
||||
embeddings = embeddings.repeat(repeat_factor, 1)
|
||||
embeddings = embeddings[:total_expected]
|
||||
token_split_sizes = split_sizes
|
||||
else:
|
||||
return []
|
||||
|
||||
return list(torch.split(embeddings, token_split_sizes, dim=0))
|
||||
|
||||
def _embed_audio(self, **kwargs) -> list[torch.Tensor] | None:
|
||||
self.check_version("5.13.0", "audio models support")
|
||||
input_features: torch.Tensor | None = kwargs.pop("input_features", None)
|
||||
if input_features is None:
|
||||
input_features = kwargs.pop("input_values", None)
|
||||
if input_features is None:
|
||||
return None
|
||||
|
||||
num_audio_tokens = kwargs.pop("num_audio_tokens")
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
context = nullcontext()
|
||||
if current_platform.is_rocm():
|
||||
context = torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
)
|
||||
with context:
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
audio_embeddings = audio_output.pooler_output
|
||||
|
||||
split_sizes = num_audio_tokens.flatten().tolist()
|
||||
return self._split_embeddings(audio_embeddings, split_sizes)
|
||||
|
||||
def _embed_vision(self, **kwargs) -> list[torch.Tensor] | torch.Tensor | None:
|
||||
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
|
||||
image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
|
||||
# Model might use `image_patches` instead of `pixel_values`
|
||||
@@ -379,84 +625,57 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
num_image_patches = kwargs.pop("num_image_patches")
|
||||
|
||||
if pixel_values is not None:
|
||||
context = nullcontext()
|
||||
if current_platform.is_rocm():
|
||||
# ROCm: Force math SDP backend for vision encoder to avoid accuracy issues
|
||||
# with flash_sdp and mem_efficient_sdp
|
||||
if current_platform.is_rocm():
|
||||
# TODO: [ROCm] Fix accuracy issues with flash backend
|
||||
logger.debug(
|
||||
"ROCm platform detected. Forcing math SDP backend "
|
||||
"for vision encoder. Currently ROCm platform has "
|
||||
"accuracy issues with `flash_sdp` and"
|
||||
"`mem_efficient_sdp` backends. See issue: "
|
||||
"https://github.com/vllm-project/vllm/issues/30167"
|
||||
)
|
||||
with torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
):
|
||||
vision_embeddings = self.model.get_image_features(
|
||||
pixel_values, **kwargs
|
||||
)
|
||||
else:
|
||||
vision_embeddings = self.model.get_image_features(
|
||||
pixel_values, **kwargs
|
||||
)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
# After v5 is settled, we can enable qwen3-vl with several outputs
|
||||
# from `self.get_image_features`
|
||||
if isinstance(vision_embeddings, tuple):
|
||||
vision_embeddings = vision_embeddings[0]
|
||||
elif isinstance(vision_embeddings, dict):
|
||||
vision_embeddings = vision_embeddings.pooler_output
|
||||
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
split_sizes = num_image_patches.flatten().tolist()
|
||||
total_patches = sum(split_sizes)
|
||||
|
||||
# Flatten to 2D: [total_tokens, hidden_dim]
|
||||
if vision_embeddings.ndim == 3:
|
||||
vision_embeddings = vision_embeddings.view(
|
||||
-1, vision_embeddings.shape[-1]
|
||||
)
|
||||
|
||||
total_tokens = vision_embeddings.shape[0]
|
||||
if total_tokens == total_patches:
|
||||
# Direct match: num_image_patches are actual token counts
|
||||
# (e.g., Qwen2.5-VL style)
|
||||
token_split_sizes = split_sizes
|
||||
elif total_patches > 0 and total_tokens % total_patches == 0:
|
||||
# Uniform expansion: each patch expands to N tokens
|
||||
# (e.g., Idefics3 style)
|
||||
tokens_per_patch = total_tokens // total_patches
|
||||
token_split_sizes = [s * tokens_per_patch for s in split_sizes]
|
||||
elif total_patches > 0:
|
||||
# Mismatch (profiling with dummy data) - pad/truncate
|
||||
if total_tokens == 0:
|
||||
raise ValueError(
|
||||
"Vision encoder returned empty embeddings. "
|
||||
f"Expected {total_patches} patches from "
|
||||
f"num_image_patches={split_sizes}"
|
||||
)
|
||||
if total_tokens < total_patches:
|
||||
repeat_factor = (
|
||||
total_patches + total_tokens - 1
|
||||
) // total_tokens
|
||||
vision_embeddings = vision_embeddings.repeat(repeat_factor, 1)
|
||||
vision_embeddings = vision_embeddings[:total_patches]
|
||||
token_split_sizes = split_sizes
|
||||
else:
|
||||
return []
|
||||
|
||||
return list(torch.split(vision_embeddings, token_split_sizes, dim=0))
|
||||
|
||||
return vision_embeddings
|
||||
else:
|
||||
# TODO: [ROCm] Fix accuracy issues with flash backend
|
||||
logger.debug(
|
||||
"No pixel values or image embeddings provided for multimodal embedding."
|
||||
"ROCm platform detected. Forcing math SDP backend "
|
||||
"for vision encoder. Currently ROCm platform has "
|
||||
"accuracy issues with `flash_sdp` and"
|
||||
"`mem_efficient_sdp` backends. See issue: "
|
||||
"https://github.com/vllm-project/vllm/issues/30167"
|
||||
)
|
||||
return None
|
||||
context = torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
)
|
||||
with context:
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
# After v5 is settled, we can enable qwen3-vl with several outputs
|
||||
# from `self.get_image_features`
|
||||
if isinstance(vision_embeddings, tuple):
|
||||
vision_embeddings = vision_embeddings[0]
|
||||
elif isinstance(vision_embeddings, dict):
|
||||
vision_embeddings = vision_embeddings.pooler_output
|
||||
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
split_sizes = num_image_patches.flatten().tolist()
|
||||
return self._split_embeddings(vision_embeddings, split_sizes)
|
||||
|
||||
return vision_embeddings
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
embeddings: tuple[torch.Tensor, ...] = ()
|
||||
if "input_features" in kwargs or "input_values" in kwargs:
|
||||
audio_embeddings = self._embed_audio(**kwargs)
|
||||
if audio_embeddings is not None:
|
||||
embeddings += tuple(audio_embeddings)
|
||||
if (
|
||||
"pixel_values" in kwargs
|
||||
or "image_embeds" in kwargs
|
||||
or "image_patches" in kwargs
|
||||
):
|
||||
vision_embeddings = self._embed_vision(**kwargs)
|
||||
if vision_embeddings is not None:
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
embeddings += (vision_embeddings,)
|
||||
else:
|
||||
embeddings += tuple(vision_embeddings)
|
||||
return embeddings
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user