feat[vLLM × v5]: Add audio support for the Transformers backend (#39330)
Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
ca0defa343
commit
b9b6306ebe
@@ -664,6 +664,10 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
"""Transformers modeling backend mixin for multi-modal models."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
@@ -27,6 +28,7 @@ from vllm.config.utils import getattr_iter
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import SupportsMRoPE, SupportsMultiModal
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
@@ -36,6 +38,7 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.multimodal.parse import (
|
||||
ImageProcessorItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
@@ -59,11 +62,90 @@ _MODALITY_TO_TOKEN_TYPE_ID = {"image": 1, "video": 2, "audio": 3}
|
||||
|
||||
|
||||
class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
def _get_audio_processor(self) -> Any:
|
||||
# TODO: drop feature_extractor branch once huggingface/transformers#44394 lands.
|
||||
return getattr_iter(
|
||||
self.get_hf_processor(), ("audio_processor", "feature_extractor")
|
||||
)
|
||||
|
||||
def _is_audio_model(self) -> bool:
|
||||
return self._get_audio_processor() is not None
|
||||
|
||||
def _is_image_model(self) -> bool:
|
||||
return hasattr(self.get_hf_processor(), "image_processor")
|
||||
|
||||
def _is_video_model(self) -> bool:
|
||||
return hasattr(self.get_hf_processor(), "video_processor")
|
||||
|
||||
def _get_audio_token_id(self) -> int:
|
||||
processor = self.get_hf_processor()
|
||||
if hasattr(processor, "audio_token_id"):
|
||||
return processor.audio_token_id
|
||||
config = self.get_hf_config()
|
||||
val = getattr_iter(config, ("audio_token_id", "audio_token_index"))
|
||||
if val is not None:
|
||||
return val
|
||||
if hasattr(processor, "audio_token"):
|
||||
tokenizer = self.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
if processor.audio_token in vocab:
|
||||
return vocab[processor.audio_token]
|
||||
raise ValueError("Cannot find audio_token_id on processor or model config")
|
||||
|
||||
def _get_audio_sampling_rate(self) -> float:
|
||||
sub = self._get_audio_processor()
|
||||
if sub is not None and hasattr(sub, "sampling_rate"):
|
||||
return sub.sampling_rate
|
||||
return 16000.0
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
if self._is_audio_model():
|
||||
return MultiModalDataParser(
|
||||
target_sr=self._get_audio_sampling_rate(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
return super().get_data_parser()
|
||||
|
||||
def get_supported_mm_limits(self):
|
||||
return {"image": None}
|
||||
limits = {}
|
||||
if self._is_audio_model():
|
||||
limits["audio"] = None
|
||||
if self._is_image_model():
|
||||
limits["image"] = None
|
||||
if not limits:
|
||||
raise ValueError(
|
||||
f"Unable to detect a supported modality on "
|
||||
f"{type(self.get_hf_processor()).__name__}. "
|
||||
"Checked `_is_audio_model` and `_is_image_model`."
|
||||
)
|
||||
return limits
|
||||
|
||||
def get_mm_max_tokens_per_item(self, seq_len, mm_counts):
|
||||
return {"image": self.get_max_image_tokens()}
|
||||
result = {}
|
||||
if self._is_audio_model():
|
||||
result["audio"] = self.get_max_audio_tokens()
|
||||
if self._is_image_model():
|
||||
result["image"] = self.get_max_image_tokens()
|
||||
if not result:
|
||||
raise ValueError(
|
||||
f"Unable to detect a supported modality on "
|
||||
f"{type(self.get_hf_processor()).__name__}. "
|
||||
"Checked `_is_audio_model` and `_is_image_model`."
|
||||
)
|
||||
return result
|
||||
|
||||
def get_max_audio_tokens(self) -> int:
|
||||
config = self.get_hf_config()
|
||||
audio_config_names = ("audio_config", "encoder_config")
|
||||
names = ("max_source_positions", "max_position_embeddings", "max_pos_emb")
|
||||
audio_config = getattr_iter(config, audio_config_names, default=config)
|
||||
val = getattr_iter(audio_config, names)
|
||||
if val is not None:
|
||||
return int(val)
|
||||
raise ValueError(
|
||||
f"Unable to get max input length from {type(audio_config).__name__}. "
|
||||
f"The following attribute names were checked: {names}."
|
||||
)
|
||||
|
||||
def get_max_image_tokens(self) -> int:
|
||||
width, height = self.get_max_image_size()
|
||||
@@ -82,14 +164,19 @@ class MultiModalProcessingInfo(BaseProcessingInfo):
|
||||
|
||||
class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
|
||||
processor = self.info.get_hf_processor()
|
||||
if "gemma3" in processor.__class__.__name__.lower():
|
||||
image_token = processor.boi_token
|
||||
else:
|
||||
image_token = getattr(processor, "image_token", "")
|
||||
return image_token * num_images
|
||||
text = ""
|
||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||
processor = self.info.get_hf_processor()
|
||||
audio_token = getattr(processor, "audio_token", "")
|
||||
text += audio_token * num_audios
|
||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||
processor = self.info.get_hf_processor()
|
||||
if "gemma3" in processor.__class__.__name__.lower():
|
||||
image_token = processor.boi_token
|
||||
else:
|
||||
image_token = getattr(processor, "image_token", "")
|
||||
text += image_token * num_images
|
||||
return text
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
@@ -97,20 +184,28 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, "BaseDummyOptions"],
|
||||
) -> MultiModalDataDict:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
|
||||
target_width, target_height = self.info.get_max_image_size()
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
data: MultiModalDataDict = {}
|
||||
if self.info._is_audio_model() and (num_audios := mm_counts.get("audio", 0)):
|
||||
sampling_rate = self.info._get_audio_sampling_rate()
|
||||
sub = self.info._get_audio_processor()
|
||||
chunk_length = getattr(sub, "chunk_length", None) if sub else None
|
||||
if chunk_length is None:
|
||||
chunk_length = 30
|
||||
audio_len = int(chunk_length * sampling_rate)
|
||||
data["audio"] = self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=mm_options.get("audio"),
|
||||
)
|
||||
if self.info._is_image_model() and (num_images := mm_counts.get("image", 0)):
|
||||
target_width, target_height = self.info.get_max_image_size()
|
||||
data["image"] = self._get_dummy_images(
|
||||
width=target_width,
|
||||
height=target_height,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides,
|
||||
),
|
||||
}
|
||||
overrides=mm_options.get("image"),
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
@@ -142,21 +237,39 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
# HF Processors always return a mask but vLLM doesn't need it
|
||||
hf_inputs.pop("attention_mask", None)
|
||||
num_image_patches = hf_inputs.get("num_image_patches")
|
||||
mm_fields = {
|
||||
key: MultiModalFieldConfig.flat_from_sizes("image", num_image_patches)
|
||||
for key in hf_inputs
|
||||
}
|
||||
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_image_patches
|
||||
)
|
||||
|
||||
# Keep these as batched, as they always have batch size as first dim
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
mm_fields: dict[str, MultiModalFieldConfig] = {}
|
||||
if self.info._is_audio_model():
|
||||
num_audio_tokens = hf_inputs.get("num_audio_tokens")
|
||||
mm_fields.update(
|
||||
{
|
||||
key: MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", num_audio_tokens
|
||||
)
|
||||
for key in hf_inputs
|
||||
}
|
||||
)
|
||||
mm_fields["num_audio_tokens"] = MultiModalFieldConfig.batched("audio")
|
||||
if self.info._is_image_model():
|
||||
num_image_patches = hf_inputs.get("num_image_patches")
|
||||
mm_fields.update(
|
||||
{
|
||||
key: MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_image_patches
|
||||
)
|
||||
for key in hf_inputs
|
||||
}
|
||||
)
|
||||
mm_fields["image_embeds"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"image", num_image_patches
|
||||
)
|
||||
|
||||
# Keep these as batched, as they always have batch size as first dim
|
||||
mm_fields["image_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["video_grid_thw"] = MultiModalFieldConfig.batched("image")
|
||||
mm_fields["num_image_patches"] = MultiModalFieldConfig.batched(
|
||||
"image", keep_on_cpu=True
|
||||
)
|
||||
return mm_fields
|
||||
|
||||
def _get_hf_mm_data(
|
||||
@@ -164,13 +277,93 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
mm_items: MultiModalDataItems,
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
"""
|
||||
In contrast to the base class, this method always adds
|
||||
`return_mm_token_type_ids` to the processor data
|
||||
In contrast to the base class, this method requests
|
||||
`return_mm_token_type_ids` and remaps the `audios` key to `audio` for
|
||||
audio models.
|
||||
"""
|
||||
processor_data, passthrough_data = super()._get_hf_mm_data(mm_items)
|
||||
if self.info._is_audio_model() and "audios" in processor_data:
|
||||
processor_data["audio"] = processor_data.pop("audios")
|
||||
processor_data["return_mm_token_type_ids"] = True
|
||||
return processor_data, passthrough_data
|
||||
|
||||
def _apply_audio(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
audio_token_id = self.info._get_audio_token_id()
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = prompt_tensor == audio_token_id
|
||||
|
||||
if not is_audio.any():
|
||||
return {}
|
||||
|
||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||
transitions = padded.int().diff()
|
||||
starts = torch.where(transitions == 1)[0]
|
||||
ends = torch.where(transitions == -1)[0]
|
||||
lengths = ends - starts
|
||||
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=s.item(),
|
||||
length=ln.item(),
|
||||
is_embed=torch.ones(ln.item(), dtype=torch.bool),
|
||||
)
|
||||
for s, ln in zip(starts, lengths)
|
||||
]
|
||||
processed_data["num_audio_tokens"] = lengths
|
||||
return {"audio": ranges}
|
||||
|
||||
def _apply_vision(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
if mm_token_type_ids is None:
|
||||
return {}
|
||||
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
# We can infer vLLM style placeholder from token type ids, if we split
|
||||
# it for each input `mm_data`.
|
||||
mm_positions = torch.where(mm_token_type_ids == 1)[1]
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = []
|
||||
for item_idx in range(len(images)):
|
||||
image_size = images.get_image_size(item_idx)
|
||||
image_sizes.append((image_size.height, image_size.width))
|
||||
|
||||
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if split_sizes:
|
||||
chunked_mm_positions = torch.split(mm_positions, split_sizes)
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
|
||||
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
|
||||
)
|
||||
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
|
||||
processed_data["num_image_patches"] = torch.tensor(
|
||||
mm_tokens_per_modality["num_image_patches"]
|
||||
)
|
||||
return mm_placeholders
|
||||
|
||||
def apply(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
@@ -198,8 +391,8 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. We have a lot of logic tied to
|
||||
# `mm_tokens_per_modality` below
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
prompt_ids, processed_data, _ = self._apply_hf_processor_text_mm(
|
||||
prompt_text=prompt,
|
||||
mm_items=mm_items,
|
||||
@@ -207,52 +400,33 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
|
||||
# For gemma3 we check `token_type_ids` as the key
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||
|
||||
# We can infer vLLM style placeholder from token type ids, if we split
|
||||
# it for each input `mm_data`.
|
||||
mm_positions = torch.where(mm_token_type_ids == 1)[1]
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = []
|
||||
for item_idx in range(len(images)):
|
||||
image_size = images.get_image_size(item_idx)
|
||||
image_sizes.append((image_size.height, image_size.width))
|
||||
|
||||
mm_tokens_per_modality = hf_processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
mm_placeholders = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if split_sizes:
|
||||
chunked_mm_positions = torch.split(mm_positions, split_sizes)
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_token_type_ids[0].bool()]
|
||||
chunked_mm_tokens = torch.split(mm_tokens, split_sizes)
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
is_embed=(mm_tokens == hf_processor.image_token_id).bool(),
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
if self.info._is_audio_model():
|
||||
mm_placeholders.update(self._apply_audio(prompt_ids, processed_data))
|
||||
if self.info._is_image_model():
|
||||
mm_placeholders.update(
|
||||
self._apply_vision(
|
||||
prompt_ids,
|
||||
processed_data,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_token_type_ids,
|
||||
)
|
||||
for positions, mm_tokens in zip(chunked_mm_positions, chunked_mm_tokens)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
)
|
||||
|
||||
processed_data["num_image_patches"] = torch.tensor(
|
||||
mm_tokens_per_modality["num_image_patches"]
|
||||
)
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
processed_data,
|
||||
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
@@ -364,7 +538,79 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
return LanguageModel(self)
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
"""
|
||||
Get the module prefix in multimodal models
|
||||
"""
|
||||
for name in ("language_model", "text_model"):
|
||||
if getattr(self.model, name, None) is not None:
|
||||
return MultiModelKeys.from_string_field(language_model=f"model.{name}")
|
||||
raise ValueError(
|
||||
"Could not locate the language model submodule for LoRA support"
|
||||
)
|
||||
|
||||
def _split_embeddings(
|
||||
self, embeddings: torch.Tensor, split_sizes: list[int]
|
||||
) -> list[torch.Tensor]:
|
||||
total_expected = sum(split_sizes)
|
||||
|
||||
# Flatten to 2D: [total_tokens, hidden_dim]
|
||||
if embeddings.ndim == 3:
|
||||
embeddings = embeddings.view(-1, embeddings.shape[-1])
|
||||
|
||||
total_tokens = embeddings.shape[0]
|
||||
if total_tokens == total_expected:
|
||||
# Direct match: split_sizes are actual token counts
|
||||
token_split_sizes = split_sizes
|
||||
elif total_expected > 0 and total_tokens % total_expected == 0:
|
||||
# Uniform expansion: each item expands to N tokens
|
||||
tokens_per_item = total_tokens // total_expected
|
||||
token_split_sizes = [s * tokens_per_item for s in split_sizes]
|
||||
elif total_expected > 0:
|
||||
# Mismatch (profiling with dummy data) - pad/truncate
|
||||
if total_tokens == 0:
|
||||
raise ValueError(
|
||||
"Encoder returned empty embeddings. "
|
||||
f"Expected {total_expected} tokens from "
|
||||
f"split_sizes={split_sizes}"
|
||||
)
|
||||
if total_tokens < total_expected:
|
||||
repeat_factor = (total_expected + total_tokens - 1) // total_tokens
|
||||
embeddings = embeddings.repeat(repeat_factor, 1)
|
||||
embeddings = embeddings[:total_expected]
|
||||
token_split_sizes = split_sizes
|
||||
else:
|
||||
return []
|
||||
|
||||
return list(torch.split(embeddings, token_split_sizes, dim=0))
|
||||
|
||||
def _embed_audio(self, **kwargs) -> list[torch.Tensor] | None:
|
||||
self.check_version("5.13.0", "audio models support")
|
||||
input_features: torch.Tensor | None = kwargs.pop("input_features", None)
|
||||
if input_features is None:
|
||||
input_features = kwargs.pop("input_values", None)
|
||||
if input_features is None:
|
||||
return None
|
||||
|
||||
num_audio_tokens = kwargs.pop("num_audio_tokens")
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
context = nullcontext()
|
||||
if current_platform.is_rocm():
|
||||
context = torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
)
|
||||
with context:
|
||||
audio_output = self.model.get_audio_features(
|
||||
input_features, return_dict=True, **kwargs
|
||||
)
|
||||
audio_embeddings = audio_output.pooler_output
|
||||
|
||||
split_sizes = num_audio_tokens.flatten().tolist()
|
||||
return self._split_embeddings(audio_embeddings, split_sizes)
|
||||
|
||||
def _embed_vision(self, **kwargs) -> list[torch.Tensor] | torch.Tensor | None:
|
||||
pixel_values: torch.Tensor | None = kwargs.pop("pixel_values", None)
|
||||
image_embeds: torch.Tensor | None = kwargs.pop("image_embeds", None)
|
||||
# Model might use `image_patches` instead of `pixel_values`
|
||||
@@ -379,84 +625,57 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE):
|
||||
|
||||
num_image_patches = kwargs.pop("num_image_patches")
|
||||
|
||||
if pixel_values is not None:
|
||||
context = nullcontext()
|
||||
if current_platform.is_rocm():
|
||||
# ROCm: Force math SDP backend for vision encoder to avoid accuracy issues
|
||||
# with flash_sdp and mem_efficient_sdp
|
||||
if current_platform.is_rocm():
|
||||
# TODO: [ROCm] Fix accuracy issues with flash backend
|
||||
logger.debug(
|
||||
"ROCm platform detected. Forcing math SDP backend "
|
||||
"for vision encoder. Currently ROCm platform has "
|
||||
"accuracy issues with `flash_sdp` and"
|
||||
"`mem_efficient_sdp` backends. See issue: "
|
||||
"https://github.com/vllm-project/vllm/issues/30167"
|
||||
)
|
||||
with torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
):
|
||||
vision_embeddings = self.model.get_image_features(
|
||||
pixel_values, **kwargs
|
||||
)
|
||||
else:
|
||||
vision_embeddings = self.model.get_image_features(
|
||||
pixel_values, **kwargs
|
||||
)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
# After v5 is settled, we can enable qwen3-vl with several outputs
|
||||
# from `self.get_image_features`
|
||||
if isinstance(vision_embeddings, tuple):
|
||||
vision_embeddings = vision_embeddings[0]
|
||||
elif isinstance(vision_embeddings, dict):
|
||||
vision_embeddings = vision_embeddings.pooler_output
|
||||
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
split_sizes = num_image_patches.flatten().tolist()
|
||||
total_patches = sum(split_sizes)
|
||||
|
||||
# Flatten to 2D: [total_tokens, hidden_dim]
|
||||
if vision_embeddings.ndim == 3:
|
||||
vision_embeddings = vision_embeddings.view(
|
||||
-1, vision_embeddings.shape[-1]
|
||||
)
|
||||
|
||||
total_tokens = vision_embeddings.shape[0]
|
||||
if total_tokens == total_patches:
|
||||
# Direct match: num_image_patches are actual token counts
|
||||
# (e.g., Qwen2.5-VL style)
|
||||
token_split_sizes = split_sizes
|
||||
elif total_patches > 0 and total_tokens % total_patches == 0:
|
||||
# Uniform expansion: each patch expands to N tokens
|
||||
# (e.g., Idefics3 style)
|
||||
tokens_per_patch = total_tokens // total_patches
|
||||
token_split_sizes = [s * tokens_per_patch for s in split_sizes]
|
||||
elif total_patches > 0:
|
||||
# Mismatch (profiling with dummy data) - pad/truncate
|
||||
if total_tokens == 0:
|
||||
raise ValueError(
|
||||
"Vision encoder returned empty embeddings. "
|
||||
f"Expected {total_patches} patches from "
|
||||
f"num_image_patches={split_sizes}"
|
||||
)
|
||||
if total_tokens < total_patches:
|
||||
repeat_factor = (
|
||||
total_patches + total_tokens - 1
|
||||
) // total_tokens
|
||||
vision_embeddings = vision_embeddings.repeat(repeat_factor, 1)
|
||||
vision_embeddings = vision_embeddings[:total_patches]
|
||||
token_split_sizes = split_sizes
|
||||
else:
|
||||
return []
|
||||
|
||||
return list(torch.split(vision_embeddings, token_split_sizes, dim=0))
|
||||
|
||||
return vision_embeddings
|
||||
else:
|
||||
# TODO: [ROCm] Fix accuracy issues with flash backend
|
||||
logger.debug(
|
||||
"No pixel values or image embeddings provided for multimodal embedding."
|
||||
"ROCm platform detected. Forcing math SDP backend "
|
||||
"for vision encoder. Currently ROCm platform has "
|
||||
"accuracy issues with `flash_sdp` and"
|
||||
"`mem_efficient_sdp` backends. See issue: "
|
||||
"https://github.com/vllm-project/vllm/issues/30167"
|
||||
)
|
||||
return None
|
||||
context = torch.nn.attention.sdpa_kernel(
|
||||
backends=[torch.nn.attention.SDPBackend.MATH]
|
||||
)
|
||||
with context:
|
||||
vision_embeddings = self.model.get_image_features(pixel_values, **kwargs)
|
||||
|
||||
# Transformers `v5`, `self.get_image_features` returns a tuple
|
||||
# containing the features and optionally attentions/hidden_states
|
||||
# After v5 is settled, we can enable qwen3-vl with several outputs
|
||||
# from `self.get_image_features`
|
||||
if isinstance(vision_embeddings, tuple):
|
||||
vision_embeddings = vision_embeddings[0]
|
||||
elif isinstance(vision_embeddings, dict):
|
||||
vision_embeddings = vision_embeddings.pooler_output
|
||||
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
split_sizes = num_image_patches.flatten().tolist()
|
||||
return self._split_embeddings(vision_embeddings, split_sizes)
|
||||
|
||||
return vision_embeddings
|
||||
|
||||
def embed_multimodal(self, **kwargs):
|
||||
embeddings: tuple[torch.Tensor, ...] = ()
|
||||
if "input_features" in kwargs or "input_values" in kwargs:
|
||||
audio_embeddings = self._embed_audio(**kwargs)
|
||||
if audio_embeddings is not None:
|
||||
embeddings += tuple(audio_embeddings)
|
||||
if (
|
||||
"pixel_values" in kwargs
|
||||
or "image_embeds" in kwargs
|
||||
or "image_patches" in kwargs
|
||||
):
|
||||
vision_embeddings = self._embed_vision(**kwargs)
|
||||
if vision_embeddings is not None:
|
||||
if isinstance(vision_embeddings, torch.Tensor):
|
||||
embeddings += (vision_embeddings,)
|
||||
else:
|
||||
embeddings += tuple(vision_embeddings)
|
||||
return embeddings
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user