[Model] Remove AyaVision, MusicFlamingo (#47263)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-01 10:39:33 +00:00
committed by GitHub
parent cc56379e28
commit a22e0dfc69
12 changed files with 3 additions and 1422 deletions
-445
View File
@@ -1,445 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
import torch
from torch import nn
from transformers import BatchFeature, GotOcr2ImageProcessor
from transformers.activations import ACT2FN
from transformers.image_processing_utils import get_size_dict
from transformers.models.aya_vision import AyaVisionConfig
from transformers.models.aya_vision.processing_aya_vision import AyaVisionProcessor
from transformers.models.got_ocr2.image_processing_got_ocr2 import (
get_optimal_tiled_canvas,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_layer_index,
init_vllm_registered_model,
maybe_prefix,
)
class AyaVisionImagePixelInputs(TensorSchema):
"""
Dimensions:
- np: The total number of patches over each image over each prompt in
the batch
- c: Number of channels
- h: Height of each image patch
- w: Width of each image patch
- bn: Batch size * number of images
"""
type: Literal["pixel_values"]
pixel_values: Annotated[
torch.Tensor,
TensorShape("np", 3, "h", "w"),
]
num_patches: Annotated[
torch.Tensor,
TensorShape("bn"),
]
class AyaVisionMultiModalProjector(nn.Module):
def __init__(self, config: AyaVisionConfig):
super().__init__()
self.config = config
self.downsample_factor = config.downsample_factor
self.alignment_intermediate_size = getattr(
config, "alignment_intermediate_size", config.text_config.hidden_size
)
self.layernorm = nn.LayerNorm(
config.vision_config.hidden_size * (config.downsample_factor**2),
eps=config.adapter_layer_norm_eps,
)
self.linear_1 = nn.Linear(
config.vision_config.hidden_size * (config.downsample_factor**2),
self.alignment_intermediate_size,
bias=True,
)
self.act = ACT2FN["silu"] # SwiGLU uses SiLU activation
# For SwiGLU, project down to half size since we split intermediate dim
self.linear_2 = nn.Linear(
self.alignment_intermediate_size // 2,
config.text_config.hidden_size,
bias=True,
)
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
image_features = self.pixel_shuffle(image_features)
image_features = self.layernorm(image_features)
hidden_states = self.linear_1(image_features)
# Split along last dimension and apply SwiGLU
x, gate = hidden_states.chunk(2, dim=-1)
hidden_states = self.act(gate) * x
hidden_states = self.linear_2(hidden_states)
return hidden_states
def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor: # B, S, D
batch_size, seq_length, _ = image_features.shape
height = width = int(seq_length**0.5)
image_features = image_features.reshape(
image_features.shape[0], width, height, -1
)
channels = image_features.shape[-1]
image_features = image_features.reshape(
batch_size,
width,
int(height / self.downsample_factor),
int(channels * self.downsample_factor),
)
image_features = image_features.permute(0, 2, 1, 3)
image_features = image_features.reshape(
batch_size,
int(height / self.downsample_factor),
int(width / self.downsample_factor),
-1,
)
image_features = image_features.permute(0, 2, 1, 3)
return image_features
class AyaVisionProcessingInfo(BaseProcessingInfo):
def get_hf_config(self) -> AyaVisionConfig:
return self.ctx.get_hf_config(AyaVisionConfig)
def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:
return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor:
return self.get_hf_processor(**kwargs).image_processor
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_image_size_with_most_features(self) -> ImageSize:
image_processor = self.get_image_processor()
height = image_processor.size["height"]
width = image_processor.size["width"]
max_patches = image_processor.max_patches
return ImageSize(height=height * max_patches, width=width * max_patches)
def get_num_patches(
self,
*,
image_width: int,
image_height: int,
size: dict,
min_patches: int,
max_patches: int,
) -> int:
"""
Calculate the number of patches needed for a given image based on size
constraints. This method replicates and adjusts the logic from:
transformers/models/got_ocr2/image_processing_got_ocr2
"""
size = get_size_dict(size, default_to_square=False)
num_columns, num_rows = get_optimal_tiled_canvas(
(image_height, image_width),
(size["height"], size["width"]),
min_patches,
max_patches,
)
num_blocks = num_columns * num_rows
return num_blocks if num_blocks == 1 else num_blocks + 1
class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
processor = self.info.get_hf_processor()
image_token = processor.image_token
return image_token * num_images
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_size = self.info.get_image_size_with_most_features()
image_overrides = mm_options.get("image")
return {
"image": self._get_dummy_images(
width=image_size.width,
height=image_size.height,
num_images=num_images,
overrides=image_overrides,
)
}
class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingInfo]):
def _call_hf_processor(
self,
prompt: str,
mm_data: Mapping[str, object],
mm_kwargs: Mapping[str, object],
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
processed_outputs = super()._call_hf_processor(
prompt,
mm_data,
mm_kwargs,
tok_kwargs,
)
hf_processor = self.info.get_hf_processor(**mm_kwargs)
image_processor = hf_processor.image_processor
# HF processor pops the `num_patches` kwarg, which is needed by vLLM
if (images := mm_data.get("images")) is not None:
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
parsed_images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
parsed_images.get_image_size(i) for i in range(len(parsed_images))
]
num_patches = [
self.info.get_num_patches(
image_width=image_size.width,
image_height=image_size.height,
size=image_processor.size,
min_patches=image_processor.min_patches,
max_patches=image_processor.max_patches,
)
for image_size in image_sizes
]
processed_outputs["num_patches"] = torch.tensor(num_patches)
return processed_outputs
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
num_patches = hf_inputs.get("num_patches", torch.empty(0))
return dict(
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
num_patches=MultiModalFieldConfig.batched("image"),
image_embeds=MultiModalFieldConfig.batched("image"),
)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_token = hf_processor.image_token
img_patch_token = hf_processor.img_patch_token
image_processor = hf_processor.image_processor
def get_replacement(item_idx: int):
images = mm_items.get_items("image", ImageProcessorItems)
image_size: ImageSize = images.get_image_size(item_idx)
num_patches = self.info.get_num_patches(
image_width=image_size.width,
image_height=image_size.height,
size=image_processor.size,
min_patches=image_processor.min_patches,
max_patches=image_processor.max_patches,
)
repl = hf_processor._prompt_split_image(num_patches=num_patches)
return PromptUpdateDetails.select_text(repl, img_patch_token)
return [
PromptReplacement(
modality="image",
target=image_token,
replacement=get_replacement,
)
]
def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:
feature_layers = hf_config.vision_feature_layer
num_hidden_layers = hf_config.vision_config.num_hidden_layers
# If we have one feature layer, initialize up to that layer
if isinstance(feature_layers, int):
return get_layer_index(feature_layers, num_hidden_layers)
# If we have multiple feature layers, initialize up to the deepest m
elif isinstance(feature_layers, (list, tuple)):
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
raise TypeError(
f"vision_layer_feature type: {type(feature_layers)} is not supported"
)
@MULTIMODAL_REGISTRY.register_processor(
AyaVisionMultiModalProcessor,
info=AyaVisionProcessingInfo,
dummy_inputs=AyaVisionDummyInputsBuilder,
)
class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
# mapping for new names in checkpoint saved after transformers v4.52
"model.language_model.": "language_model.model.",
"model.vision_tower.": "vision_tower.",
"model.multi_modal_projector.": "multi_modal_projector.",
"lm_head.": "language_model.lm_head.",
}
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
raise ValueError("Only image modality is supported")
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config: AyaVisionConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
num_hidden_layers = _get_num_hidden_layers(config)
self.config = config
self.quant_config = quant_config
self.multimodal_config = multimodal_config
with self._mark_tower_model(vllm_config, "image"):
self.vision_tower = SiglipVisionModel(
config.vision_config,
quant_config,
num_hidden_layers_override=num_hidden_layers,
prefix=maybe_prefix(prefix, "vision_model"),
)
self.multi_modal_projector = AyaVisionMultiModalProjector(config)
with self._mark_language_model(vllm_config):
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config.text_config,
prefix=maybe_prefix(prefix, "model"),
# Cohere2ForCausalLM and CohereForCausalLM are the same on vllm
architectures=["Cohere2ForCausalLM"],
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
@property
def dtype(self):
return next(self.parameters()).dtype
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _image_pixels_to_features(
self,
vision_tower: SiglipVisionModel,
pixel_values: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
return vision_tower(
pixel_values.to(dtype=vision_tower.dtype),
feature_select_strategy=self.config.vision_feature_select_strategy,
)
def _process_image_input(
self, image_input: AyaVisionImagePixelInputs, **kwargs
) -> list[torch.Tensor]:
pixel_values = image_input["pixel_values"]
num_patches = image_input["num_patches"]
image_features = self._image_pixels_to_features(
self.vision_tower, pixel_values=pixel_values
)
image_embeds = self.multi_modal_projector(image_features)
return [e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())]
def _parse_and_validate_image_input(
self, **kwargs: object
) -> AyaVisionImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
num_patches = kwargs.pop("num_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
assert image_embeds is None, "Aya Vision does not support image_embeds."
if pixel_values is None:
return None
return AyaVisionImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
num_patches=num_patches,
resolve_bindings={
"h": self.config.vision_config.image_size,
"w": self.config.vision_config.image_size,
},
)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
return []
return self._process_image_input(image_input, **kwargs)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
hidden_states = self.language_model.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
-449
View File
@@ -1,449 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2026 The vLLM team.
# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
# reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from collections.abc import Callable, Mapping, Sequence
from math import pi
from typing import Annotated, Any, Optional, TypeAlias
import torch
from torch import Tensor, broadcast_tensors, nn
from transformers import BatchFeature
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
from transformers.models.musicflamingo import (
MusicFlamingoConfig,
MusicFlamingoProcessor,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItems,
)
from vllm.multimodal.parse import (
DictEmbeddingItems,
ModalityData,
ModalityDataItems,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.processing import (
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.utils.tensor_schema import TensorShape
from .audioflamingo3 import (
AudioFlamingo3DummyInputsBuilder,
AudioFlamingo3EmbeddingInputs,
AudioFlamingo3Encoder,
AudioFlamingo3FeatureInputs,
AudioFlamingo3ForConditionalGeneration,
AudioFlamingo3MultiModalDataParser,
AudioFlamingo3MultiModalProcessor,
AudioFlamingo3MultiModalProjector,
AudioFlamingo3ProcessingInfo,
_audioflamingo3_field_config,
_count_audio_tokens_from_mask,
)
def rotate_half(x):
x = x.reshape(*x.shape[:-1], -1, 2)
x1, x2 = x.unbind(dim=-1)
x = torch.stack((-x2, x1), dim=-1)
return x.flatten(-2)
def apply_rotary_time_emb(hidden_states, cos, sin):
original_dtype = hidden_states.dtype
hidden_states = hidden_states.to(torch.float64)
cos = cos.to(hidden_states)
sin = sin.to(hidden_states)
rot_dim = cos.shape[-1]
if rot_dim > hidden_states.shape[-1]:
raise ValueError(
f"feature dimension {hidden_states.shape[-1]} is not of "
f"sufficient size to rotate in all the positions {rot_dim}"
)
rotated = hidden_states[..., :rot_dim]
passthrough = hidden_states[..., rot_dim:]
rotated = (rotated * cos) + (rotate_half(rotated) * sin)
return torch.cat((rotated, passthrough), dim=-1).to(original_dtype)
class MusicFlamingoRotaryEmbedding(nn.Module):
inv_freq: torch.Tensor
def __init__(self, config: MusicFlamingoConfig, device=None):
super().__init__()
self.max_seq_len_cached = config.max_position_embeddings
self.original_max_seq_len = config.max_position_embeddings
self.config = config
self.rope_type = self.config.rope_parameters["rope_type"]
rope_init_fn: Callable = self.compute_default_rope_parameters
if self.rope_type != "default":
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
self.register_buffer("inv_freq", inv_freq, persistent=False)
self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
position_angles = self._compute_position_angles(self.inv_freq)
self.register_buffer("position_angles", position_angles, persistent=False)
@staticmethod
def compute_default_rope_parameters(
config: MusicFlamingoConfig | None = None,
device: Optional["torch.device"] = None,
seq_len: int | None = None,
) -> tuple["torch.Tensor", float]:
del seq_len
base = config.rope_parameters["rope_theta"]
partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
head_dim = getattr(config, "head_dim", None) or (
config.hidden_size // config.num_attention_heads
)
dim = int(head_dim * partial_rotary_factor)
attention_factor = 1.0
inv_freq = 1.0 / (
base
** (
torch.arange(0, dim, 2, dtype=torch.int64).to(
device=device,
dtype=torch.float,
)
/ dim
)
)
return inv_freq, attention_factor
def _compute_position_angles(self, inv_freq):
positions = torch.arange(
int(self.max_seq_len_cached),
device=inv_freq.device,
dtype=inv_freq.dtype,
)
positions = positions / self.max_seq_len_cached * (2 * pi)
position_angles = positions.unsqueeze(-1) * inv_freq
position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)
return position_angles.to(dtype=inv_freq.dtype)
def _restore_fp32_rope_buffers(self) -> None:
rope_init_fn: Callable = self.compute_default_rope_parameters
if self.rope_type != "default":
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
inv_freq, self.attention_scaling = rope_init_fn(
self.config, self.inv_freq.device
)
self.inv_freq = inv_freq
self.original_inv_freq = inv_freq.clone()
self.position_angles = self._compute_position_angles(inv_freq)
def _apply(self, fn):
super()._apply(fn)
self._restore_fp32_rope_buffers()
return self
@torch.no_grad()
def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:
window_starts = timestamps[:, 0].to(
device=self.inv_freq.device,
dtype=self.inv_freq.dtype,
)
window_duration = self.config.audio_frame_step * 4 * seq_len
window_positions = (
torch.round(window_starts / window_duration) / self.max_seq_len_cached
)
window_freqs = window_positions.unsqueeze(-1) * self.inv_freq
window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)
window_freqs = window_freqs[:, None, :]
time_freqs = self.position_angles[:seq_len][None, :, :]
window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)
freqs = torch.cat((window_freqs, time_freqs), dim=-1)
angle = (-timestamps * 2 * pi).to(freqs)
freqs = freqs * angle.unsqueeze(-1)
return freqs.cos(), freqs.sin()
class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):
rote_timestamps: Annotated[
torch.Tensor | None,
TensorShape(
"num_chunks",
"num_audio_time_steps",
dynamic_dims={"num_audio_time_steps"},
),
]
MusicFlamingoEmbeddingInputs = AudioFlamingo3EmbeddingInputs
MusicFlamingoInputs: TypeAlias = (
MusicFlamingoFeatureInputs | MusicFlamingoEmbeddingInputs
)
class MusicFlamingoEncoder(AudioFlamingo3Encoder):
pass
class MusicFlamingoMultiModalProjector(AudioFlamingo3MultiModalProjector):
pass
class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):
def get_hf_config(self) -> MusicFlamingoConfig:
return self.ctx.get_hf_config(MusicFlamingoConfig)
def get_hf_processor(self, **kwargs: object) -> MusicFlamingoProcessor:
return self.ctx.get_hf_processor(MusicFlamingoProcessor, **kwargs)
def get_data_parser(self) -> MultiModalDataParser:
feature_extractor = self.get_feature_extractor()
return MusicFlamingoMultiModalDataParser(
target_sr=feature_extractor.sampling_rate,
audio_resample_method="soxr",
expected_hidden_size=self._get_expected_hidden_size(),
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": 1}
class MusicFlamingoDummyInputsBuilder(AudioFlamingo3DummyInputsBuilder):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
hf_processor = self.info.get_hf_processor()
return hf_processor.audio_token * num_audios
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Mapping[str, BaseDummyOptions],
) -> MultiModalDataDict:
hf_processor = self.info.get_hf_processor()
feature_extractor = self.info.get_feature_extractor()
sampling_rate = feature_extractor.sampling_rate
audio_len = int(hf_processor.max_audio_len * sampling_rate)
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
return {
"audio": self._get_dummy_audios(
length=audio_len,
num_audios=num_audios,
overrides=audio_overrides,
)
}
def _musicflamingo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
fields = dict(_audioflamingo3_field_config(hf_inputs))
chunk_counts = hf_inputs.get("chunk_counts")
if chunk_counts is not None:
fields["rote_timestamps"] = MultiModalFieldConfig.flat_from_sizes(
"audio", chunk_counts, dim=0
)
else:
fields["rote_timestamps"] = MultiModalFieldConfig.batched("audio")
return fields
class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
def _parse_audio_data(
self,
data: dict[str, torch.Tensor] | ModalityData[Any],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
return DictEmbeddingItems(
data,
modality="audio",
required_fields={"audio_embeds"},
fields_factory=_musicflamingo_field_config,
)
return super()._parse_audio_data(data)
class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return _musicflamingo_field_config(hf_inputs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
vocab = tokenizer.get_vocab()
audio_token = processor.audio_token
audio_token_id = vocab.get(audio_token, processor.audio_token_id)
audio_bos_token = processor.audio_bos_token
audio_bos_token_id = vocab.get(audio_bos_token, processor.audio_bos_token_id)
audio_eos_token = processor.audio_eos_token
audio_eos_token_id = vocab.get(audio_eos_token, processor.audio_eos_token_id)
out_mm_data = out_mm_kwargs.get_data()
feature_attention_mask = out_mm_data.get("feature_attention_mask")
chunk_counts = out_mm_data.get("chunk_counts")
def get_replacement_musicflamingo(item_idx: int):
if feature_attention_mask is not None:
num_features = _count_audio_tokens_from_mask(
feature_attention_mask,
chunk_counts,
item_idx,
)
else:
audio_embeds = out_mm_data["audio_embeds"][item_idx]
num_features = audio_embeds.shape[0]
if num_features == 0:
raise ValueError("Audio is too short")
full_tokens = [
audio_bos_token_id,
*([audio_token_id] * int(num_features)),
audio_eos_token_id,
]
return PromptUpdateDetails.select_token_id(
full_tokens,
embed_token_id=audio_token_id,
)
return [
PromptReplacement(
modality="audio",
target=audio_token,
replacement=get_replacement_musicflamingo,
)
]
@MULTIMODAL_REGISTRY.register_processor(
MusicFlamingoMultiModalProcessor,
info=MusicFlamingoProcessingInfo,
dummy_inputs=MusicFlamingoDummyInputsBuilder,
)
class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGeneration):
"""vLLM MusicFlamingo model aligned with HF modular_musicflamingo."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.audio_tower = MusicFlamingoEncoder(self.config.audio_config)
self.multi_modal_projector = MusicFlamingoMultiModalProjector(self.config)
self.pos_emb = MusicFlamingoRotaryEmbedding(self.config)
def _parse_and_validate_audio_input(
self, **kwargs: object
) -> MusicFlamingoInputs | None:
rote_timestamps = kwargs.pop("rote_timestamps", None)
audio_input = super()._parse_and_validate_audio_input(**kwargs)
if audio_input is None or audio_input["type"] == "audio_embeds":
return audio_input
return MusicFlamingoFeatureInputs(
type="audio_features",
input_features=audio_input["input_features"],
feature_attention_mask=audio_input["feature_attention_mask"],
chunk_counts=audio_input["chunk_counts"],
rote_timestamps=rote_timestamps,
)
def _build_audio_timestamps(
self,
chunk_counts: list[int],
seq_len: int,
device: torch.device,
) -> torch.Tensor:
audio_embed_frame_step = self.config.audio_frame_step * 4
frame_offsets = (
torch.arange(seq_len, device=device, dtype=torch.float32)
* audio_embed_frame_step
)
if not chunk_counts:
return frame_offsets.new_empty((0, seq_len))
window_indices = torch.cat(
[
torch.arange(count, device=device, dtype=torch.float32)
for count in chunk_counts
]
)
return (
window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step
+ frame_offsets
)
def _process_audio_input(
self, audio_input: MusicFlamingoInputs
) -> torch.Tensor | tuple[torch.Tensor, ...]:
if audio_input["type"] == "audio_embeds":
return super()._process_audio_input(audio_input)
rote_timestamps = audio_input["rote_timestamps"]
(
input_features,
feature_attention_mask,
chunk_counts,
) = self._normalize_audio_feature_inputs(audio_input)
hidden_states = self._encode_audio_features(
input_features,
feature_attention_mask,
)
if rote_timestamps is None:
rote_timestamps = self._build_audio_timestamps(
chunk_counts,
seq_len=hidden_states.shape[-2],
device=hidden_states.device,
)
elif isinstance(rote_timestamps, list):
rote_timestamps = torch.cat(rote_timestamps, dim=0)
cos, sin = self.pos_emb(
rote_timestamps.to(hidden_states.device),
seq_len=hidden_states.shape[-2],
)
hidden_states = apply_rotary_time_emb(hidden_states, cos, sin)
audio_features = self.multi_modal_projector(hidden_states)
return self._group_audio_embeddings(
audio_features,
feature_attention_mask,
chunk_counts,
)
+2 -8
View File
@@ -337,14 +337,6 @@ _MULTIMODAL_MODELS = {
"audioflamingo3",
"AudioFlamingo3ForConditionalGeneration",
),
"MusicFlamingoForConditionalGeneration": (
"musicflamingo",
"MusicFlamingoForConditionalGeneration",
),
"AyaVisionForConditionalGeneration": (
"aya_vision",
"AyaVisionForConditionalGeneration",
),
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
@@ -733,6 +725,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"TarsierForConditionalGeneration": "0.24.0",
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
"MantisForConditionalGeneration": "0.24.0",
"MusicFlamingoForConditionalGeneration": "0.24.0",
"AyaVisionForConditionalGeneration": "0.24.0",
}
_OOT_SUPPORTED_MODELS = {