diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 099ddb2e52..9a86a57502 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -532,7 +532,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- | | `AriaForConditionalGeneration` | Aria | T + I+ | `rhymes-ai/Aria` | | | | `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ | -| `AyaVisionForConditionalGeneration` | Aya Vision | T + I+ | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ | | `BagelForConditionalGeneration` | BAGEL | T + I+ | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | `BeeForConditionalGeneration` | Bee-8B | T + IE+ | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ | | `Blip2ForConditionalGeneration` | BLIP-2 | T + IE | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ | @@ -594,7 +593,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`^, `allenai/MolmoWeb-8B`^ | ✅︎ | ✅︎ | | `MossAudioModel` | MOSS-Audio | T + A+ | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ | | `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ | -| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ | | `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ | | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ | | `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + IE+ + VE+ | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ | diff --git a/examples/generate/multimodal/audio_language_offline.py b/examples/generate/multimodal/audio_language_offline.py index 12a38cf41c..fc20e8fed1 100644 --- a/examples/generate/multimodal/audio_language_offline.py +++ b/examples/generate/multimodal/audio_language_offline.py @@ -91,44 +91,6 @@ def run_cohere_asr(question: str, audio_count: int) -> ModelRequestData: ) -# MusicFlamingo -def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData: - model_name = "nvidia/music-flamingo-2601-hf" - engine_args = EngineArgs( - model=model_name, - max_model_len=4096, - max_num_seqs=2, - limit_mm_per_prompt={"audio": audio_count}, - enforce_eager=True, - ) - - # MusicFlamingo prompt placeholders use ; vLLM's MusicFlamingo - # multimodal processor expands each one into <|sound_bos|> + audio tokens + - # <|sound_eos|> based on extracted audio feature lengths. - audio_placeholder = "" * audio_count - system_prompt = ( - "You are Music Flamingo, a multimodal assistant for language and music. " - "On each turn you receive an audio clip which contains music and optional " - "text, you will receive at least one or both; use your world knowledge and " - "reasoning to help the user with any task. Interpret the entirety of the " - "content any input music--regardlenss of whether the user calls it audio, " - "music, or sound." - ) - - prompt = ( - "<|im_start|>system\n" - f"{system_prompt}<|im_end|>\n" - "<|im_start|>user\n" - f"{audio_placeholder}{question}<|im_end|>\n" - "<|im_start|>assistant\n" - ) - - return ModelRequestData( - engine_args=engine_args, - prompt=prompt, - ) - - # Gemma3N def run_gemma3n(question: str, audio_count: int) -> ModelRequestData: model_name = "google/gemma-3n-E2B-it" @@ -565,7 +527,6 @@ model_example_map = { "kimi_audio": run_kimi_audio, "midashenglm": run_midashenglm, "minicpmo": run_minicpmo, - "musicflamingo": run_musicflamingo, "phi4_mm": run_phi4mm, "qwen2_audio": run_qwen2_audio, "qwen2_5_omni": run_qwen2_5_omni, diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py index c3bec9d5fd..c354142774 100644 --- a/examples/generate/multimodal/vision_language_multi_image_offline.py +++ b/examples/generate/multimodal/vision_language_multi_image_offline.py @@ -74,39 +74,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData: ) -def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData: - model_name = "CohereLabs/aya-vision-8b" - - engine_args = EngineArgs( - model=model_name, - max_num_seqs=2, - limit_mm_per_prompt={"image": len(image_urls)}, - ) - - placeholders = [{"type": "image", "image": url} for url in image_urls] - messages = [ - { - "role": "user", - "content": [ - *placeholders, - {"type": "text", "text": question}, - ], - } - ] - - processor = AutoProcessor.from_pretrained(model_name) - - prompt = processor.apply_chat_template( - messages, tokenize=False, add_generation_prompt=True - ) - - return ModelRequestData( - engine_args=engine_args, - prompt=prompt, - image_data=[fetch_image(url) for url in image_urls], - ) - - def load_bee(question: str, image_urls: list[str]) -> ModelRequestData: model_name = "Open-Bee/Bee-8B-RL" @@ -1420,7 +1387,6 @@ def load_molmo2(question: str, image_urls: list[str]) -> ModelRequestData: model_example_map = { "aria": load_aria, - "aya_vision": load_aya_vision, "bee": load_bee, "command_a_vision": load_command_a_vision, "deepseek_vl_v2": load_deepseek_vl2, diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py index 46765ca038..a1eb6cc538 100644 --- a/examples/generate/multimodal/vision_language_offline.py +++ b/examples/generate/multimodal/vision_language_offline.py @@ -68,28 +68,6 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData: ) -# Aya Vision -def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData: - assert modality == "image" - model_name = "CohereLabs/aya-vision-8b" - - engine_args = EngineArgs( - model=model_name, - max_model_len=2048, - max_num_seqs=2, - mm_processor_kwargs={"crop_to_patches": True}, - limit_mm_per_prompt={modality: 1}, - ) - prompts = [ - f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" - for question in questions - ] - return ModelRequestData( - engine_args=engine_args, - prompts=prompts, - ) - - # Bee-8B def run_bee(questions: list[str], modality: str) -> ModelRequestData: assert modality == "image" @@ -2327,7 +2305,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData: model_example_map = { "aria": run_aria, - "aya_vision": run_aya_vision, "bagel": run_bagel, "cheers": run_cheers, "bee": run_bee, diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index e245d2b899..ff532fd878 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -326,39 +326,6 @@ VLM_TEST_SETTINGS = { large_gpu_mark(min_gb=64), ], ), - "aya_vision": VLMTestInfo( - models=["CohereLabs/aya-vision-8b"], - test_type=(VLMTestType.IMAGE), - prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501 - single_image_prompts=IMAGE_ASSETS.prompts( - { - "stop_sign": "What's the content in the center of the image?", - "cherry_blossom": "What is the season?", - } - ), - multi_image_prompt="Describe the two images in detail.", - max_model_len=4096, - max_num_seqs=2, - auto_cls=AutoModelForImageTextToText, - vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}}, - ), - "aya_vision-multi_image": VLMTestInfo( - models=["CohereLabs/aya-vision-8b"], - test_type=(VLMTestType.MULTI_IMAGE), - prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501 - single_image_prompts=IMAGE_ASSETS.prompts( - { - "stop_sign": "What's the content in the center of the image?", - "cherry_blossom": "What is the season?", - } - ), - multi_image_prompt="Describe the two images in detail.", - max_model_len=4096, - max_num_seqs=2, - auto_cls=AutoModelForImageTextToText, - vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}}, - marks=[large_gpu_mark(min_gb=32)], - ), "blip2": VLMTestInfo( models=["Salesforce/blip2-opt-2.7b"], test_type=VLMTestType.IMAGE, diff --git a/tests/models/multimodal/generation/test_musicflamingo.py b/tests/models/multimodal/generation/test_musicflamingo.py deleted file mode 100644 index 625fbd775d..0000000000 --- a/tests/models/multimodal/generation/test_musicflamingo.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -import os - -import pytest - -from tests.models.registry import HF_EXAMPLE_MODELS -from vllm import LLM, SamplingParams - -MODEL_NAME = "nvidia/music-flamingo-2601-hf" -SINGLE_CONVERSATION = [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Describe this track in full detail - tell me the " - "genre, tempo, and key, then dive into the instruments, " - "production style, and overall mood it creates.", - }, - { - "type": "audio_url", - "audio_url": { - "url": "https://huggingface.co/datasets/nvidia/AudioSkills/" - "resolve/main/assets/song_1.mp3", - }, - }, - ], - } -] -BATCHED_CONVERSATIONS = [ - SINGLE_CONVERSATION, - [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Generate a structured lyric sheet from the input music.", - }, - { - "type": "audio_url", - "audio_url": { - "url": "https://huggingface.co/datasets/nvidia/" - "AudioSkills/resolve/main/assets/song_2.mp3", - }, - }, - ], - } - ], -] - - -def get_fixture_path(filename): - return os.path.join( - os.path.dirname(__file__), "../../fixtures/musicflamingo", filename - ) - - -def load_expected_fixture(filename): - fixture_path = get_fixture_path(filename) - with open(fixture_path) as f: - return json.load(f) - - -def assert_output_matches(output, expected_text, expected_token_ids): - generated = output.outputs[0] - assert generated.text == expected_text - actual_token_ids = list(generated.token_ids) - assert ( - actual_token_ids == expected_token_ids - or actual_token_ids == expected_token_ids[:-1] - or actual_token_ids[:-1] == expected_token_ids - ) - - -@pytest.fixture(scope="module") -def llm(): - model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration") - model_info.check_transformers_version(on_fail="skip") - - try: - llm = LLM( - model=MODEL_NAME, - dtype="bfloat16", - enforce_eager=True, - max_model_len=8192, - limit_mm_per_prompt={"audio": 1}, - ) - except Exception as e: - pytest.skip(f"Failed to load model {MODEL_NAME}: {e}") - - # ROCm may compile decoder kernels on the first inference pass; warm up - # once so exact fixture assertions cover the steady-state path. - llm.chat( - messages=SINGLE_CONVERSATION, - sampling_params=SamplingParams(temperature=0.0, max_tokens=1), - use_tqdm=False, - ) - - return llm - - -def test_single_generation(llm): - expected = load_expected_fixture("expected_results_single.json") - - outputs = llm.chat( - messages=SINGLE_CONVERSATION, - sampling_params=SamplingParams(temperature=0.0, max_tokens=50), - ) - - assert_output_matches( - outputs[0], - expected["transcriptions"][0], - expected["token_ids"][0], - ) - - -def test_batched_generation(llm): - expected = load_expected_fixture("expected_results_batched.json") - - outputs = llm.chat( - messages=BATCHED_CONVERSATIONS, - sampling_params=SamplingParams(temperature=0.0, max_tokens=50), - ) - - for i, output in enumerate(outputs): - assert_output_matches( - output, - expected["transcriptions"][i], - expected["token_ids"][i], - ) - - -def test_single_and_batched_generation_match(llm): - sampling_params = SamplingParams(temperature=0.0, max_tokens=50) - - single_output = llm.chat( - messages=SINGLE_CONVERSATION, - sampling_params=sampling_params, - )[0] - batched_output = llm.chat( - messages=BATCHED_CONVERSATIONS, - sampling_params=sampling_params, - )[0] - - assert single_output.outputs[0].text == batched_output.outputs[0].text - assert list(single_output.outputs[0].token_ids) == list( - batched_output.outputs[0].token_ids - ) diff --git a/tests/models/multimodal/processing/test_musicflamingo.py b/tests/models/multimodal/processing/test_musicflamingo.py deleted file mode 100644 index bb3d200d6f..0000000000 --- a/tests/models/multimodal/processing/test_musicflamingo.py +++ /dev/null @@ -1,228 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2026 The vLLM team. -# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights -# reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from importlib.metadata import version -from unittest.mock import MagicMock - -import numpy as np -import pytest -import torch -from packaging.version import Version -from transformers import PretrainedConfig - -from tests.models.registry import HF_EXAMPLE_MODELS - - -class MockMusicFlamingoConfig(PretrainedConfig): - model_type = "musicflamingo" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.audio_config = PretrainedConfig() - self.text_config = PretrainedConfig() - - -class MockMusicFlamingoProcessor: - def __init__(self): - self.audio_token = "" - self.audio_token_id = 12345 - self.audio_bos_token = "<|sound_bos|>" - self.audio_bos_token_id = 12346 - self.audio_eos_token = "<|sound_eos|>" - self.audio_eos_token_id = 12347 - self.max_audio_len = 1200 - self.feature_extractor = MockFeatureExtractor() - - def __call__(self, text=None, audio=None, **kwargs): - return { - "input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long), - "input_features": torch.zeros((3, 80, 3000)), - "input_features_mask": torch.ones((3, 3000), dtype=torch.long), - } - - -class MockFeatureExtractor: - def __init__(self): - self.sampling_rate = 16000 - self.chunk_length = 30 - - -@pytest.fixture -def mock_ctx(): - config = MockMusicFlamingoConfig() - - ctx = MagicMock() - ctx.get_hf_config.return_value = config - ctx.get_hf_processor.return_value = MockMusicFlamingoProcessor() - ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor( - **data, **kwargs - ) - ctx.model_config.hf_config = config - return ctx - - -@pytest.fixture(autouse=True) -def check_transformers_version(): - model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration") - model_info.check_transformers_version(on_fail="skip") - - -def test_musicflamingo_chunk_counting_without_rote_timestamps(mock_ctx): - from vllm.model_executor.models.musicflamingo import ( - MusicFlamingoDummyInputsBuilder, - MusicFlamingoMultiModalProcessor, - MusicFlamingoProcessingInfo, - ) - - info = MusicFlamingoProcessingInfo(mock_ctx) - processor = MusicFlamingoMultiModalProcessor( - info, MusicFlamingoDummyInputsBuilder(info) - ) - - sr = 16000 - audio_1 = np.zeros(30 * sr) - audio_2 = np.zeros(45 * sr) - - mm_data = {"audio": [audio_1, audio_2]} - prompt = "<|user|>Listen.<|end|>" - - processed = processor._call_hf_processor(prompt, mm_data, {}, {}) - - chunk_counts = processed["chunk_counts"] - - assert chunk_counts.tolist() == [1, 2] - assert "rote_timestamps" not in processed - assert processed["feature_attention_mask"].shape == (3, 3000) - - -def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx): - from vllm.model_executor.models.musicflamingo import ( - MusicFlamingoDummyInputsBuilder, - MusicFlamingoProcessingInfo, - ) - - info = MusicFlamingoProcessingInfo(mock_ctx) - builder = MusicFlamingoDummyInputsBuilder(info) - - assert builder.get_dummy_text({"audio": 2}) == "" - - -@pytest.mark.skipif( - Version(version("transformers")) >= Version("5.5"), - reason="transformers v5.5 added native MusicFlamingoForConditionalGeneration " - "with a different get_audio_features signature (requires input_ids)", -) -def test_musicflamingo_audio_feature_pipeline_matches_hf_small_config(): - from transformers.models.musicflamingo import ( - modeling_musicflamingo as hf_musicflamingo_modeling, - ) - from transformers.models.musicflamingo.configuration_musicflamingo import ( - MusicFlamingoConfig, - ) - - from vllm.model_executor.models.audioflamingo3 import ( - _build_audio_encoder_attention_mask, - _flatten_valid_audio_embeddings, - ) - from vllm.model_executor.models.musicflamingo import ( - MusicFlamingoEncoder, - MusicFlamingoMultiModalProjector, - MusicFlamingoRotaryEmbedding, - apply_rotary_time_emb, - ) - - text_config = { - "model_type": "qwen2", - "intermediate_size": 64, - "initializer_range": 0.02, - "hidden_size": 32, - "max_position_embeddings": 1024, - "num_hidden_layers": 2, - "num_attention_heads": 4, - "num_key_value_heads": 2, - "vocab_size": 128, - "pad_token_id": 1, - "use_mrope": False, - } - audio_config = { - "hidden_size": 16, - "num_attention_heads": 4, - "intermediate_size": 32, - "num_hidden_layers": 2, - "num_mel_bins": 80, - "max_source_positions": 1500, - "dropout": 0.0, - "attention_dropout": 0.0, - "activation_dropout": 0.0, - "encoder_layerdrop": 0.0, - } - - torch.manual_seed(0) - config = MusicFlamingoConfig( - text_config=text_config, - audio_config=audio_config, - audio_token_id=0, - head_dim=8, - rope_parameters={"rope_type": "default", "rope_theta": 2048}, - ) - hf_model = hf_musicflamingo_modeling.MusicFlamingoForConditionalGeneration( - config - ).eval() - - vllm_encoder = MusicFlamingoEncoder(config.audio_config).eval() - vllm_encoder.load_state_dict(hf_model.audio_tower.state_dict()) - - vllm_projector = MusicFlamingoMultiModalProjector(config).eval() - vllm_projector.load_state_dict(hf_model.multi_modal_projector.state_dict()) - - vllm_rope = MusicFlamingoRotaryEmbedding(config).eval() - vllm_rope.load_state_dict(hf_model.pos_emb.state_dict(), strict=False) - - input_features = torch.randn(3, 80, 3000) - feature_attention_mask = torch.zeros(3, 3000, dtype=torch.bool) - feature_attention_mask[0, :3000] = True - feature_attention_mask[1, :2500] = True - feature_attention_mask[2, :1500] = True - rote_timestamps = ( - torch.arange(750, dtype=torch.float32).unsqueeze(0).repeat(3, 1) * 0.04 - ) - - hf_output = hf_model.get_audio_features( - input_features, - feature_attention_mask, - rote_timestamps=rote_timestamps, - return_dict=True, - ).pooler_output - vllm_attention_mask = _build_audio_encoder_attention_mask( - feature_attention_mask, - dtype=vllm_encoder.conv1.weight.dtype, - device=vllm_encoder.conv1.weight.device, - ) - vllm_hidden_states = vllm_encoder( - input_features, - attention_mask=vllm_attention_mask, - ) - cos, sin = vllm_rope(rote_timestamps, seq_len=vllm_hidden_states.shape[-2]) - vllm_hidden_states = apply_rotary_time_emb(vllm_hidden_states, cos, sin) - vllm_output, _ = _flatten_valid_audio_embeddings( - vllm_projector(vllm_hidden_states), - feature_attention_mask, - ) - - torch.testing.assert_close(vllm_output, hf_output) diff --git a/tests/models/registry.py b/tests/models/registry.py index 34f8ca4f82..da2dc8bb9d 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -780,14 +780,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "vllm": "Needs https://github.com/huggingface/transformers/pull/43538" }, ), - "MusicFlamingoForConditionalGeneration": _HfExamplesInfo( - "nvidia/music-flamingo-2601-hf", - min_transformers_version="5.5.0", - transformers_version_reason={ - "vllm": "Needs https://github.com/huggingface/transformers/pull/43538" - }, - ), - "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"), "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"), "BeeForConditionalGeneration": _HfExamplesInfo( "Open-Bee/Bee-8B-RL", diff --git a/vllm/model_executor/models/aya_vision.py b/vllm/model_executor/models/aya_vision.py deleted file mode 100644 index f4c9bbee9e..0000000000 --- a/vllm/model_executor/models/aya_vision.py +++ /dev/null @@ -1,445 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Adapted from https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision -from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal - -import torch -from torch import nn -from transformers import BatchFeature, GotOcr2ImageProcessor -from transformers.activations import ACT2FN -from transformers.image_processing_utils import get_size_dict -from transformers.models.aya_vision import AyaVisionConfig -from transformers.models.aya_vision.processing_aya_vision import AyaVisionProcessor -from transformers.models.got_ocr2.image_processing_got_ocr2 import ( - get_optimal_tiled_canvas, -) - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.inputs import MultiModalDataDict -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - MultiModalFieldConfig, - MultiModalKwargsItems, -) -from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems -from vllm.multimodal.processing import ( - BaseDummyInputsBuilder, - BaseMultiModalProcessor, - BaseProcessingInfo, - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, -) -from vllm.sequence import IntermediateTensors -from vllm.utils.tensor_schema import TensorSchema, TensorShape - -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP -from .siglip import SiglipVisionModel -from .utils import ( - AutoWeightsLoader, - WeightsMapper, - get_layer_index, - init_vllm_registered_model, - maybe_prefix, -) - - -class AyaVisionImagePixelInputs(TensorSchema): - """ - Dimensions: - - np: The total number of patches over each image over each prompt in - the batch - - c: Number of channels - - h: Height of each image patch - - w: Width of each image patch - - bn: Batch size * number of images - """ - - type: Literal["pixel_values"] - - pixel_values: Annotated[ - torch.Tensor, - TensorShape("np", 3, "h", "w"), - ] - - num_patches: Annotated[ - torch.Tensor, - TensorShape("bn"), - ] - - -class AyaVisionMultiModalProjector(nn.Module): - def __init__(self, config: AyaVisionConfig): - super().__init__() - self.config = config - self.downsample_factor = config.downsample_factor - self.alignment_intermediate_size = getattr( - config, "alignment_intermediate_size", config.text_config.hidden_size - ) - self.layernorm = nn.LayerNorm( - config.vision_config.hidden_size * (config.downsample_factor**2), - eps=config.adapter_layer_norm_eps, - ) - - self.linear_1 = nn.Linear( - config.vision_config.hidden_size * (config.downsample_factor**2), - self.alignment_intermediate_size, - bias=True, - ) - - self.act = ACT2FN["silu"] # SwiGLU uses SiLU activation - # For SwiGLU, project down to half size since we split intermediate dim - self.linear_2 = nn.Linear( - self.alignment_intermediate_size // 2, - config.text_config.hidden_size, - bias=True, - ) - - def forward(self, image_features: torch.Tensor) -> torch.Tensor: - image_features = self.pixel_shuffle(image_features) - image_features = self.layernorm(image_features) - hidden_states = self.linear_1(image_features) - - # Split along last dimension and apply SwiGLU - x, gate = hidden_states.chunk(2, dim=-1) - hidden_states = self.act(gate) * x - - hidden_states = self.linear_2(hidden_states) - return hidden_states - - def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor: # B, S, D - batch_size, seq_length, _ = image_features.shape - height = width = int(seq_length**0.5) - image_features = image_features.reshape( - image_features.shape[0], width, height, -1 - ) - channels = image_features.shape[-1] - image_features = image_features.reshape( - batch_size, - width, - int(height / self.downsample_factor), - int(channels * self.downsample_factor), - ) - image_features = image_features.permute(0, 2, 1, 3) - image_features = image_features.reshape( - batch_size, - int(height / self.downsample_factor), - int(width / self.downsample_factor), - -1, - ) - image_features = image_features.permute(0, 2, 1, 3) - return image_features - - -class AyaVisionProcessingInfo(BaseProcessingInfo): - def get_hf_config(self) -> AyaVisionConfig: - return self.ctx.get_hf_config(AyaVisionConfig) - - def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor: - return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs) - - def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor: - return self.get_hf_processor(**kwargs).image_processor - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"image": None} - - def get_image_size_with_most_features(self) -> ImageSize: - image_processor = self.get_image_processor() - height = image_processor.size["height"] - width = image_processor.size["width"] - max_patches = image_processor.max_patches - return ImageSize(height=height * max_patches, width=width * max_patches) - - def get_num_patches( - self, - *, - image_width: int, - image_height: int, - size: dict, - min_patches: int, - max_patches: int, - ) -> int: - """ - Calculate the number of patches needed for a given image based on size - constraints. This method replicates and adjusts the logic from: - transformers/models/got_ocr2/image_processing_got_ocr2 - """ - size = get_size_dict(size, default_to_square=False) - num_columns, num_rows = get_optimal_tiled_canvas( - (image_height, image_width), - (size["height"], size["width"]), - min_patches, - max_patches, - ) - num_blocks = num_columns * num_rows - return num_blocks if num_blocks == 1 else num_blocks + 1 - - -class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo]): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_images = mm_counts.get("image", 0) - - processor = self.info.get_hf_processor() - image_token = processor.image_token - - return image_token * num_images - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], - ) -> MultiModalDataDict: - num_images = mm_counts.get("image", 0) - image_size = self.info.get_image_size_with_most_features() - - image_overrides = mm_options.get("image") - - return { - "image": self._get_dummy_images( - width=image_size.width, - height=image_size.height, - num_images=num_images, - overrides=image_overrides, - ) - } - - -class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingInfo]): - def _call_hf_processor( - self, - prompt: str, - mm_data: Mapping[str, object], - mm_kwargs: Mapping[str, object], - tok_kwargs: Mapping[str, object], - ) -> BatchFeature: - processed_outputs = super()._call_hf_processor( - prompt, - mm_data, - mm_kwargs, - tok_kwargs, - ) - hf_processor = self.info.get_hf_processor(**mm_kwargs) - image_processor = hf_processor.image_processor - - # HF processor pops the `num_patches` kwarg, which is needed by vLLM - if (images := mm_data.get("images")) is not None: - mm_items = self.info.parse_mm_data({"image": images}, validate=False) - parsed_images = mm_items.get_items("image", ImageProcessorItems) - image_sizes = [ - parsed_images.get_image_size(i) for i in range(len(parsed_images)) - ] - - num_patches = [ - self.info.get_num_patches( - image_width=image_size.width, - image_height=image_size.height, - size=image_processor.size, - min_patches=image_processor.min_patches, - max_patches=image_processor.max_patches, - ) - for image_size in image_sizes - ] - processed_outputs["num_patches"] = torch.tensor(num_patches) - - return processed_outputs - - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - num_patches = hf_inputs.get("num_patches", torch.empty(0)) - return dict( - pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches), - num_patches=MultiModalFieldConfig.batched("image"), - image_embeds=MultiModalFieldConfig.batched("image"), - ) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - image_token = hf_processor.image_token - img_patch_token = hf_processor.img_patch_token - image_processor = hf_processor.image_processor - - def get_replacement(item_idx: int): - images = mm_items.get_items("image", ImageProcessorItems) - image_size: ImageSize = images.get_image_size(item_idx) - num_patches = self.info.get_num_patches( - image_width=image_size.width, - image_height=image_size.height, - size=image_processor.size, - min_patches=image_processor.min_patches, - max_patches=image_processor.max_patches, - ) - repl = hf_processor._prompt_split_image(num_patches=num_patches) - - return PromptUpdateDetails.select_text(repl, img_patch_token) - - return [ - PromptReplacement( - modality="image", - target=image_token, - replacement=get_replacement, - ) - ] - - -def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int: - feature_layers = hf_config.vision_feature_layer - num_hidden_layers = hf_config.vision_config.num_hidden_layers - # If we have one feature layer, initialize up to that layer - if isinstance(feature_layers, int): - return get_layer_index(feature_layers, num_hidden_layers) - # If we have multiple feature layers, initialize up to the deepest m - elif isinstance(feature_layers, (list, tuple)): - return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers) - raise TypeError( - f"vision_layer_feature type: {type(feature_layers)} is not supported" - ) - - -@MULTIMODAL_REGISTRY.register_processor( - AyaVisionMultiModalProcessor, - info=AyaVisionProcessingInfo, - dummy_inputs=AyaVisionDummyInputsBuilder, -) -class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_prefix={ - # mapping for new names in checkpoint saved after transformers v4.52 - "model.language_model.": "language_model.model.", - "model.vision_tower.": "vision_tower.", - "model.multi_modal_projector.": "multi_modal_projector.", - "lm_head.": "language_model.lm_head.", - } - ) - - @classmethod - def get_placeholder_str(cls, modality: str, i: int) -> str | None: - if modality.startswith("image"): - return "" - - raise ValueError("Only image modality is supported") - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__() - config: AyaVisionConfig = vllm_config.model_config.hf_config - quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config - num_hidden_layers = _get_num_hidden_layers(config) - self.config = config - self.quant_config = quant_config - self.multimodal_config = multimodal_config - - with self._mark_tower_model(vllm_config, "image"): - self.vision_tower = SiglipVisionModel( - config.vision_config, - quant_config, - num_hidden_layers_override=num_hidden_layers, - prefix=maybe_prefix(prefix, "vision_model"), - ) - self.multi_modal_projector = AyaVisionMultiModalProjector(config) - - with self._mark_language_model(vllm_config): - self.language_model = init_vllm_registered_model( - vllm_config=vllm_config, - hf_config=config.text_config, - prefix=maybe_prefix(prefix, "model"), - # Cohere2ForCausalLM and CohereForCausalLM are the same on vllm - architectures=["Cohere2ForCausalLM"], - ) - - self.make_empty_intermediate_tensors = ( - self.language_model.make_empty_intermediate_tensors - ) - - @property - def dtype(self): - return next(self.parameters()).dtype - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - def _image_pixels_to_features( - self, - vision_tower: SiglipVisionModel, - pixel_values: torch.Tensor, - ) -> torch.Tensor | tuple[torch.Tensor, ...]: - return vision_tower( - pixel_values.to(dtype=vision_tower.dtype), - feature_select_strategy=self.config.vision_feature_select_strategy, - ) - - def _process_image_input( - self, image_input: AyaVisionImagePixelInputs, **kwargs - ) -> list[torch.Tensor]: - pixel_values = image_input["pixel_values"] - num_patches = image_input["num_patches"] - image_features = self._image_pixels_to_features( - self.vision_tower, pixel_values=pixel_values - ) - image_embeds = self.multi_modal_projector(image_features) - return [e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())] - - def _parse_and_validate_image_input( - self, **kwargs: object - ) -> AyaVisionImagePixelInputs | None: - pixel_values = kwargs.pop("pixel_values", None) - num_patches = kwargs.pop("num_patches", None) - image_embeds = kwargs.pop("image_embeds", None) - assert image_embeds is None, "Aya Vision does not support image_embeds." - - if pixel_values is None: - return None - - return AyaVisionImagePixelInputs( - type="pixel_values", - pixel_values=pixel_values, - num_patches=num_patches, - resolve_bindings={ - "h": self.config.vision_config.image_size, - "w": self.config.vision_config.image_size, - }, - ) - - def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: - image_input = self._parse_and_validate_image_input(**kwargs) - if image_input is None: - return [] - - return self._process_image_input(image_input, **kwargs) - - def forward( - self, - input_ids: torch.Tensor | None, - positions: torch.Tensor, - intermediate_tensors: IntermediateTensors | None = None, - inputs_embeds: torch.Tensor | None = None, - **kwargs: object, - ) -> torch.Tensor | IntermediateTensors: - if intermediate_tensors is not None: - inputs_embeds = None - - hidden_states = self.language_model.model( - input_ids=input_ids, - positions=positions, - intermediate_tensors=intermediate_tensors, - inputs_embeds=inputs_embeds, - ) - return hidden_states - - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - return self.language_model.compute_logits(hidden_states) diff --git a/vllm/model_executor/models/musicflamingo.py b/vllm/model_executor/models/musicflamingo.py deleted file mode 100644 index 509121695f..0000000000 --- a/vllm/model_executor/models/musicflamingo.py +++ /dev/null @@ -1,449 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Copyright 2026 The vLLM team. -# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights -# reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections.abc import Callable, Mapping, Sequence -from math import pi -from typing import Annotated, Any, Optional, TypeAlias - -import torch -from torch import Tensor, broadcast_tensors, nn -from transformers import BatchFeature -from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS -from transformers.models.musicflamingo import ( - MusicFlamingoConfig, - MusicFlamingoProcessor, -) - -from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions -from vllm.inputs import MultiModalDataDict -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.inputs import ( - MultiModalFieldConfig, - MultiModalKwargsItems, -) -from vllm.multimodal.parse import ( - DictEmbeddingItems, - ModalityData, - ModalityDataItems, - MultiModalDataItems, - MultiModalDataParser, -) -from vllm.multimodal.processing import ( - PromptReplacement, - PromptUpdate, - PromptUpdateDetails, -) -from vllm.utils.tensor_schema import TensorShape - -from .audioflamingo3 import ( - AudioFlamingo3DummyInputsBuilder, - AudioFlamingo3EmbeddingInputs, - AudioFlamingo3Encoder, - AudioFlamingo3FeatureInputs, - AudioFlamingo3ForConditionalGeneration, - AudioFlamingo3MultiModalDataParser, - AudioFlamingo3MultiModalProcessor, - AudioFlamingo3MultiModalProjector, - AudioFlamingo3ProcessingInfo, - _audioflamingo3_field_config, - _count_audio_tokens_from_mask, -) - - -def rotate_half(x): - x = x.reshape(*x.shape[:-1], -1, 2) - x1, x2 = x.unbind(dim=-1) - x = torch.stack((-x2, x1), dim=-1) - return x.flatten(-2) - - -def apply_rotary_time_emb(hidden_states, cos, sin): - original_dtype = hidden_states.dtype - hidden_states = hidden_states.to(torch.float64) - cos = cos.to(hidden_states) - sin = sin.to(hidden_states) - rot_dim = cos.shape[-1] - if rot_dim > hidden_states.shape[-1]: - raise ValueError( - f"feature dimension {hidden_states.shape[-1]} is not of " - f"sufficient size to rotate in all the positions {rot_dim}" - ) - - rotated = hidden_states[..., :rot_dim] - passthrough = hidden_states[..., rot_dim:] - rotated = (rotated * cos) + (rotate_half(rotated) * sin) - return torch.cat((rotated, passthrough), dim=-1).to(original_dtype) - - -class MusicFlamingoRotaryEmbedding(nn.Module): - inv_freq: torch.Tensor - - def __init__(self, config: MusicFlamingoConfig, device=None): - super().__init__() - self.max_seq_len_cached = config.max_position_embeddings - self.original_max_seq_len = config.max_position_embeddings - - self.config = config - self.rope_type = self.config.rope_parameters["rope_type"] - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn(self.config, device) - - self.register_buffer("inv_freq", inv_freq, persistent=False) - self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) - position_angles = self._compute_position_angles(self.inv_freq) - self.register_buffer("position_angles", position_angles, persistent=False) - - @staticmethod - def compute_default_rope_parameters( - config: MusicFlamingoConfig | None = None, - device: Optional["torch.device"] = None, - seq_len: int | None = None, - ) -> tuple["torch.Tensor", float]: - del seq_len - base = config.rope_parameters["rope_theta"] - partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0) - head_dim = getattr(config, "head_dim", None) or ( - config.hidden_size // config.num_attention_heads - ) - dim = int(head_dim * partial_rotary_factor) - attention_factor = 1.0 - - inv_freq = 1.0 / ( - base - ** ( - torch.arange(0, dim, 2, dtype=torch.int64).to( - device=device, - dtype=torch.float, - ) - / dim - ) - ) - return inv_freq, attention_factor - - def _compute_position_angles(self, inv_freq): - positions = torch.arange( - int(self.max_seq_len_cached), - device=inv_freq.device, - dtype=inv_freq.dtype, - ) - positions = positions / self.max_seq_len_cached * (2 * pi) - position_angles = positions.unsqueeze(-1) * inv_freq - position_angles = torch.repeat_interleave(position_angles, 2, dim=-1) - return position_angles.to(dtype=inv_freq.dtype) - - def _restore_fp32_rope_buffers(self) -> None: - rope_init_fn: Callable = self.compute_default_rope_parameters - if self.rope_type != "default": - rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] - inv_freq, self.attention_scaling = rope_init_fn( - self.config, self.inv_freq.device - ) - - self.inv_freq = inv_freq - self.original_inv_freq = inv_freq.clone() - self.position_angles = self._compute_position_angles(inv_freq) - - def _apply(self, fn): - super()._apply(fn) - self._restore_fp32_rope_buffers() - return self - - @torch.no_grad() - def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]: - window_starts = timestamps[:, 0].to( - device=self.inv_freq.device, - dtype=self.inv_freq.dtype, - ) - window_duration = self.config.audio_frame_step * 4 * seq_len - window_positions = ( - torch.round(window_starts / window_duration) / self.max_seq_len_cached - ) - window_freqs = window_positions.unsqueeze(-1) * self.inv_freq - window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1) - - window_freqs = window_freqs[:, None, :] - time_freqs = self.position_angles[:seq_len][None, :, :] - window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs) - freqs = torch.cat((window_freqs, time_freqs), dim=-1) - angle = (-timestamps * 2 * pi).to(freqs) - freqs = freqs * angle.unsqueeze(-1) - return freqs.cos(), freqs.sin() - - -class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs): - rote_timestamps: Annotated[ - torch.Tensor | None, - TensorShape( - "num_chunks", - "num_audio_time_steps", - dynamic_dims={"num_audio_time_steps"}, - ), - ] - - -MusicFlamingoEmbeddingInputs = AudioFlamingo3EmbeddingInputs - -MusicFlamingoInputs: TypeAlias = ( - MusicFlamingoFeatureInputs | MusicFlamingoEmbeddingInputs -) - - -class MusicFlamingoEncoder(AudioFlamingo3Encoder): - pass - - -class MusicFlamingoMultiModalProjector(AudioFlamingo3MultiModalProjector): - pass - - -class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo): - def get_hf_config(self) -> MusicFlamingoConfig: - return self.ctx.get_hf_config(MusicFlamingoConfig) - - def get_hf_processor(self, **kwargs: object) -> MusicFlamingoProcessor: - return self.ctx.get_hf_processor(MusicFlamingoProcessor, **kwargs) - - def get_data_parser(self) -> MultiModalDataParser: - feature_extractor = self.get_feature_extractor() - return MusicFlamingoMultiModalDataParser( - target_sr=feature_extractor.sampling_rate, - audio_resample_method="soxr", - expected_hidden_size=self._get_expected_hidden_size(), - ) - - def get_supported_mm_limits(self) -> Mapping[str, int | None]: - return {"audio": 1} - - -class MusicFlamingoDummyInputsBuilder(AudioFlamingo3DummyInputsBuilder): - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: - num_audios = mm_counts.get("audio", 0) - hf_processor = self.info.get_hf_processor() - return hf_processor.audio_token * num_audios - - def get_dummy_mm_data( - self, - seq_len: int, - mm_counts: Mapping[str, int], - mm_options: Mapping[str, BaseDummyOptions], - ) -> MultiModalDataDict: - hf_processor = self.info.get_hf_processor() - feature_extractor = self.info.get_feature_extractor() - sampling_rate = feature_extractor.sampling_rate - audio_len = int(hf_processor.max_audio_len * sampling_rate) - num_audios = mm_counts.get("audio", 0) - audio_overrides = mm_options.get("audio") - - return { - "audio": self._get_dummy_audios( - length=audio_len, - num_audios=num_audios, - overrides=audio_overrides, - ) - } - - -def _musicflamingo_field_config(hf_inputs: Mapping[str, torch.Tensor]): - fields = dict(_audioflamingo3_field_config(hf_inputs)) - chunk_counts = hf_inputs.get("chunk_counts") - if chunk_counts is not None: - fields["rote_timestamps"] = MultiModalFieldConfig.flat_from_sizes( - "audio", chunk_counts, dim=0 - ) - else: - fields["rote_timestamps"] = MultiModalFieldConfig.batched("audio") - return fields - - -class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser): - def _parse_audio_data( - self, - data: dict[str, torch.Tensor] | ModalityData[Any], - ) -> ModalityDataItems[Any, Any] | None: - if isinstance(data, dict): - return DictEmbeddingItems( - data, - modality="audio", - required_fields={"audio_embeds"}, - fields_factory=_musicflamingo_field_config, - ) - return super()._parse_audio_data(data) - - -class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor): - def _get_mm_fields_config( - self, - hf_inputs: BatchFeature, - hf_processor_mm_kwargs: Mapping[str, object], - ) -> Mapping[str, MultiModalFieldConfig]: - return _musicflamingo_field_config(hf_inputs) - - def _get_prompt_updates( - self, - mm_items: MultiModalDataItems, - hf_processor_mm_kwargs: Mapping[str, object], - out_mm_kwargs: MultiModalKwargsItems, - ) -> Sequence[PromptUpdate]: - processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) - tokenizer = self.info.get_tokenizer() - vocab = tokenizer.get_vocab() - - audio_token = processor.audio_token - audio_token_id = vocab.get(audio_token, processor.audio_token_id) - - audio_bos_token = processor.audio_bos_token - audio_bos_token_id = vocab.get(audio_bos_token, processor.audio_bos_token_id) - - audio_eos_token = processor.audio_eos_token - audio_eos_token_id = vocab.get(audio_eos_token, processor.audio_eos_token_id) - - out_mm_data = out_mm_kwargs.get_data() - feature_attention_mask = out_mm_data.get("feature_attention_mask") - chunk_counts = out_mm_data.get("chunk_counts") - - def get_replacement_musicflamingo(item_idx: int): - if feature_attention_mask is not None: - num_features = _count_audio_tokens_from_mask( - feature_attention_mask, - chunk_counts, - item_idx, - ) - else: - audio_embeds = out_mm_data["audio_embeds"][item_idx] - num_features = audio_embeds.shape[0] - - if num_features == 0: - raise ValueError("Audio is too short") - - full_tokens = [ - audio_bos_token_id, - *([audio_token_id] * int(num_features)), - audio_eos_token_id, - ] - - return PromptUpdateDetails.select_token_id( - full_tokens, - embed_token_id=audio_token_id, - ) - - return [ - PromptReplacement( - modality="audio", - target=audio_token, - replacement=get_replacement_musicflamingo, - ) - ] - - -@MULTIMODAL_REGISTRY.register_processor( - MusicFlamingoMultiModalProcessor, - info=MusicFlamingoProcessingInfo, - dummy_inputs=MusicFlamingoDummyInputsBuilder, -) -class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGeneration): - """vLLM MusicFlamingo model aligned with HF modular_musicflamingo.""" - - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): - super().__init__(vllm_config=vllm_config, prefix=prefix) - self.audio_tower = MusicFlamingoEncoder(self.config.audio_config) - self.multi_modal_projector = MusicFlamingoMultiModalProjector(self.config) - self.pos_emb = MusicFlamingoRotaryEmbedding(self.config) - - def _parse_and_validate_audio_input( - self, **kwargs: object - ) -> MusicFlamingoInputs | None: - rote_timestamps = kwargs.pop("rote_timestamps", None) - audio_input = super()._parse_and_validate_audio_input(**kwargs) - if audio_input is None or audio_input["type"] == "audio_embeds": - return audio_input - - return MusicFlamingoFeatureInputs( - type="audio_features", - input_features=audio_input["input_features"], - feature_attention_mask=audio_input["feature_attention_mask"], - chunk_counts=audio_input["chunk_counts"], - rote_timestamps=rote_timestamps, - ) - - def _build_audio_timestamps( - self, - chunk_counts: list[int], - seq_len: int, - device: torch.device, - ) -> torch.Tensor: - audio_embed_frame_step = self.config.audio_frame_step * 4 - frame_offsets = ( - torch.arange(seq_len, device=device, dtype=torch.float32) - * audio_embed_frame_step - ) - - if not chunk_counts: - return frame_offsets.new_empty((0, seq_len)) - - window_indices = torch.cat( - [ - torch.arange(count, device=device, dtype=torch.float32) - for count in chunk_counts - ] - ) - return ( - window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step - + frame_offsets - ) - - def _process_audio_input( - self, audio_input: MusicFlamingoInputs - ) -> torch.Tensor | tuple[torch.Tensor, ...]: - if audio_input["type"] == "audio_embeds": - return super()._process_audio_input(audio_input) - - rote_timestamps = audio_input["rote_timestamps"] - ( - input_features, - feature_attention_mask, - chunk_counts, - ) = self._normalize_audio_feature_inputs(audio_input) - hidden_states = self._encode_audio_features( - input_features, - feature_attention_mask, - ) - if rote_timestamps is None: - rote_timestamps = self._build_audio_timestamps( - chunk_counts, - seq_len=hidden_states.shape[-2], - device=hidden_states.device, - ) - elif isinstance(rote_timestamps, list): - rote_timestamps = torch.cat(rote_timestamps, dim=0) - - cos, sin = self.pos_emb( - rote_timestamps.to(hidden_states.device), - seq_len=hidden_states.shape[-2], - ) - hidden_states = apply_rotary_time_emb(hidden_states, cos, sin) - audio_features = self.multi_modal_projector(hidden_states) - - return self._group_audio_embeddings( - audio_features, - feature_attention_mask, - chunk_counts, - ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 5603a0bb1b..79519c8057 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -337,14 +337,6 @@ _MULTIMODAL_MODELS = { "audioflamingo3", "AudioFlamingo3ForConditionalGeneration", ), - "MusicFlamingoForConditionalGeneration": ( - "musicflamingo", - "MusicFlamingoForConditionalGeneration", - ), - "AyaVisionForConditionalGeneration": ( - "aya_vision", - "AyaVisionForConditionalGeneration", - ), "BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"), "BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"), "Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"), @@ -733,6 +725,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = { "TarsierForConditionalGeneration": "0.24.0", "Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4 "MantisForConditionalGeneration": "0.24.0", + "MusicFlamingoForConditionalGeneration": "0.24.0", + "AyaVisionForConditionalGeneration": "0.24.0", } _OOT_SUPPORTED_MODELS = { diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index e372834d68..3b39c91109 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -345,7 +345,7 @@ class ModelArchConfigConvertorBase: max_len_key = key derived_max_model_len = min(derived_max_model_len, max_len) - # For Command-R / Cohere, Cohere2 / Aya Vision models + # For Command-R / Cohere, Cohere2 models if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None): max_len_key = "model_max_length" derived_max_model_len = tmp_max_len