diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 099ddb2e52..9a86a57502 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -532,7 +532,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
| `AriaForConditionalGeneration` | Aria | T + I+ | `rhymes-ai/Aria` | | |
| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |
-| `AyaVisionForConditionalGeneration` | Aya Vision | T + I+ | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
| `BagelForConditionalGeneration` | BAGEL | T + I+ | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
| `BeeForConditionalGeneration` | Bee-8B | T + IE+ | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
| `Blip2ForConditionalGeneration` | BLIP-2 | T + IE | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
@@ -594,7 +593,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `Molmo2ForConditionalGeneration` | Molmo2 | T + I+ / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`^, `allenai/MolmoWeb-8B`^ | ✅︎ | ✅︎ |
| `MossAudioModel` | MOSS-Audio | T + A+ | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ |
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
-| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
| `NVLM_D_Model` | NVLM-D 1.0 | T + I+ | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + IE+ | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
| `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + IE+ + VE+ | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
diff --git a/examples/generate/multimodal/audio_language_offline.py b/examples/generate/multimodal/audio_language_offline.py
index 12a38cf41c..fc20e8fed1 100644
--- a/examples/generate/multimodal/audio_language_offline.py
+++ b/examples/generate/multimodal/audio_language_offline.py
@@ -91,44 +91,6 @@ def run_cohere_asr(question: str, audio_count: int) -> ModelRequestData:
)
-# MusicFlamingo
-def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData:
- model_name = "nvidia/music-flamingo-2601-hf"
- engine_args = EngineArgs(
- model=model_name,
- max_model_len=4096,
- max_num_seqs=2,
- limit_mm_per_prompt={"audio": audio_count},
- enforce_eager=True,
- )
-
- # MusicFlamingo prompt placeholders use ; vLLM's MusicFlamingo
- # multimodal processor expands each one into <|sound_bos|> + audio tokens +
- # <|sound_eos|> based on extracted audio feature lengths.
- audio_placeholder = "" * audio_count
- system_prompt = (
- "You are Music Flamingo, a multimodal assistant for language and music. "
- "On each turn you receive an audio clip which contains music and optional "
- "text, you will receive at least one or both; use your world knowledge and "
- "reasoning to help the user with any task. Interpret the entirety of the "
- "content any input music--regardlenss of whether the user calls it audio, "
- "music, or sound."
- )
-
- prompt = (
- "<|im_start|>system\n"
- f"{system_prompt}<|im_end|>\n"
- "<|im_start|>user\n"
- f"{audio_placeholder}{question}<|im_end|>\n"
- "<|im_start|>assistant\n"
- )
-
- return ModelRequestData(
- engine_args=engine_args,
- prompt=prompt,
- )
-
-
# Gemma3N
def run_gemma3n(question: str, audio_count: int) -> ModelRequestData:
model_name = "google/gemma-3n-E2B-it"
@@ -565,7 +527,6 @@ model_example_map = {
"kimi_audio": run_kimi_audio,
"midashenglm": run_midashenglm,
"minicpmo": run_minicpmo,
- "musicflamingo": run_musicflamingo,
"phi4_mm": run_phi4mm,
"qwen2_audio": run_qwen2_audio,
"qwen2_5_omni": run_qwen2_5_omni,
diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py
index c3bec9d5fd..c354142774 100644
--- a/examples/generate/multimodal/vision_language_multi_image_offline.py
+++ b/examples/generate/multimodal/vision_language_multi_image_offline.py
@@ -74,39 +74,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
)
-def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:
- model_name = "CohereLabs/aya-vision-8b"
-
- engine_args = EngineArgs(
- model=model_name,
- max_num_seqs=2,
- limit_mm_per_prompt={"image": len(image_urls)},
- )
-
- placeholders = [{"type": "image", "image": url} for url in image_urls]
- messages = [
- {
- "role": "user",
- "content": [
- *placeholders,
- {"type": "text", "text": question},
- ],
- }
- ]
-
- processor = AutoProcessor.from_pretrained(model_name)
-
- prompt = processor.apply_chat_template(
- messages, tokenize=False, add_generation_prompt=True
- )
-
- return ModelRequestData(
- engine_args=engine_args,
- prompt=prompt,
- image_data=[fetch_image(url) for url in image_urls],
- )
-
-
def load_bee(question: str, image_urls: list[str]) -> ModelRequestData:
model_name = "Open-Bee/Bee-8B-RL"
@@ -1420,7 +1387,6 @@ def load_molmo2(question: str, image_urls: list[str]) -> ModelRequestData:
model_example_map = {
"aria": load_aria,
- "aya_vision": load_aya_vision,
"bee": load_bee,
"command_a_vision": load_command_a_vision,
"deepseek_vl_v2": load_deepseek_vl2,
diff --git a/examples/generate/multimodal/vision_language_offline.py b/examples/generate/multimodal/vision_language_offline.py
index 46765ca038..a1eb6cc538 100644
--- a/examples/generate/multimodal/vision_language_offline.py
+++ b/examples/generate/multimodal/vision_language_offline.py
@@ -68,28 +68,6 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:
)
-# Aya Vision
-def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
- assert modality == "image"
- model_name = "CohereLabs/aya-vision-8b"
-
- engine_args = EngineArgs(
- model=model_name,
- max_model_len=2048,
- max_num_seqs=2,
- mm_processor_kwargs={"crop_to_patches": True},
- limit_mm_per_prompt={modality: 1},
- )
- prompts = [
- f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
- for question in questions
- ]
- return ModelRequestData(
- engine_args=engine_args,
- prompts=prompts,
- )
-
-
# Bee-8B
def run_bee(questions: list[str], modality: str) -> ModelRequestData:
assert modality == "image"
@@ -2327,7 +2305,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
model_example_map = {
"aria": run_aria,
- "aya_vision": run_aya_vision,
"bagel": run_bagel,
"cheers": run_cheers,
"bee": run_bee,
diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py
index e245d2b899..ff532fd878 100644
--- a/tests/models/multimodal/generation/test_common.py
+++ b/tests/models/multimodal/generation/test_common.py
@@ -326,39 +326,6 @@ VLM_TEST_SETTINGS = {
large_gpu_mark(min_gb=64),
],
),
- "aya_vision": VLMTestInfo(
- models=["CohereLabs/aya-vision-8b"],
- test_type=(VLMTestType.IMAGE),
- prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
- single_image_prompts=IMAGE_ASSETS.prompts(
- {
- "stop_sign": "What's the content in the center of the image?",
- "cherry_blossom": "What is the season?",
- }
- ),
- multi_image_prompt="Describe the two images in detail.",
- max_model_len=4096,
- max_num_seqs=2,
- auto_cls=AutoModelForImageTextToText,
- vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},
- ),
- "aya_vision-multi_image": VLMTestInfo(
- models=["CohereLabs/aya-vision-8b"],
- test_type=(VLMTestType.MULTI_IMAGE),
- prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
- single_image_prompts=IMAGE_ASSETS.prompts(
- {
- "stop_sign": "What's the content in the center of the image?",
- "cherry_blossom": "What is the season?",
- }
- ),
- multi_image_prompt="Describe the two images in detail.",
- max_model_len=4096,
- max_num_seqs=2,
- auto_cls=AutoModelForImageTextToText,
- vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},
- marks=[large_gpu_mark(min_gb=32)],
- ),
"blip2": VLMTestInfo(
models=["Salesforce/blip2-opt-2.7b"],
test_type=VLMTestType.IMAGE,
diff --git a/tests/models/multimodal/generation/test_musicflamingo.py b/tests/models/multimodal/generation/test_musicflamingo.py
deleted file mode 100644
index 625fbd775d..0000000000
--- a/tests/models/multimodal/generation/test_musicflamingo.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import json
-import os
-
-import pytest
-
-from tests.models.registry import HF_EXAMPLE_MODELS
-from vllm import LLM, SamplingParams
-
-MODEL_NAME = "nvidia/music-flamingo-2601-hf"
-SINGLE_CONVERSATION = [
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": "Describe this track in full detail - tell me the "
- "genre, tempo, and key, then dive into the instruments, "
- "production style, and overall mood it creates.",
- },
- {
- "type": "audio_url",
- "audio_url": {
- "url": "https://huggingface.co/datasets/nvidia/AudioSkills/"
- "resolve/main/assets/song_1.mp3",
- },
- },
- ],
- }
-]
-BATCHED_CONVERSATIONS = [
- SINGLE_CONVERSATION,
- [
- {
- "role": "user",
- "content": [
- {
- "type": "text",
- "text": "Generate a structured lyric sheet from the input music.",
- },
- {
- "type": "audio_url",
- "audio_url": {
- "url": "https://huggingface.co/datasets/nvidia/"
- "AudioSkills/resolve/main/assets/song_2.mp3",
- },
- },
- ],
- }
- ],
-]
-
-
-def get_fixture_path(filename):
- return os.path.join(
- os.path.dirname(__file__), "../../fixtures/musicflamingo", filename
- )
-
-
-def load_expected_fixture(filename):
- fixture_path = get_fixture_path(filename)
- with open(fixture_path) as f:
- return json.load(f)
-
-
-def assert_output_matches(output, expected_text, expected_token_ids):
- generated = output.outputs[0]
- assert generated.text == expected_text
- actual_token_ids = list(generated.token_ids)
- assert (
- actual_token_ids == expected_token_ids
- or actual_token_ids == expected_token_ids[:-1]
- or actual_token_ids[:-1] == expected_token_ids
- )
-
-
-@pytest.fixture(scope="module")
-def llm():
- model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration")
- model_info.check_transformers_version(on_fail="skip")
-
- try:
- llm = LLM(
- model=MODEL_NAME,
- dtype="bfloat16",
- enforce_eager=True,
- max_model_len=8192,
- limit_mm_per_prompt={"audio": 1},
- )
- except Exception as e:
- pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
-
- # ROCm may compile decoder kernels on the first inference pass; warm up
- # once so exact fixture assertions cover the steady-state path.
- llm.chat(
- messages=SINGLE_CONVERSATION,
- sampling_params=SamplingParams(temperature=0.0, max_tokens=1),
- use_tqdm=False,
- )
-
- return llm
-
-
-def test_single_generation(llm):
- expected = load_expected_fixture("expected_results_single.json")
-
- outputs = llm.chat(
- messages=SINGLE_CONVERSATION,
- sampling_params=SamplingParams(temperature=0.0, max_tokens=50),
- )
-
- assert_output_matches(
- outputs[0],
- expected["transcriptions"][0],
- expected["token_ids"][0],
- )
-
-
-def test_batched_generation(llm):
- expected = load_expected_fixture("expected_results_batched.json")
-
- outputs = llm.chat(
- messages=BATCHED_CONVERSATIONS,
- sampling_params=SamplingParams(temperature=0.0, max_tokens=50),
- )
-
- for i, output in enumerate(outputs):
- assert_output_matches(
- output,
- expected["transcriptions"][i],
- expected["token_ids"][i],
- )
-
-
-def test_single_and_batched_generation_match(llm):
- sampling_params = SamplingParams(temperature=0.0, max_tokens=50)
-
- single_output = llm.chat(
- messages=SINGLE_CONVERSATION,
- sampling_params=sampling_params,
- )[0]
- batched_output = llm.chat(
- messages=BATCHED_CONVERSATIONS,
- sampling_params=sampling_params,
- )[0]
-
- assert single_output.outputs[0].text == batched_output.outputs[0].text
- assert list(single_output.outputs[0].token_ids) == list(
- batched_output.outputs[0].token_ids
- )
diff --git a/tests/models/multimodal/processing/test_musicflamingo.py b/tests/models/multimodal/processing/test_musicflamingo.py
deleted file mode 100644
index bb3d200d6f..0000000000
--- a/tests/models/multimodal/processing/test_musicflamingo.py
+++ /dev/null
@@ -1,228 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# Copyright 2026 The vLLM team.
-# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
-# reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from importlib.metadata import version
-from unittest.mock import MagicMock
-
-import numpy as np
-import pytest
-import torch
-from packaging.version import Version
-from transformers import PretrainedConfig
-
-from tests.models.registry import HF_EXAMPLE_MODELS
-
-
-class MockMusicFlamingoConfig(PretrainedConfig):
- model_type = "musicflamingo"
-
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
- self.audio_config = PretrainedConfig()
- self.text_config = PretrainedConfig()
-
-
-class MockMusicFlamingoProcessor:
- def __init__(self):
- self.audio_token = ""
- self.audio_token_id = 12345
- self.audio_bos_token = "<|sound_bos|>"
- self.audio_bos_token_id = 12346
- self.audio_eos_token = "<|sound_eos|>"
- self.audio_eos_token_id = 12347
- self.max_audio_len = 1200
- self.feature_extractor = MockFeatureExtractor()
-
- def __call__(self, text=None, audio=None, **kwargs):
- return {
- "input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
- "input_features": torch.zeros((3, 80, 3000)),
- "input_features_mask": torch.ones((3, 3000), dtype=torch.long),
- }
-
-
-class MockFeatureExtractor:
- def __init__(self):
- self.sampling_rate = 16000
- self.chunk_length = 30
-
-
-@pytest.fixture
-def mock_ctx():
- config = MockMusicFlamingoConfig()
-
- ctx = MagicMock()
- ctx.get_hf_config.return_value = config
- ctx.get_hf_processor.return_value = MockMusicFlamingoProcessor()
- ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
- **data, **kwargs
- )
- ctx.model_config.hf_config = config
- return ctx
-
-
-@pytest.fixture(autouse=True)
-def check_transformers_version():
- model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration")
- model_info.check_transformers_version(on_fail="skip")
-
-
-def test_musicflamingo_chunk_counting_without_rote_timestamps(mock_ctx):
- from vllm.model_executor.models.musicflamingo import (
- MusicFlamingoDummyInputsBuilder,
- MusicFlamingoMultiModalProcessor,
- MusicFlamingoProcessingInfo,
- )
-
- info = MusicFlamingoProcessingInfo(mock_ctx)
- processor = MusicFlamingoMultiModalProcessor(
- info, MusicFlamingoDummyInputsBuilder(info)
- )
-
- sr = 16000
- audio_1 = np.zeros(30 * sr)
- audio_2 = np.zeros(45 * sr)
-
- mm_data = {"audio": [audio_1, audio_2]}
- prompt = "<|user|>Listen.<|end|>"
-
- processed = processor._call_hf_processor(prompt, mm_data, {}, {})
-
- chunk_counts = processed["chunk_counts"]
-
- assert chunk_counts.tolist() == [1, 2]
- assert "rote_timestamps" not in processed
- assert processed["feature_attention_mask"].shape == (3, 3000)
-
-
-def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):
- from vllm.model_executor.models.musicflamingo import (
- MusicFlamingoDummyInputsBuilder,
- MusicFlamingoProcessingInfo,
- )
-
- info = MusicFlamingoProcessingInfo(mock_ctx)
- builder = MusicFlamingoDummyInputsBuilder(info)
-
- assert builder.get_dummy_text({"audio": 2}) == ""
-
-
-@pytest.mark.skipif(
- Version(version("transformers")) >= Version("5.5"),
- reason="transformers v5.5 added native MusicFlamingoForConditionalGeneration "
- "with a different get_audio_features signature (requires input_ids)",
-)
-def test_musicflamingo_audio_feature_pipeline_matches_hf_small_config():
- from transformers.models.musicflamingo import (
- modeling_musicflamingo as hf_musicflamingo_modeling,
- )
- from transformers.models.musicflamingo.configuration_musicflamingo import (
- MusicFlamingoConfig,
- )
-
- from vllm.model_executor.models.audioflamingo3 import (
- _build_audio_encoder_attention_mask,
- _flatten_valid_audio_embeddings,
- )
- from vllm.model_executor.models.musicflamingo import (
- MusicFlamingoEncoder,
- MusicFlamingoMultiModalProjector,
- MusicFlamingoRotaryEmbedding,
- apply_rotary_time_emb,
- )
-
- text_config = {
- "model_type": "qwen2",
- "intermediate_size": 64,
- "initializer_range": 0.02,
- "hidden_size": 32,
- "max_position_embeddings": 1024,
- "num_hidden_layers": 2,
- "num_attention_heads": 4,
- "num_key_value_heads": 2,
- "vocab_size": 128,
- "pad_token_id": 1,
- "use_mrope": False,
- }
- audio_config = {
- "hidden_size": 16,
- "num_attention_heads": 4,
- "intermediate_size": 32,
- "num_hidden_layers": 2,
- "num_mel_bins": 80,
- "max_source_positions": 1500,
- "dropout": 0.0,
- "attention_dropout": 0.0,
- "activation_dropout": 0.0,
- "encoder_layerdrop": 0.0,
- }
-
- torch.manual_seed(0)
- config = MusicFlamingoConfig(
- text_config=text_config,
- audio_config=audio_config,
- audio_token_id=0,
- head_dim=8,
- rope_parameters={"rope_type": "default", "rope_theta": 2048},
- )
- hf_model = hf_musicflamingo_modeling.MusicFlamingoForConditionalGeneration(
- config
- ).eval()
-
- vllm_encoder = MusicFlamingoEncoder(config.audio_config).eval()
- vllm_encoder.load_state_dict(hf_model.audio_tower.state_dict())
-
- vllm_projector = MusicFlamingoMultiModalProjector(config).eval()
- vllm_projector.load_state_dict(hf_model.multi_modal_projector.state_dict())
-
- vllm_rope = MusicFlamingoRotaryEmbedding(config).eval()
- vllm_rope.load_state_dict(hf_model.pos_emb.state_dict(), strict=False)
-
- input_features = torch.randn(3, 80, 3000)
- feature_attention_mask = torch.zeros(3, 3000, dtype=torch.bool)
- feature_attention_mask[0, :3000] = True
- feature_attention_mask[1, :2500] = True
- feature_attention_mask[2, :1500] = True
- rote_timestamps = (
- torch.arange(750, dtype=torch.float32).unsqueeze(0).repeat(3, 1) * 0.04
- )
-
- hf_output = hf_model.get_audio_features(
- input_features,
- feature_attention_mask,
- rote_timestamps=rote_timestamps,
- return_dict=True,
- ).pooler_output
- vllm_attention_mask = _build_audio_encoder_attention_mask(
- feature_attention_mask,
- dtype=vllm_encoder.conv1.weight.dtype,
- device=vllm_encoder.conv1.weight.device,
- )
- vllm_hidden_states = vllm_encoder(
- input_features,
- attention_mask=vllm_attention_mask,
- )
- cos, sin = vllm_rope(rote_timestamps, seq_len=vllm_hidden_states.shape[-2])
- vllm_hidden_states = apply_rotary_time_emb(vllm_hidden_states, cos, sin)
- vllm_output, _ = _flatten_valid_audio_embeddings(
- vllm_projector(vllm_hidden_states),
- feature_attention_mask,
- )
-
- torch.testing.assert_close(vllm_output, hf_output)
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 34f8ca4f82..da2dc8bb9d 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -780,14 +780,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
},
),
- "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
- "nvidia/music-flamingo-2601-hf",
- min_transformers_version="5.5.0",
- transformers_version_reason={
- "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
- },
- ),
- "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
"BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
"BeeForConditionalGeneration": _HfExamplesInfo(
"Open-Bee/Bee-8B-RL",
diff --git a/vllm/model_executor/models/aya_vision.py b/vllm/model_executor/models/aya_vision.py
deleted file mode 100644
index f4c9bbee9e..0000000000
--- a/vllm/model_executor/models/aya_vision.py
+++ /dev/null
@@ -1,445 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-# Adapted from https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision
-from collections.abc import Iterable, Mapping, Sequence
-from typing import Annotated, Literal
-
-import torch
-from torch import nn
-from transformers import BatchFeature, GotOcr2ImageProcessor
-from transformers.activations import ACT2FN
-from transformers.image_processing_utils import get_size_dict
-from transformers.models.aya_vision import AyaVisionConfig
-from transformers.models.aya_vision.processing_aya_vision import AyaVisionProcessor
-from transformers.models.got_ocr2.image_processing_got_ocr2 import (
- get_optimal_tiled_canvas,
-)
-
-from vllm.config import VllmConfig
-from vllm.config.multimodal import BaseDummyOptions
-from vllm.inputs import MultiModalDataDict
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalFieldConfig,
- MultiModalKwargsItems,
-)
-from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
-from vllm.multimodal.processing import (
- BaseDummyInputsBuilder,
- BaseMultiModalProcessor,
- BaseProcessingInfo,
- PromptReplacement,
- PromptUpdate,
- PromptUpdateDetails,
-)
-from vllm.sequence import IntermediateTensors
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
-
-from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
-from .siglip import SiglipVisionModel
-from .utils import (
- AutoWeightsLoader,
- WeightsMapper,
- get_layer_index,
- init_vllm_registered_model,
- maybe_prefix,
-)
-
-
-class AyaVisionImagePixelInputs(TensorSchema):
- """
- Dimensions:
- - np: The total number of patches over each image over each prompt in
- the batch
- - c: Number of channels
- - h: Height of each image patch
- - w: Width of each image patch
- - bn: Batch size * number of images
- """
-
- type: Literal["pixel_values"]
-
- pixel_values: Annotated[
- torch.Tensor,
- TensorShape("np", 3, "h", "w"),
- ]
-
- num_patches: Annotated[
- torch.Tensor,
- TensorShape("bn"),
- ]
-
-
-class AyaVisionMultiModalProjector(nn.Module):
- def __init__(self, config: AyaVisionConfig):
- super().__init__()
- self.config = config
- self.downsample_factor = config.downsample_factor
- self.alignment_intermediate_size = getattr(
- config, "alignment_intermediate_size", config.text_config.hidden_size
- )
- self.layernorm = nn.LayerNorm(
- config.vision_config.hidden_size * (config.downsample_factor**2),
- eps=config.adapter_layer_norm_eps,
- )
-
- self.linear_1 = nn.Linear(
- config.vision_config.hidden_size * (config.downsample_factor**2),
- self.alignment_intermediate_size,
- bias=True,
- )
-
- self.act = ACT2FN["silu"] # SwiGLU uses SiLU activation
- # For SwiGLU, project down to half size since we split intermediate dim
- self.linear_2 = nn.Linear(
- self.alignment_intermediate_size // 2,
- config.text_config.hidden_size,
- bias=True,
- )
-
- def forward(self, image_features: torch.Tensor) -> torch.Tensor:
- image_features = self.pixel_shuffle(image_features)
- image_features = self.layernorm(image_features)
- hidden_states = self.linear_1(image_features)
-
- # Split along last dimension and apply SwiGLU
- x, gate = hidden_states.chunk(2, dim=-1)
- hidden_states = self.act(gate) * x
-
- hidden_states = self.linear_2(hidden_states)
- return hidden_states
-
- def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor: # B, S, D
- batch_size, seq_length, _ = image_features.shape
- height = width = int(seq_length**0.5)
- image_features = image_features.reshape(
- image_features.shape[0], width, height, -1
- )
- channels = image_features.shape[-1]
- image_features = image_features.reshape(
- batch_size,
- width,
- int(height / self.downsample_factor),
- int(channels * self.downsample_factor),
- )
- image_features = image_features.permute(0, 2, 1, 3)
- image_features = image_features.reshape(
- batch_size,
- int(height / self.downsample_factor),
- int(width / self.downsample_factor),
- -1,
- )
- image_features = image_features.permute(0, 2, 1, 3)
- return image_features
-
-
-class AyaVisionProcessingInfo(BaseProcessingInfo):
- def get_hf_config(self) -> AyaVisionConfig:
- return self.ctx.get_hf_config(AyaVisionConfig)
-
- def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:
- return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
-
- def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor:
- return self.get_hf_processor(**kwargs).image_processor
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"image": None}
-
- def get_image_size_with_most_features(self) -> ImageSize:
- image_processor = self.get_image_processor()
- height = image_processor.size["height"]
- width = image_processor.size["width"]
- max_patches = image_processor.max_patches
- return ImageSize(height=height * max_patches, width=width * max_patches)
-
- def get_num_patches(
- self,
- *,
- image_width: int,
- image_height: int,
- size: dict,
- min_patches: int,
- max_patches: int,
- ) -> int:
- """
- Calculate the number of patches needed for a given image based on size
- constraints. This method replicates and adjusts the logic from:
- transformers/models/got_ocr2/image_processing_got_ocr2
- """
- size = get_size_dict(size, default_to_square=False)
- num_columns, num_rows = get_optimal_tiled_canvas(
- (image_height, image_width),
- (size["height"], size["width"]),
- min_patches,
- max_patches,
- )
- num_blocks = num_columns * num_rows
- return num_blocks if num_blocks == 1 else num_blocks + 1
-
-
-class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo]):
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_images = mm_counts.get("image", 0)
-
- processor = self.info.get_hf_processor()
- image_token = processor.image_token
-
- return image_token * num_images
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions],
- ) -> MultiModalDataDict:
- num_images = mm_counts.get("image", 0)
- image_size = self.info.get_image_size_with_most_features()
-
- image_overrides = mm_options.get("image")
-
- return {
- "image": self._get_dummy_images(
- width=image_size.width,
- height=image_size.height,
- num_images=num_images,
- overrides=image_overrides,
- )
- }
-
-
-class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingInfo]):
- def _call_hf_processor(
- self,
- prompt: str,
- mm_data: Mapping[str, object],
- mm_kwargs: Mapping[str, object],
- tok_kwargs: Mapping[str, object],
- ) -> BatchFeature:
- processed_outputs = super()._call_hf_processor(
- prompt,
- mm_data,
- mm_kwargs,
- tok_kwargs,
- )
- hf_processor = self.info.get_hf_processor(**mm_kwargs)
- image_processor = hf_processor.image_processor
-
- # HF processor pops the `num_patches` kwarg, which is needed by vLLM
- if (images := mm_data.get("images")) is not None:
- mm_items = self.info.parse_mm_data({"image": images}, validate=False)
- parsed_images = mm_items.get_items("image", ImageProcessorItems)
- image_sizes = [
- parsed_images.get_image_size(i) for i in range(len(parsed_images))
- ]
-
- num_patches = [
- self.info.get_num_patches(
- image_width=image_size.width,
- image_height=image_size.height,
- size=image_processor.size,
- min_patches=image_processor.min_patches,
- max_patches=image_processor.max_patches,
- )
- for image_size in image_sizes
- ]
- processed_outputs["num_patches"] = torch.tensor(num_patches)
-
- return processed_outputs
-
- def _get_mm_fields_config(
- self,
- hf_inputs: BatchFeature,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- num_patches = hf_inputs.get("num_patches", torch.empty(0))
- return dict(
- pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
- num_patches=MultiModalFieldConfig.batched("image"),
- image_embeds=MultiModalFieldConfig.batched("image"),
- )
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, object],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
- image_token = hf_processor.image_token
- img_patch_token = hf_processor.img_patch_token
- image_processor = hf_processor.image_processor
-
- def get_replacement(item_idx: int):
- images = mm_items.get_items("image", ImageProcessorItems)
- image_size: ImageSize = images.get_image_size(item_idx)
- num_patches = self.info.get_num_patches(
- image_width=image_size.width,
- image_height=image_size.height,
- size=image_processor.size,
- min_patches=image_processor.min_patches,
- max_patches=image_processor.max_patches,
- )
- repl = hf_processor._prompt_split_image(num_patches=num_patches)
-
- return PromptUpdateDetails.select_text(repl, img_patch_token)
-
- return [
- PromptReplacement(
- modality="image",
- target=image_token,
- replacement=get_replacement,
- )
- ]
-
-
-def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:
- feature_layers = hf_config.vision_feature_layer
- num_hidden_layers = hf_config.vision_config.num_hidden_layers
- # If we have one feature layer, initialize up to that layer
- if isinstance(feature_layers, int):
- return get_layer_index(feature_layers, num_hidden_layers)
- # If we have multiple feature layers, initialize up to the deepest m
- elif isinstance(feature_layers, (list, tuple)):
- return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
- raise TypeError(
- f"vision_layer_feature type: {type(feature_layers)} is not supported"
- )
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- AyaVisionMultiModalProcessor,
- info=AyaVisionProcessingInfo,
- dummy_inputs=AyaVisionDummyInputsBuilder,
-)
-class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
- hf_to_vllm_mapper = WeightsMapper(
- orig_to_new_prefix={
- # mapping for new names in checkpoint saved after transformers v4.52
- "model.language_model.": "language_model.model.",
- "model.vision_tower.": "vision_tower.",
- "model.multi_modal_projector.": "multi_modal_projector.",
- "lm_head.": "language_model.lm_head.",
- }
- )
-
- @classmethod
- def get_placeholder_str(cls, modality: str, i: int) -> str | None:
- if modality.startswith("image"):
- return ""
-
- raise ValueError("Only image modality is supported")
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config: AyaVisionConfig = vllm_config.model_config.hf_config
- quant_config = vllm_config.quant_config
- multimodal_config = vllm_config.model_config.multimodal_config
- num_hidden_layers = _get_num_hidden_layers(config)
- self.config = config
- self.quant_config = quant_config
- self.multimodal_config = multimodal_config
-
- with self._mark_tower_model(vllm_config, "image"):
- self.vision_tower = SiglipVisionModel(
- config.vision_config,
- quant_config,
- num_hidden_layers_override=num_hidden_layers,
- prefix=maybe_prefix(prefix, "vision_model"),
- )
- self.multi_modal_projector = AyaVisionMultiModalProjector(config)
-
- with self._mark_language_model(vllm_config):
- self.language_model = init_vllm_registered_model(
- vllm_config=vllm_config,
- hf_config=config.text_config,
- prefix=maybe_prefix(prefix, "model"),
- # Cohere2ForCausalLM and CohereForCausalLM are the same on vllm
- architectures=["Cohere2ForCausalLM"],
- )
-
- self.make_empty_intermediate_tensors = (
- self.language_model.make_empty_intermediate_tensors
- )
-
- @property
- def dtype(self):
- return next(self.parameters()).dtype
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-
- def _image_pixels_to_features(
- self,
- vision_tower: SiglipVisionModel,
- pixel_values: torch.Tensor,
- ) -> torch.Tensor | tuple[torch.Tensor, ...]:
- return vision_tower(
- pixel_values.to(dtype=vision_tower.dtype),
- feature_select_strategy=self.config.vision_feature_select_strategy,
- )
-
- def _process_image_input(
- self, image_input: AyaVisionImagePixelInputs, **kwargs
- ) -> list[torch.Tensor]:
- pixel_values = image_input["pixel_values"]
- num_patches = image_input["num_patches"]
- image_features = self._image_pixels_to_features(
- self.vision_tower, pixel_values=pixel_values
- )
- image_embeds = self.multi_modal_projector(image_features)
- return [e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())]
-
- def _parse_and_validate_image_input(
- self, **kwargs: object
- ) -> AyaVisionImagePixelInputs | None:
- pixel_values = kwargs.pop("pixel_values", None)
- num_patches = kwargs.pop("num_patches", None)
- image_embeds = kwargs.pop("image_embeds", None)
- assert image_embeds is None, "Aya Vision does not support image_embeds."
-
- if pixel_values is None:
- return None
-
- return AyaVisionImagePixelInputs(
- type="pixel_values",
- pixel_values=pixel_values,
- num_patches=num_patches,
- resolve_bindings={
- "h": self.config.vision_config.image_size,
- "w": self.config.vision_config.image_size,
- },
- )
-
- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
- image_input = self._parse_and_validate_image_input(**kwargs)
- if image_input is None:
- return []
-
- return self._process_image_input(image_input, **kwargs)
-
- def forward(
- self,
- input_ids: torch.Tensor | None,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- **kwargs: object,
- ) -> torch.Tensor | IntermediateTensors:
- if intermediate_tensors is not None:
- inputs_embeds = None
-
- hidden_states = self.language_model.model(
- input_ids=input_ids,
- positions=positions,
- intermediate_tensors=intermediate_tensors,
- inputs_embeds=inputs_embeds,
- )
- return hidden_states
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- return self.language_model.compute_logits(hidden_states)
diff --git a/vllm/model_executor/models/musicflamingo.py b/vllm/model_executor/models/musicflamingo.py
deleted file mode 100644
index 509121695f..0000000000
--- a/vllm/model_executor/models/musicflamingo.py
+++ /dev/null
@@ -1,449 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# Copyright 2026 The vLLM team.
-# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
-# reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from collections.abc import Callable, Mapping, Sequence
-from math import pi
-from typing import Annotated, Any, Optional, TypeAlias
-
-import torch
-from torch import Tensor, broadcast_tensors, nn
-from transformers import BatchFeature
-from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
-from transformers.models.musicflamingo import (
- MusicFlamingoConfig,
- MusicFlamingoProcessor,
-)
-
-from vllm.config import VllmConfig
-from vllm.config.multimodal import BaseDummyOptions
-from vllm.inputs import MultiModalDataDict
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalFieldConfig,
- MultiModalKwargsItems,
-)
-from vllm.multimodal.parse import (
- DictEmbeddingItems,
- ModalityData,
- ModalityDataItems,
- MultiModalDataItems,
- MultiModalDataParser,
-)
-from vllm.multimodal.processing import (
- PromptReplacement,
- PromptUpdate,
- PromptUpdateDetails,
-)
-from vllm.utils.tensor_schema import TensorShape
-
-from .audioflamingo3 import (
- AudioFlamingo3DummyInputsBuilder,
- AudioFlamingo3EmbeddingInputs,
- AudioFlamingo3Encoder,
- AudioFlamingo3FeatureInputs,
- AudioFlamingo3ForConditionalGeneration,
- AudioFlamingo3MultiModalDataParser,
- AudioFlamingo3MultiModalProcessor,
- AudioFlamingo3MultiModalProjector,
- AudioFlamingo3ProcessingInfo,
- _audioflamingo3_field_config,
- _count_audio_tokens_from_mask,
-)
-
-
-def rotate_half(x):
- x = x.reshape(*x.shape[:-1], -1, 2)
- x1, x2 = x.unbind(dim=-1)
- x = torch.stack((-x2, x1), dim=-1)
- return x.flatten(-2)
-
-
-def apply_rotary_time_emb(hidden_states, cos, sin):
- original_dtype = hidden_states.dtype
- hidden_states = hidden_states.to(torch.float64)
- cos = cos.to(hidden_states)
- sin = sin.to(hidden_states)
- rot_dim = cos.shape[-1]
- if rot_dim > hidden_states.shape[-1]:
- raise ValueError(
- f"feature dimension {hidden_states.shape[-1]} is not of "
- f"sufficient size to rotate in all the positions {rot_dim}"
- )
-
- rotated = hidden_states[..., :rot_dim]
- passthrough = hidden_states[..., rot_dim:]
- rotated = (rotated * cos) + (rotate_half(rotated) * sin)
- return torch.cat((rotated, passthrough), dim=-1).to(original_dtype)
-
-
-class MusicFlamingoRotaryEmbedding(nn.Module):
- inv_freq: torch.Tensor
-
- def __init__(self, config: MusicFlamingoConfig, device=None):
- super().__init__()
- self.max_seq_len_cached = config.max_position_embeddings
- self.original_max_seq_len = config.max_position_embeddings
-
- self.config = config
- self.rope_type = self.config.rope_parameters["rope_type"]
- rope_init_fn: Callable = self.compute_default_rope_parameters
- if self.rope_type != "default":
- rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
- inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
-
- self.register_buffer("inv_freq", inv_freq, persistent=False)
- self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
- position_angles = self._compute_position_angles(self.inv_freq)
- self.register_buffer("position_angles", position_angles, persistent=False)
-
- @staticmethod
- def compute_default_rope_parameters(
- config: MusicFlamingoConfig | None = None,
- device: Optional["torch.device"] = None,
- seq_len: int | None = None,
- ) -> tuple["torch.Tensor", float]:
- del seq_len
- base = config.rope_parameters["rope_theta"]
- partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
- head_dim = getattr(config, "head_dim", None) or (
- config.hidden_size // config.num_attention_heads
- )
- dim = int(head_dim * partial_rotary_factor)
- attention_factor = 1.0
-
- inv_freq = 1.0 / (
- base
- ** (
- torch.arange(0, dim, 2, dtype=torch.int64).to(
- device=device,
- dtype=torch.float,
- )
- / dim
- )
- )
- return inv_freq, attention_factor
-
- def _compute_position_angles(self, inv_freq):
- positions = torch.arange(
- int(self.max_seq_len_cached),
- device=inv_freq.device,
- dtype=inv_freq.dtype,
- )
- positions = positions / self.max_seq_len_cached * (2 * pi)
- position_angles = positions.unsqueeze(-1) * inv_freq
- position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)
- return position_angles.to(dtype=inv_freq.dtype)
-
- def _restore_fp32_rope_buffers(self) -> None:
- rope_init_fn: Callable = self.compute_default_rope_parameters
- if self.rope_type != "default":
- rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
- inv_freq, self.attention_scaling = rope_init_fn(
- self.config, self.inv_freq.device
- )
-
- self.inv_freq = inv_freq
- self.original_inv_freq = inv_freq.clone()
- self.position_angles = self._compute_position_angles(inv_freq)
-
- def _apply(self, fn):
- super()._apply(fn)
- self._restore_fp32_rope_buffers()
- return self
-
- @torch.no_grad()
- def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:
- window_starts = timestamps[:, 0].to(
- device=self.inv_freq.device,
- dtype=self.inv_freq.dtype,
- )
- window_duration = self.config.audio_frame_step * 4 * seq_len
- window_positions = (
- torch.round(window_starts / window_duration) / self.max_seq_len_cached
- )
- window_freqs = window_positions.unsqueeze(-1) * self.inv_freq
- window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)
-
- window_freqs = window_freqs[:, None, :]
- time_freqs = self.position_angles[:seq_len][None, :, :]
- window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)
- freqs = torch.cat((window_freqs, time_freqs), dim=-1)
- angle = (-timestamps * 2 * pi).to(freqs)
- freqs = freqs * angle.unsqueeze(-1)
- return freqs.cos(), freqs.sin()
-
-
-class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):
- rote_timestamps: Annotated[
- torch.Tensor | None,
- TensorShape(
- "num_chunks",
- "num_audio_time_steps",
- dynamic_dims={"num_audio_time_steps"},
- ),
- ]
-
-
-MusicFlamingoEmbeddingInputs = AudioFlamingo3EmbeddingInputs
-
-MusicFlamingoInputs: TypeAlias = (
- MusicFlamingoFeatureInputs | MusicFlamingoEmbeddingInputs
-)
-
-
-class MusicFlamingoEncoder(AudioFlamingo3Encoder):
- pass
-
-
-class MusicFlamingoMultiModalProjector(AudioFlamingo3MultiModalProjector):
- pass
-
-
-class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):
- def get_hf_config(self) -> MusicFlamingoConfig:
- return self.ctx.get_hf_config(MusicFlamingoConfig)
-
- def get_hf_processor(self, **kwargs: object) -> MusicFlamingoProcessor:
- return self.ctx.get_hf_processor(MusicFlamingoProcessor, **kwargs)
-
- def get_data_parser(self) -> MultiModalDataParser:
- feature_extractor = self.get_feature_extractor()
- return MusicFlamingoMultiModalDataParser(
- target_sr=feature_extractor.sampling_rate,
- audio_resample_method="soxr",
- expected_hidden_size=self._get_expected_hidden_size(),
- )
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"audio": 1}
-
-
-class MusicFlamingoDummyInputsBuilder(AudioFlamingo3DummyInputsBuilder):
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_audios = mm_counts.get("audio", 0)
- hf_processor = self.info.get_hf_processor()
- return hf_processor.audio_token * num_audios
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions],
- ) -> MultiModalDataDict:
- hf_processor = self.info.get_hf_processor()
- feature_extractor = self.info.get_feature_extractor()
- sampling_rate = feature_extractor.sampling_rate
- audio_len = int(hf_processor.max_audio_len * sampling_rate)
- num_audios = mm_counts.get("audio", 0)
- audio_overrides = mm_options.get("audio")
-
- return {
- "audio": self._get_dummy_audios(
- length=audio_len,
- num_audios=num_audios,
- overrides=audio_overrides,
- )
- }
-
-
-def _musicflamingo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
- fields = dict(_audioflamingo3_field_config(hf_inputs))
- chunk_counts = hf_inputs.get("chunk_counts")
- if chunk_counts is not None:
- fields["rote_timestamps"] = MultiModalFieldConfig.flat_from_sizes(
- "audio", chunk_counts, dim=0
- )
- else:
- fields["rote_timestamps"] = MultiModalFieldConfig.batched("audio")
- return fields
-
-
-class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
- def _parse_audio_data(
- self,
- data: dict[str, torch.Tensor] | ModalityData[Any],
- ) -> ModalityDataItems[Any, Any] | None:
- if isinstance(data, dict):
- return DictEmbeddingItems(
- data,
- modality="audio",
- required_fields={"audio_embeds"},
- fields_factory=_musicflamingo_field_config,
- )
- return super()._parse_audio_data(data)
-
-
-class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
- def _get_mm_fields_config(
- self,
- hf_inputs: BatchFeature,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- return _musicflamingo_field_config(hf_inputs)
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, object],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
- tokenizer = self.info.get_tokenizer()
- vocab = tokenizer.get_vocab()
-
- audio_token = processor.audio_token
- audio_token_id = vocab.get(audio_token, processor.audio_token_id)
-
- audio_bos_token = processor.audio_bos_token
- audio_bos_token_id = vocab.get(audio_bos_token, processor.audio_bos_token_id)
-
- audio_eos_token = processor.audio_eos_token
- audio_eos_token_id = vocab.get(audio_eos_token, processor.audio_eos_token_id)
-
- out_mm_data = out_mm_kwargs.get_data()
- feature_attention_mask = out_mm_data.get("feature_attention_mask")
- chunk_counts = out_mm_data.get("chunk_counts")
-
- def get_replacement_musicflamingo(item_idx: int):
- if feature_attention_mask is not None:
- num_features = _count_audio_tokens_from_mask(
- feature_attention_mask,
- chunk_counts,
- item_idx,
- )
- else:
- audio_embeds = out_mm_data["audio_embeds"][item_idx]
- num_features = audio_embeds.shape[0]
-
- if num_features == 0:
- raise ValueError("Audio is too short")
-
- full_tokens = [
- audio_bos_token_id,
- *([audio_token_id] * int(num_features)),
- audio_eos_token_id,
- ]
-
- return PromptUpdateDetails.select_token_id(
- full_tokens,
- embed_token_id=audio_token_id,
- )
-
- return [
- PromptReplacement(
- modality="audio",
- target=audio_token,
- replacement=get_replacement_musicflamingo,
- )
- ]
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- MusicFlamingoMultiModalProcessor,
- info=MusicFlamingoProcessingInfo,
- dummy_inputs=MusicFlamingoDummyInputsBuilder,
-)
-class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGeneration):
- """vLLM MusicFlamingo model aligned with HF modular_musicflamingo."""
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__(vllm_config=vllm_config, prefix=prefix)
- self.audio_tower = MusicFlamingoEncoder(self.config.audio_config)
- self.multi_modal_projector = MusicFlamingoMultiModalProjector(self.config)
- self.pos_emb = MusicFlamingoRotaryEmbedding(self.config)
-
- def _parse_and_validate_audio_input(
- self, **kwargs: object
- ) -> MusicFlamingoInputs | None:
- rote_timestamps = kwargs.pop("rote_timestamps", None)
- audio_input = super()._parse_and_validate_audio_input(**kwargs)
- if audio_input is None or audio_input["type"] == "audio_embeds":
- return audio_input
-
- return MusicFlamingoFeatureInputs(
- type="audio_features",
- input_features=audio_input["input_features"],
- feature_attention_mask=audio_input["feature_attention_mask"],
- chunk_counts=audio_input["chunk_counts"],
- rote_timestamps=rote_timestamps,
- )
-
- def _build_audio_timestamps(
- self,
- chunk_counts: list[int],
- seq_len: int,
- device: torch.device,
- ) -> torch.Tensor:
- audio_embed_frame_step = self.config.audio_frame_step * 4
- frame_offsets = (
- torch.arange(seq_len, device=device, dtype=torch.float32)
- * audio_embed_frame_step
- )
-
- if not chunk_counts:
- return frame_offsets.new_empty((0, seq_len))
-
- window_indices = torch.cat(
- [
- torch.arange(count, device=device, dtype=torch.float32)
- for count in chunk_counts
- ]
- )
- return (
- window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step
- + frame_offsets
- )
-
- def _process_audio_input(
- self, audio_input: MusicFlamingoInputs
- ) -> torch.Tensor | tuple[torch.Tensor, ...]:
- if audio_input["type"] == "audio_embeds":
- return super()._process_audio_input(audio_input)
-
- rote_timestamps = audio_input["rote_timestamps"]
- (
- input_features,
- feature_attention_mask,
- chunk_counts,
- ) = self._normalize_audio_feature_inputs(audio_input)
- hidden_states = self._encode_audio_features(
- input_features,
- feature_attention_mask,
- )
- if rote_timestamps is None:
- rote_timestamps = self._build_audio_timestamps(
- chunk_counts,
- seq_len=hidden_states.shape[-2],
- device=hidden_states.device,
- )
- elif isinstance(rote_timestamps, list):
- rote_timestamps = torch.cat(rote_timestamps, dim=0)
-
- cos, sin = self.pos_emb(
- rote_timestamps.to(hidden_states.device),
- seq_len=hidden_states.shape[-2],
- )
- hidden_states = apply_rotary_time_emb(hidden_states, cos, sin)
- audio_features = self.multi_modal_projector(hidden_states)
-
- return self._group_audio_embeddings(
- audio_features,
- feature_attention_mask,
- chunk_counts,
- )
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 5603a0bb1b..79519c8057 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -337,14 +337,6 @@ _MULTIMODAL_MODELS = {
"audioflamingo3",
"AudioFlamingo3ForConditionalGeneration",
),
- "MusicFlamingoForConditionalGeneration": (
- "musicflamingo",
- "MusicFlamingoForConditionalGeneration",
- ),
- "AyaVisionForConditionalGeneration": (
- "aya_vision",
- "AyaVisionForConditionalGeneration",
- ),
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
@@ -733,6 +725,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"TarsierForConditionalGeneration": "0.24.0",
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
"MantisForConditionalGeneration": "0.24.0",
+ "MusicFlamingoForConditionalGeneration": "0.24.0",
+ "AyaVisionForConditionalGeneration": "0.24.0",
}
_OOT_SUPPORTED_MODELS = {
diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py
index e372834d68..3b39c91109 100644
--- a/vllm/transformers_utils/model_arch_config_convertor.py
+++ b/vllm/transformers_utils/model_arch_config_convertor.py
@@ -345,7 +345,7 @@ class ModelArchConfigConvertorBase:
max_len_key = key
derived_max_model_len = min(derived_max_model_len, max_len)
- # For Command-R / Cohere, Cohere2 / Aya Vision models
+ # For Command-R / Cohere, Cohere2 models
if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None):
max_len_key = "model_max_length"
derived_max_model_len = tmp_max_len