[Model] Remove AyaVision, MusicFlamingo (#47263)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -532,7 +532,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
|
||||
| `AriaForConditionalGeneration` | Aria | T + I<sup>+</sup> | `rhymes-ai/Aria` | | |
|
||||
| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |
|
||||
| `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
|
||||
| `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
|
||||
| `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
|
||||
| `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
|
||||
@@ -594,7 +593,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
|
||||
| `Molmo2ForConditionalGeneration` | Molmo2 | T + I<sup>+</sup> / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`<sup>^</sup>, `allenai/MolmoWeb-8B`<sup>^</sup> | ✅︎ | ✅︎ |
|
||||
| `MossAudioModel` | MOSS-Audio | T + A<sup>+</sup> | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ |
|
||||
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
|
||||
| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
|
||||
| `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
|
||||
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
|
||||
| `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
|
||||
|
||||
@@ -91,44 +91,6 @@ def run_cohere_asr(question: str, audio_count: int) -> ModelRequestData:
|
||||
)
|
||||
|
||||
|
||||
# MusicFlamingo
|
||||
def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData:
|
||||
model_name = "nvidia/music-flamingo-2601-hf"
|
||||
engine_args = EngineArgs(
|
||||
model=model_name,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
limit_mm_per_prompt={"audio": audio_count},
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
# MusicFlamingo prompt placeholders use <sound>; vLLM's MusicFlamingo
|
||||
# multimodal processor expands each one into <|sound_bos|> + audio tokens +
|
||||
# <|sound_eos|> based on extracted audio feature lengths.
|
||||
audio_placeholder = "<sound>" * audio_count
|
||||
system_prompt = (
|
||||
"You are Music Flamingo, a multimodal assistant for language and music. "
|
||||
"On each turn you receive an audio clip which contains music and optional "
|
||||
"text, you will receive at least one or both; use your world knowledge and "
|
||||
"reasoning to help the user with any task. Interpret the entirety of the "
|
||||
"content any input music--regardlenss of whether the user calls it audio, "
|
||||
"music, or sound."
|
||||
)
|
||||
|
||||
prompt = (
|
||||
"<|im_start|>system\n"
|
||||
f"{system_prompt}<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
f"{audio_placeholder}{question}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
return ModelRequestData(
|
||||
engine_args=engine_args,
|
||||
prompt=prompt,
|
||||
)
|
||||
|
||||
|
||||
# Gemma3N
|
||||
def run_gemma3n(question: str, audio_count: int) -> ModelRequestData:
|
||||
model_name = "google/gemma-3n-E2B-it"
|
||||
@@ -565,7 +527,6 @@ model_example_map = {
|
||||
"kimi_audio": run_kimi_audio,
|
||||
"midashenglm": run_midashenglm,
|
||||
"minicpmo": run_minicpmo,
|
||||
"musicflamingo": run_musicflamingo,
|
||||
"phi4_mm": run_phi4mm,
|
||||
"qwen2_audio": run_qwen2_audio,
|
||||
"qwen2_5_omni": run_qwen2_5_omni,
|
||||
|
||||
@@ -74,39 +74,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||
)
|
||||
|
||||
|
||||
def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||
model_name = "CohereLabs/aya-vision-8b"
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model=model_name,
|
||||
max_num_seqs=2,
|
||||
limit_mm_per_prompt={"image": len(image_urls)},
|
||||
)
|
||||
|
||||
placeholders = [{"type": "image", "image": url} for url in image_urls]
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*placeholders,
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
processor = AutoProcessor.from_pretrained(model_name)
|
||||
|
||||
prompt = processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
|
||||
return ModelRequestData(
|
||||
engine_args=engine_args,
|
||||
prompt=prompt,
|
||||
image_data=[fetch_image(url) for url in image_urls],
|
||||
)
|
||||
|
||||
|
||||
def load_bee(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||
model_name = "Open-Bee/Bee-8B-RL"
|
||||
|
||||
@@ -1420,7 +1387,6 @@ def load_molmo2(question: str, image_urls: list[str]) -> ModelRequestData:
|
||||
|
||||
model_example_map = {
|
||||
"aria": load_aria,
|
||||
"aya_vision": load_aya_vision,
|
||||
"bee": load_bee,
|
||||
"command_a_vision": load_command_a_vision,
|
||||
"deepseek_vl_v2": load_deepseek_vl2,
|
||||
|
||||
@@ -68,28 +68,6 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:
|
||||
)
|
||||
|
||||
|
||||
# Aya Vision
|
||||
def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
|
||||
assert modality == "image"
|
||||
model_name = "CohereLabs/aya-vision-8b"
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model=model_name,
|
||||
max_model_len=2048,
|
||||
max_num_seqs=2,
|
||||
mm_processor_kwargs={"crop_to_patches": True},
|
||||
limit_mm_per_prompt={modality: 1},
|
||||
)
|
||||
prompts = [
|
||||
f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
|
||||
for question in questions
|
||||
]
|
||||
return ModelRequestData(
|
||||
engine_args=engine_args,
|
||||
prompts=prompts,
|
||||
)
|
||||
|
||||
|
||||
# Bee-8B
|
||||
def run_bee(questions: list[str], modality: str) -> ModelRequestData:
|
||||
assert modality == "image"
|
||||
@@ -2327,7 +2305,6 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
|
||||
|
||||
model_example_map = {
|
||||
"aria": run_aria,
|
||||
"aya_vision": run_aya_vision,
|
||||
"bagel": run_bagel,
|
||||
"cheers": run_cheers,
|
||||
"bee": run_bee,
|
||||
|
||||
@@ -326,39 +326,6 @@ VLM_TEST_SETTINGS = {
|
||||
large_gpu_mark(min_gb=64),
|
||||
],
|
||||
),
|
||||
"aya_vision": VLMTestInfo(
|
||||
models=["CohereLabs/aya-vision-8b"],
|
||||
test_type=(VLMTestType.IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>What's the content in the center of the image?",
|
||||
"cherry_blossom": "<image>What is the season?",
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<image><image>Describe the two images in detail.",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},
|
||||
),
|
||||
"aya_vision-multi_image": VLMTestInfo(
|
||||
models=["CohereLabs/aya-vision-8b"],
|
||||
test_type=(VLMTestType.MULTI_IMAGE),
|
||||
prompt_formatter=lambda img_prompt: f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{img_prompt}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", # noqa: E501
|
||||
single_image_prompts=IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<image>What's the content in the center of the image?",
|
||||
"cherry_blossom": "<image>What is the season?",
|
||||
}
|
||||
),
|
||||
multi_image_prompt="<image><image>Describe the two images in detail.",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
vllm_runner_kwargs={"mm_processor_kwargs": {"crop_to_patches": True}},
|
||||
marks=[large_gpu_mark(min_gb=32)],
|
||||
),
|
||||
"blip2": VLMTestInfo(
|
||||
models=["Salesforce/blip2-opt-2.7b"],
|
||||
test_type=VLMTestType.IMAGE,
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
MODEL_NAME = "nvidia/music-flamingo-2601-hf"
|
||||
SINGLE_CONVERSATION = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this track in full detail - tell me the "
|
||||
"genre, tempo, and key, then dive into the instruments, "
|
||||
"production style, and overall mood it creates.",
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "https://huggingface.co/datasets/nvidia/AudioSkills/"
|
||||
"resolve/main/assets/song_1.mp3",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
BATCHED_CONVERSATIONS = [
|
||||
SINGLE_CONVERSATION,
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Generate a structured lyric sheet from the input music.",
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "https://huggingface.co/datasets/nvidia/"
|
||||
"AudioSkills/resolve/main/assets/song_2.mp3",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
def get_fixture_path(filename):
|
||||
return os.path.join(
|
||||
os.path.dirname(__file__), "../../fixtures/musicflamingo", filename
|
||||
)
|
||||
|
||||
|
||||
def load_expected_fixture(filename):
|
||||
fixture_path = get_fixture_path(filename)
|
||||
with open(fixture_path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def assert_output_matches(output, expected_text, expected_token_ids):
|
||||
generated = output.outputs[0]
|
||||
assert generated.text == expected_text
|
||||
actual_token_ids = list(generated.token_ids)
|
||||
assert (
|
||||
actual_token_ids == expected_token_ids
|
||||
or actual_token_ids == expected_token_ids[:-1]
|
||||
or actual_token_ids[:-1] == expected_token_ids
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
try:
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
max_model_len=8192,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
|
||||
|
||||
# ROCm may compile decoder kernels on the first inference pass; warm up
|
||||
# once so exact fixture assertions cover the steady-state path.
|
||||
llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
sampling_params=SamplingParams(temperature=0.0, max_tokens=1),
|
||||
use_tqdm=False,
|
||||
)
|
||||
|
||||
return llm
|
||||
|
||||
|
||||
def test_single_generation(llm):
|
||||
expected = load_expected_fixture("expected_results_single.json")
|
||||
|
||||
outputs = llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
sampling_params=SamplingParams(temperature=0.0, max_tokens=50),
|
||||
)
|
||||
|
||||
assert_output_matches(
|
||||
outputs[0],
|
||||
expected["transcriptions"][0],
|
||||
expected["token_ids"][0],
|
||||
)
|
||||
|
||||
|
||||
def test_batched_generation(llm):
|
||||
expected = load_expected_fixture("expected_results_batched.json")
|
||||
|
||||
outputs = llm.chat(
|
||||
messages=BATCHED_CONVERSATIONS,
|
||||
sampling_params=SamplingParams(temperature=0.0, max_tokens=50),
|
||||
)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
assert_output_matches(
|
||||
output,
|
||||
expected["transcriptions"][i],
|
||||
expected["token_ids"][i],
|
||||
)
|
||||
|
||||
|
||||
def test_single_and_batched_generation_match(llm):
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=50)
|
||||
|
||||
single_output = llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
sampling_params=sampling_params,
|
||||
)[0]
|
||||
batched_output = llm.chat(
|
||||
messages=BATCHED_CONVERSATIONS,
|
||||
sampling_params=sampling_params,
|
||||
)[0]
|
||||
|
||||
assert single_output.outputs[0].text == batched_output.outputs[0].text
|
||||
assert list(single_output.outputs[0].token_ids) == list(
|
||||
batched_output.outputs[0].token_ids
|
||||
)
|
||||
@@ -1,228 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2026 The vLLM team.
|
||||
# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
|
||||
# reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from importlib.metadata import version
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
|
||||
|
||||
class MockMusicFlamingoConfig(PretrainedConfig):
|
||||
model_type = "musicflamingo"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.audio_config = PretrainedConfig()
|
||||
self.text_config = PretrainedConfig()
|
||||
|
||||
|
||||
class MockMusicFlamingoProcessor:
|
||||
def __init__(self):
|
||||
self.audio_token = "<sound>"
|
||||
self.audio_token_id = 12345
|
||||
self.audio_bos_token = "<|sound_bos|>"
|
||||
self.audio_bos_token_id = 12346
|
||||
self.audio_eos_token = "<|sound_eos|>"
|
||||
self.audio_eos_token_id = 12347
|
||||
self.max_audio_len = 1200
|
||||
self.feature_extractor = MockFeatureExtractor()
|
||||
|
||||
def __call__(self, text=None, audio=None, **kwargs):
|
||||
return {
|
||||
"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
|
||||
"input_features": torch.zeros((3, 80, 3000)),
|
||||
"input_features_mask": torch.ones((3, 3000), dtype=torch.long),
|
||||
}
|
||||
|
||||
|
||||
class MockFeatureExtractor:
|
||||
def __init__(self):
|
||||
self.sampling_rate = 16000
|
||||
self.chunk_length = 30
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_ctx():
|
||||
config = MockMusicFlamingoConfig()
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.get_hf_config.return_value = config
|
||||
ctx.get_hf_processor.return_value = MockMusicFlamingoProcessor()
|
||||
ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
|
||||
**data, **kwargs
|
||||
)
|
||||
ctx.model_config.hf_config = config
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def check_transformers_version():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("MusicFlamingoForConditionalGeneration")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
|
||||
def test_musicflamingo_chunk_counting_without_rote_timestamps(mock_ctx):
|
||||
from vllm.model_executor.models.musicflamingo import (
|
||||
MusicFlamingoDummyInputsBuilder,
|
||||
MusicFlamingoMultiModalProcessor,
|
||||
MusicFlamingoProcessingInfo,
|
||||
)
|
||||
|
||||
info = MusicFlamingoProcessingInfo(mock_ctx)
|
||||
processor = MusicFlamingoMultiModalProcessor(
|
||||
info, MusicFlamingoDummyInputsBuilder(info)
|
||||
)
|
||||
|
||||
sr = 16000
|
||||
audio_1 = np.zeros(30 * sr)
|
||||
audio_2 = np.zeros(45 * sr)
|
||||
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
prompt = "<|user|>Listen.<|end|>"
|
||||
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
assert chunk_counts.tolist() == [1, 2]
|
||||
assert "rote_timestamps" not in processed
|
||||
assert processed["feature_attention_mask"].shape == (3, 3000)
|
||||
|
||||
|
||||
def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):
|
||||
from vllm.model_executor.models.musicflamingo import (
|
||||
MusicFlamingoDummyInputsBuilder,
|
||||
MusicFlamingoProcessingInfo,
|
||||
)
|
||||
|
||||
info = MusicFlamingoProcessingInfo(mock_ctx)
|
||||
builder = MusicFlamingoDummyInputsBuilder(info)
|
||||
|
||||
assert builder.get_dummy_text({"audio": 2}) == "<sound><sound>"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
Version(version("transformers")) >= Version("5.5"),
|
||||
reason="transformers v5.5 added native MusicFlamingoForConditionalGeneration "
|
||||
"with a different get_audio_features signature (requires input_ids)",
|
||||
)
|
||||
def test_musicflamingo_audio_feature_pipeline_matches_hf_small_config():
|
||||
from transformers.models.musicflamingo import (
|
||||
modeling_musicflamingo as hf_musicflamingo_modeling,
|
||||
)
|
||||
from transformers.models.musicflamingo.configuration_musicflamingo import (
|
||||
MusicFlamingoConfig,
|
||||
)
|
||||
|
||||
from vllm.model_executor.models.audioflamingo3 import (
|
||||
_build_audio_encoder_attention_mask,
|
||||
_flatten_valid_audio_embeddings,
|
||||
)
|
||||
from vllm.model_executor.models.musicflamingo import (
|
||||
MusicFlamingoEncoder,
|
||||
MusicFlamingoMultiModalProjector,
|
||||
MusicFlamingoRotaryEmbedding,
|
||||
apply_rotary_time_emb,
|
||||
)
|
||||
|
||||
text_config = {
|
||||
"model_type": "qwen2",
|
||||
"intermediate_size": 64,
|
||||
"initializer_range": 0.02,
|
||||
"hidden_size": 32,
|
||||
"max_position_embeddings": 1024,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 4,
|
||||
"num_key_value_heads": 2,
|
||||
"vocab_size": 128,
|
||||
"pad_token_id": 1,
|
||||
"use_mrope": False,
|
||||
}
|
||||
audio_config = {
|
||||
"hidden_size": 16,
|
||||
"num_attention_heads": 4,
|
||||
"intermediate_size": 32,
|
||||
"num_hidden_layers": 2,
|
||||
"num_mel_bins": 80,
|
||||
"max_source_positions": 1500,
|
||||
"dropout": 0.0,
|
||||
"attention_dropout": 0.0,
|
||||
"activation_dropout": 0.0,
|
||||
"encoder_layerdrop": 0.0,
|
||||
}
|
||||
|
||||
torch.manual_seed(0)
|
||||
config = MusicFlamingoConfig(
|
||||
text_config=text_config,
|
||||
audio_config=audio_config,
|
||||
audio_token_id=0,
|
||||
head_dim=8,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": 2048},
|
||||
)
|
||||
hf_model = hf_musicflamingo_modeling.MusicFlamingoForConditionalGeneration(
|
||||
config
|
||||
).eval()
|
||||
|
||||
vllm_encoder = MusicFlamingoEncoder(config.audio_config).eval()
|
||||
vllm_encoder.load_state_dict(hf_model.audio_tower.state_dict())
|
||||
|
||||
vllm_projector = MusicFlamingoMultiModalProjector(config).eval()
|
||||
vllm_projector.load_state_dict(hf_model.multi_modal_projector.state_dict())
|
||||
|
||||
vllm_rope = MusicFlamingoRotaryEmbedding(config).eval()
|
||||
vllm_rope.load_state_dict(hf_model.pos_emb.state_dict(), strict=False)
|
||||
|
||||
input_features = torch.randn(3, 80, 3000)
|
||||
feature_attention_mask = torch.zeros(3, 3000, dtype=torch.bool)
|
||||
feature_attention_mask[0, :3000] = True
|
||||
feature_attention_mask[1, :2500] = True
|
||||
feature_attention_mask[2, :1500] = True
|
||||
rote_timestamps = (
|
||||
torch.arange(750, dtype=torch.float32).unsqueeze(0).repeat(3, 1) * 0.04
|
||||
)
|
||||
|
||||
hf_output = hf_model.get_audio_features(
|
||||
input_features,
|
||||
feature_attention_mask,
|
||||
rote_timestamps=rote_timestamps,
|
||||
return_dict=True,
|
||||
).pooler_output
|
||||
vllm_attention_mask = _build_audio_encoder_attention_mask(
|
||||
feature_attention_mask,
|
||||
dtype=vllm_encoder.conv1.weight.dtype,
|
||||
device=vllm_encoder.conv1.weight.device,
|
||||
)
|
||||
vllm_hidden_states = vllm_encoder(
|
||||
input_features,
|
||||
attention_mask=vllm_attention_mask,
|
||||
)
|
||||
cos, sin = vllm_rope(rote_timestamps, seq_len=vllm_hidden_states.shape[-2])
|
||||
vllm_hidden_states = apply_rotary_time_emb(vllm_hidden_states, cos, sin)
|
||||
vllm_output, _ = _flatten_valid_audio_embeddings(
|
||||
vllm_projector(vllm_hidden_states),
|
||||
feature_attention_mask,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(vllm_output, hf_output)
|
||||
@@ -780,14 +780,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
|
||||
},
|
||||
),
|
||||
"MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/music-flamingo-2601-hf",
|
||||
min_transformers_version="5.5.0",
|
||||
transformers_version_reason={
|
||||
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
|
||||
},
|
||||
),
|
||||
"AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
|
||||
"BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
|
||||
"BeeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Open-Bee/Bee-8B-RL",
|
||||
|
||||
@@ -1,445 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import BatchFeature, GotOcr2ImageProcessor
|
||||
from transformers.activations import ACT2FN
|
||||
from transformers.image_processing_utils import get_size_dict
|
||||
from transformers.models.aya_vision import AyaVisionConfig
|
||||
from transformers.models.aya_vision.processing_aya_vision import AyaVisionProcessor
|
||||
from transformers.models.got_ocr2.image_processing_got_ocr2 import (
|
||||
get_optimal_tiled_canvas,
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .siglip import SiglipVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_layer_index,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class AyaVisionImagePixelInputs(TensorSchema):
|
||||
"""
|
||||
Dimensions:
|
||||
- np: The total number of patches over each image over each prompt in
|
||||
the batch
|
||||
- c: Number of channels
|
||||
- h: Height of each image patch
|
||||
- w: Width of each image patch
|
||||
- bn: Batch size * number of images
|
||||
"""
|
||||
|
||||
type: Literal["pixel_values"]
|
||||
|
||||
pixel_values: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("np", 3, "h", "w"),
|
||||
]
|
||||
|
||||
num_patches: Annotated[
|
||||
torch.Tensor,
|
||||
TensorShape("bn"),
|
||||
]
|
||||
|
||||
|
||||
class AyaVisionMultiModalProjector(nn.Module):
|
||||
def __init__(self, config: AyaVisionConfig):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.downsample_factor = config.downsample_factor
|
||||
self.alignment_intermediate_size = getattr(
|
||||
config, "alignment_intermediate_size", config.text_config.hidden_size
|
||||
)
|
||||
self.layernorm = nn.LayerNorm(
|
||||
config.vision_config.hidden_size * (config.downsample_factor**2),
|
||||
eps=config.adapter_layer_norm_eps,
|
||||
)
|
||||
|
||||
self.linear_1 = nn.Linear(
|
||||
config.vision_config.hidden_size * (config.downsample_factor**2),
|
||||
self.alignment_intermediate_size,
|
||||
bias=True,
|
||||
)
|
||||
|
||||
self.act = ACT2FN["silu"] # SwiGLU uses SiLU activation
|
||||
# For SwiGLU, project down to half size since we split intermediate dim
|
||||
self.linear_2 = nn.Linear(
|
||||
self.alignment_intermediate_size // 2,
|
||||
config.text_config.hidden_size,
|
||||
bias=True,
|
||||
)
|
||||
|
||||
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
|
||||
image_features = self.pixel_shuffle(image_features)
|
||||
image_features = self.layernorm(image_features)
|
||||
hidden_states = self.linear_1(image_features)
|
||||
|
||||
# Split along last dimension and apply SwiGLU
|
||||
x, gate = hidden_states.chunk(2, dim=-1)
|
||||
hidden_states = self.act(gate) * x
|
||||
|
||||
hidden_states = self.linear_2(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def pixel_shuffle(self, image_features: torch.Tensor) -> torch.Tensor: # B, S, D
|
||||
batch_size, seq_length, _ = image_features.shape
|
||||
height = width = int(seq_length**0.5)
|
||||
image_features = image_features.reshape(
|
||||
image_features.shape[0], width, height, -1
|
||||
)
|
||||
channels = image_features.shape[-1]
|
||||
image_features = image_features.reshape(
|
||||
batch_size,
|
||||
width,
|
||||
int(height / self.downsample_factor),
|
||||
int(channels * self.downsample_factor),
|
||||
)
|
||||
image_features = image_features.permute(0, 2, 1, 3)
|
||||
image_features = image_features.reshape(
|
||||
batch_size,
|
||||
int(height / self.downsample_factor),
|
||||
int(width / self.downsample_factor),
|
||||
-1,
|
||||
)
|
||||
image_features = image_features.permute(0, 2, 1, 3)
|
||||
return image_features
|
||||
|
||||
|
||||
class AyaVisionProcessingInfo(BaseProcessingInfo):
|
||||
def get_hf_config(self) -> AyaVisionConfig:
|
||||
return self.ctx.get_hf_config(AyaVisionConfig)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> AyaVisionProcessor:
|
||||
return self.ctx.get_hf_processor(AyaVisionProcessor, **kwargs)
|
||||
|
||||
def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor:
|
||||
return self.get_hf_processor(**kwargs).image_processor
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"image": None}
|
||||
|
||||
def get_image_size_with_most_features(self) -> ImageSize:
|
||||
image_processor = self.get_image_processor()
|
||||
height = image_processor.size["height"]
|
||||
width = image_processor.size["width"]
|
||||
max_patches = image_processor.max_patches
|
||||
return ImageSize(height=height * max_patches, width=width * max_patches)
|
||||
|
||||
def get_num_patches(
|
||||
self,
|
||||
*,
|
||||
image_width: int,
|
||||
image_height: int,
|
||||
size: dict,
|
||||
min_patches: int,
|
||||
max_patches: int,
|
||||
) -> int:
|
||||
"""
|
||||
Calculate the number of patches needed for a given image based on size
|
||||
constraints. This method replicates and adjusts the logic from:
|
||||
transformers/models/got_ocr2/image_processing_got_ocr2
|
||||
"""
|
||||
size = get_size_dict(size, default_to_square=False)
|
||||
num_columns, num_rows = get_optimal_tiled_canvas(
|
||||
(image_height, image_width),
|
||||
(size["height"], size["width"]),
|
||||
min_patches,
|
||||
max_patches,
|
||||
)
|
||||
num_blocks = num_columns * num_rows
|
||||
return num_blocks if num_blocks == 1 else num_blocks + 1
|
||||
|
||||
|
||||
class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
|
||||
processor = self.info.get_hf_processor()
|
||||
image_token = processor.image_token
|
||||
|
||||
return image_token * num_images
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
num_images = mm_counts.get("image", 0)
|
||||
image_size = self.info.get_image_size_with_most_features()
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
width=image_size.width,
|
||||
height=image_size.height,
|
||||
num_images=num_images,
|
||||
overrides=image_overrides,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
class AyaVisionMultiModalProcessor(BaseMultiModalProcessor[AyaVisionProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
mm_data: Mapping[str, object],
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
processed_outputs = super()._call_hf_processor(
|
||||
prompt,
|
||||
mm_data,
|
||||
mm_kwargs,
|
||||
tok_kwargs,
|
||||
)
|
||||
hf_processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
image_processor = hf_processor.image_processor
|
||||
|
||||
# HF processor pops the `num_patches` kwarg, which is needed by vLLM
|
||||
if (images := mm_data.get("images")) is not None:
|
||||
mm_items = self.info.parse_mm_data({"image": images}, validate=False)
|
||||
parsed_images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
parsed_images.get_image_size(i) for i in range(len(parsed_images))
|
||||
]
|
||||
|
||||
num_patches = [
|
||||
self.info.get_num_patches(
|
||||
image_width=image_size.width,
|
||||
image_height=image_size.height,
|
||||
size=image_processor.size,
|
||||
min_patches=image_processor.min_patches,
|
||||
max_patches=image_processor.max_patches,
|
||||
)
|
||||
for image_size in image_sizes
|
||||
]
|
||||
processed_outputs["num_patches"] = torch.tensor(num_patches)
|
||||
|
||||
return processed_outputs
|
||||
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
num_patches = hf_inputs.get("num_patches", torch.empty(0))
|
||||
return dict(
|
||||
pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
|
||||
num_patches=MultiModalFieldConfig.batched("image"),
|
||||
image_embeds=MultiModalFieldConfig.batched("image"),
|
||||
)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
image_token = hf_processor.image_token
|
||||
img_patch_token = hf_processor.img_patch_token
|
||||
image_processor = hf_processor.image_processor
|
||||
|
||||
def get_replacement(item_idx: int):
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_size: ImageSize = images.get_image_size(item_idx)
|
||||
num_patches = self.info.get_num_patches(
|
||||
image_width=image_size.width,
|
||||
image_height=image_size.height,
|
||||
size=image_processor.size,
|
||||
min_patches=image_processor.min_patches,
|
||||
max_patches=image_processor.max_patches,
|
||||
)
|
||||
repl = hf_processor._prompt_split_image(num_patches=num_patches)
|
||||
|
||||
return PromptUpdateDetails.select_text(repl, img_patch_token)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="image",
|
||||
target=image_token,
|
||||
replacement=get_replacement,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:
|
||||
feature_layers = hf_config.vision_feature_layer
|
||||
num_hidden_layers = hf_config.vision_config.num_hidden_layers
|
||||
# If we have one feature layer, initialize up to that layer
|
||||
if isinstance(feature_layers, int):
|
||||
return get_layer_index(feature_layers, num_hidden_layers)
|
||||
# If we have multiple feature layers, initialize up to the deepest m
|
||||
elif isinstance(feature_layers, (list, tuple)):
|
||||
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
raise TypeError(
|
||||
f"vision_layer_feature type: {type(feature_layers)} is not supported"
|
||||
)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
AyaVisionMultiModalProcessor,
|
||||
info=AyaVisionProcessingInfo,
|
||||
dummy_inputs=AyaVisionDummyInputsBuilder,
|
||||
)
|
||||
class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
# mapping for new names in checkpoint saved after transformers v4.52
|
||||
"model.language_model.": "language_model.model.",
|
||||
"model.vision_tower.": "vision_tower.",
|
||||
"model.multi_modal_projector.": "multi_modal_projector.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("image"):
|
||||
return "<image>"
|
||||
|
||||
raise ValueError("Only image modality is supported")
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config: AyaVisionConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
num_hidden_layers = _get_num_hidden_layers(config)
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.multimodal_config = multimodal_config
|
||||
|
||||
with self._mark_tower_model(vllm_config, "image"):
|
||||
self.vision_tower = SiglipVisionModel(
|
||||
config.vision_config,
|
||||
quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers,
|
||||
prefix=maybe_prefix(prefix, "vision_model"),
|
||||
)
|
||||
self.multi_modal_projector = AyaVisionMultiModalProjector(config)
|
||||
|
||||
with self._mark_language_model(vllm_config):
|
||||
self.language_model = init_vllm_registered_model(
|
||||
vllm_config=vllm_config,
|
||||
hf_config=config.text_config,
|
||||
prefix=maybe_prefix(prefix, "model"),
|
||||
# Cohere2ForCausalLM and CohereForCausalLM are the same on vllm
|
||||
architectures=["Cohere2ForCausalLM"],
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return next(self.parameters()).dtype
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def _image_pixels_to_features(
|
||||
self,
|
||||
vision_tower: SiglipVisionModel,
|
||||
pixel_values: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
return vision_tower(
|
||||
pixel_values.to(dtype=vision_tower.dtype),
|
||||
feature_select_strategy=self.config.vision_feature_select_strategy,
|
||||
)
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: AyaVisionImagePixelInputs, **kwargs
|
||||
) -> list[torch.Tensor]:
|
||||
pixel_values = image_input["pixel_values"]
|
||||
num_patches = image_input["num_patches"]
|
||||
image_features = self._image_pixels_to_features(
|
||||
self.vision_tower, pixel_values=pixel_values
|
||||
)
|
||||
image_embeds = self.multi_modal_projector(image_features)
|
||||
return [e.flatten(0, 2) for e in image_embeds.split(num_patches.tolist())]
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self, **kwargs: object
|
||||
) -> AyaVisionImagePixelInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
num_patches = kwargs.pop("num_patches", None)
|
||||
image_embeds = kwargs.pop("image_embeds", None)
|
||||
assert image_embeds is None, "Aya Vision does not support image_embeds."
|
||||
|
||||
if pixel_values is None:
|
||||
return None
|
||||
|
||||
return AyaVisionImagePixelInputs(
|
||||
type="pixel_values",
|
||||
pixel_values=pixel_values,
|
||||
num_patches=num_patches,
|
||||
resolve_bindings={
|
||||
"h": self.config.vision_config.image_size,
|
||||
"w": self.config.vision_config.image_size,
|
||||
},
|
||||
)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
if image_input is None:
|
||||
return []
|
||||
|
||||
return self._process_image_input(image_input, **kwargs)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs: object,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if intermediate_tensors is not None:
|
||||
inputs_embeds = None
|
||||
|
||||
hidden_states = self.language_model.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
return self.language_model.compute_logits(hidden_states)
|
||||
@@ -1,449 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2026 The vLLM team.
|
||||
# Copyright 2026 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
|
||||
# reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from math import pi
|
||||
from typing import Annotated, Any, Optional, TypeAlias
|
||||
|
||||
import torch
|
||||
from torch import Tensor, broadcast_tensors, nn
|
||||
from transformers import BatchFeature
|
||||
from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
|
||||
from transformers.models.musicflamingo import (
|
||||
MusicFlamingoConfig,
|
||||
MusicFlamingoProcessor,
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItems,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
DictEmbeddingItems,
|
||||
ModalityData,
|
||||
ModalityDataItems,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.utils.tensor_schema import TensorShape
|
||||
|
||||
from .audioflamingo3 import (
|
||||
AudioFlamingo3DummyInputsBuilder,
|
||||
AudioFlamingo3EmbeddingInputs,
|
||||
AudioFlamingo3Encoder,
|
||||
AudioFlamingo3FeatureInputs,
|
||||
AudioFlamingo3ForConditionalGeneration,
|
||||
AudioFlamingo3MultiModalDataParser,
|
||||
AudioFlamingo3MultiModalProcessor,
|
||||
AudioFlamingo3MultiModalProjector,
|
||||
AudioFlamingo3ProcessingInfo,
|
||||
_audioflamingo3_field_config,
|
||||
_count_audio_tokens_from_mask,
|
||||
)
|
||||
|
||||
|
||||
def rotate_half(x):
|
||||
x = x.reshape(*x.shape[:-1], -1, 2)
|
||||
x1, x2 = x.unbind(dim=-1)
|
||||
x = torch.stack((-x2, x1), dim=-1)
|
||||
return x.flatten(-2)
|
||||
|
||||
|
||||
def apply_rotary_time_emb(hidden_states, cos, sin):
|
||||
original_dtype = hidden_states.dtype
|
||||
hidden_states = hidden_states.to(torch.float64)
|
||||
cos = cos.to(hidden_states)
|
||||
sin = sin.to(hidden_states)
|
||||
rot_dim = cos.shape[-1]
|
||||
if rot_dim > hidden_states.shape[-1]:
|
||||
raise ValueError(
|
||||
f"feature dimension {hidden_states.shape[-1]} is not of "
|
||||
f"sufficient size to rotate in all the positions {rot_dim}"
|
||||
)
|
||||
|
||||
rotated = hidden_states[..., :rot_dim]
|
||||
passthrough = hidden_states[..., rot_dim:]
|
||||
rotated = (rotated * cos) + (rotate_half(rotated) * sin)
|
||||
return torch.cat((rotated, passthrough), dim=-1).to(original_dtype)
|
||||
|
||||
|
||||
class MusicFlamingoRotaryEmbedding(nn.Module):
|
||||
inv_freq: torch.Tensor
|
||||
|
||||
def __init__(self, config: MusicFlamingoConfig, device=None):
|
||||
super().__init__()
|
||||
self.max_seq_len_cached = config.max_position_embeddings
|
||||
self.original_max_seq_len = config.max_position_embeddings
|
||||
|
||||
self.config = config
|
||||
self.rope_type = self.config.rope_parameters["rope_type"]
|
||||
rope_init_fn: Callable = self.compute_default_rope_parameters
|
||||
if self.rope_type != "default":
|
||||
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
inv_freq, self.attention_scaling = rope_init_fn(self.config, device)
|
||||
|
||||
self.register_buffer("inv_freq", inv_freq, persistent=False)
|
||||
self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False)
|
||||
position_angles = self._compute_position_angles(self.inv_freq)
|
||||
self.register_buffer("position_angles", position_angles, persistent=False)
|
||||
|
||||
@staticmethod
|
||||
def compute_default_rope_parameters(
|
||||
config: MusicFlamingoConfig | None = None,
|
||||
device: Optional["torch.device"] = None,
|
||||
seq_len: int | None = None,
|
||||
) -> tuple["torch.Tensor", float]:
|
||||
del seq_len
|
||||
base = config.rope_parameters["rope_theta"]
|
||||
partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
|
||||
head_dim = getattr(config, "head_dim", None) or (
|
||||
config.hidden_size // config.num_attention_heads
|
||||
)
|
||||
dim = int(head_dim * partial_rotary_factor)
|
||||
attention_factor = 1.0
|
||||
|
||||
inv_freq = 1.0 / (
|
||||
base
|
||||
** (
|
||||
torch.arange(0, dim, 2, dtype=torch.int64).to(
|
||||
device=device,
|
||||
dtype=torch.float,
|
||||
)
|
||||
/ dim
|
||||
)
|
||||
)
|
||||
return inv_freq, attention_factor
|
||||
|
||||
def _compute_position_angles(self, inv_freq):
|
||||
positions = torch.arange(
|
||||
int(self.max_seq_len_cached),
|
||||
device=inv_freq.device,
|
||||
dtype=inv_freq.dtype,
|
||||
)
|
||||
positions = positions / self.max_seq_len_cached * (2 * pi)
|
||||
position_angles = positions.unsqueeze(-1) * inv_freq
|
||||
position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)
|
||||
return position_angles.to(dtype=inv_freq.dtype)
|
||||
|
||||
def _restore_fp32_rope_buffers(self) -> None:
|
||||
rope_init_fn: Callable = self.compute_default_rope_parameters
|
||||
if self.rope_type != "default":
|
||||
rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
|
||||
inv_freq, self.attention_scaling = rope_init_fn(
|
||||
self.config, self.inv_freq.device
|
||||
)
|
||||
|
||||
self.inv_freq = inv_freq
|
||||
self.original_inv_freq = inv_freq.clone()
|
||||
self.position_angles = self._compute_position_angles(inv_freq)
|
||||
|
||||
def _apply(self, fn):
|
||||
super()._apply(fn)
|
||||
self._restore_fp32_rope_buffers()
|
||||
return self
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:
|
||||
window_starts = timestamps[:, 0].to(
|
||||
device=self.inv_freq.device,
|
||||
dtype=self.inv_freq.dtype,
|
||||
)
|
||||
window_duration = self.config.audio_frame_step * 4 * seq_len
|
||||
window_positions = (
|
||||
torch.round(window_starts / window_duration) / self.max_seq_len_cached
|
||||
)
|
||||
window_freqs = window_positions.unsqueeze(-1) * self.inv_freq
|
||||
window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)
|
||||
|
||||
window_freqs = window_freqs[:, None, :]
|
||||
time_freqs = self.position_angles[:seq_len][None, :, :]
|
||||
window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)
|
||||
freqs = torch.cat((window_freqs, time_freqs), dim=-1)
|
||||
angle = (-timestamps * 2 * pi).to(freqs)
|
||||
freqs = freqs * angle.unsqueeze(-1)
|
||||
return freqs.cos(), freqs.sin()
|
||||
|
||||
|
||||
class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):
|
||||
rote_timestamps: Annotated[
|
||||
torch.Tensor | None,
|
||||
TensorShape(
|
||||
"num_chunks",
|
||||
"num_audio_time_steps",
|
||||
dynamic_dims={"num_audio_time_steps"},
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
MusicFlamingoEmbeddingInputs = AudioFlamingo3EmbeddingInputs
|
||||
|
||||
MusicFlamingoInputs: TypeAlias = (
|
||||
MusicFlamingoFeatureInputs | MusicFlamingoEmbeddingInputs
|
||||
)
|
||||
|
||||
|
||||
class MusicFlamingoEncoder(AudioFlamingo3Encoder):
|
||||
pass
|
||||
|
||||
|
||||
class MusicFlamingoMultiModalProjector(AudioFlamingo3MultiModalProjector):
|
||||
pass
|
||||
|
||||
|
||||
class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):
|
||||
def get_hf_config(self) -> MusicFlamingoConfig:
|
||||
return self.ctx.get_hf_config(MusicFlamingoConfig)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object) -> MusicFlamingoProcessor:
|
||||
return self.ctx.get_hf_processor(MusicFlamingoProcessor, **kwargs)
|
||||
|
||||
def get_data_parser(self) -> MultiModalDataParser:
|
||||
feature_extractor = self.get_feature_extractor()
|
||||
return MusicFlamingoMultiModalDataParser(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
audio_resample_method="soxr",
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": 1}
|
||||
|
||||
|
||||
class MusicFlamingoDummyInputsBuilder(AudioFlamingo3DummyInputsBuilder):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
return hf_processor.audio_token * num_audios
|
||||
|
||||
def get_dummy_mm_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Mapping[str, BaseDummyOptions],
|
||||
) -> MultiModalDataDict:
|
||||
hf_processor = self.info.get_hf_processor()
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
audio_len = int(hf_processor.max_audio_len * sampling_rate)
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
audio_overrides = mm_options.get("audio")
|
||||
|
||||
return {
|
||||
"audio": self._get_dummy_audios(
|
||||
length=audio_len,
|
||||
num_audios=num_audios,
|
||||
overrides=audio_overrides,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _musicflamingo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
fields = dict(_audioflamingo3_field_config(hf_inputs))
|
||||
chunk_counts = hf_inputs.get("chunk_counts")
|
||||
if chunk_counts is not None:
|
||||
fields["rote_timestamps"] = MultiModalFieldConfig.flat_from_sizes(
|
||||
"audio", chunk_counts, dim=0
|
||||
)
|
||||
else:
|
||||
fields["rote_timestamps"] = MultiModalFieldConfig.batched("audio")
|
||||
return fields
|
||||
|
||||
|
||||
class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[Any],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="audio",
|
||||
required_fields={"audio_embeds"},
|
||||
fields_factory=_musicflamingo_field_config,
|
||||
)
|
||||
return super()._parse_audio_data(data)
|
||||
|
||||
|
||||
class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return _musicflamingo_field_config(hf_inputs)
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
audio_token = processor.audio_token
|
||||
audio_token_id = vocab.get(audio_token, processor.audio_token_id)
|
||||
|
||||
audio_bos_token = processor.audio_bos_token
|
||||
audio_bos_token_id = vocab.get(audio_bos_token, processor.audio_bos_token_id)
|
||||
|
||||
audio_eos_token = processor.audio_eos_token
|
||||
audio_eos_token_id = vocab.get(audio_eos_token, processor.audio_eos_token_id)
|
||||
|
||||
out_mm_data = out_mm_kwargs.get_data()
|
||||
feature_attention_mask = out_mm_data.get("feature_attention_mask")
|
||||
chunk_counts = out_mm_data.get("chunk_counts")
|
||||
|
||||
def get_replacement_musicflamingo(item_idx: int):
|
||||
if feature_attention_mask is not None:
|
||||
num_features = _count_audio_tokens_from_mask(
|
||||
feature_attention_mask,
|
||||
chunk_counts,
|
||||
item_idx,
|
||||
)
|
||||
else:
|
||||
audio_embeds = out_mm_data["audio_embeds"][item_idx]
|
||||
num_features = audio_embeds.shape[0]
|
||||
|
||||
if num_features == 0:
|
||||
raise ValueError("Audio is too short")
|
||||
|
||||
full_tokens = [
|
||||
audio_bos_token_id,
|
||||
*([audio_token_id] * int(num_features)),
|
||||
audio_eos_token_id,
|
||||
]
|
||||
|
||||
return PromptUpdateDetails.select_token_id(
|
||||
full_tokens,
|
||||
embed_token_id=audio_token_id,
|
||||
)
|
||||
|
||||
return [
|
||||
PromptReplacement(
|
||||
modality="audio",
|
||||
target=audio_token,
|
||||
replacement=get_replacement_musicflamingo,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MusicFlamingoMultiModalProcessor,
|
||||
info=MusicFlamingoProcessingInfo,
|
||||
dummy_inputs=MusicFlamingoDummyInputsBuilder,
|
||||
)
|
||||
class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGeneration):
|
||||
"""vLLM MusicFlamingo model aligned with HF modular_musicflamingo."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
self.audio_tower = MusicFlamingoEncoder(self.config.audio_config)
|
||||
self.multi_modal_projector = MusicFlamingoMultiModalProjector(self.config)
|
||||
self.pos_emb = MusicFlamingoRotaryEmbedding(self.config)
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
self, **kwargs: object
|
||||
) -> MusicFlamingoInputs | None:
|
||||
rote_timestamps = kwargs.pop("rote_timestamps", None)
|
||||
audio_input = super()._parse_and_validate_audio_input(**kwargs)
|
||||
if audio_input is None or audio_input["type"] == "audio_embeds":
|
||||
return audio_input
|
||||
|
||||
return MusicFlamingoFeatureInputs(
|
||||
type="audio_features",
|
||||
input_features=audio_input["input_features"],
|
||||
feature_attention_mask=audio_input["feature_attention_mask"],
|
||||
chunk_counts=audio_input["chunk_counts"],
|
||||
rote_timestamps=rote_timestamps,
|
||||
)
|
||||
|
||||
def _build_audio_timestamps(
|
||||
self,
|
||||
chunk_counts: list[int],
|
||||
seq_len: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
audio_embed_frame_step = self.config.audio_frame_step * 4
|
||||
frame_offsets = (
|
||||
torch.arange(seq_len, device=device, dtype=torch.float32)
|
||||
* audio_embed_frame_step
|
||||
)
|
||||
|
||||
if not chunk_counts:
|
||||
return frame_offsets.new_empty((0, seq_len))
|
||||
|
||||
window_indices = torch.cat(
|
||||
[
|
||||
torch.arange(count, device=device, dtype=torch.float32)
|
||||
for count in chunk_counts
|
||||
]
|
||||
)
|
||||
return (
|
||||
window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step
|
||||
+ frame_offsets
|
||||
)
|
||||
|
||||
def _process_audio_input(
|
||||
self, audio_input: MusicFlamingoInputs
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
if audio_input["type"] == "audio_embeds":
|
||||
return super()._process_audio_input(audio_input)
|
||||
|
||||
rote_timestamps = audio_input["rote_timestamps"]
|
||||
(
|
||||
input_features,
|
||||
feature_attention_mask,
|
||||
chunk_counts,
|
||||
) = self._normalize_audio_feature_inputs(audio_input)
|
||||
hidden_states = self._encode_audio_features(
|
||||
input_features,
|
||||
feature_attention_mask,
|
||||
)
|
||||
if rote_timestamps is None:
|
||||
rote_timestamps = self._build_audio_timestamps(
|
||||
chunk_counts,
|
||||
seq_len=hidden_states.shape[-2],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
elif isinstance(rote_timestamps, list):
|
||||
rote_timestamps = torch.cat(rote_timestamps, dim=0)
|
||||
|
||||
cos, sin = self.pos_emb(
|
||||
rote_timestamps.to(hidden_states.device),
|
||||
seq_len=hidden_states.shape[-2],
|
||||
)
|
||||
hidden_states = apply_rotary_time_emb(hidden_states, cos, sin)
|
||||
audio_features = self.multi_modal_projector(hidden_states)
|
||||
|
||||
return self._group_audio_embeddings(
|
||||
audio_features,
|
||||
feature_attention_mask,
|
||||
chunk_counts,
|
||||
)
|
||||
@@ -337,14 +337,6 @@ _MULTIMODAL_MODELS = {
|
||||
"audioflamingo3",
|
||||
"AudioFlamingo3ForConditionalGeneration",
|
||||
),
|
||||
"MusicFlamingoForConditionalGeneration": (
|
||||
"musicflamingo",
|
||||
"MusicFlamingoForConditionalGeneration",
|
||||
),
|
||||
"AyaVisionForConditionalGeneration": (
|
||||
"aya_vision",
|
||||
"AyaVisionForConditionalGeneration",
|
||||
),
|
||||
"BagelForConditionalGeneration": ("bagel", "BagelForConditionalGeneration"),
|
||||
"BeeForConditionalGeneration": ("bee", "BeeForConditionalGeneration"),
|
||||
"Blip2ForConditionalGeneration": ("blip2", "Blip2ForConditionalGeneration"),
|
||||
@@ -733,6 +725,8 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
|
||||
"TarsierForConditionalGeneration": "0.24.0",
|
||||
"Tarsier2ForConditionalGeneration": "0.23.0", # last version with Transformers v4
|
||||
"MantisForConditionalGeneration": "0.24.0",
|
||||
"MusicFlamingoForConditionalGeneration": "0.24.0",
|
||||
"AyaVisionForConditionalGeneration": "0.24.0",
|
||||
}
|
||||
|
||||
_OOT_SUPPORTED_MODELS = {
|
||||
|
||||
@@ -345,7 +345,7 @@ class ModelArchConfigConvertorBase:
|
||||
max_len_key = key
|
||||
derived_max_model_len = min(derived_max_model_len, max_len)
|
||||
|
||||
# For Command-R / Cohere, Cohere2 / Aya Vision models
|
||||
# For Command-R / Cohere, Cohere2 models
|
||||
if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None):
|
||||
max_len_key = "model_max_length"
|
||||
derived_max_model_len = tmp_max_len
|
||||
|
||||
Reference in New Issue
Block a user