diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md
index 040107c11e..96d5ec25c0 100644
--- a/docs/models/supported_models.md
+++ b/docs/models/supported_models.md
@@ -711,7 +711,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `PaliGemmaForConditionalGeneration` | PaliGemma, PaliGemma 2 | T + IE | `google/paligemma-3b-pt-224`, `google/paligemma-3b-mix-224`, `google/paligemma2-3b-ft-docci-448`, etc. | | ✅︎ |
| `Phi3VForCausalLM` | Phi-3-Vision, Phi-3.5-Vision | T + IE+ | `microsoft/Phi-3-vision-128k-instruct`, `microsoft/Phi-3.5-vision-instruct`, etc. | | ✅︎ |
| `Phi4MMForCausalLM` | Phi-4-multimodal | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ |
-| `Phi4MultimodalForCausalLM` | Phi-4-multimodal (HF Transformers) | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct` (with revision `refs/pr/70`), etc. | ✅︎ | ✅︎ |
| `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I+ | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | | ✅︎ |
| `QwenVLForConditionalGeneration`^ | Qwen-VL | T + IE+ | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ |
| `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A+ | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ |
diff --git a/tests/models/multimodal/generation/test_phi4_multimodal.py b/tests/models/multimodal/generation/test_phi4_multimodal.py
deleted file mode 100644
index 6245622171..0000000000
--- a/tests/models/multimodal/generation/test_phi4_multimodal.py
+++ /dev/null
@@ -1,281 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import os
-from collections.abc import Sequence
-
-import librosa
-import pytest
-from huggingface_hub import snapshot_download
-
-from vllm.assets.image import ImageAsset
-from vllm.lora.request import LoRARequest
-from vllm.multimodal.image import rescale_image_size
-
-from ....conftest import (
- IMAGE_ASSETS,
- HfRunner,
- PromptAudioInput,
- PromptImageInput,
- VllmRunner,
-)
-from ....utils import large_gpu_test
-from ...utils import check_logprobs_close
-
-HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
- {
- "stop_sign": "<|user|>\n<|image|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
- "cherry_blossom": "<|user|>\n<|image|>\nPlease infer the season with reason in details.<|end|>\n<|assistant|>\n", # noqa: E501
- }
-)
-HF_MULTIIMAGE_IMAGE_PROMPT = (
- "<|user|>\n<|image|>\n<|image|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
-)
-
-model_path = snapshot_download(
- "microsoft/Phi-4-multimodal-instruct", revision="refs/pr/70"
-)
-# Since the vision-lora and speech-lora co-exist with the base model,
-# we have to manually specify the path of the lora weights.
-vision_lora_path = os.path.join(model_path, "vision-lora")
-speech_question = os.path.join(
- model_path, "examples", "what_is_shown_in_this_image.wav"
-)
-models = [model_path]
-
-target_dtype = "half"
-
-
-def run_test(
- hf_runner: type[HfRunner],
- vllm_runner: type[VllmRunner],
- inputs: Sequence[tuple[list[str], PromptImageInput, PromptAudioInput | None]],
- model: str,
- *,
- max_model_len: int,
- dtype: str,
- max_tokens: int,
- num_logprobs: int,
- mm_limit: int,
- tensor_parallel_size: int,
- distributed_executor_backend: str | None = None,
-):
- """Inference result should be the same between hf and vllm.
-
- All the image fixtures for the test are from IMAGE_ASSETS.
- For huggingface runner, we provide the PIL images as input.
- For vllm runner, we provide MultiModalDataDict objects
- and corresponding MultiModalConfig as input.
- Note, the text input is also adjusted to abide by vllm contract.
- The text output is sanitized to be able to compare with hf.
- """
- # NOTE: take care of the order. run vLLM first, and then run HF.
- # vLLM needs a fresh new process without cuda initialization.
- # if we run HF first, the cuda initialization will be done and it
- # will hurt multiprocessing backend with fork method (the default method).
- # max_model_len should be greater than image_feature_size
- with vllm_runner(
- model,
- task="generate",
- max_model_len=max_model_len,
- max_num_seqs=2,
- dtype=dtype,
- limit_mm_per_prompt={"image": mm_limit},
- tensor_parallel_size=tensor_parallel_size,
- distributed_executor_backend=distributed_executor_backend,
- enable_lora=True,
- max_lora_rank=320,
- gpu_memory_utilization=0.8, # set to 0.8 to avoid OOM in CI
- enforce_eager=True,
- trust_remote_code=False,
- ) as vllm_model:
- lora_request = LoRARequest("vision", 1, vision_lora_path)
- vllm_outputs_per_case = [
- vllm_model.generate_greedy_logprobs(
- prompts,
- max_tokens,
- num_logprobs=num_logprobs,
- images=images,
- audios=audios,
- lora_request=lora_request,
- )
- for prompts, images, audios in inputs
- ]
-
- with hf_runner(model, dtype=dtype) as hf_model:
- hf_model.model.load_adapter(
- vision_lora_path,
- adapter_name="vision",
- )
- hf_processor = hf_model.processor
- eos_token_id = hf_processor.tokenizer.eos_token_id
- hf_outputs_per_case = [
- hf_model.generate_greedy_logprobs_limit(
- prompts,
- max_tokens,
- num_logprobs=num_logprobs,
- images=images,
- audios=audios,
- eos_token_id=eos_token_id,
- )
- for prompts, images, audios in inputs
- ]
-
- for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
- check_logprobs_close(
- outputs_0_lst=hf_outputs,
- outputs_1_lst=vllm_outputs,
- name_0="hf",
- name_1="vllm",
- )
-
-
-@pytest.mark.parametrize("model", models)
-@pytest.mark.parametrize(
- "size_factors",
- [
- # No image
- [],
- # Single-scale
- [1.0],
- # Single-scale, batched
- [1.0, 1.0, 1.0],
- # Multi-scale
- [0.25, 0.5, 1.0],
- ],
-)
-@pytest.mark.parametrize("dtype", [target_dtype])
-@pytest.mark.parametrize("max_model_len", [12800])
-@pytest.mark.parametrize("max_tokens", [128])
-@pytest.mark.parametrize("num_logprobs", [10])
-def test_models(
- hf_runner,
- vllm_runner,
- image_assets,
- model,
- size_factors,
- dtype: str,
- max_model_len: int,
- max_tokens: int,
- num_logprobs: int,
-) -> None:
- images = [asset.pil_image for asset in image_assets]
-
- inputs_per_image = [
- (
- [prompt for _ in size_factors],
- [rescale_image_size(image, factor) for factor in size_factors],
- None,
- )
- for image, prompt in zip(images, HF_IMAGE_PROMPTS)
- ]
-
- run_test(
- hf_runner,
- vllm_runner,
- inputs_per_image,
- model,
- dtype=dtype,
- max_model_len=max_model_len,
- max_tokens=max_tokens,
- num_logprobs=num_logprobs,
- mm_limit=1,
- tensor_parallel_size=1,
- )
-
-
-@large_gpu_test(min_gb=48)
-@pytest.mark.parametrize("model", models)
-@pytest.mark.parametrize(
- "size_factors",
- [
- # No image
- # [],
- # Single-scale
- [1.0],
- # Single-scale, batched
- [1.0, 1.0, 1.0],
- # Multi-scale
- [0.25, 0.5, 1.0],
- ],
-)
-@pytest.mark.parametrize("dtype", [target_dtype])
-@pytest.mark.parametrize("max_model_len", [25600])
-@pytest.mark.parametrize("max_tokens", [128])
-@pytest.mark.parametrize("num_logprobs", [10])
-def test_multi_images_models(
- hf_runner,
- vllm_runner,
- image_assets,
- model,
- size_factors,
- dtype: str,
- max_model_len: int,
- max_tokens: int,
- num_logprobs: int,
-) -> None:
- images = [asset.pil_image for asset in image_assets]
-
- inputs_per_case = [
- (
- [HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
- [
- [rescale_image_size(image, factor) for image in images]
- for factor in size_factors
- ],
- None,
- ),
- ]
-
- run_test(
- hf_runner,
- vllm_runner,
- inputs_per_case,
- model,
- dtype=dtype,
- max_model_len=max_model_len,
- max_tokens=max_tokens,
- num_logprobs=num_logprobs,
- mm_limit=2,
- tensor_parallel_size=1,
- )
-
-
-@pytest.mark.parametrize("model", models)
-@pytest.mark.parametrize("dtype", [target_dtype])
-@pytest.mark.parametrize("max_model_len", [12800])
-@pytest.mark.parametrize("max_tokens", [128])
-@pytest.mark.parametrize("num_logprobs", [10])
-def test_vision_speech_models(
- hf_runner,
- vllm_runner,
- model,
- dtype: str,
- max_model_len: int,
- max_tokens: int,
- num_logprobs: int,
-) -> None:
- # use the example speech question so that the model outputs are reasonable
- audio = librosa.load(speech_question, sr=16000)
- image = ImageAsset("cherry_blossom").pil_image.convert("RGB")
-
- inputs_vision_speech = [
- (
- ["<|user|><|image|><|audio|><|end|><|assistant|>"],
- [image],
- [audio],
- ),
- ]
-
- run_test(
- hf_runner,
- vllm_runner,
- inputs_vision_speech,
- model,
- dtype=dtype,
- max_model_len=max_model_len,
- max_tokens=max_tokens,
- num_logprobs=num_logprobs,
- mm_limit=1,
- tensor_parallel_size=1,
- )
diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py
index 8ef1fba8df..6b9d388f2b 100644
--- a/tests/models/multimodal/processing/test_common.py
+++ b/tests/models/multimodal/processing/test_common.py
@@ -396,28 +396,6 @@ def test_processing_correctness(
)
-# Phi4MultimodalForCausalLM share same model repo with original format
-# Phi4MMForCausalLM, so we add it as a separate test case
-# Remove this test after conversion PR merged:
-# https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/70
-@pytest.mark.parametrize("model_arch", ["Phi4MultimodalForCausalLM"])
-@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
-@pytest.mark.parametrize("num_batches", [32])
-@pytest.mark.parametrize("simplify_rate", [1.0])
-def test_processing_correctness_phi4_multimodal(
- model_arch: str,
- hit_rate: float,
- num_batches: int,
- simplify_rate: float,
-):
- _test_processing_correctness(
- model_arch,
- hit_rate=hit_rate,
- num_batches=num_batches,
- simplify_rate=simplify_rate,
- )
-
-
def _assert_inputs_equal(
a: MultiModalInputs,
b: MultiModalInputs,
diff --git a/tests/models/registry.py b/tests/models/registry.py
index bf88bac209..b9f9945eb5 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -771,10 +771,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Phi4MMForCausalLM": _HfExamplesInfo(
"microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
),
- "Phi4MultimodalForCausalLM": _HfExamplesInfo(
- "microsoft/Phi-4-multimodal-instruct",
- revision="refs/pr/70",
- ),
"PixtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Pixtral-12B-2409",
extras={
diff --git a/vllm/model_executor/models/phi4_multimodal.py b/vllm/model_executor/models/phi4_multimodal.py
deleted file mode 100644
index 0f1230a55b..0000000000
--- a/vllm/model_executor/models/phi4_multimodal.py
+++ /dev/null
@@ -1,1447 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import math
-from collections.abc import Iterable, Mapping, Sequence
-from typing import Annotated, Any, Literal, TypeAlias
-
-import numpy as np
-import torch
-import torch.nn as nn
-import torch.nn.functional as F
-from transformers import (
- BatchFeature,
- Phi4MultimodalAudioConfig,
- Phi4MultimodalConfig,
- Phi4MultimodalFeatureExtractor,
- Phi4MultimodalImageProcessorFast,
-)
-from transformers import Phi4MultimodalProcessor as Phi4MMProcessor
-from transformers.models.phi4_multimodal.modeling_phi4_multimodal import (
- Phi4MultimodalAudioConvModule,
- Phi4MultimodalAudioNemoConvSubsampling,
- Phi4MultimodalAudioRelativeAttentionBias,
- adaptive_enc_mask,
- unfold_tensor,
-)
-
-from vllm.config import VllmConfig
-from vllm.config.multimodal import BaseDummyOptions
-from vllm.distributed import (
- divide,
- get_tensor_model_parallel_rank,
- get_tensor_model_parallel_world_size,
-)
-from vllm.model_executor.layers.activation import MulAndSilu, get_act_fn
-from vllm.model_executor.layers.linear import (
- ColumnParallelLinear,
- MergedColumnParallelLinear,
- QKVParallelLinear,
- RowParallelLinear,
-)
-from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.model_executor.models.module_mapping import MultiModelKeys
-from vllm.multimodal import MULTIMODAL_REGISTRY
-from vllm.multimodal.inputs import (
- MultiModalDataDict,
- MultiModalFieldConfig,
- MultiModalKwargsItems,
- NestedTensors,
-)
-from vllm.multimodal.parse import (
- AudioProcessorItems,
- ImageEmbeddingItems,
- ImageProcessorItems,
- ImageSize,
- MultiModalDataItems,
- MultiModalDataParser,
-)
-from vllm.multimodal.processing import (
- BaseMultiModalProcessor,
- BaseProcessingInfo,
- PromptReplacement,
- PromptUpdate,
-)
-from vllm.multimodal.profiling import BaseDummyInputsBuilder
-from vllm.sequence import IntermediateTensors
-from vllm.utils.tensor_schema import TensorSchema, TensorShape
-
-from .idefics2_vision_model import Idefics2VisionTransformer
-from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
-from .utils import (
- AutoWeightsLoader,
- WeightsMapper,
- init_vllm_registered_model,
- maybe_prefix,
-)
-
-_AUDIO_MAX_SOUNDFILE_SIZE = 241_000
-
-
-def _get_padding_size(
- orig_width: int, orig_height: int, target_height: int, target_width: int
-):
- ratio_width = target_width / orig_width
- ratio_height = target_height / orig_height
-
- if ratio_width < ratio_height:
- padding_width = 0
- padding_height = target_height - int(orig_height * ratio_width)
- else:
- padding_width = target_width - int(orig_width * ratio_height)
- padding_height = 0
- return padding_height, padding_width
-
-
-class Phi4MMProjector(nn.Module):
- def __init__(self, input_size: int, hidden_size: int):
- super().__init__()
- self.up = ColumnParallelLinear(input_size, hidden_size)
- self.down = RowParallelLinear(hidden_size, hidden_size)
- self.act = get_act_fn("gelu")
-
- def forward(self, x: torch.Tensor) -> torch.Tensor:
- x, _ = self.up(x)
- x = self.act(x)
- x, _ = self.down(x)
- return x
-
-
-class Phi4MMImageEmbedding(nn.Module):
- """Image embedding."""
-
- def __init__(self, config: Phi4MultimodalConfig):
- super().__init__()
- self.config = config
- self.layer_idx = config.vision_config.feature_layer
- self.crop_size = config.vision_config.crop_size
- self.image_dim_out = config.vision_config.hidden_size
-
- n_patches = config.vision_config.image_size // config.vision_config.patch_size
- if n_patches % 2 != 0:
- self.img_processor_padding = nn.ReflectionPad2d((0, 1, 0, 1))
- n_patches += 1
- self.num_img_tokens = (n_patches // 2) ** 2
-
- num_hidden_layers = (
- config.vision_config.num_hidden_layers + self.layer_idx + 1
- if self.layer_idx < 0
- else self.layer_idx + 1
- )
- self.img_processor = Idefics2VisionTransformer(
- config.vision_config,
- require_post_norm=False,
- num_hidden_layers_override=num_hidden_layers,
- )
- self.image_token_compression = nn.AvgPool2d(kernel_size=2, stride=2)
- self.img_projection = Phi4MMProjector(self.image_dim_out, config.hidden_size)
- self.global_img_feature_extensor = nn.Parameter(
- torch.zeros([1, 1, self.image_dim_out])
- )
- self.sub_img_feature_extensor = nn.Parameter(
- torch.zeros([1, 1, 1, self.image_dim_out])
- )
-
- def get_img_features(
- self,
- img_embeds: torch.FloatTensor,
- attention_mask: torch.Tensor | None = None,
- ) -> torch.FloatTensor:
- img_feature = self.img_processor(
- img_embeds, patch_attention_mask=attention_mask
- )
-
- patch_feature = img_feature
- # reshape to 2D tensor
- width = int(math.sqrt(patch_feature.size(1)))
- patch_feature = patch_feature.view(-1, width, width, patch_feature.size(-1))
- # convert to NCHW
- patch_feature = patch_feature.permute(0, 3, 1, 2)
- if getattr(self, "img_processor_padding", None) is not None:
- patch_feature = self.img_processor_padding(patch_feature)
- patch_feature = self.image_token_compression(patch_feature)
- # convert to NHWC
- patch_feature = patch_feature.permute(0, 2, 3, 1)
- patch_feature = patch_feature.view(
- -1, patch_feature.size(1) * patch_feature.size(2), patch_feature.size(-1)
- )
- return patch_feature
-
- def forward(
- self,
- image_pixel_values: torch.FloatTensor,
- image_sizes: torch.Tensor | None = None,
- image_attention_mask: torch.Tensor | None = None,
- ) -> torch.FloatTensor:
- image_pixel_values = image_pixel_values.to(
- self.img_processor.embeddings.patch_embedding.weight.dtype
- )
-
- target_device = self.img_projection.up.bias.device
- target_dtype = self.img_projection.up.bias.dtype
-
- batch_size = image_pixel_values.shape[0]
-
- img_features = self.get_img_features(
- image_pixel_values.flatten(0, 1),
- attention_mask=image_attention_mask.flatten(0, 1).to(
- dtype=bool, device=target_device
- ),
- )
- base_feat_size = int(np.sqrt(img_features.shape[1]))
- img_features = img_features.view(
- batch_size, -1, base_feat_size**2, self.image_dim_out
- )
- image_sizes = image_sizes.view(-1, 2)
-
- output_imgs = []
- for idx in range(batch_size):
- height, width = image_sizes[idx]
- height_ratio = height // self.crop_size
- width_ratio = width // self.crop_size
- area_ratio = height_ratio * width_ratio
-
- global_img = img_features[idx, :1]
- global_img = global_img.reshape(
- 1, base_feat_size, base_feat_size, self.image_dim_out
- ).contiguous()
- temporary_extensor = self.sub_img_feature_extensor.repeat(
- 1, base_feat_size, 1, 1
- )
- global_img = torch.cat([global_img, temporary_extensor], dim=2).reshape(
- 1, -1, self.image_dim_out
- )
-
- sub_img = img_features[idx, 1:]
- sub_img = sub_img[:area_ratio]
- sub_img = (
- sub_img.reshape(
- height_ratio,
- width_ratio,
- base_feat_size,
- base_feat_size,
- self.image_dim_out,
- )
- .transpose(1, 2)
- .reshape(
- 1,
- height_ratio * base_feat_size,
- width_ratio * base_feat_size,
- self.image_dim_out,
- )
- .contiguous()
- )
-
- if image_attention_mask is not None:
- reshaped_image_attention_mask = (
- image_attention_mask[idx, 1 : area_ratio + 1, 0::2, 0::2]
- .reshape(height_ratio, width_ratio, base_feat_size, base_feat_size)
- .transpose(1, 2)
- .reshape(
- 1, height_ratio * base_feat_size, width_ratio * base_feat_size
- )
- )
- useful_height = int(reshaped_image_attention_mask[0, :, 0].sum().item())
- useful_width = int(reshaped_image_attention_mask[0, 0, :].sum().item())
- sub_img = sub_img[:, :useful_height, :useful_width]
- temporary_extensor = self.sub_img_feature_extensor.repeat(
- 1, useful_height, 1, 1
- )
- else:
- temporary_extensor = self.sub_img_feature_extensor.repeat(
- 1, height_ratio * base_feat_size, 1, 1
- )
-
- sub_img = torch.cat([sub_img, temporary_extensor], dim=2).reshape(
- 1, -1, self.image_dim_out
- )
-
- # Merge global and sub
- output_imgs.append(
- torch.cat(
- [sub_img, self.global_img_feature_extensor, global_img], dim=1
- )
- )
-
- img_set_tensor = []
- for output_img in output_imgs:
- output_img = output_img.to(device=target_device, dtype=target_dtype)
- img_feature_proj = self.img_projection(output_img)
- img_set_tensor.append(img_feature_proj.flatten(0, 1))
-
- return img_set_tensor
-
-
-class Phi4MultimodalAudioMLP(nn.Module):
- def __init__(
- self,
- config: Phi4MultimodalAudioConfig,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- self.layer_norm = nn.LayerNorm(config.hidden_size)
- self.act_fn = MulAndSilu()
- self.gate_up_proj = MergedColumnParallelLinear(
- config.hidden_size,
- [config.intermediate_size] * 2,
- bias=True,
- quant_config=quant_config,
- prefix=f"{prefix}.gate_up_proj",
- )
- self.down_proj = RowParallelLinear(
- config.intermediate_size,
- config.hidden_size,
- bias=True,
- quant_config=quant_config,
- prefix=f"{prefix}.down_proj",
- )
-
- def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
- hidden_states = self.layer_norm(hidden_states)
- hidden_states, _ = self.gate_up_proj(hidden_states)
- hidden_states = self.act_fn(hidden_states)
- hidden_states, _ = self.down_proj(hidden_states)
- return hidden_states
-
-
-class Phi4MultimodalAudioAttention(nn.Module):
- def __init__(
- self,
- config: Phi4MultimodalAudioConfig,
- quant_config: QuantizationConfig | None = None,
- prefix: str = "",
- ):
- super().__init__()
- self.config = config
- self.embed_dim = config.hidden_size
- self.total_num_heads = config.num_attention_heads
- self.head_dim = self.embed_dim // self.total_num_heads
- if self.head_dim * self.total_num_heads != self.embed_dim:
- raise ValueError(
- "embed_dim must be divisible by num_heads "
- f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
- f" {self.num_heads})."
- )
- self.scale = self.head_dim**-0.5
-
- self.qkv_proj = QKVParallelLinear(
- hidden_size=self.embed_dim,
- head_size=self.head_dim,
- total_num_heads=self.total_num_heads,
- quant_config=quant_config,
- prefix=f"{prefix}.qkv_proj",
- )
-
- self.o_proj = RowParallelLinear(
- input_size=self.embed_dim,
- output_size=self.embed_dim,
- quant_config=quant_config,
- prefix=f"{prefix}.out_proj",
- )
-
- self.tp_size = get_tensor_model_parallel_world_size()
- self.tp_rank = get_tensor_model_parallel_rank()
- self.num_heads = divide(self.total_num_heads, self.tp_size)
-
- def split_attn_mask(self, attention_mask: torch.Tensor) -> torch.Tensor:
- start_idx = self.num_heads * self.tp_rank
- end_idx = self.num_heads * (self.tp_rank + 1)
- return attention_mask[:, start_idx:end_idx]
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- attention_mask: torch.Tensor,
- ) -> torch.Tensor:
- qkv_states, _ = self.qkv_proj(hidden_states)
- query, key, value = qkv_states.chunk(3, dim=-1)
-
- bsz, seq_len, _ = query.size()
- query = query.view(bsz, seq_len, self.num_heads, self.head_dim)
- key = key.view(bsz, seq_len, self.num_heads, self.head_dim)
- value = value.view(bsz, seq_len, self.num_heads, self.head_dim)
- query, key, value = (x.transpose(1, 2) for x in (query, key, value))
-
- attention_mask = self.split_attn_mask(attention_mask)
- out = F.scaled_dot_product_attention(
- query,
- key,
- value,
- scale=self.scale,
- attn_mask=attention_mask,
- )
- out = out.transpose(1, 2).reshape(bsz, seq_len, -1)
-
- attn_output, _ = self.o_proj(out)
-
- return attn_output
-
-
-class Phi4MultimodalAudioConformerEncoderLayer(nn.Module):
- def __init__(self, config: Phi4MultimodalAudioConfig):
- super().__init__()
-
- self.feed_forward_in = Phi4MultimodalAudioMLP(config)
- self.self_attn = Phi4MultimodalAudioAttention(config)
- self.conv = Phi4MultimodalAudioConvModule(config)
- self.feed_forward_out = Phi4MultimodalAudioMLP(config)
- self.layer_norm_att = nn.LayerNorm(config.hidden_size)
- self.layer_norm = nn.LayerNorm(config.hidden_size)
-
- def forward(
- self,
- hidden_states: torch.Tensor,
- attention_mask: torch.Tensor,
- ) -> torch.Tensor:
- residual = hidden_states + 0.5 * self.feed_forward_in(hidden_states)
- hidden_states = self.layer_norm_att(residual)
-
- hidden_states = residual + self.self_attn(hidden_states, attention_mask)
- hidden_states = hidden_states + self.conv(hidden_states)
- hidden_states = hidden_states + 0.5 * self.feed_forward_out(hidden_states)
-
- out = self.layer_norm(hidden_states)
-
- return out
-
-
-class Phi4MMAudioMeanVarianceNormLayer(nn.Module):
- """Mean/variance normalization layer.
-
- Will subtract mean and multiply input by inverted standard deviation.
- Typically used as a very first layer in a model.
-
- Args:
- config: [Phi4MultimodalAudioConfig](https://huggingface.co/docs/transformers/model_doc/phi4_multimodal#transformers.Phi4MultimodalAudioConfig)
- object containing model parameters.
- """
-
- def __init__(self, config: Phi4MultimodalAudioConfig):
- super().__init__()
- self.global_mean = nn.Parameter(torch.zeros(config.input_size))
- self.global_invstd = nn.Parameter(torch.ones(config.input_size))
-
- def forward(self, input_: torch.Tensor) -> torch.Tensor:
- """MeanVarianceNormLayer Forward
-
- Args:
- input_: torch.Tensor
- input tensor.
- """
- return (input_ - self.global_mean) * self.global_invstd
-
-
-class Phi4MultimodalAudioModel(nn.Module):
- def __init__(self, config: Phi4MultimodalAudioConfig):
- super().__init__()
- self.config = config
-
- self.encoder_embedding = Phi4MMAudioMeanVarianceNormLayer(config)
- self.embed = Phi4MultimodalAudioNemoConvSubsampling(config)
- self.relative_attention_bias_layer = Phi4MultimodalAudioRelativeAttentionBias(
- config
- )
- self.encoders = nn.ModuleList(
- [
- Phi4MultimodalAudioConformerEncoderLayer(config)
- for _ in range(config.num_blocks)
- ]
- )
-
- def _streaming_mask(
- self,
- seq_len: int,
- batch_size: int,
- chunk_size: int,
- left_chunk: int,
- ):
- # Create mask matrix for streaming
- # S stores start index. if chunksize is 18, s is [0,18,36,....]
- chunk_start_idx = np.arange(0, seq_len, chunk_size)
-
- enc_streaming_mask = (
- adaptive_enc_mask(seq_len, chunk_start_idx, left_window=left_chunk)
- .unsqueeze(0)
- .expand([batch_size, -1, -1])
- )
- return enc_streaming_mask
-
- def forward_embeddings(
- self,
- hidden_states: torch.Tensor,
- masks: torch.Tensor,
- ):
- """Forwarding the inputs through the top embedding layers"""
- seq_len = math.ceil(hidden_states.shape[1] / self.config.time_reduction)
- if seq_len <= 0:
- raise ValueError(
- f"Sequence length after time reduction is invalid: {seq_len}."
- "Your input feature is too short."
- )
-
- batch_size = hidden_states.shape[0]
-
- enc_streaming_mask = self._streaming_mask(
- seq_len, batch_size, self.config.chunk_size, self.config.left_chunk
- )
- enc_streaming_mask = enc_streaming_mask.to(hidden_states.device)
-
- hidden_states, masks = self.embed(hidden_states, masks)
-
- streaming_mask = enc_streaming_mask
- if streaming_mask is not None and masks is not None:
- hs_mask = masks & streaming_mask
- elif masks is not None:
- hs_mask = masks
- else:
- hs_mask = streaming_mask
-
- return hidden_states, hs_mask, masks
-
- def calculate_hs_mask(
- self, hidden_states: torch.Tensor, device: torch.device, mask: torch.Tensor
- ):
- max_audio_length = hidden_states.shape[1]
- batch_size = hidden_states.shape[0]
- enc_streaming_mask = self._streaming_mask(
- max_audio_length, batch_size, self.config.chunk_size, self.config.left_chunk
- )
- enc_streaming_mask = enc_streaming_mask.to(device)
- if mask is None:
- return enc_streaming_mask
-
- feature_lens = mask.sum(1)
- padding_length = feature_lens
- pad_mask = torch.arange(0, max_audio_length, device=device).expand(
- padding_length.size(0), -1
- ) < padding_length.unsqueeze(1)
- pad_mask = pad_mask.unsqueeze(1)
- pad_mask = pad_mask & enc_streaming_mask
- return pad_mask
-
- def forward(self, hidden_states: torch.Tensor, mask: torch.Tensor | None = None):
- hidden_states = self.encoder_embedding(hidden_states)
- hidden_states, hs_mask, mask = self.forward_embeddings(hidden_states, mask)
-
- unfolded = False
- bs, seq_len, _ = hidden_states.shape
- max_seq_len = 500 # maximum position for absolute positional encoding
- if seq_len > max_seq_len:
- # audio sequence is longer than max_seq_len,
- # unfold it into chunks of max_seq_len
- unfolded = True
- # the unfold op will drop residual frames,
- # pad it to the multiple of max_seq_len
- if seq_len % max_seq_len > 0:
- chunk_pad_size = max_seq_len - (seq_len % max_seq_len)
- else:
- chunk_pad_size = 0
- if chunk_pad_size > 0:
- hidden_states_pad = F.pad(
- hidden_states, (0, 0, 0, chunk_pad_size), "constant", 0
- )
- hidden_states = hidden_states_pad.to(hidden_states.device)
-
- hidden_states = unfold_tensor(hidden_states, max_seq_len)
- masks_unfold = None
- if mask is not None:
- # revise hs_mask here because the previous calculated hs_mask
- # did not consider extra pad
- subsampled_pad_mask = mask.squeeze(1) # [bz, subsampled_unmask_seq_len]
- extra_padded_subsamlped_pad_mask = F.pad(
- subsampled_pad_mask, (0, chunk_pad_size), "constant", False
- ) # extra padding to the pad mask
- extra_padded_subsamlped_pad_mask = (
- extra_padded_subsamlped_pad_mask.unsqueeze(-1).float()
- )
- masks_unfold = unfold_tensor(
- extra_padded_subsamlped_pad_mask, max_seq_len
- ) # unfold the pad mask like we did to the input tensor
- masks_unfold = masks_unfold.squeeze(
- -1
- ).bool() # unfold op does not support bool tensor
- hs_mask = self.calculate_hs_mask(
- hidden_states, hidden_states.device, masks_unfold
- ) # calculate hs_mask based on the unfolded pad mask
-
- relative_attention_bias = self.relative_attention_bias_layer(hidden_states)
- attention_mask = hs_mask.unsqueeze(1) + relative_attention_bias
-
- for layer in self.encoders:
- hidden_states = layer(hidden_states, attention_mask)
-
- if unfolded:
- embed_dim = hidden_states.shape[-1]
- hidden_states = hidden_states.reshape(bs, -1, embed_dim)
- # if we ever padded before unfolding, we need to remove the padding
- if chunk_pad_size > 0:
- hidden_states = hidden_states[:, :-chunk_pad_size, :]
-
- return hidden_states
-
-
-class Phi4MMAudioEmbedding(nn.Module):
- def __init__(self, config: Phi4MultimodalConfig):
- super().__init__()
- self.config = config
- self.layer_idx = config.audio_config.feature_layer
-
- self.encoder = Phi4MultimodalAudioModel(config.audio_config)
-
- audio_config = config.audio_config
- proj_input_size = audio_config.hidden_size * audio_config.downsample_rate
- self.vision_speech_projection = Phi4MMProjector(
- proj_input_size, config.hidden_size
- )
- self.speech_projection = Phi4MMProjector(proj_input_size, config.hidden_size)
-
- def get_projection(
- self,
- audio_projection_mode: Literal["speech", "vision"],
- ) -> Phi4MMProjector:
- if audio_projection_mode == "speech":
- return self.speech_projection
- elif audio_projection_mode == "vision":
- return self.vision_speech_projection
-
- def forward(
- self,
- audio_input_features: torch.FloatTensor,
- audio_embed_sizes=None,
- audio_attention_mask=None,
- audio_projection_mode="speech",
- ) -> torch.FloatTensor:
- audio_projection = self.get_projection(audio_projection_mode)
-
- target_device = audio_projection.up.bias.device
- target_dtype = audio_projection.up.bias.dtype
-
- audio_input_features = audio_input_features.to(
- device=target_device, dtype=target_dtype
- )
-
- audio_encoder_hidden_states = self.encoder(
- audio_input_features, audio_attention_mask
- )
- audio_embeds = audio_projection(audio_encoder_hidden_states)
-
- return audio_embeds.flatten(0, 1)
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- stacked_params_mapping = [
- # (param_name, shard_name, shard_id)
- ("qkv_proj", "q_proj", "q"),
- ("qkv_proj", "k_proj", "k"),
- ("qkv_proj", "v_proj", "v"),
- ]
- params_dict = dict(self.named_parameters())
- loaded_params: set[str] = set()
-
- for name, loaded_weight in weights:
- for param_name, weight_name, shard_id in stacked_params_mapping:
- if weight_name not in name:
- continue
- name = name.replace(weight_name, param_name)
- param = params_dict[name]
- weight_loader = param.weight_loader
- weight_loader(param, loaded_weight, shard_id)
- break
- else:
- param = params_dict[name]
- weight_loader = getattr(param, "weight_loader", default_weight_loader)
- weight_loader(param, loaded_weight)
- loaded_params.add(name)
- return loaded_params
-
-
-class Phi4MMImagePixelInputs(TensorSchema):
- """
- Dimensions:
- - bn: Batch size * number of images
- - p: Number of patches (1 + num_patches)
- - c: Number of channels (3)
- - h: Height of each image patch
- - w: Width of each image patch
- - nc: Number of crops
- - H_mask: Height of attention mask
- - W_mask: Width of attention mask
- """
-
- type: Literal["pixel_values"]
-
- pixel_values: Annotated[
- torch.Tensor | list[torch.Tensor],
- TensorShape(
- "bn", "p", 3, "h", "w", dynamic_dims={"p"}
- ), # may be different per batch and image
- ]
-
- image_sizes: Annotated[
- torch.Tensor,
- TensorShape("bn", 2), # (height, width)
- ]
-
- num_img_tokens: Annotated[
- list[int],
- TensorShape("bn"),
- ]
-
- image_attention_mask: Annotated[
- torch.Tensor,
- TensorShape("bn", "nc", 32, 32), # H_mask, W_mask
- ]
-
-
-class Phi4MMImageEmbeddingInputs(TensorSchema):
- """
- Dimensions:
- - bn: Batch size * number of images
- - f: Image feature size
- - h: Hidden size (must match language model backbone)
- """
-
- type: Literal["image_embeds"]
-
- data: Annotated[
- torch.Tensor | list[torch.Tensor],
- TensorShape("bn", "f", "h"),
- ]
-
-
-class Phi4MMAudioFeatureInputs(TensorSchema):
- """
- Dimensions:
- - bn: Batch size * number of audios
- - f: Number of Mel filterbank bins (80)
- - t: Time frames (M)
- """
-
- type: Literal["audio_features"]
-
- audio_features: Annotated[
- torch.Tensor | list[torch.Tensor],
- TensorShape("bn", "t", 80, dynamic_dims={"t"}),
- ]
-
-
-class Phi4MMAudioEmbeddingInputs(TensorSchema):
- """
- Dimensions:
- - b: Batch size
- - n: Number of audios
- - f: Audio feature size
- - h: Hidden size (must match language model backbone)
- """
-
- type: Literal["audio_embeds"]
-
- data: Annotated[
- NestedTensors,
- TensorShape("b", "n", "f", "h"),
- ]
-
-
-Phi4MMImageInput: TypeAlias = Phi4MMImagePixelInputs | Phi4MMImageEmbeddingInputs
-Phi4MMAudioInputs: TypeAlias = Phi4MMAudioFeatureInputs | Phi4MMAudioEmbeddingInputs
-
-
-def cat_with_pad(tensors, dim, padding_value=0):
- """
- cat along dim, while pad to max for all other dims
- """
- ndim = tensors[0].dim()
- assert all(t.dim() == ndim for t in tensors[1:]), (
- "All tensors must have the same number of dimensions"
- )
-
- out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)]
- out_size[dim] = sum(t.shape[dim] for t in tensors)
- output = tensors[0].new_full(out_size, padding_value)
-
- index = 0
- for t in tensors:
- # Create a slice list where every dimension except dim is full slice
- slices = [slice(0, t.shape[d]) for d in range(ndim)]
- # Update only the concat dimension slice
- slices[dim] = slice(index, index + t.shape[dim])
-
- output[slices] = t
- index += t.shape[dim]
-
- return output
-
-
-class Phi4MMProcessingInfo(BaseProcessingInfo):
- def get_hf_config(self) -> Phi4MultimodalConfig:
- return self.ctx.get_hf_config(Phi4MultimodalConfig)
-
- def get_hf_processor(self, **kwargs: object) -> Phi4MMProcessor:
- return self.ctx.get_hf_processor(Phi4MMProcessor, **kwargs)
-
- def get_feature_extractor(self, **kwargs: object) -> Phi4MultimodalFeatureExtractor:
- return self.get_hf_processor(**kwargs).audio_processor
-
- def get_image_processor(
- self,
- processor: Phi4MMProcessor | None = None,
- ) -> Phi4MultimodalImageProcessorFast:
- if processor is None:
- processor = self.get_hf_processor()
- return processor.image_processor
-
- def get_dynamic_hd(
- self,
- processor: Phi4MMProcessor | None = None,
- ) -> int:
- return self.get_image_processor(processor).dynamic_hd
-
- def get_supported_mm_limits(self) -> Mapping[str, int | None]:
- return {"audio": None, "image": None}
-
- def _find_target_aspect_ratio(
- self,
- orig_width: int,
- orig_height: int,
- image_size: int,
- max_num: int,
- min_num: int,
- ):
- w_crop_num = math.ceil(orig_width / float(image_size))
- h_crop_num = math.ceil(orig_height / float(image_size))
- if w_crop_num * h_crop_num > max_num:
- aspect_ratio = orig_width / orig_height
-
- # calculate the existing image aspect ratio
- target_ratios = set(
- (i, j)
- for i in range(1, max_num + 1)
- for j in range(1, max_num + 1)
- if i * j <= max_num and i * j >= min_num
- )
- target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
-
- # find the closest aspect ratio to the target
- image_processor = self.get_image_processor()
- target_aspect_ratio = image_processor.find_closest_aspect_ratio(
- aspect_ratio,
- target_ratios,
- orig_width,
- orig_height,
- image_size,
- )
-
- # calculate the target width and height
- target_width = image_size * target_aspect_ratio[0]
- target_height = image_size * target_aspect_ratio[1]
- else:
- target_width = image_size * w_crop_num
- target_height = image_size * h_crop_num
- target_aspect_ratio = (w_crop_num, h_crop_num)
- return target_aspect_ratio, target_height, target_width
-
- def _compute_num_image_tokens(
- self,
- orig_width: int,
- orig_height: int,
- dynamic_hd_size: int,
- vit_image_size: int,
- vit_patch_size: int,
- token_compression_factor: int = 2,
- ):
- """
- compute the number of tokens an image is expected to take up considering
- the image encoder architecture and exclude output features containing
- only padding pixels
-
- for siglip, vit_image_size=448, vit_patch_size=14, so output will be
- 32x32 feature map
- NOTE right now, Phi4MM uses hard-coded token_compression_factor=2
- """
- assert vit_image_size % vit_patch_size == 0, (
- "vit_image_size must be divisible by vit_patch_size"
- )
- assert vit_image_size // vit_patch_size % token_compression_factor == 0, (
- "vit_image_size // vit_patch_size must be divisible by "
- "token_compression_factor"
- )
-
- target_aspect_ratio, target_height, target_width = (
- self._find_target_aspect_ratio(
- orig_width, orig_height, vit_image_size, dynamic_hd_size, min_num=1
- )
- )
- assert target_aspect_ratio[0] * vit_image_size == target_width, (
- f"{target_aspect_ratio[0]} * {vit_image_size} != {target_width}"
- )
- assert target_aspect_ratio[1] * vit_image_size == target_height, (
- f"{target_aspect_ratio[1]} * {vit_image_size} != {target_height}"
- )
- assert (
- target_height % vit_image_size == 0 and target_width % vit_image_size == 0
- )
-
- padding_height, padding_width = _get_padding_size(
- orig_width, orig_height, target_height, target_width
- )
- assert padding_width == 0 or padding_height == 0, (
- "padding_width or padding_height must be 0"
- )
-
- target_feat_width = target_width // vit_patch_size
- target_feat_height = target_height // vit_patch_size
- if padding_width >= vit_patch_size:
- assert padding_height == 0, "padding_height not 0"
- non_pad_feat_width = target_feat_width - math.floor(
- padding_width / vit_patch_size
- )
- non_pad_feat_height = target_feat_height
- elif padding_height >= vit_patch_size:
- assert padding_width == 0, "padding_width not 0"
- non_pad_feat_height = target_feat_height - math.floor(
- padding_height / vit_patch_size
- )
- non_pad_feat_width = target_feat_width
- else:
- # small padding shorter than a vit patch
- non_pad_feat_width = target_feat_width
- non_pad_feat_height = target_feat_height
-
- feat_width = non_pad_feat_width // token_compression_factor
- feat_height = non_pad_feat_height // token_compression_factor
- # NOTE it's possible that the non-padding feature is not divisible
- if non_pad_feat_width % token_compression_factor != 0:
- feat_width += 1
- if non_pad_feat_height % token_compression_factor != 0:
- feat_height += 1
- num_hd_patch_tokens = feat_width * feat_height
- num_hd_newline_tokens = feat_height
- vit_feature_size = vit_image_size // vit_patch_size
- num_global_image_tokens = (vit_feature_size // token_compression_factor) ** 2
- num_sep_tokens = 1
- num_global_image_newline_tokens = vit_feature_size // token_compression_factor
-
- return (
- num_global_image_tokens
- + num_sep_tokens
- + num_hd_patch_tokens
- + num_hd_newline_tokens
- + num_global_image_newline_tokens
- )
-
- def get_num_image_tokens(
- self,
- *,
- image_width: int,
- image_height: int,
- processor: Phi4MMProcessor | None = None,
- ) -> int:
- hf_config = self.get_hf_config()
- vision_config = hf_config.vision_config
- vit_image_size = vision_config.image_size
- vit_patch_size = vision_config.patch_size
-
- dynamic_hd_size = self.get_dynamic_hd(processor=processor)
-
- # we use default `token_compression_factor=2`,
- # since it's not in HF vision config.
- image_num_tokens = self._compute_num_image_tokens(
- image_width,
- image_height,
- dynamic_hd_size=dynamic_hd_size,
- vit_image_size=vit_image_size,
- vit_patch_size=vit_patch_size,
- )
-
- return image_num_tokens
-
- def get_image_size_with_most_features(
- self,
- processor: Phi4MMProcessor | None = None,
- ) -> ImageSize:
- vit_image_size = self.get_hf_config().vision_config.image_size
-
- max_side = vit_image_size * self.get_dynamic_hd(processor=processor)
- return ImageSize(height=max_side, width=vit_image_size)
-
- def get_audio_num_frames(self, audio_len: int, sr: float) -> int:
- """
- Compute the output size of the `extract_features` method.
-
- Args:
- audio_len (int): Length of the input waveform in samples.
- sr (float): Sampling rate of the waveform, either 16000 or 8000.
-
- Returns:
- tuple (int, int): Output size as (T, D), where:
- T: Number of time frames.
- D: Number of Mel filterbank bins (80).
- """
-
- # Resample to 16000 or 8000 if needed
- if sr > 16000:
- audio_len //= sr // 16000
- elif 8000 <= sr < 16000:
- # We'll resample to 16K from 8K
- audio_len *= 2
- elif sr < 8000:
- raise RuntimeError(f"Unsupported sample rate {sr}")
-
- # Spectrogram parameters for 16 kHz
- win_length = 400 # Frame length in samples
- hop_length = 160 # Frame shift in samples
-
- # Calculate number of frames (T)
- num_frames = (audio_len - win_length) // hop_length + 1
- if num_frames < 1:
- raise ValueError("Waveform too short for given parameters.")
-
- # Return time frames (T)
- return num_frames
-
- def _compute_audio_embed_size(self, audio_frames: int) -> int:
- """
- Compute the size of audio embeddings from the number of audio frames.
- """
- # `_compute_audio_embed_size` in audio_processor use torch for
- # computation, therefore we re-implement it to use pythonic
- # numeric computation to avoid extra tensor conversion.
- audio_processor = self.get_feature_extractor()
- audio_compression_rate = audio_processor.audio_compression_rate
- audio_downsample_rate = audio_processor.audio_downsample_rate
-
- integer = audio_frames // audio_compression_rate
- remainder = audio_frames % audio_compression_rate
- result = integer + int(remainder > 0)
-
- integer = result // audio_downsample_rate
- remainder = result % audio_downsample_rate
- result = integer + int(remainder > 0) # qformer compression
-
- return result
-
-
-class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
- def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
- num_audios = mm_counts.get("audio", 0)
- num_images = mm_counts.get("image", 0)
-
- tokenizer = self.info.get_tokenizer()
- image_tokens: str = tokenizer.image_token * num_images
- audio_tokens: str = tokenizer.audio_token * num_audios
-
- return image_tokens + audio_tokens
-
- def get_dummy_mm_data(
- self,
- seq_len: int,
- mm_counts: Mapping[str, int],
- mm_options: Mapping[str, BaseDummyOptions] | None = None,
- ) -> MultiModalDataDict:
- num_audios = mm_counts.get("audio", 0)
- num_images = mm_counts.get("image", 0)
-
- target_width, target_height = self.info.get_image_size_with_most_features()
-
- image_overrides = mm_options.get("image") if mm_options else None
- audio_overrides = mm_options.get("audio") if mm_options else None
-
- mm_data = {
- "image": self._get_dummy_images(
- width=target_width,
- height=target_height,
- num_images=num_images,
- overrides=image_overrides,
- ),
- "audio": self._get_dummy_audios(
- length=_AUDIO_MAX_SOUNDFILE_SIZE,
- num_audios=num_audios,
- overrides=audio_overrides,
- ),
- }
-
- return mm_data
-
-
-class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
- def _get_data_parser(self) -> MultiModalDataParser:
- feature_extractor = self.info.get_feature_extractor()
- return MultiModalDataParser(target_sr=feature_extractor.sampling_rate)
-
- def _call_hf_processor(
- self,
- prompt: str,
- mm_data: Mapping[str, object],
- mm_kwargs: Mapping[str, object],
- tok_kwargs: Mapping[str, object],
- ) -> BatchFeature:
- if not mm_data:
- prompt_ids = self.info.get_tokenizer().encode(prompt)
- prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
- return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
-
- audio_data = mm_data.pop("audios", [])
- if audio_data:
- mm_data["audio"] = audio_data
-
- processed_outputs = super()._call_hf_processor(
- prompt, mm_data, mm_kwargs, tok_kwargs
- )
-
- if "image_pixel_values" in processed_outputs:
- num_img_tokens = [
- self.info.get_num_image_tokens(
- image_width=img_size[0], image_height=img_size[1]
- )
- for img_size in processed_outputs["image_sizes"]
- ]
- processed_outputs["num_img_tokens"] = num_img_tokens
-
- if audio_data:
- audio_features = processed_outputs["audio_input_features"]
- sr = self.info.get_feature_extractor(**mm_kwargs).sampling_rate
- feature_sizes = [
- self.info.get_audio_num_frames(len(audio), sr) for audio in audio_data
- ]
- processed_outputs["audio_input_features"] = [
- audio_features[idx, :size] for idx, size in enumerate(feature_sizes)
- ]
-
- return processed_outputs
-
- def _get_mm_fields_config(
- self,
- hf_inputs: BatchFeature,
- hf_processor_mm_kwargs: Mapping[str, object],
- ) -> Mapping[str, MultiModalFieldConfig]:
- return dict(
- image_pixel_values=MultiModalFieldConfig.batched("image"),
- image_attention_mask=MultiModalFieldConfig.batched("image"),
- image_sizes=MultiModalFieldConfig.batched("image"),
- num_img_tokens=MultiModalFieldConfig.batched("image"),
- audio_input_features=MultiModalFieldConfig.batched("audio"),
- )
-
- def _get_prompt_updates(
- self,
- mm_items: MultiModalDataItems,
- hf_processor_mm_kwargs: Mapping[str, Any],
- out_mm_kwargs: MultiModalKwargsItems,
- ) -> Sequence[PromptUpdate]:
- tokenizer = self.info.get_tokenizer()
- image_token_id: int = tokenizer.vocab[tokenizer.image_token]
- audio_token_id: int = tokenizer.vocab[tokenizer.audio_token]
-
- hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
- audio_processor = self.info.get_feature_extractor(**hf_processor_mm_kwargs)
-
- def get_image_replacement_phi4mm(item_idx: int):
- images = mm_items.get_items(
- "image", (ImageEmbeddingItems, ImageProcessorItems)
- )
-
- if isinstance(images, ImageEmbeddingItems):
- num_image_tokens = images.get_feature_size(item_idx)
- else:
- image_size = images.get_image_size(item_idx)
- num_image_tokens = self.info.get_num_image_tokens(
- image_width=image_size.width,
- image_height=image_size.height,
- processor=hf_processor,
- )
-
- return [image_token_id] * num_image_tokens
-
- def get_audio_replacement_phi4mm(item_idx: int):
- audios = mm_items.get_items("audio", AudioProcessorItems)
- # TODO(Isotr0py): support embedding inputs
- audio_len = audios.get_audio_length(item_idx)
- audio_frames = self.info.get_audio_num_frames(
- audio_len, audio_processor.sampling_rate
- )
- audio_embed_size = self.info._compute_audio_embed_size(audio_frames)
-
- return [audio_token_id] * audio_embed_size
-
- return [
- PromptReplacement(
- modality="audio",
- target=[audio_token_id],
- replacement=get_audio_replacement_phi4mm,
- ),
- PromptReplacement(
- modality="image",
- target=[image_token_id],
- replacement=get_image_replacement_phi4mm,
- ),
- ]
-
-
-@MULTIMODAL_REGISTRY.register_processor(
- Phi4MMMultiModalProcessor,
- info=Phi4MMProcessingInfo,
- dummy_inputs=Phi4MMDummyInputsBuilder,
-)
-class Phi4MultimodalForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
- """
- Implements the Phi-4-multimodal-instruct model in vLLM.
- """
-
- merge_by_field_config = True
-
- packed_modules_mapping = {
- "qkv_proj": [
- "qkv_proj",
- ],
- "gate_up_proj": [
- "gate_up_proj",
- ],
- }
-
- hf_to_vllm_mapper = WeightsMapper(
- orig_to_new_prefix={
- # Multimodal embedding
- "model.embed_tokens_extend.": "",
- # LLM backbone
- "model.": "language_model.model.",
- },
- orig_to_new_substr={
- # projection
- ".img_projection_": ".img_projection.",
- ".up_proj_for_speech.": ".speech_projection.up.",
- ".up_proj_for_vision_speech.": ".vision_speech_projection.up.",
- ".down_proj_for_speech.": ".speech_projection.down.",
- ".down_proj_for_vision_speech.": ".vision_speech_projection.down.",
- },
- )
-
- @classmethod
- def get_placeholder_str(cls, modality: str, i: int) -> str | None:
- if modality.startswith("image"):
- return "<|image|>"
- if modality.startswith("audio"):
- return "<|audio|>"
-
- raise ValueError("Only image or audio modality is supported")
-
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
- super().__init__()
- config = vllm_config.model_config.hf_config
- multimodal_config = vllm_config.model_config.multimodal_config
- self.config = config
- self.multimodal_config = multimodal_config
-
- # TODO: Optionally initializes these for supporting input embeddings.
- self.image_embed = Phi4MMImageEmbedding(
- config,
- # prefix=maybe_prefix(prefix, "image_embed"),
- )
- self.audio_embed = Phi4MMAudioEmbedding(
- config,
- # prefix=maybe_prefix(prefix, "audio_embed"),
- )
-
- self.language_model = init_vllm_registered_model(
- vllm_config=vllm_config,
- prefix=maybe_prefix(prefix, "language_model"),
- architectures=["Phi3ForCausalLM"],
- )
-
- self.make_empty_intermediate_tensors = (
- self.language_model.make_empty_intermediate_tensors
- )
-
- def _parse_and_validate_audio_input(
- self, **kwargs: object
- ) -> Phi4MMAudioInputs | None:
- """
- Parse and validate the audio input to the model. This handles both
- audio features and audio embeddings, but only the former is used for
- now.
-
- Args:
- kwargs (object): Keyword arguments.
-
- Returns:
- Optional[Phi4MMAudioInputs]: Parsed and validated audio inputs.
- """
- audio_features = kwargs.pop("audio_input_features", None)
- audio_embeds = kwargs.pop("audio_embeds", None)
-
- if audio_features is None and audio_embeds is None:
- return None
-
- if audio_features is not None:
- return Phi4MMAudioFeatureInputs(
- type="audio_features",
- audio_features=audio_features,
- )
-
- if audio_embeds is not None:
- return Phi4MMAudioEmbeddingInputs(type="audio_embeds", data=audio_embeds)
-
- raise AssertionError("This line should be unreachable.")
-
- def _process_audio_input(
- self, audio_input: Phi4MMAudioInputs, audio_projection_mode: str
- ) -> NestedTensors:
- """
- Create the audio embeddings from the audio input, where the audio input
- is pairs of audio features and audio embed lengths. The audio input is
- created by `input_mapper_for_phi4mm_audio`.
-
- Args:
- audio_input (Phi4MMAudioInputs): Audio input.
-
- Returns:
- NestedTensors: Audio embeddings
- """
- if audio_input["type"] == "audio_embeds":
- return audio_input["data"]
-
- audio_features = audio_input["audio_features"]
- # (e.g. multiple examples) and the second dim is the multi-audio dim
- # (e.g. multiple audios in the same example)
-
- dtype = next(self.audio_embed.parameters()).dtype
- audio_embeds = [
- self.audio_embed(
- features.unsqueeze(0).to(dtype),
- audio_projection_mode=audio_projection_mode,
- )
- for features in audio_features
- ]
- return audio_embeds
-
- def _parse_and_validate_image_input(
- self, **kwargs: object
- ) -> Phi4MMImagePixelInputs | None:
- pixel_values = kwargs.get("image_pixel_values")
- if pixel_values is None:
- return None
-
- image_sizes = kwargs.get("image_sizes")
- image_attention_mask = kwargs.get("image_attention_mask")
- num_img_tokens = kwargs.get("num_img_tokens")
- assert (
- image_sizes is not None
- and image_attention_mask is not None
- and num_img_tokens is not None
- ), "Missing image inputs"
-
- return Phi4MMImagePixelInputs(
- type="pixel_values",
- pixel_values=pixel_values,
- image_sizes=image_sizes,
- image_attention_mask=image_attention_mask,
- num_img_tokens=num_img_tokens,
- )
-
- def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
- modalities = {}
-
- # Preserve the order of modalities if there are multiple of them
- # from the order of kwargs.
- for input_key in kwargs:
- if (
- input_key in ("image_pixel_values", "image_embeds")
- and "images" not in modalities
- ):
- modalities["images"] = self._parse_and_validate_image_input(**kwargs)
- if (
- input_key in ("audio_input_features", "audio_embeds")
- and "audios" not in modalities
- ):
- modalities["audios"] = self._parse_and_validate_audio_input(**kwargs)
-
- return modalities
-
- def _process_image_input(
- self, image_input: Phi4MMImagePixelInputs
- ) -> list[torch.Tensor]:
- if image_input["type"] == "image_embeds":
- image_embeds = image_input["image_embeds"].type(self.visual.dtype)
- else:
- dtype = next(self.image_embed.parameters()).dtype
- pixel_values = image_input["pixel_values"].to(dtype)
- image_sizes = image_input["image_sizes"]
- image_attention_mask = image_input["image_attention_mask"]
- image_embeds = self.image_embed(
- pixel_values, image_sizes, image_attention_mask
- )
- return image_embeds
-
- def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
- modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
- if not modalities:
- return []
-
- # The result multimodal_embeddings is tuple of tensors, with each
- # tensor corresponding to a multimodal data item (image or video).
- multimodal_embeddings: tuple[torch.Tensor, ...] = ()
-
- # NOTE: It is important to iterate over the keys in this dictionary
- # to preserve the order of the modalities.
- audio_projection_mode = "speech"
- for modality in modalities:
- # make sure process images first
- if modality == "images":
- audio_projection_mode = "vision"
- image_input = modalities["images"]
- image_embeddings = self._process_image_input(image_input)
- multimodal_embeddings += tuple(image_embeddings)
- if modality == "audios":
- audio_input = modalities["audios"]
- audio_embeddings = self._process_audio_input(
- audio_input, audio_projection_mode=audio_projection_mode
- )
- multimodal_embeddings += tuple(audio_embeddings)
-
- return multimodal_embeddings
-
- def forward(
- self,
- input_ids: torch.Tensor,
- positions: torch.Tensor,
- intermediate_tensors: IntermediateTensors | None = None,
- inputs_embeds: torch.Tensor | None = None,
- **kwargs: object,
- ) -> torch.Tensor:
- if intermediate_tensors is not None:
- inputs_embeds = None
-
- hidden_states = self.language_model(
- input_ids,
- positions,
- intermediate_tensors,
- inputs_embeds=inputs_embeds,
- )
-
- return hidden_states
-
- def compute_logits(
- self,
- hidden_states: torch.Tensor,
- ) -> torch.Tensor | None:
- return self.language_model.compute_logits(hidden_states)
-
- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
- loader = AutoWeightsLoader(self)
- return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-
- def get_mm_mapping(self) -> MultiModelKeys:
- """
- Get the module prefix in multimodal models
- """
- return MultiModelKeys.from_string_field(
- language_model="language_model.",
- connector=[
- "img_projection",
- "vision_speech_projection",
- "speech_projection",
- ],
- tower_model=["image_embed", "audio_embed"],
- )
-
- def get_language_model(self) -> torch.nn.Module:
- return self.language_model
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index d3b6268e76..a4a964bc7c 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -374,7 +374,6 @@ _MULTIMODAL_MODELS = {
),
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
- "Phi4MultimodalForCausalLM": ("phi4_multimodal", "Phi4MultimodalForCausalLM"), # noqa: E501
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
"QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), # noqa: E501
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
@@ -507,6 +506,7 @@ _PREVIOUSLY_SUPPORTED_MODELS = {
"MotifForCausalLM": "0.10.2",
"Phi3SmallForCausalLM": "0.9.2",
"Phi4FlashForCausalLM": "0.10.2",
+ "Phi4MultimodalForCausalLM": "0.12.0",
# encoder-decoder models except whisper
# have been removed for V0 deprecation.
"BartModel": "0.10.2",