[VLM] Clean up Phi-4-MM ViT implementation (#14812)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -60,7 +60,7 @@ class AriaVisionTransformer(Idefics3VisionTransformer, SupportsQuant):
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__(config, quant_config, prefix)
|
||||
super().__init__(config, quant_config=quant_config, prefix=prefix)
|
||||
# Unlike Idefics3VisionTransformer which uses LayerNorm after the
|
||||
# final layer, Aria omits this normalization, so we replace it with an
|
||||
# Identity layer
|
||||
@@ -512,7 +512,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
self.config = config
|
||||
self.vision_tower = AriaVisionTransformer(
|
||||
config.vision_config,
|
||||
quant_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.vision_tower",
|
||||
)
|
||||
self.multi_modal_projector = AriaProjector(config)
|
||||
|
||||
@@ -113,7 +113,7 @@ class Idefics2VisionAttention(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Idefics2Config,
|
||||
config: Idefics2VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
@@ -164,7 +164,7 @@ class Idefics2VisionMLP(nn.Module):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Idefics2Config,
|
||||
config: Idefics2VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
@@ -249,16 +249,24 @@ class Idefics2Encoder(nn.Module):
|
||||
self,
|
||||
config: Idefics2Config,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
*,
|
||||
num_hidden_layers_override: Optional[int] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.config = config
|
||||
|
||||
if num_hidden_layers_override is None:
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
else:
|
||||
num_hidden_layers = num_hidden_layers_override
|
||||
|
||||
self.layers = nn.ModuleList([
|
||||
Idefics2EncoderLayer(config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.layers.{layer_idx}")
|
||||
for layer_idx in range(config.num_hidden_layers)
|
||||
for layer_idx in range(num_hidden_layers)
|
||||
])
|
||||
|
||||
def forward(
|
||||
@@ -287,6 +295,9 @@ class Idefics2VisionTransformer(nn.Module):
|
||||
self,
|
||||
config: Idefics2VisionConfig,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
*,
|
||||
num_hidden_layers_override: Optional[int] = None,
|
||||
require_post_norm: bool = True,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -294,11 +305,24 @@ class Idefics2VisionTransformer(nn.Module):
|
||||
embed_dim = config.hidden_size
|
||||
self.config = config
|
||||
self.embeddings = Idefics2VisionEmbeddings(config)
|
||||
self.encoder = Idefics2Encoder(config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.encoder")
|
||||
self.post_layernorm = nn.LayerNorm(embed_dim,
|
||||
eps=config.layer_norm_eps)
|
||||
self.encoder = Idefics2Encoder(
|
||||
config,
|
||||
quant_config=quant_config,
|
||||
num_hidden_layers_override=num_hidden_layers_override,
|
||||
prefix=f"{prefix}.encoder")
|
||||
|
||||
num_hidden_layers = config.num_hidden_layers
|
||||
if len(self.encoder.layers) > config.num_hidden_layers:
|
||||
raise ValueError(
|
||||
f"The original encoder only has {num_hidden_layers} "
|
||||
f"layers, but you requested {len(self.encoder.layers)} layers."
|
||||
)
|
||||
|
||||
self.require_post_norm = require_post_norm
|
||||
self.post_layernorm = nn.LayerNorm(
|
||||
embed_dim,
|
||||
eps=config.layer_norm_eps,
|
||||
) if require_post_norm else nn.Identity()
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embeddings
|
||||
@@ -328,7 +352,24 @@ class Idefics2VisionTransformer(nn.Module):
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: Set[str] = set()
|
||||
layer_count = len(self.encoder.layers)
|
||||
|
||||
for name, loaded_weight in weights:
|
||||
# skip pooling header
|
||||
if name.startswith("head."):
|
||||
continue
|
||||
|
||||
# post_layernorm is optional
|
||||
if (name.startswith("post_layernorm.")
|
||||
and not self.require_post_norm):
|
||||
continue
|
||||
|
||||
# omit layers when num_hidden_layers_override is set
|
||||
if name.startswith("encoder.layers."):
|
||||
layer_idx = int(name.split(".")[2])
|
||||
if layer_idx >= layer_count:
|
||||
continue
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
@@ -11,7 +11,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torchvision.transforms as T
|
||||
from PIL import Image
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PretrainedConfig, SiglipVisionConfig
|
||||
from transformers.utils import logging
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -32,10 +32,10 @@ from vllm.multimodal.inputs import MultiModalInputs, NestedTensors
|
||||
from vllm.sequence import IntermediateTensors, SequenceData
|
||||
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
|
||||
|
||||
from .idefics2_vision_model import Idefics2VisionTransformer
|
||||
from .interfaces import SupportsLoRA, SupportsMultiModal
|
||||
from .phi4mm_audio import AudioEmbedding
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
from .vision_siglip_navit import get_siglip_vision_model
|
||||
|
||||
# <|endoftext10|> (see vocab.json in hf model)
|
||||
_IMAGE_PLACEHOLDER_TOKEN_ID = 200010
|
||||
@@ -339,6 +339,33 @@ def preprocess(images, dynamic_hd_size, vit_resolution, vit_patch_size):
|
||||
return data
|
||||
|
||||
|
||||
def get_navit_vision_model(layer_idx: int = -1, **kwargs):
|
||||
vision_config = {
|
||||
"hidden_size": 1152,
|
||||
"image_size": 448,
|
||||
"intermediate_size": 4304,
|
||||
"model_type": "siglip_vision_model",
|
||||
"num_attention_heads": 16,
|
||||
"num_hidden_layers": 27,
|
||||
"patch_size": 14,
|
||||
}
|
||||
|
||||
model_config = SiglipVisionConfig(**vision_config, **kwargs)
|
||||
if layer_idx < 0:
|
||||
num_hidden_layers = model_config.num_hidden_layers \
|
||||
+ layer_idx + 1
|
||||
else:
|
||||
num_hidden_layers = layer_idx + 1
|
||||
|
||||
vision_model = Idefics2VisionTransformer(
|
||||
config=model_config,
|
||||
require_post_norm=False,
|
||||
num_hidden_layers_override=num_hidden_layers,
|
||||
)
|
||||
|
||||
return vision_model
|
||||
|
||||
|
||||
class Phi4MMImageEncoder(nn.Module):
|
||||
"""Image embedding."""
|
||||
|
||||
@@ -362,8 +389,7 @@ class Phi4MMImageEncoder(nn.Module):
|
||||
self.layer_idx = -2
|
||||
self.type_feature = 'patch'
|
||||
|
||||
self.img_processor = get_siglip_vision_model(
|
||||
_flash_attn_2_enabled=True)
|
||||
self.img_processor = get_navit_vision_model(layer_idx=self.layer_idx)
|
||||
|
||||
pe_weight = self.img_processor.embeddings.position_embedding.weight
|
||||
L, D = pe_weight.size()
|
||||
@@ -430,16 +456,11 @@ class Phi4MMImageEncoder(nn.Module):
|
||||
def get_img_features(self,
|
||||
img_embeds: torch.FloatTensor,
|
||||
attention_mask=None) -> torch.FloatTensor:
|
||||
LAYER_IDX = self.layer_idx
|
||||
TYPE_FEATURE = self.type_feature
|
||||
|
||||
img_processor_output = self.img_processor(
|
||||
img_embeds,
|
||||
output_hidden_states=True,
|
||||
patch_attention_mask=attention_mask)
|
||||
img_feature = img_processor_output.hidden_states[LAYER_IDX]
|
||||
img_feature = self.img_processor(img_embeds,
|
||||
patch_attention_mask=attention_mask)
|
||||
|
||||
if TYPE_FEATURE == "patch":
|
||||
if self.type_feature == "patch":
|
||||
patch_feature = img_feature
|
||||
|
||||
use_token_compression = self.image_token_compression is not None
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user