[Model] Add tower and connector LoRA support for LFM2-VL (#51498)
Signed-off-by: zupengwang <71580390+zupengwang@users.noreply.github.com>
This commit is contained in:
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -37,9 +38,10 @@ class Siglip2VisionEmbeddings(nn.Module):
|
||||
self.config = config
|
||||
self.embed_dim = config.hidden_size
|
||||
self.patch_size = config.patch_size
|
||||
self.patch_embedding = nn.Linear(
|
||||
in_features=config.num_channels * self.patch_size * self.patch_size,
|
||||
out_features=self.embed_dim,
|
||||
self.patch_embedding = ReplicatedLinear(
|
||||
input_size=config.num_channels * self.patch_size * self.patch_size,
|
||||
output_size=self.embed_dim,
|
||||
return_bias=False,
|
||||
)
|
||||
self.num_patches = config.num_patches
|
||||
self.position_embedding_size = int(self.num_patches**0.5)
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncCalculator,
|
||||
@@ -499,16 +500,20 @@ class Lfm2VLMultiModalProjector(nn.Module):
|
||||
self.projector_use_layernorm = config.projector_use_layernorm
|
||||
if self.projector_use_layernorm:
|
||||
self.layer_norm = nn.LayerNorm(in_channels)
|
||||
self.linear_1 = nn.Linear(
|
||||
self.linear_1 = ReplicatedLinear(
|
||||
in_channels,
|
||||
config.projector_hidden_size,
|
||||
bias=config.projector_bias,
|
||||
prefix=maybe_prefix(prefix, "linear_1"),
|
||||
return_bias=False,
|
||||
)
|
||||
self.act = ACT2FN[config.projector_hidden_act]
|
||||
self.linear_2 = nn.Linear(
|
||||
self.linear_2 = ReplicatedLinear(
|
||||
config.projector_hidden_size,
|
||||
config.text_config.hidden_size,
|
||||
bias=config.projector_bias,
|
||||
prefix=maybe_prefix(prefix, "linear_2"),
|
||||
return_bias=False,
|
||||
)
|
||||
|
||||
def forward(
|
||||
@@ -1259,3 +1264,13 @@ class Lfm2VLForConditionalGeneration(
|
||||
connector="multi_modal_projector",
|
||||
tower_model="vision_tower",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||
downsample_factor = self.config.downsample_factor
|
||||
|
||||
return num_image_tokens * downsample_factor**2
|
||||
|
||||
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||
downsample_factor = self.config.downsample_factor
|
||||
|
||||
return num_vision_tokens // downsample_factor**2
|
||||
|
||||
Reference in New Issue
Block a user