[Model] Add tower and connector LoRA support for LFM2-VL (#51498)

Signed-off-by: zupengwang <71580390+zupengwang@users.noreply.github.com>
This commit is contained in:
Zupeng Wang
2026-08-20 20:33:43 +08:00
committed by GitHub
parent 6259572b28
commit df1376907b
2 changed files with 22 additions and 5 deletions
+5 -3
View File
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -37,9 +38,10 @@ class Siglip2VisionEmbeddings(nn.Module):
self.config = config
self.embed_dim = config.hidden_size
self.patch_size = config.patch_size
self.patch_embedding = nn.Linear(
in_features=config.num_channels * self.patch_size * self.patch_size,
out_features=self.embed_dim,
self.patch_embedding = ReplicatedLinear(
input_size=config.num_channels * self.patch_size * self.patch_size,
output_size=self.embed_dim,
return_bias=False,
)
self.num_patches = config.num_patches
self.position_embedding_size = int(self.num_patches**0.5)
+17 -2
View File
@@ -22,6 +22,7 @@ from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.forward_context import set_forward_context
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncCalculator,
@@ -499,16 +500,20 @@ class Lfm2VLMultiModalProjector(nn.Module):
self.projector_use_layernorm = config.projector_use_layernorm
if self.projector_use_layernorm:
self.layer_norm = nn.LayerNorm(in_channels)
self.linear_1 = nn.Linear(
self.linear_1 = ReplicatedLinear(
in_channels,
config.projector_hidden_size,
bias=config.projector_bias,
prefix=maybe_prefix(prefix, "linear_1"),
return_bias=False,
)
self.act = ACT2FN[config.projector_hidden_act]
self.linear_2 = nn.Linear(
self.linear_2 = ReplicatedLinear(
config.projector_hidden_size,
config.text_config.hidden_size,
bias=config.projector_bias,
prefix=maybe_prefix(prefix, "linear_2"),
return_bias=False,
)
def forward(
@@ -1259,3 +1264,13 @@ class Lfm2VLForConditionalGeneration(
connector="multi_modal_projector",
tower_model="vision_tower",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
downsample_factor = self.config.downsample_factor
return num_image_tokens * downsample_factor**2
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
downsample_factor = self.config.downsample_factor
return num_vision_tokens // downsample_factor**2