diff --git a/vllm/model_executor/models/lfm2_siglip2.py b/vllm/model_executor/models/lfm2_siglip2.py index f1679af813..31e7c0f144 100644 --- a/vllm/model_executor/models/lfm2_siglip2.py +++ b/vllm/model_executor/models/lfm2_siglip2.py @@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, + ReplicatedLinear, RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig @@ -37,9 +38,10 @@ class Siglip2VisionEmbeddings(nn.Module): self.config = config self.embed_dim = config.hidden_size self.patch_size = config.patch_size - self.patch_embedding = nn.Linear( - in_features=config.num_channels * self.patch_size * self.patch_size, - out_features=self.embed_dim, + self.patch_embedding = ReplicatedLinear( + input_size=config.num_channels * self.patch_size * self.patch_size, + output_size=self.embed_dim, + return_bias=False, ) self.num_patches = config.num_patches self.position_embedding_size = int(self.num_patches**0.5) diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index ce60f2d236..d247fef4d7 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -22,6 +22,7 @@ from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions from vllm.forward_context import set_forward_context from vllm.inputs import MultiModalDataDict +from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateCopyFunc, MambaStateCopyFuncCalculator, @@ -499,16 +500,20 @@ class Lfm2VLMultiModalProjector(nn.Module): self.projector_use_layernorm = config.projector_use_layernorm if self.projector_use_layernorm: self.layer_norm = nn.LayerNorm(in_channels) - self.linear_1 = nn.Linear( + self.linear_1 = ReplicatedLinear( in_channels, config.projector_hidden_size, bias=config.projector_bias, + prefix=maybe_prefix(prefix, "linear_1"), + return_bias=False, ) self.act = ACT2FN[config.projector_hidden_act] - self.linear_2 = nn.Linear( + self.linear_2 = ReplicatedLinear( config.projector_hidden_size, config.text_config.hidden_size, bias=config.projector_bias, + prefix=maybe_prefix(prefix, "linear_2"), + return_bias=False, ) def forward( @@ -1259,3 +1264,13 @@ class Lfm2VLForConditionalGeneration( connector="multi_modal_projector", tower_model="vision_tower", ) + + def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: + downsample_factor = self.config.downsample_factor + + return num_image_tokens * downsample_factor**2 + + def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: + downsample_factor = self.config.downsample_factor + + return num_vision_tokens // downsample_factor**2