[LoRA][Gemma4] Support vision tower LoRA (#42662)

Signed-off-by: bk-201 <joy25810@foxmail.com>
Signed-off-by: linitra24 <Joy25810@foxmail.com>
Signed-off-by: linitra24 <joy25810@gmail.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: linitra24 <joy25810@gmail.com>
This commit is contained in:
linitra24
2026-08-13 22:07:43 +08:00
committed by GitHub
co-authored by Jee Jee Li linitra24
parent 8e1131e1d1
commit 95c9144424
10 changed files with 309 additions and 48 deletions
+23 -1
View File
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.utils import WeightsMapper
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
from vllm.multimodal.registry import _ProcessorFactories
from vllm.sequence import IntermediateTensors
from vllm.tasks import ScoreType
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
"""
...
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: "MultiModalKwargsItem | None",
num_mm_embeds: int,
) -> tuple[int, int | None]:
"""
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
MM LoRA uses these counts to build adapter mappings for the tower and
connector forwards. Models with multiple modalities can override this
when each modality has different encoder padding or pooling behavior.
"""
del modality, mm_kwargs
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
return (
num_encoder_tokens,
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
)
@overload
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...