[LoRA][Gemma4] Support vision tower LoRA (#42662)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: linitra24 <Joy25810@foxmail.com> Signed-off-by: linitra24 <joy25810@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: linitra24 <joy25810@gmail.com>
This commit is contained in:
co-authored by
Jee Jee Li
linitra24
parent
8e1131e1d1
commit
95c9144424
@@ -48,7 +48,7 @@ if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalKwargsItem
|
||||
from vllm.multimodal.registry import _ProcessorFactories
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import ScoreType
|
||||
@@ -385,6 +385,28 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
|
||||
"""
|
||||
...
|
||||
|
||||
def get_mm_lora_token_counts(
|
||||
self,
|
||||
*,
|
||||
modality: str,
|
||||
mm_kwargs: "MultiModalKwargsItem | None",
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
"""
|
||||
Return ``(tower_tokens, connector_tokens)`` for multimodal LoRA mappings.
|
||||
|
||||
MM LoRA uses these counts to build adapter mappings for the tower and
|
||||
connector forwards. Models with multiple modalities can override this
|
||||
when each modality has different encoder padding or pooling behavior.
|
||||
"""
|
||||
del modality, mm_kwargs
|
||||
num_encoder_tokens = self.get_num_mm_encoder_tokens(num_mm_embeds)
|
||||
num_connector_tokens = self.get_num_mm_connector_tokens(num_encoder_tokens)
|
||||
return (
|
||||
num_encoder_tokens,
|
||||
num_connector_tokens if isinstance(num_connector_tokens, int) else None,
|
||||
)
|
||||
|
||||
@overload
|
||||
def embed_input_ids(self, input_ids: Tensor) -> Tensor: ...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user