[LoRA][Refactor] Unify multimodal LoRA token count hooks (#55071)

Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
This commit is contained in:
linitra24
2026-09-14 13:19:15 +00:00
committed by GitHub
parent 6623fe5b4e
commit 9d4d9aa5bc
23 changed files with 276 additions and 295 deletions
+10 -17
View File
@@ -31,7 +31,7 @@ import torch
import torch.nn as nn
from torch import Tensor
from transformers.models.whisper.tokenization_whisper import LANGUAGES
from typing_extensions import Self, TypeIs
from typing_extensions import Self, TypeIs, deprecated
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -420,23 +420,16 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol):
yield
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
"""
Implement this function to enable LoRA support
for the tower module of the multi-modal model.
Given the number of image tokens, output the number of
multi-modal encoder tokens.
"""
...
@deprecated(
"get_num_mm_encoder_tokens is deprecated; use get_mm_lora_token_counts instead."
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int: ...
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
"""
Implement this function to enable LoRA support
for the connector module of the multi-modal model.
Given the number of vision tokens, output the number of
multi-modal connector tokens.
"""
...
@deprecated(
"get_num_mm_connector_tokens is deprecated; use "
"get_mm_lora_token_counts instead."
)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int: ...
def get_mm_lora_token_counts(
self,