From e85d1b69cf2f1c6101cfc7c799bb0c457cacc4b3 Mon Sep 17 00:00:00 2001 From: linitra24 Date: Fri, 21 Aug 2026 17:45:54 +0800 Subject: [PATCH] [Bugfix][LoRA] Use an explicit capability flag for tower connector LoRA (#53092) Signed-off-by: linitra24 --- vllm/lora/model_manager.py | 4 ++-- vllm/model_executor/models/blip2.py | 2 ++ vllm/model_executor/models/dots_ocr.py | 1 + vllm/model_executor/models/gemma3_mm.py | 2 ++ vllm/model_executor/models/gemma4_mm.py | 1 + vllm/model_executor/models/glm4_1v.py | 1 + vllm/model_executor/models/idefics3.py | 2 ++ vllm/model_executor/models/interfaces.py | 6 ++++++ vllm/model_executor/models/internvl.py | 1 + vllm/model_executor/models/keye.py | 2 ++ vllm/model_executor/models/llava.py | 2 ++ vllm/model_executor/models/llava_next.py | 1 + vllm/model_executor/models/llava_next_video.py | 2 ++ vllm/model_executor/models/mllama4.py | 1 + vllm/model_executor/models/paligemma.py | 2 ++ vllm/model_executor/models/pixtral.py | 2 ++ vllm/model_executor/models/qwen2_5_vl.py | 1 + vllm/model_executor/models/qwen2_vl.py | 1 + vllm/model_executor/models/qwen3_asr.py | 1 + vllm/model_executor/models/qwen3_vl.py | 1 + vllm/model_executor/models/ultravox.py | 2 ++ 21 files changed, 36 insertions(+), 2 deletions(-) diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index 48e31a5c03..d19bf311a1 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -201,8 +201,8 @@ class LoRAModelManager: self.punica_wrapper_mapping[lm_prefix] = llm_punica_wrapper # First, determine if the model supports tower connector LoRA. - self.supports_tower_connector_lora = self.supports_mm and hasattr( - self.model, "get_num_mm_encoder_tokens" + self.supports_tower_connector_lora = ( + self.supports_mm and self.model.supports_tower_connector_lora ) # Then, handle the case where the feature is disabled in the config. diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index 2f11b1dee0..6b1be9860a 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -525,6 +525,8 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]): class Blip2ForConditionalGeneration( nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsQuant ): + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index c7bea3b780..e91d6ce94c 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -618,6 +618,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA "fc13": ["fc1", "fc3"], } supports_encoder_tp_data = True + supports_tower_connector_lora = True @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index 31d3e82575..2176b023fa 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -524,6 +524,8 @@ class Gemma3ForConditionalGeneration( } ) + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index e993b1922f..15774a9ad2 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -978,6 +978,7 @@ class Gemma4ForConditionalGeneration( # window instead of dropping them (which would make image attention # causal-only for images larger than the sliding window). mm_prefix_clamp_sliding_window: bool = True + supports_tower_connector_lora = True packed_modules_mapping = { "qkv_proj": [ diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index c9ab921a8f..9bcca2f8bf 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -1755,6 +1755,7 @@ class Glm4vForConditionalGeneration( ) supports_encoder_tp_data = True + supports_tower_connector_lora = True @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: diff --git a/vllm/model_executor/models/idefics3.py b/vllm/model_executor/models/idefics3.py index ab6c64f46f..226bfbff79 100644 --- a/vllm/model_executor/models/idefics3.py +++ b/vllm/model_executor/models/idefics3.py @@ -503,6 +503,8 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo ], } + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index c9d8b47ec1..d9f84be396 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -167,6 +167,12 @@ class SupportsMultiModal(SupportsMultiModalEmbeddings, Protocol): `multimodal_config.mm_device_do_normalize`. """ + supports_tower_connector_lora: ClassVar[bool] = False + """ + A flag that indicates whether this model supports + `lora_config.enable_tower_connector_lora`. + """ + requires_raw_input_tokens: ClassVar[bool] = False """ A flag that indicates this model processes input id tokens diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index f818ce9dfd..3895533439 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -553,6 +553,7 @@ class InternVLChatModel( SupportsEncoderCudaGraph, ): supports_encoder_tp_data = True + supports_tower_connector_lora = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix=dict.fromkeys( diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index dd894e60b1..7a504e83b7 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -1235,6 +1235,8 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal): } ) + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/llava.py b/vllm/model_executor/models/llava.py index 9ef36e1824..3ddb60452e 100644 --- a/vllm/model_executor/models/llava.py +++ b/vllm/model_executor/models/llava.py @@ -520,6 +520,8 @@ class LlavaForConditionalGeneration( } ) + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py index 42a2df9824..fec9d6af5e 100644 --- a/vllm/model_executor/models/llava_next.py +++ b/vllm/model_executor/models/llava_next.py @@ -231,6 +231,7 @@ class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]): class LlavaNextForConditionalGeneration( nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP ): + supports_tower_connector_lora = True packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], diff --git a/vllm/model_executor/models/llava_next_video.py b/vllm/model_executor/models/llava_next_video.py index bb9c95017d..7794183734 100644 --- a/vllm/model_executor/models/llava_next_video.py +++ b/vllm/model_executor/models/llava_next_video.py @@ -323,6 +323,8 @@ class LlavaNextVideoForConditionalGeneration( } ) + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("video"): diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 04748465e0..ca862596ef 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -740,6 +740,7 @@ class Llama4ForConditionalGeneration( } supports_encoder_tp_data = True + supports_tower_connector_lora = True @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index 8d01f4219b..1e80e8d818 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -274,6 +274,8 @@ class PaliGemmaForConditionalGeneration( } ) + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index efc142f5f7..48b3310d30 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -309,6 +309,8 @@ class PixtralForConditionalGeneration( "gate_up_proj": ["gate_proj", "up_proj"], } + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 4ca28f7002..396a892cab 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -1256,6 +1256,7 @@ class Qwen2_5_VLForConditionalGeneration( supports_encoder_tp_data = True supports_mm_device_do_normalize = True + supports_tower_connector_lora = True def iter_mm_grid_thw( self, mm_features: list[MultiModalFeatureSpec] diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 49e026534c..700fefe585 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -1194,6 +1194,7 @@ class Qwen2VLForConditionalGeneration( supports_encoder_tp_data = True supports_mm_device_do_normalize = True + supports_tower_connector_lora = True def iter_mm_grid_thw( self, mm_features: list[MultiModalFeatureSpec] diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 301c08cee0..c06d074d4f 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -361,6 +361,7 @@ class Qwen3ASRForConditionalGeneration( } supported_languages = ISO639_1_SUPPORTED_LANGS + supports_tower_connector_lora = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 05a464af27..bc31091cfe 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1753,6 +1753,7 @@ class Qwen3VLForConditionalGeneration( } supports_encoder_tp_data = True + supports_tower_connector_lora = True supported_video_pruning_methods = ("evs", "vidcom2") diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index 1622175981..81e97ce4ce 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -610,6 +610,8 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA): } ) + supports_tower_connector_lora = True + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("audio"):