[Core] Initialize LoRA support for tower and connector in multi-modal models (#26674)

Signed-off-by: bk-201 <joy25810@foxmail.com>
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com>
Co-authored-by: bk-201 <joy25810@foxmail.com>
Co-authored-by: prashanth058 <prashanth.dannamaneni@uipath.com>
Co-authored-by: Anexdeus <5142168@mail.ru>
This commit is contained in:
Jee Jee Li
2025-12-26 04:48:20 -08:00
committed by GitHub
co-authored by bk-201 prashanth058 Anexdeus
parent 0b544e6476
commit ce1eafd1a5
20 changed files with 635 additions and 80 deletions
+79 -6
View File
@@ -54,6 +54,7 @@ from vllm.forward_context import (
set_forward_context,
)
from vllm.logger import init_logger
from vllm.lora.layers import LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.rotary_embedding import (
MRotaryEmbedding,
@@ -79,7 +80,11 @@ from vllm.model_executor.models.interfaces_base import (
is_text_generation_model,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import BatchedTensorInputs, MultiModalKwargsItem
from vllm.multimodal.inputs import (
BatchedTensorInputs,
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.multimodal.utils import group_mm_kwargs_by_modality
from vllm.pooling_params import PoolingParams
from vllm.sampling_params import SamplingType
@@ -2095,7 +2100,11 @@ class GPUModelRunner(
def _batch_mm_inputs_from_scheduler(
self,
scheduler_output: "SchedulerOutput",
) -> tuple[list[str], list[MultiModalKwargsItem]]:
) -> tuple[
list[str],
list[MultiModalKwargsItem],
list[tuple[str, PlaceholderRange]],
]:
"""Batch multimodal inputs from scheduled encoder inputs.
Args:
@@ -2103,16 +2112,20 @@ class GPUModelRunner(
inputs.
Returns:
A tuple of (mm_hashes, mm_kwargs) where:
A tuple of (mm_hashes, mm_kwargs, mm_lora_refs) where:
- mm_hashes: List of multimodal hashes for each item
- mm_kwargs: List of multimodal kwargs for each item
- mm_lora_refs: List of (req_id, placeholder_range) for each item
"""
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
if not scheduled_encoder_inputs:
return [], []
return [], [], []
mm_hashes = list[str]()
mm_kwargs = list[MultiModalKwargsItem]()
# Multimodal LoRA reference info to map each multimodal item
# back to its request & position
mm_lora_refs = list[tuple[str, PlaceholderRange]]()
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
req_state = self.requests[req_id]
@@ -2123,13 +2136,16 @@ class GPUModelRunner(
mm_hashes.append(mm_feature.identifier)
mm_kwargs.append(mm_feature.data)
mm_lora_refs.append((req_id, mm_feature.mm_position))
return mm_hashes, mm_kwargs
return mm_hashes, mm_kwargs, mm_lora_refs
def _execute_mm_encoder(
self, scheduler_output: "SchedulerOutput"
) -> list[torch.Tensor]:
mm_hashes, mm_kwargs = self._batch_mm_inputs_from_scheduler(scheduler_output)
mm_hashes, mm_kwargs, mm_lora_refs = self._batch_mm_inputs_from_scheduler(
scheduler_output
)
if not mm_kwargs:
return []
@@ -2142,6 +2158,63 @@ class GPUModelRunner(
# multimodal inputs. The proper solution should be reordering the
# encoder outputs.
model = cast(SupportsMultiModal, self.model)
if self.lora_config and self.lora_manager.supports_tower_connector_lora():
# Build LoRA mappings independently for encoder inputs
# (encoder batch structure is different from main batch)
prompt_lora_mapping = []
token_lora_mapping = []
lora_requests = set()
encoder_token_counts = []
for req_id, pos_info in mm_lora_refs:
req_idx = self.input_batch.req_id_to_index[req_id]
lora_id = int(self.input_batch.request_lora_mapping[req_idx])
# Prefer pos_info.get_num_embeds to count precise MM embedding tokens.
num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]
pos_info.get_num_embeds
)
prompt_lora_mapping.append(lora_id)
token_lora_mapping.extend([lora_id] * num_tokens)
encoder_token_counts.append(num_tokens)
if lora_id > 0:
lora_request = self.input_batch.lora_id_to_lora_request.get(lora_id)
if lora_request is not None:
lora_requests.add(lora_request)
# Set tower adapter mapping
tower_mapping = LoRAMapping(
tuple(token_lora_mapping),
tuple(prompt_lora_mapping),
is_prefill=True,
type=LoRAMappingType.TOWER,
)
self.lora_manager.set_active_adapters(lora_requests, tower_mapping)
if hasattr(self.model, "get_num_mm_connector_tokens"):
post_op_counts = [
self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined]
for num_tokens in encoder_token_counts
]
connector_token_mapping = np.repeat(
np.array(prompt_lora_mapping, dtype=np.int32),
np.array(post_op_counts, dtype=np.int32),
)
connector_mapping = LoRAMapping(
index_mapping=tuple(connector_token_mapping.tolist()),
prompt_mapping=tuple(prompt_lora_mapping),
is_prefill=True,
type=LoRAMappingType.CONNECTOR,
)
self.lora_manager.set_active_adapters(
lora_requests,
connector_mapping,
)
encoder_outputs: list[torch.Tensor] = []
for modality, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
mm_kwargs,