[Core] Initialize LoRA support for tower and connector in multi-modal models (#26674)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: bk-201 <joy25810@foxmail.com> Co-authored-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: Anexdeus <5142168@mail.ru>
This commit is contained in:
co-authored by
bk-201
prashanth058
Anexdeus
parent
0b544e6476
commit
ce1eafd1a5
@@ -54,6 +54,7 @@ from vllm.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.rotary_embedding import (
|
||||
MRotaryEmbedding,
|
||||
@@ -79,7 +80,11 @@ from vllm.model_executor.models.interfaces_base import (
|
||||
is_text_generation_model,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import BatchedTensorInputs, MultiModalKwargsItem
|
||||
from vllm.multimodal.inputs import (
|
||||
BatchedTensorInputs,
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.utils import group_mm_kwargs_by_modality
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sampling_params import SamplingType
|
||||
@@ -2095,7 +2100,11 @@ class GPUModelRunner(
|
||||
def _batch_mm_inputs_from_scheduler(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
) -> tuple[list[str], list[MultiModalKwargsItem]]:
|
||||
) -> tuple[
|
||||
list[str],
|
||||
list[MultiModalKwargsItem],
|
||||
list[tuple[str, PlaceholderRange]],
|
||||
]:
|
||||
"""Batch multimodal inputs from scheduled encoder inputs.
|
||||
|
||||
Args:
|
||||
@@ -2103,16 +2112,20 @@ class GPUModelRunner(
|
||||
inputs.
|
||||
|
||||
Returns:
|
||||
A tuple of (mm_hashes, mm_kwargs) where:
|
||||
A tuple of (mm_hashes, mm_kwargs, mm_lora_refs) where:
|
||||
- mm_hashes: List of multimodal hashes for each item
|
||||
- mm_kwargs: List of multimodal kwargs for each item
|
||||
- mm_lora_refs: List of (req_id, placeholder_range) for each item
|
||||
"""
|
||||
scheduled_encoder_inputs = scheduler_output.scheduled_encoder_inputs
|
||||
if not scheduled_encoder_inputs:
|
||||
return [], []
|
||||
return [], [], []
|
||||
|
||||
mm_hashes = list[str]()
|
||||
mm_kwargs = list[MultiModalKwargsItem]()
|
||||
# Multimodal LoRA reference info to map each multimodal item
|
||||
# back to its request & position
|
||||
mm_lora_refs = list[tuple[str, PlaceholderRange]]()
|
||||
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
|
||||
req_state = self.requests[req_id]
|
||||
|
||||
@@ -2123,13 +2136,16 @@ class GPUModelRunner(
|
||||
|
||||
mm_hashes.append(mm_feature.identifier)
|
||||
mm_kwargs.append(mm_feature.data)
|
||||
mm_lora_refs.append((req_id, mm_feature.mm_position))
|
||||
|
||||
return mm_hashes, mm_kwargs
|
||||
return mm_hashes, mm_kwargs, mm_lora_refs
|
||||
|
||||
def _execute_mm_encoder(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> list[torch.Tensor]:
|
||||
mm_hashes, mm_kwargs = self._batch_mm_inputs_from_scheduler(scheduler_output)
|
||||
mm_hashes, mm_kwargs, mm_lora_refs = self._batch_mm_inputs_from_scheduler(
|
||||
scheduler_output
|
||||
)
|
||||
|
||||
if not mm_kwargs:
|
||||
return []
|
||||
@@ -2142,6 +2158,63 @@ class GPUModelRunner(
|
||||
# multimodal inputs. The proper solution should be reordering the
|
||||
# encoder outputs.
|
||||
model = cast(SupportsMultiModal, self.model)
|
||||
|
||||
if self.lora_config and self.lora_manager.supports_tower_connector_lora():
|
||||
# Build LoRA mappings independently for encoder inputs
|
||||
# (encoder batch structure is different from main batch)
|
||||
prompt_lora_mapping = []
|
||||
token_lora_mapping = []
|
||||
lora_requests = set()
|
||||
encoder_token_counts = []
|
||||
|
||||
for req_id, pos_info in mm_lora_refs:
|
||||
req_idx = self.input_batch.req_id_to_index[req_id]
|
||||
lora_id = int(self.input_batch.request_lora_mapping[req_idx])
|
||||
|
||||
# Prefer pos_info.get_num_embeds to count precise MM embedding tokens.
|
||||
num_tokens = self.model.get_num_mm_encoder_tokens( # type: ignore[attr-defined]
|
||||
pos_info.get_num_embeds
|
||||
)
|
||||
prompt_lora_mapping.append(lora_id)
|
||||
token_lora_mapping.extend([lora_id] * num_tokens)
|
||||
encoder_token_counts.append(num_tokens)
|
||||
|
||||
if lora_id > 0:
|
||||
lora_request = self.input_batch.lora_id_to_lora_request.get(lora_id)
|
||||
if lora_request is not None:
|
||||
lora_requests.add(lora_request)
|
||||
|
||||
# Set tower adapter mapping
|
||||
tower_mapping = LoRAMapping(
|
||||
tuple(token_lora_mapping),
|
||||
tuple(prompt_lora_mapping),
|
||||
is_prefill=True,
|
||||
type=LoRAMappingType.TOWER,
|
||||
)
|
||||
self.lora_manager.set_active_adapters(lora_requests, tower_mapping)
|
||||
|
||||
if hasattr(self.model, "get_num_mm_connector_tokens"):
|
||||
post_op_counts = [
|
||||
self.model.get_num_mm_connector_tokens(num_tokens) # type: ignore[attr-defined]
|
||||
for num_tokens in encoder_token_counts
|
||||
]
|
||||
|
||||
connector_token_mapping = np.repeat(
|
||||
np.array(prompt_lora_mapping, dtype=np.int32),
|
||||
np.array(post_op_counts, dtype=np.int32),
|
||||
)
|
||||
connector_mapping = LoRAMapping(
|
||||
index_mapping=tuple(connector_token_mapping.tolist()),
|
||||
prompt_mapping=tuple(prompt_lora_mapping),
|
||||
is_prefill=True,
|
||||
type=LoRAMappingType.CONNECTOR,
|
||||
)
|
||||
|
||||
self.lora_manager.set_active_adapters(
|
||||
lora_requests,
|
||||
connector_mapping,
|
||||
)
|
||||
|
||||
encoder_outputs: list[torch.Tensor] = []
|
||||
for modality, num_items, mm_kwargs_group in group_mm_kwargs_by_modality(
|
||||
mm_kwargs,
|
||||
|
||||
Reference in New Issue
Block a user