# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any import numpy as np from vllm.lora.layers import LoRAMapping, LoRAMappingType from vllm.lora.worker_manager import WorkerLoRAManager from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.v1.worker.gpu.lora_utils import LoraState from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache def set_active_mm_loras( model: Any, lora_manager: WorkerLoRAManager, encoder_cache: EncoderCache | None, req_id_to_index: dict[str, int], lora_state: LoraState, scheduled_encoder_inputs: dict[str, list[int]], ) -> None: if ( not scheduled_encoder_inputs or encoder_cache is None or not lora_manager.supports_tower_connector_lora() ): return prompt_lora_mapping: list[int] = [] token_lora_mapping: list[int] = [] lora_requests = set() encoder_token_counts: list[int] = [] connector_token_counts: list[int] = [] mm_items: list[tuple[int, MultiModalFeatureSpec]] = [] for req_id, encoder_input_ids in scheduled_encoder_inputs.items(): req_idx = req_id_to_index.get(req_id) if req_idx is None: continue lora_id = int(lora_state.lora_ids[req_idx]) mm_features = encoder_cache.mm_features[req_id] mm_items.extend((lora_id, mm_features[i]) for i in encoder_input_ids) if lora_id > 0: lora_request = lora_state.lora_requests.get(req_id) if lora_request is not None: lora_requests.add(lora_request) # Runs items sorted by modality. mm_items.sort(key=lambda x: x[1].modality) for lora_id, mm_feature in mm_items: pos_info = mm_feature.mm_position tower_tokens, connector_tokens = model.get_mm_lora_token_counts( modality=mm_feature.modality, mm_kwargs=mm_feature.data, num_mm_embeds=pos_info.get_num_embeds(), ) prompt_lora_mapping.append(lora_id) token_lora_mapping.extend([lora_id] * tower_tokens) encoder_token_counts.append(tower_tokens) connector_token_counts.append(connector_tokens) if not prompt_lora_mapping: return lora_manager.set_active_adapters( lora_requests, LoRAMapping( tuple(token_lora_mapping), tuple(prompt_lora_mapping), is_prefill=True, type=LoRAMappingType.TOWER, ), ) mm_mapping = model.get_mm_mapping() if hasattr(model, "get_mm_mapping") else None if ( mm_mapping is None or not mm_mapping.connector or not all(count is not None for count in connector_token_counts) ): return connector_token_mapping = np.repeat( np.array(prompt_lora_mapping, dtype=np.int32), np.array(connector_token_counts, dtype=np.int32), ) lora_manager.set_active_adapters( lora_requests, LoRAMapping( index_mapping=tuple(connector_token_mapping.tolist()), prompt_mapping=tuple(prompt_lora_mapping), is_prefill=True, type=LoRAMappingType.CONNECTOR, ), )