[Core] Initialize LoRA support for tower and connector in multi-modal models (#26674)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: bk-201 <joy25810@foxmail.com> Co-authored-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: Anexdeus <5142168@mail.ru>
This commit is contained in:
co-authored by
bk-201
prashanth058
Anexdeus
parent
0b544e6476
commit
ce1eafd1a5
@@ -406,6 +406,24 @@ class InputProcessor:
|
||||
mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)]
|
||||
return mm_uuids
|
||||
|
||||
def _get_mm_identifier(
|
||||
self,
|
||||
mm_hash: str,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> str:
|
||||
"""
|
||||
When enable_tower_connector_lora is True, multi-modal embeddings
|
||||
vary depending on the LoRA request. Therefore, the mm_hash must be
|
||||
generated based on the LoRA request to prevent incorrect cache hits.
|
||||
"""
|
||||
if (
|
||||
lora_request is None
|
||||
or self.lora_config is None
|
||||
or not self.lora_config.enable_tower_connector_lora
|
||||
):
|
||||
return mm_hash
|
||||
return f"{lora_request.lora_name}:{mm_hash}"
|
||||
|
||||
@staticmethod
|
||||
def assign_request_id(request: EngineCoreRequest):
|
||||
"""Replace the externally supplied request ID with an internal request ID
|
||||
@@ -539,7 +557,10 @@ class InputProcessor:
|
||||
MultiModalFeatureSpec(
|
||||
data=decoder_mm_inputs[modality][idx],
|
||||
modality=modality,
|
||||
identifier=decoder_mm_hashes[modality][idx],
|
||||
identifier=self._get_mm_identifier(
|
||||
decoder_mm_hashes[modality][idx],
|
||||
lora_request,
|
||||
),
|
||||
mm_position=decoder_mm_positions[modality][idx],
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user