[Core] Initialize LoRA support for tower and connector in multi-modal models (#26674)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: bk-201 <joy25810@foxmail.com> Co-authored-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: Anexdeus <5142168@mail.ru>
This commit is contained in:
co-authored by
bk-201
prashanth058
Anexdeus
parent
0b544e6476
commit
ce1eafd1a5
@@ -714,3 +714,21 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo
|
||||
connector="model.connector",
|
||||
tower_model="model.vision_model",
|
||||
)
|
||||
|
||||
def get_num_mm_encoder_tokens(
|
||||
self,
|
||||
num_image_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
scale_factor = hf_config.scale_factor
|
||||
|
||||
return num_image_tokens * scale_factor**2
|
||||
|
||||
def get_num_mm_connector_tokens(
|
||||
self,
|
||||
num_vision_tokens: int,
|
||||
) -> int:
|
||||
hf_config = self.config
|
||||
scale_factor = hf_config.scale_factor
|
||||
|
||||
return num_vision_tokens // scale_factor**2
|
||||
|
||||
Reference in New Issue
Block a user