[Core] Initialize LoRA support for tower and connector in multi-modal models (#26674)
Signed-off-by: bk-201 <joy25810@foxmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: bk-201 <joy25810@foxmail.com> Co-authored-by: prashanth058 <prashanth.dannamaneni@uipath.com> Co-authored-by: Anexdeus <5142168@mail.ru>
This commit is contained in:
co-authored by
bk-201
prashanth058
Anexdeus
parent
0b544e6476
commit
ce1eafd1a5
@@ -483,6 +483,7 @@ class EngineArgs:
|
||||
fully_sharded_loras: bool = LoRAConfig.fully_sharded_loras
|
||||
max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
|
||||
lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
|
||||
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
|
||||
|
||||
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
|
||||
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
|
||||
@@ -996,6 +997,10 @@ class EngineArgs:
|
||||
"--lora-dtype",
|
||||
**lora_kwargs["lora_dtype"],
|
||||
)
|
||||
lora_group.add_argument(
|
||||
"--enable-tower-connector-lora",
|
||||
**lora_kwargs["enable_tower_connector_lora"],
|
||||
)
|
||||
lora_group.add_argument("--max-cpu-loras", **lora_kwargs["max_cpu_loras"])
|
||||
lora_group.add_argument(
|
||||
"--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
|
||||
@@ -1631,6 +1636,7 @@ class EngineArgs:
|
||||
default_mm_loras=self.default_mm_loras,
|
||||
fully_sharded_loras=self.fully_sharded_loras,
|
||||
lora_dtype=self.lora_dtype,
|
||||
enable_tower_connector_lora=self.enable_tower_connector_lora,
|
||||
max_cpu_loras=self.max_cpu_loras
|
||||
if self.max_cpu_loras and self.max_cpu_loras > 0
|
||||
else None,
|
||||
@@ -1639,6 +1645,19 @@ class EngineArgs:
|
||||
else None
|
||||
)
|
||||
|
||||
if (
|
||||
lora_config is not None
|
||||
and lora_config.enable_tower_connector_lora
|
||||
and self.mm_processor_cache_gb != 0
|
||||
):
|
||||
raise ValueError(
|
||||
"Currently, enable_tower_connector_lora is "
|
||||
"incompatible with the multi-modal processor cache. "
|
||||
"When enable_tower_connector_lora is set, "
|
||||
"mm_processor_cache_gb must be 0, got %s",
|
||||
self.mm_processor_cache_gb,
|
||||
)
|
||||
|
||||
if (
|
||||
lora_config is not None
|
||||
and speculative_config is not None
|
||||
|
||||
Reference in New Issue
Block a user