Reduce the kernel overhead when num of active loras is smaller than max loras. Multiple cuda graphs are captured for each num of active-loras. (#32005)
Signed-off-by: Yu Gong <yu3.gong@gmail.com>
This commit is contained in:
@@ -485,6 +485,7 @@ class EngineArgs:
|
||||
max_cpu_loras: int | None = LoRAConfig.max_cpu_loras
|
||||
lora_dtype: str | torch.dtype | None = LoRAConfig.lora_dtype
|
||||
enable_tower_connector_lora: bool = LoRAConfig.enable_tower_connector_lora
|
||||
specialize_active_lora: bool = LoRAConfig.specialize_active_lora
|
||||
|
||||
ray_workers_use_nsight: bool = ParallelConfig.ray_workers_use_nsight
|
||||
num_gpu_blocks_override: int | None = CacheConfig.num_gpu_blocks_override
|
||||
@@ -1026,6 +1027,9 @@ class EngineArgs:
|
||||
"--fully-sharded-loras", **lora_kwargs["fully_sharded_loras"]
|
||||
)
|
||||
lora_group.add_argument("--default-mm-loras", **lora_kwargs["default_mm_loras"])
|
||||
lora_group.add_argument(
|
||||
"--specialize-active-lora", **lora_kwargs["specialize_active_lora"]
|
||||
)
|
||||
|
||||
# Observability arguments
|
||||
observability_kwargs = get_kwargs(ObservabilityConfig)
|
||||
@@ -1657,6 +1661,7 @@ class EngineArgs:
|
||||
fully_sharded_loras=self.fully_sharded_loras,
|
||||
lora_dtype=self.lora_dtype,
|
||||
enable_tower_connector_lora=self.enable_tower_connector_lora,
|
||||
specialize_active_lora=self.specialize_active_lora,
|
||||
max_cpu_loras=self.max_cpu_loras
|
||||
if self.max_cpu_loras and self.max_cpu_loras > 0
|
||||
else None,
|
||||
|
||||
Reference in New Issue
Block a user