[LoRA] Add support for pinning lora adapters in the LRU cache (#5603)

This commit is contained in:
rohithkrn
2024-06-21 15:42:46 -07:00
committed by GitHub
parent 7187507301
commit f5dda63eb5
13 changed files with 171 additions and 5 deletions
+26
View File
@@ -525,6 +525,12 @@ class LoRAModelManager:
self.long_lora_context.offsets_by_lora_id.pop(lora_id, None)
return bool(self._registered_loras.pop(lora_id, None))
def pin_lora(self, lora_id: int) -> bool:
"""Pin a LoRAModel in the manager cache."""
raise NotImplementedError(
"Pinning is not supported in LoRAModelManager."
"Use LRUCacheLoRAModelManager for pinning") # type: ignore
# TODO see if this can be vectorized
def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
(base_indices, sampler_indices, sampler_indices_padded,
@@ -777,6 +783,26 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
return True
return False
def pin_lora(self, lora_id: int) -> bool:
"""Pin a LoRAModel in the manager cache."""
self._pin_lora_in_cpu_cache(lora_id)
self._pin_lora_in_gpu_cache(lora_id)
return True
def _pin_lora_in_cpu_cache(self, lora_id: int):
try:
self._registered_loras.pin(lora_id)
except ValueError as err:
raise ValueError("Pinning failed. "
f"LoRA {lora_id} is not registered.") from err
def _pin_lora_in_gpu_cache(self, lora_id: int):
if lora_id not in self._active_loras:
# move lora to gpu if not already active
self.activate_lora(lora_id)
self._active_loras.pin(lora_id)
def create_lora_manager(
model: nn.Module,
+3
View File
@@ -221,6 +221,9 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
def remove_lora(self, lora_id: int) -> bool:
return self._lora_manager.remove_lora(lora_id)
def pin_lora(self, lora_id: int) -> bool:
return self._lora_manager.pin_lora(lora_id)
def remove_all_loras(self):
self._lora_manager.remove_all_loras()