[LoRA] Add support for pinning lora adapters in the LRU cache (#5603)
This commit is contained in:
@@ -525,6 +525,12 @@ class LoRAModelManager:
|
||||
self.long_lora_context.offsets_by_lora_id.pop(lora_id, None)
|
||||
return bool(self._registered_loras.pop(lora_id, None))
|
||||
|
||||
def pin_lora(self, lora_id: int) -> bool:
|
||||
"""Pin a LoRAModel in the manager cache."""
|
||||
raise NotImplementedError(
|
||||
"Pinning is not supported in LoRAModelManager."
|
||||
"Use LRUCacheLoRAModelManager for pinning") # type: ignore
|
||||
|
||||
# TODO see if this can be vectorized
|
||||
def _set_lora_mapping(self, mapping: LoRAMapping) -> None:
|
||||
(base_indices, sampler_indices, sampler_indices_padded,
|
||||
@@ -777,6 +783,26 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
|
||||
return True
|
||||
return False
|
||||
|
||||
def pin_lora(self, lora_id: int) -> bool:
|
||||
"""Pin a LoRAModel in the manager cache."""
|
||||
self._pin_lora_in_cpu_cache(lora_id)
|
||||
self._pin_lora_in_gpu_cache(lora_id)
|
||||
return True
|
||||
|
||||
def _pin_lora_in_cpu_cache(self, lora_id: int):
|
||||
try:
|
||||
self._registered_loras.pin(lora_id)
|
||||
except ValueError as err:
|
||||
raise ValueError("Pinning failed. "
|
||||
f"LoRA {lora_id} is not registered.") from err
|
||||
|
||||
def _pin_lora_in_gpu_cache(self, lora_id: int):
|
||||
if lora_id not in self._active_loras:
|
||||
# move lora to gpu if not already active
|
||||
self.activate_lora(lora_id)
|
||||
|
||||
self._active_loras.pin(lora_id)
|
||||
|
||||
|
||||
def create_lora_manager(
|
||||
model: nn.Module,
|
||||
|
||||
@@ -221,6 +221,9 @@ class WorkerLoRAManager(AbstractWorkerLoRAManager):
|
||||
def remove_lora(self, lora_id: int) -> bool:
|
||||
return self._lora_manager.remove_lora(lora_id)
|
||||
|
||||
def pin_lora(self, lora_id: int) -> bool:
|
||||
return self._lora_manager.pin_lora(lora_id)
|
||||
|
||||
def remove_all_loras(self):
|
||||
self._lora_manager.remove_all_loras()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user