[MyPy] Fix mypy incompatible assignment errors in LRUCacheLoRAModelManager (#44657)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
@@ -98,14 +98,17 @@ class LoRAModelManager:
|
||||
f"No supported LoRA modules found in {self.model.__class__.__name__}."
|
||||
)
|
||||
|
||||
self._registered_adapters: dict[int, LoRAModel] = {}
|
||||
# Dict instead of a set for compatibility with LRUCache.
|
||||
self._active_adapters: dict[int, None] = {}
|
||||
self.adapter_type = "LoRA"
|
||||
self.lora_config = lora_config
|
||||
self.device = device
|
||||
self.max_num_seqs = max_num_seqs
|
||||
assert self.capacity >= self.lora_slots
|
||||
self._registered_adapters: AdapterLRUCache[LoRAModel] = AdapterLRUCache(
|
||||
self.capacity, self.deactivate_adapter
|
||||
)
|
||||
self._active_adapters: AdapterLRUCache[None] = AdapterLRUCache(
|
||||
self.lora_slots, self._deactivate_adapter
|
||||
)
|
||||
self.max_num_batched_tokens = math.ceil(max_num_batched_tokens / 8) * 8
|
||||
self.lora_index_to_id: list[int | None] = [None] * self.lora_slots
|
||||
self.vocab_size = vocab_size
|
||||
@@ -1156,50 +1159,15 @@ class LoRAModelManager:
|
||||
return True
|
||||
|
||||
def list_adapters(self) -> dict[int, LoRAModel]:
|
||||
return dict(self._registered_adapters)
|
||||
return dict(self._registered_adapters.cache)
|
||||
|
||||
def get_adapter(self, adapter_id: int) -> LoRAModel | None:
|
||||
return self._registered_adapters.get(adapter_id)
|
||||
|
||||
|
||||
class LoRALRUCache(AdapterLRUCache[LoRAModel]):
|
||||
def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int], object]):
|
||||
super().__init__(capacity, deactivate_lora_fn)
|
||||
|
||||
|
||||
class LRUCacheLoRAModelManager(LoRAModelManager):
|
||||
"""A model manager that manages multiple LoRAs with LRU cache."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: SupportsLoRAModel,
|
||||
max_num_seqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
vocab_size: int,
|
||||
lora_config: LoRAConfig,
|
||||
device: torch.device,
|
||||
vllm_config: VllmConfig,
|
||||
):
|
||||
super().__init__(
|
||||
model,
|
||||
max_num_seqs,
|
||||
max_num_batched_tokens,
|
||||
vocab_size,
|
||||
lora_config,
|
||||
device,
|
||||
vllm_config,
|
||||
)
|
||||
self._registered_adapters: LoRALRUCache = LoRALRUCache( # type: ignore[assignment]
|
||||
self.capacity, self.deactivate_adapter
|
||||
)
|
||||
self._active_adapters: LoRALRUCache = LoRALRUCache( # type: ignore[assignment]
|
||||
self.lora_slots, self._deactivate_adapter
|
||||
)
|
||||
|
||||
def list_adapters(self) -> dict[int, LoRAModel]:
|
||||
"""List all registered LoRAModels."""
|
||||
return dict(self._registered_adapters.cache)
|
||||
|
||||
def add_adapter(self, lora: LoRAModel) -> bool:
|
||||
"""Add a LoRAModel to the manager."""
|
||||
logger.debug("Adding lora. Model id: %d, int id: %d", lora.id, lora.id)
|
||||
|
||||
Reference in New Issue
Block a user