[MyPy] Fix mypy for vllm/lora (#41722)
Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>
This commit is contained in:
+17
-10
@@ -34,6 +34,7 @@ from vllm.lora.utils import (
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.models import (
|
||||
SupportsLoRA,
|
||||
SupportsMultiModal,
|
||||
is_pooling_model,
|
||||
supports_multimodal,
|
||||
)
|
||||
@@ -50,6 +51,12 @@ T = TypeVar("T")
|
||||
DEFAULT_LANGUAGE_WRAPPER_KEY = "language_model"
|
||||
|
||||
|
||||
class SupportsLoRAModel(nn.Module, SupportsLoRA): ...
|
||||
|
||||
|
||||
class SupportsLoRAMultiModalModel(SupportsLoRAModel, SupportsMultiModal): ...
|
||||
|
||||
|
||||
class AdapterLRUCache(LRUCache[int, T]):
|
||||
def __init__(self, capacity: int, deactivate_fn: Callable[[int], object]):
|
||||
super().__init__(capacity)
|
||||
@@ -66,13 +73,13 @@ class LoRAModelManager:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: SupportsLoRA,
|
||||
model: SupportsLoRAModel,
|
||||
max_num_seqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
vocab_size: int,
|
||||
lora_config: LoRAConfig,
|
||||
device: torch.device,
|
||||
vllm_config: VllmConfig | None = None,
|
||||
vllm_config: VllmConfig,
|
||||
):
|
||||
"""Create a LoRAModelManager and adapter for a given model.
|
||||
|
||||
@@ -85,7 +92,7 @@ class LoRAModelManager:
|
||||
vocab_size: the vocab size of the model.
|
||||
lora_config: the LoRA configuration.
|
||||
"""
|
||||
self.model: SupportsLoRA = model
|
||||
self.model: SupportsLoRAModel = model
|
||||
self.supported_lora_modules = get_supported_lora_modules(self.model)
|
||||
assert self.supported_lora_modules, (
|
||||
f"No supported LoRA modules found in {self.model.__class__.__name__}."
|
||||
@@ -106,7 +113,6 @@ class LoRAModelManager:
|
||||
self.is_pooling_model = is_pooling_model(self.model)
|
||||
self.packed_modules: dict[str, list[str]] = {}
|
||||
self.modules: dict[str, BaseLayerWithLoRA] = {}
|
||||
# Dict instead of a set for compatibility with LRUCache.
|
||||
self._last_mapping: LoRAMapping | None = None
|
||||
is_moe = is_moe_model(self.model)
|
||||
self._is_moe = is_moe
|
||||
@@ -272,6 +278,7 @@ class LoRAModelManager:
|
||||
|
||||
@property
|
||||
def capacity(self) -> int:
|
||||
assert self.lora_config.max_cpu_loras is not None
|
||||
return self.lora_config.max_cpu_loras
|
||||
|
||||
@property
|
||||
@@ -1156,7 +1163,7 @@ class LoRAModelManager:
|
||||
|
||||
|
||||
class LoRALRUCache(AdapterLRUCache[LoRAModel]):
|
||||
def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int], bool]):
|
||||
def __init__(self, capacity: int, deactivate_lora_fn: Callable[[int], object]):
|
||||
super().__init__(capacity, deactivate_lora_fn)
|
||||
|
||||
|
||||
@@ -1165,13 +1172,13 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: nn.Module,
|
||||
model: SupportsLoRAModel,
|
||||
max_num_seqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
vocab_size: int,
|
||||
lora_config: LoRAConfig,
|
||||
device: torch.device,
|
||||
vllm_config: VllmConfig | None = None,
|
||||
vllm_config: VllmConfig,
|
||||
):
|
||||
super().__init__(
|
||||
model,
|
||||
@@ -1182,10 +1189,10 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
|
||||
device,
|
||||
vllm_config,
|
||||
)
|
||||
self._registered_adapters: LoRALRUCache = LoRALRUCache(
|
||||
self._registered_adapters: LoRALRUCache = LoRALRUCache( # type: ignore[assignment]
|
||||
self.capacity, self.deactivate_adapter
|
||||
)
|
||||
self._active_adapters: LoRALRUCache = LoRALRUCache(
|
||||
self._active_adapters: LoRALRUCache = LoRALRUCache( # type: ignore[assignment]
|
||||
self.lora_slots, self._deactivate_adapter
|
||||
)
|
||||
|
||||
@@ -1248,7 +1255,7 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
|
||||
|
||||
|
||||
def create_lora_manager(
|
||||
model: nn.Module,
|
||||
model: SupportsLoRAModel,
|
||||
max_num_seqs: int,
|
||||
max_num_batched_tokens: int,
|
||||
vocab_size: int,
|
||||
|
||||
Reference in New Issue
Block a user