[Core] LoRA V1 - Add add/pin/list/remove_lora functions (#13705)

This commit is contained in:
Varun Sundar Rabindranath
2025-02-25 00:18:02 -08:00
committed by GitHub
parent 4d251ad00e
commit 03f48b3db6
8 changed files with 270 additions and 22 deletions
+15 -3
View File
@@ -2,7 +2,7 @@
import asyncio
import os
from typing import AsyncGenerator, List, Mapping, Optional, Type, Union
from typing import AsyncGenerator, List, Mapping, Optional, Set, Type, Union
import numpy as np
@@ -392,9 +392,21 @@ class AsyncLLM(EngineClient):
async def wake_up(self) -> None:
await self.engine_core.wake_up_async()
async def add_lora(self, lora_request: LoRARequest) -> None:
async def add_lora(self, lora_request: LoRARequest) -> bool:
"""Load a new LoRA adapter into the engine for future requests."""
await self.engine_core.add_lora_async(lora_request)
return await self.engine_core.add_lora_async(lora_request)
async def remove_lora(self, lora_id: int) -> bool:
"""Remove an already loaded LoRA adapter."""
return await self.engine_core.remove_lora_async(lora_id)
async def list_loras(self) -> Set[int]:
"""List all registered adapters."""
return await self.engine_core.list_loras_async()
async def pin_lora(self, lora_id: int) -> bool:
"""Prevent an adapter from being evicted."""
return await self.engine_core.pin_lora_async(lora_id)
@property
def is_running(self) -> bool:
+12 -3
View File
@@ -7,7 +7,7 @@ import time
from concurrent.futures import Future
from inspect import isclass, signature
from multiprocessing.connection import Connection
from typing import Any, List, Optional, Tuple, Type
from typing import Any, List, Optional, Set, Tuple, Type
import msgspec
import psutil
@@ -222,8 +222,17 @@ class EngineCore:
def execute_dummy_batch(self):
self.model_executor.collective_rpc("execute_dummy_batch")
def add_lora(self, lora_request: LoRARequest) -> None:
self.model_executor.add_lora(lora_request)
def add_lora(self, lora_request: LoRARequest) -> bool:
return self.model_executor.add_lora(lora_request)
def remove_lora(self, lora_id: int) -> bool:
return self.model_executor.remove_lora(lora_id)
def list_loras(self) -> Set[int]:
return self.model_executor.list_loras()
def pin_lora(self, lora_id: int) -> bool:
return self.model_executor.pin_lora(lora_id)
class EngineCoreProc(EngineCore):
+54 -9
View File
@@ -10,7 +10,7 @@ from abc import ABC, abstractmethod
from concurrent.futures import Future
from dataclasses import dataclass
from threading import Thread
from typing import Any, Dict, List, Optional, Type, Union
from typing import Any, Dict, List, Optional, Set, Type, Union
import zmq
import zmq.asyncio
@@ -97,7 +97,16 @@ class EngineCoreClient(ABC):
def abort_requests(self, request_ids: List[str]) -> None:
raise NotImplementedError
def add_lora(self, lora_request: LoRARequest) -> None:
def add_lora(self, lora_request: LoRARequest) -> bool:
raise NotImplementedError
def remove_lora(self, lora_id: int) -> bool:
raise NotImplementedError
def list_loras(self) -> Set[int]:
raise NotImplementedError
def pin_lora(self, lora_id: int) -> bool:
raise NotImplementedError
async def get_output_async(self) -> EngineCoreOutputs:
@@ -121,7 +130,16 @@ class EngineCoreClient(ABC):
async def abort_requests_async(self, request_ids: List[str]) -> None:
raise NotImplementedError
async def add_lora_async(self, lora_request: LoRARequest) -> None:
async def add_lora_async(self, lora_request: LoRARequest) -> bool:
raise NotImplementedError
async def remove_lora_async(self, lora_id: int) -> bool:
raise NotImplementedError
async def list_loras_async(self) -> Set[int]:
raise NotImplementedError
async def pin_lora_async(self, lora_id: int) -> bool:
raise NotImplementedError
@@ -166,8 +184,17 @@ class InprocClient(EngineCoreClient):
def execute_dummy_batch(self) -> None:
self.engine_core.execute_dummy_batch()
def add_lora(self, lora_request: LoRARequest) -> None:
self.engine_core.add_lora(lora_request)
def add_lora(self, lora_request: LoRARequest) -> bool:
return self.engine_core.add_lora(lora_request)
def remove_lora(self, lora_id: int) -> bool:
return self.engine_core.remove_lora(lora_id)
def list_loras(self) -> Set[int]:
return self.engine_core.list_loras()
def pin_lora(self, lora_id: int) -> bool:
return self.engine_core.pin_lora(lora_id)
@dataclass
@@ -356,8 +383,17 @@ class SyncMPClient(MPClient):
def reset_prefix_cache(self) -> None:
self._call_utility("reset_prefix_cache")
def add_lora(self, lora_request: LoRARequest) -> None:
self._call_utility("add_lora", lora_request)
def add_lora(self, lora_request: LoRARequest) -> bool:
return self._call_utility("add_lora", lora_request)
def remove_lora(self, lora_id: int) -> bool:
return self._call_utility("remove_lora", lora_id)
def list_loras(self) -> Set[int]:
return self._call_utility("list_loras")
def pin_lora(self, lora_id: int) -> bool:
return self._call_utility("pin_lora", lora_id)
def sleep(self, level: int = 1) -> None:
self._call_utility("sleep", level)
@@ -454,5 +490,14 @@ class AsyncMPClient(MPClient):
async def execute_dummy_batch_async(self) -> None:
await self._call_utility_async("execute_dummy_batch")
async def add_lora_async(self, lora_request: LoRARequest) -> None:
await self._call_utility_async("add_lora", lora_request)
async def add_lora_async(self, lora_request: LoRARequest) -> bool:
return await self._call_utility_async("add_lora", lora_request)
async def remove_lora_async(self, lora_id: int) -> bool:
return await self._call_utility_async("remove_lora", lora_id)
async def list_loras_async(self) -> Set[int]:
return await self._call_utility_async("list_loras")
async def pin_lora_async(self, lora_id: int) -> bool:
return await self._call_utility_async("pin_lora", lora_id)
+17 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
from typing import Dict, List, Mapping, Optional, Type, Union
from typing import Dict, List, Mapping, Optional, Set, Type, Union
from typing_extensions import TypeVar
@@ -254,3 +254,19 @@ class LLMEngine:
f"found type: {type(tokenizer_group)}")
return tokenizer_group
def add_lora(self, lora_request: LoRARequest) -> bool:
"""Load a new LoRA adapter into the engine for future requests."""
return self.engine_core.add_lora(lora_request)
def remove_lora(self, lora_id: int) -> bool:
"""Remove an already loaded LoRA adapter."""
return self.engine_core.remove_lora(lora_id)
def list_loras(self) -> Set[int]:
"""List all registered adapters."""
return self.engine_core.list_loras()
def pin_lora(self, lora_id: int) -> bool:
"""Prevent an adapter from being evicted."""
return self.engine_core.pin_lora(lora_id)
+10 -1
View File
@@ -2,7 +2,7 @@
"""A GPU worker class."""
import gc
import os
from typing import TYPE_CHECKING, Optional
from typing import TYPE_CHECKING, Optional, Set
import torch
import torch.distributed
@@ -240,6 +240,15 @@ class Worker(WorkerBase):
def add_lora(self, lora_request: LoRARequest) -> bool:
return self.model_runner.add_lora(lora_request)
def remove_lora(self, lora_id: int) -> bool:
return self.model_runner.remove_lora(lora_id)
def list_loras(self) -> Set[int]:
return self.model_runner.list_loras()
def pin_lora(self, lora_id: int) -> bool:
return self.model_runner.pin_lora(lora_id)
def check_health(self) -> None:
# worker will always be healthy as long as it's running.
return
+16 -1
View File
@@ -131,4 +131,19 @@ class LoRAModelRunnerMixin:
def add_lora(self, lora_request: LoRARequest) -> bool:
if not self.lora_manager:
raise RuntimeError("LoRA is not enabled.")
return self.lora_manager.add_adapter(lora_request)
return self.lora_manager.add_adapter(lora_request)
def remove_lora(self, lora_id: int) -> bool:
if not self.lora_manager:
raise RuntimeError("LoRA is not enabled.")
return self.lora_manager.remove_adapter(lora_id)
def pin_lora(self, lora_id: int) -> bool:
if not self.lora_manager:
raise RuntimeError("LoRA is not enabled.")
return self.lora_manager.pin_adapter(lora_id)
def list_loras(self) -> Set[int]:
if not self.lora_manager:
raise RuntimeError("LoRA is not enabled.")
return self.lora_manager.list_adapters()