[V1] LoRA - Enable Serving Usecase (#12883)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
f0b2da72a8
commit
cbc40128eb
@@ -134,3 +134,4 @@ class EngineCoreRequestType(enum.Enum):
|
||||
ABORT = b'\x01'
|
||||
PROFILE = b'\x02'
|
||||
RESET_PREFIX_CACHE = b'\x03'
|
||||
ADD_LORA = b'\x04'
|
||||
|
||||
@@ -361,6 +361,10 @@ class AsyncLLM(EngineClient):
|
||||
async def reset_prefix_cache(self) -> None:
|
||||
await self.engine_core.reset_prefix_cache_async()
|
||||
|
||||
async def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
"""Load a new LoRA adapter into the engine for future requests."""
|
||||
await self.engine_core.add_lora_async(lora_request)
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return True
|
||||
@@ -376,7 +380,3 @@ class AsyncLLM(EngineClient):
|
||||
@property
|
||||
def dead_error(self) -> BaseException:
|
||||
return Exception() # TODO: implement
|
||||
|
||||
async def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
"""Load a new LoRA adapter into the engine for future requests."""
|
||||
raise NotImplementedError("LoRA not yet supported in V1")
|
||||
|
||||
+15
-3
@@ -13,6 +13,7 @@ import zmq.asyncio
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.transformers_utils.config import (
|
||||
maybe_register_config_serialize_by_value)
|
||||
from vllm.utils import get_exception_traceback, zmq_socket_ctx
|
||||
@@ -146,6 +147,9 @@ class EngineCore:
|
||||
def reset_prefix_cache(self):
|
||||
self.scheduler.reset_prefix_cache()
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
self.model_executor.add_lora(lora_request)
|
||||
|
||||
|
||||
class EngineCoreProc(EngineCore):
|
||||
"""ZMQ-wrapper for running EngineCore in background process."""
|
||||
@@ -262,12 +266,15 @@ class EngineCoreProc(EngineCore):
|
||||
self.reset_prefix_cache()
|
||||
elif request_type == EngineCoreRequestType.PROFILE:
|
||||
self.model_executor.profile(request)
|
||||
elif request_type == EngineCoreRequestType.ADD_LORA:
|
||||
self.model_executor.add_lora(request)
|
||||
|
||||
def process_input_socket(self, input_path: str):
|
||||
"""Input socket IO thread."""
|
||||
|
||||
# Msgpack serialization decoding.
|
||||
add_request_decoder = MsgpackDecoder(EngineCoreRequest)
|
||||
add_lora_decoder = MsgpackDecoder(LoRARequest)
|
||||
generic_decoder = MsgpackDecoder()
|
||||
|
||||
with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
|
||||
@@ -277,9 +284,14 @@ class EngineCoreProc(EngineCore):
|
||||
request_type = EngineCoreRequestType(bytes(type_frame.buffer))
|
||||
|
||||
# Deserialize the request data.
|
||||
decoder = add_request_decoder if (
|
||||
request_type
|
||||
== EngineCoreRequestType.ADD) else generic_decoder
|
||||
decoder = None
|
||||
if request_type == EngineCoreRequestType.ADD:
|
||||
decoder = add_request_decoder
|
||||
elif request_type == EngineCoreRequestType.ADD_LORA:
|
||||
decoder = add_lora_decoder
|
||||
else:
|
||||
decoder = generic_decoder
|
||||
|
||||
request = decoder.decode(data_frame.buffer)
|
||||
|
||||
# Push to input queue for core busy loop.
|
||||
|
||||
@@ -12,6 +12,7 @@ import zmq.asyncio
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
|
||||
make_zmq_socket)
|
||||
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
|
||||
@@ -77,6 +78,9 @@ class EngineCoreClient(ABC):
|
||||
def abort_requests(self, request_ids: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -92,6 +96,9 @@ class EngineCoreClient(ABC):
|
||||
async def abort_requests_async(self, request_ids: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def add_lora_async(self, lora_request: LoRARequest) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class InprocClient(EngineCoreClient):
|
||||
"""
|
||||
@@ -125,6 +132,9 @@ class InprocClient(EngineCoreClient):
|
||||
def reset_prefix_cache(self) -> None:
|
||||
self.engine_core.reset_prefix_cache()
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
self.engine_core.add_lora(lora_request)
|
||||
|
||||
|
||||
class MPClient(EngineCoreClient):
|
||||
"""
|
||||
@@ -242,6 +252,9 @@ class SyncMPClient(MPClient):
|
||||
def reset_prefix_cache(self) -> None:
|
||||
self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
|
||||
|
||||
|
||||
class AsyncMPClient(MPClient):
|
||||
"""Asyncio-compatible client for multi-proc EngineCore."""
|
||||
@@ -295,3 +308,6 @@ class AsyncMPClient(MPClient):
|
||||
|
||||
async def reset_prefix_cache_async(self) -> None:
|
||||
await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
|
||||
|
||||
async def add_lora_async(self, lora_request: LoRARequest) -> None:
|
||||
await self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
set_custom_all_reduce)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor import set_random_seed
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import GiB_bytes
|
||||
@@ -234,6 +235,9 @@ class Worker(WorkerBase):
|
||||
else:
|
||||
self.profiler.stop()
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> bool:
|
||||
return self.model_runner.add_lora(lora_request)
|
||||
|
||||
def check_health(self) -> None:
|
||||
# worker will always be healthy as long as it's running.
|
||||
return
|
||||
|
||||
@@ -127,3 +127,8 @@ class LoRAModelRunnerMixin:
|
||||
|
||||
# __exit__ code
|
||||
self.lora_manager.remove_all_adapters()
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> bool:
|
||||
if not self.lora_manager:
|
||||
raise RuntimeError("LoRA is not enabled.")
|
||||
return self.lora_manager.add_adapter(lora_request)
|
||||
Reference in New Issue
Block a user