[V1] LoRA - Enable Serving Usecase (#12883)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Varun Sundar Rabindranath
2025-02-14 14:21:12 +08:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent f0b2da72a8
commit cbc40128eb
7 changed files with 210 additions and 7 deletions
+1
View File
@@ -134,3 +134,4 @@ class EngineCoreRequestType(enum.Enum):
ABORT = b'\x01'
PROFILE = b'\x02'
RESET_PREFIX_CACHE = b'\x03'
ADD_LORA = b'\x04'
+4 -4
View File
@@ -361,6 +361,10 @@ class AsyncLLM(EngineClient):
async def reset_prefix_cache(self) -> None:
await self.engine_core.reset_prefix_cache_async()
async def add_lora(self, lora_request: LoRARequest) -> None:
"""Load a new LoRA adapter into the engine for future requests."""
await self.engine_core.add_lora_async(lora_request)
@property
def is_running(self) -> bool:
return True
@@ -376,7 +380,3 @@ class AsyncLLM(EngineClient):
@property
def dead_error(self) -> BaseException:
return Exception() # TODO: implement
async def add_lora(self, lora_request: LoRARequest) -> None:
"""Load a new LoRA adapter into the engine for future requests."""
raise NotImplementedError("LoRA not yet supported in V1")
+15 -3
View File
@@ -13,6 +13,7 @@ import zmq.asyncio
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.transformers_utils.config import (
maybe_register_config_serialize_by_value)
from vllm.utils import get_exception_traceback, zmq_socket_ctx
@@ -146,6 +147,9 @@ class EngineCore:
def reset_prefix_cache(self):
self.scheduler.reset_prefix_cache()
def add_lora(self, lora_request: LoRARequest) -> None:
self.model_executor.add_lora(lora_request)
class EngineCoreProc(EngineCore):
"""ZMQ-wrapper for running EngineCore in background process."""
@@ -262,12 +266,15 @@ class EngineCoreProc(EngineCore):
self.reset_prefix_cache()
elif request_type == EngineCoreRequestType.PROFILE:
self.model_executor.profile(request)
elif request_type == EngineCoreRequestType.ADD_LORA:
self.model_executor.add_lora(request)
def process_input_socket(self, input_path: str):
"""Input socket IO thread."""
# Msgpack serialization decoding.
add_request_decoder = MsgpackDecoder(EngineCoreRequest)
add_lora_decoder = MsgpackDecoder(LoRARequest)
generic_decoder = MsgpackDecoder()
with zmq_socket_ctx(input_path, zmq.constants.PULL) as socket:
@@ -277,9 +284,14 @@ class EngineCoreProc(EngineCore):
request_type = EngineCoreRequestType(bytes(type_frame.buffer))
# Deserialize the request data.
decoder = add_request_decoder if (
request_type
== EngineCoreRequestType.ADD) else generic_decoder
decoder = None
if request_type == EngineCoreRequestType.ADD:
decoder = add_request_decoder
elif request_type == EngineCoreRequestType.ADD_LORA:
decoder = add_lora_decoder
else:
decoder = generic_decoder
request = decoder.decode(data_frame.buffer)
# Push to input queue for core busy loop.
+16
View File
@@ -12,6 +12,7 @@ import zmq.asyncio
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
make_zmq_socket)
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
@@ -77,6 +78,9 @@ class EngineCoreClient(ABC):
def abort_requests(self, request_ids: List[str]) -> None:
raise NotImplementedError
def add_lora(self, lora_request: LoRARequest) -> None:
raise NotImplementedError
async def get_output_async(self) -> EngineCoreOutputs:
raise NotImplementedError
@@ -92,6 +96,9 @@ class EngineCoreClient(ABC):
async def abort_requests_async(self, request_ids: List[str]) -> None:
raise NotImplementedError
async def add_lora_async(self, lora_request: LoRARequest) -> None:
raise NotImplementedError
class InprocClient(EngineCoreClient):
"""
@@ -125,6 +132,9 @@ class InprocClient(EngineCoreClient):
def reset_prefix_cache(self) -> None:
self.engine_core.reset_prefix_cache()
def add_lora(self, lora_request: LoRARequest) -> None:
self.engine_core.add_lora(lora_request)
class MPClient(EngineCoreClient):
"""
@@ -242,6 +252,9 @@ class SyncMPClient(MPClient):
def reset_prefix_cache(self) -> None:
self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
def add_lora(self, lora_request: LoRARequest) -> None:
self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
class AsyncMPClient(MPClient):
"""Asyncio-compatible client for multi-proc EngineCore."""
@@ -295,3 +308,6 @@ class AsyncMPClient(MPClient):
async def reset_prefix_cache_async(self) -> None:
await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
async def add_lora_async(self, lora_request: LoRARequest) -> None:
await self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
+4
View File
@@ -15,6 +15,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
init_distributed_environment,
set_custom_all_reduce)
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.model_executor import set_random_seed
from vllm.platforms import current_platform
from vllm.utils import GiB_bytes
@@ -234,6 +235,9 @@ class Worker(WorkerBase):
else:
self.profiler.stop()
def add_lora(self, lora_request: LoRARequest) -> bool:
return self.model_runner.add_lora(lora_request)
def check_health(self) -> None:
# worker will always be healthy as long as it's running.
return
@@ -127,3 +127,8 @@ class LoRAModelRunnerMixin:
# __exit__ code
self.lora_manager.remove_all_adapters()
def add_lora(self, lora_request: LoRARequest) -> bool:
if not self.lora_manager:
raise RuntimeError("LoRA is not enabled.")
return self.lora_manager.add_adapter(lora_request)