[V1] LoRA - Enable Serving Usecase (#12883)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
f0b2da72a8
commit
cbc40128eb
@@ -12,6 +12,7 @@ import zmq.asyncio
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
|
||||
make_zmq_socket)
|
||||
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
|
||||
@@ -77,6 +78,9 @@ class EngineCoreClient(ABC):
|
||||
def abort_requests(self, request_ids: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_output_async(self) -> EngineCoreOutputs:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -92,6 +96,9 @@ class EngineCoreClient(ABC):
|
||||
async def abort_requests_async(self, request_ids: List[str]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def add_lora_async(self, lora_request: LoRARequest) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class InprocClient(EngineCoreClient):
|
||||
"""
|
||||
@@ -125,6 +132,9 @@ class InprocClient(EngineCoreClient):
|
||||
def reset_prefix_cache(self) -> None:
|
||||
self.engine_core.reset_prefix_cache()
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
self.engine_core.add_lora(lora_request)
|
||||
|
||||
|
||||
class MPClient(EngineCoreClient):
|
||||
"""
|
||||
@@ -242,6 +252,9 @@ class SyncMPClient(MPClient):
|
||||
def reset_prefix_cache(self) -> None:
|
||||
self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
|
||||
|
||||
def add_lora(self, lora_request: LoRARequest) -> None:
|
||||
self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
|
||||
|
||||
|
||||
class AsyncMPClient(MPClient):
|
||||
"""Asyncio-compatible client for multi-proc EngineCore."""
|
||||
@@ -295,3 +308,6 @@ class AsyncMPClient(MPClient):
|
||||
|
||||
async def reset_prefix_cache_async(self) -> None:
|
||||
await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
|
||||
|
||||
async def add_lora_async(self, lora_request: LoRARequest) -> None:
|
||||
await self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
|
||||
|
||||
Reference in New Issue
Block a user