[V1] LoRA - Enable Serving Usecase (#12883)

Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Varun Sundar Rabindranath
2025-02-14 14:21:12 +08:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent f0b2da72a8
commit cbc40128eb
7 changed files with 210 additions and 7 deletions
+16
View File
@@ -12,6 +12,7 @@ import zmq.asyncio
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.utils import (get_open_zmq_ipc_path, kill_process_tree,
make_zmq_socket)
from vllm.v1.engine import (EngineCoreOutputs, EngineCoreRequest,
@@ -77,6 +78,9 @@ class EngineCoreClient(ABC):
def abort_requests(self, request_ids: List[str]) -> None:
raise NotImplementedError
def add_lora(self, lora_request: LoRARequest) -> None:
raise NotImplementedError
async def get_output_async(self) -> EngineCoreOutputs:
raise NotImplementedError
@@ -92,6 +96,9 @@ class EngineCoreClient(ABC):
async def abort_requests_async(self, request_ids: List[str]) -> None:
raise NotImplementedError
async def add_lora_async(self, lora_request: LoRARequest) -> None:
raise NotImplementedError
class InprocClient(EngineCoreClient):
"""
@@ -125,6 +132,9 @@ class InprocClient(EngineCoreClient):
def reset_prefix_cache(self) -> None:
self.engine_core.reset_prefix_cache()
def add_lora(self, lora_request: LoRARequest) -> None:
self.engine_core.add_lora(lora_request)
class MPClient(EngineCoreClient):
"""
@@ -242,6 +252,9 @@ class SyncMPClient(MPClient):
def reset_prefix_cache(self) -> None:
self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
def add_lora(self, lora_request: LoRARequest) -> None:
self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)
class AsyncMPClient(MPClient):
"""Asyncio-compatible client for multi-proc EngineCore."""
@@ -295,3 +308,6 @@ class AsyncMPClient(MPClient):
async def reset_prefix_cache_async(self) -> None:
await self._send_input(EngineCoreRequestType.RESET_PREFIX_CACHE, None)
async def add_lora_async(self, lora_request: LoRARequest) -> None:
await self._send_input(EngineCoreRequestType.ADD_LORA, lora_request)