[Bugfix] Avoid repeatedly creating dummy data during engine startup (#17935)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1232,6 +1232,9 @@ class AsyncLLMEngine(EngineClient):
|
||||
async def stop_profile(self) -> None:
|
||||
self.engine.stop_profile()
|
||||
|
||||
async def reset_mm_cache(self) -> None:
|
||||
self.engine.reset_mm_cache()
|
||||
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
self.engine.reset_prefix_cache(device)
|
||||
|
||||
@@ -409,6 +409,9 @@ class LLMEngine:
|
||||
# the next step without re-scheduling.
|
||||
self._skip_scheduling_next_step = False
|
||||
|
||||
# Don't keep the dummy data in memory
|
||||
self.reset_mm_cache()
|
||||
|
||||
def _initialize_kv_caches(self) -> None:
|
||||
"""Initialize the KV cache in the worker(s).
|
||||
|
||||
@@ -913,6 +916,10 @@ class LLMEngine:
|
||||
"""
|
||||
return self.scheduler[virtual_engine].has_unfinished_seqs()
|
||||
|
||||
def reset_mm_cache(self) -> bool:
|
||||
"""Reset the multi-modal cache."""
|
||||
return self.input_preprocessor.mm_registry.reset_processor_cache()
|
||||
|
||||
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
|
||||
"""Reset prefix cache for all devices."""
|
||||
|
||||
|
||||
@@ -123,6 +123,10 @@ class RPCUProfileRequest(Enum):
|
||||
STOP_PROFILE = 2
|
||||
|
||||
|
||||
class RPCResetMultiModalCacheRequest(Enum):
|
||||
RESET = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class RPCResetPrefixCacheRequest:
|
||||
device: Device
|
||||
@@ -164,6 +168,7 @@ class RPCAdapterLoadedResponse:
|
||||
|
||||
RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCStartupRequest,
|
||||
RPCUProfileRequest, RPCLoadAdapterRequest,
|
||||
RPCResetMultiModalCacheRequest,
|
||||
RPCResetPrefixCacheRequest, RPCSleepRequest,
|
||||
RPCWakeUpRequest, RPCIsSleepingRequest]
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
|
||||
RPCIsSleepingResponse,
|
||||
RPCLoadAdapterRequest,
|
||||
RPCProcessRequest,
|
||||
RPCResetMultiModalCacheRequest,
|
||||
RPCResetPrefixCacheRequest,
|
||||
RPCSleepRequest, RPCStartupRequest,
|
||||
RPCStartupResponse,
|
||||
@@ -687,6 +688,13 @@ class MQLLMEngineClient(EngineClient):
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
|
||||
|
||||
async def reset_mm_cache(self) -> None:
|
||||
"""Reset the multi-modal cache"""
|
||||
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCResetMultiModalCacheRequest.RESET,
|
||||
socket=self.input_socket)
|
||||
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
"""Reset the prefix cache"""
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
|
||||
RPCIsSleepingResponse,
|
||||
RPCLoadAdapterRequest,
|
||||
RPCProcessRequest,
|
||||
RPCResetMultiModalCacheRequest,
|
||||
RPCResetPrefixCacheRequest,
|
||||
RPCSleepRequest, RPCStartupRequest,
|
||||
RPCStartupResponse,
|
||||
@@ -269,6 +270,8 @@ class MQLLMEngine:
|
||||
self.stop_profile()
|
||||
elif isinstance(request, RPCLoadAdapterRequest):
|
||||
self._handle_load_adapter_request(request)
|
||||
elif isinstance(request, RPCResetMultiModalCacheRequest):
|
||||
self.reset_mm_cache()
|
||||
elif isinstance(request, RPCResetPrefixCacheRequest):
|
||||
self.reset_prefix_cache()
|
||||
elif isinstance(request, RPCSleepRequest):
|
||||
@@ -409,6 +412,9 @@ class MQLLMEngine:
|
||||
def stop_profile(self) -> None:
|
||||
self.engine.stop_profile()
|
||||
|
||||
def reset_mm_cache(self) -> bool:
|
||||
return self.engine.reset_mm_cache()
|
||||
|
||||
def reset_prefix_cache(self) -> bool:
|
||||
return self.engine.reset_prefix_cache()
|
||||
|
||||
|
||||
@@ -278,6 +278,11 @@ class EngineClient(ABC):
|
||||
"""Start profiling the engine"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def reset_mm_cache(self) -> None:
|
||||
"""Reset the multi-modal cache"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
|
||||
Reference in New Issue
Block a user