[FEAT]Support reset prefix cache by specified device (#15003)
This commit is contained in:
@@ -35,7 +35,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.usage.usage_lib import UsageContext
|
||||
from vllm.utils import deprecate_kwargs, weak_bind
|
||||
from vllm.utils import Device, deprecate_kwargs, weak_bind
|
||||
|
||||
logger = init_logger(__name__)
|
||||
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
|
||||
@@ -1216,8 +1216,9 @@ class AsyncLLMEngine(EngineClient):
|
||||
async def stop_profile(self) -> None:
|
||||
self.engine.stop_profile()
|
||||
|
||||
async def reset_prefix_cache(self) -> None:
|
||||
self.engine.reset_prefix_cache()
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
self.engine.reset_prefix_cache(device)
|
||||
|
||||
async def sleep(self, level: int = 1) -> None:
|
||||
self.engine.sleep(level)
|
||||
|
||||
@@ -955,12 +955,12 @@ class LLMEngine:
|
||||
"""
|
||||
return self.scheduler[virtual_engine].has_unfinished_seqs()
|
||||
|
||||
def reset_prefix_cache(self) -> bool:
|
||||
def reset_prefix_cache(self, device: Optional[Device] = None) -> bool:
|
||||
"""Reset prefix cache for all devices."""
|
||||
|
||||
success = True
|
||||
for scheduler in self.scheduler:
|
||||
success = success and scheduler.reset_prefix_cache()
|
||||
success = success and scheduler.reset_prefix_cache(device)
|
||||
return success
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -13,7 +13,7 @@ from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import deprecate_kwargs
|
||||
from vllm.utils import Device, deprecate_kwargs
|
||||
|
||||
VLLM_RPC_SUCCESS_STR = "SUCCESS"
|
||||
|
||||
@@ -123,8 +123,9 @@ class RPCUProfileRequest(Enum):
|
||||
STOP_PROFILE = 2
|
||||
|
||||
|
||||
class RPCResetPrefixCacheRequest(Enum):
|
||||
RESET_PREFIX_CACHE = 1
|
||||
@dataclass
|
||||
class RPCResetPrefixCacheRequest:
|
||||
device: Device
|
||||
|
||||
|
||||
class RPCSleepRequest(Enum):
|
||||
|
||||
@@ -47,7 +47,7 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
|
||||
from vllm.utils import deprecate_kwargs
|
||||
from vllm.utils import Device, deprecate_kwargs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -684,11 +684,12 @@ class MQLLMEngineClient(EngineClient):
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCUProfileRequest.STOP_PROFILE, socket=self.input_socket)
|
||||
|
||||
async def reset_prefix_cache(self) -> None:
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
"""Reset the prefix cache"""
|
||||
|
||||
await self._send_one_way_rpc_request(
|
||||
request=RPCResetPrefixCacheRequest.RESET_PREFIX_CACHE,
|
||||
request=RPCResetPrefixCacheRequest(device),
|
||||
socket=self.input_socket)
|
||||
|
||||
async def sleep(self, level: int = 1) -> None:
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.pooling_params import PoolingParams
|
||||
from vllm.prompt_adapter.request import PromptAdapterRequest
|
||||
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import collect_from_async_generator, random_uuid
|
||||
from vllm.utils import Device, collect_from_async_generator, random_uuid
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -274,7 +274,8 @@ class EngineClient(ABC):
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def reset_prefix_cache(self) -> None:
|
||||
async def reset_prefix_cache(self,
|
||||
device: Optional[Device] = None) -> None:
|
||||
"""Reset the prefix cache"""
|
||||
...
|
||||
|
||||
|
||||
Reference in New Issue
Block a user