[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -256,9 +256,6 @@ class Worker(WorkerBase):
|
||||
buffer.data.copy_(self._sleep_saved_draft_buffers[name].data)
|
||||
self._sleep_saved_draft_buffers = {}
|
||||
|
||||
if tags is None or "kv_cache" in tags:
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
self.synchronize_device()
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
@@ -689,8 +686,12 @@ class Worker(WorkerBase):
|
||||
# related to kv cache connector (e.g. kv cache sharing layers).
|
||||
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
|
||||
|
||||
with self._maybe_get_memory_pool_context(tag="kv_cache"):
|
||||
self.model_runner.initialize_kv_cache(kv_cache_config)
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=self._maybe_get_memory_pool_context(
|
||||
tag="kv_cache"
|
||||
),
|
||||
)
|
||||
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.model_runner.init_routed_experts_capturer()
|
||||
|
||||
Reference in New Issue
Block a user