[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Ronald
2026-08-27 15:08:28 -07:00
committed by GitHub
co-authored by aoshen02
parent d262964e8c
commit 479eeb32d2
9 changed files with 228 additions and 146 deletions
+6 -4
View File
@@ -20,6 +20,7 @@ instead of embedding feature-specific logic directly.
import functools
import gc
import time
from contextlib import AbstractContextManager
from copy import deepcopy
from typing import Any, NamedTuple
@@ -524,7 +525,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return get_kv_cache_spec(self.vllm_config)
def initialize_kv_cache(
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
self,
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> None:
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
@@ -656,6 +660,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.device,
self.kernel_block_sizes,
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
)
if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
@@ -863,9 +868,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.reset_encoder_cache()
gc.collect()
def post_kv_cache_wake_up(self) -> None:
self.block_tables.init_block_table_layout_tensors()
def reset_mm_cache(self) -> None:
if self.encoder_cache is not None:
self.encoder_cache.reset_mm_cache()