[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Ronald
2026-08-27 15:08:28 -07:00
committed by GitHub
co-authored by aoshen02
parent d262964e8c
commit 479eeb32d2
9 changed files with 228 additions and 146 deletions
+10 -6
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import Any, cast
@@ -208,13 +209,16 @@ def init_kv_cache(
device: torch.device,
kernel_block_sizes: list[int],
vllm_config: VllmConfig,
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> dict[str, Any]:
kv_caches = allocate_kv_cache(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
)
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
kv_caches = allocate_kv_cache(
kv_cache_config,
device,
vllm_config.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
)
for layer_name, target in get_shared_kv_cache_layers(vllm_config).items():
kv_caches[layer_name] = kv_caches[target]
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
-5
View File
@@ -86,11 +86,6 @@ class BlockTables:
)
def init_block_table_layout_tensors(self) -> None:
# Called at init and after a CuMem kv_cache wake-up. The ptr tensors
# cache raw data_ptr() values that go stale once the underlying tensors
# are reallocated on wake; the size tensors need re-populating because
# their storage lives under the kv_cache pool tag and comes back with
# undefined contents.
self.block_table_ptrs = self._make_ptr_tensor(
[b.gpu for b in self.block_tables]
)
+6 -4
View File
@@ -20,6 +20,7 @@ instead of embedding feature-specific logic directly.
import functools
import gc
import time
from contextlib import AbstractContextManager
from copy import deepcopy
from typing import Any, NamedTuple
@@ -524,7 +525,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return get_kv_cache_spec(self.vllm_config)
def initialize_kv_cache(
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
self,
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> None:
kv_cache_config = deepcopy(kv_cache_config)
self.kv_cache_config = kv_cache_config
@@ -656,6 +660,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.device,
self.kernel_block_sizes,
self.vllm_config,
kv_cache_allocation_context=kv_cache_allocation_context,
)
if is_profiling:
self.kv_connector = NO_OP_KV_CONNECTOR
@@ -863,9 +868,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.reset_encoder_cache()
gc.collect()
def post_kv_cache_wake_up(self) -> None:
self.block_tables.init_block_table_layout_tensors()
def reset_mm_cache(self) -> None:
if self.encoder_cache is not None:
self.encoder_cache.reset_mm_cache()