[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -208,13 +209,16 @@ def init_kv_cache(
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> dict[str, Any]:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
for layer_name, target in get_shared_kv_cache_layers(vllm_config).items():
|
||||
kv_caches[layer_name] = kv_caches[target]
|
||||
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
|
||||
|
||||
@@ -86,11 +86,6 @@ class BlockTables:
|
||||
)
|
||||
|
||||
def init_block_table_layout_tensors(self) -> None:
|
||||
# Called at init and after a CuMem kv_cache wake-up. The ptr tensors
|
||||
# cache raw data_ptr() values that go stale once the underlying tensors
|
||||
# are reallocated on wake; the size tensors need re-populating because
|
||||
# their storage lives under the kv_cache pool tag and comes back with
|
||||
# undefined contents.
|
||||
self.block_table_ptrs = self._make_ptr_tensor(
|
||||
[b.gpu for b in self.block_tables]
|
||||
)
|
||||
|
||||
@@ -20,6 +20,7 @@ instead of embedding feature-specific logic directly.
|
||||
import functools
|
||||
import gc
|
||||
import time
|
||||
from contextlib import AbstractContextManager
|
||||
from copy import deepcopy
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -524,7 +525,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
is_profiling: bool = False,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> None:
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -656,6 +660,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.device,
|
||||
self.kernel_block_sizes,
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
)
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
@@ -863,9 +868,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.reset_encoder_cache()
|
||||
gc.collect()
|
||||
|
||||
def post_kv_cache_wake_up(self) -> None:
|
||||
self.block_tables.init_block_table_layout_tensors()
|
||||
|
||||
def reset_mm_cache(self) -> None:
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.reset_mm_cache()
|
||||
|
||||
Reference in New Issue
Block a user