[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -208,13 +209,16 @@ def init_kv_cache(
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> dict[str, Any]:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
for layer_name, target in get_shared_kv_cache_layers(vllm_config).items():
|
||||
kv_caches[layer_name] = kv_caches[target]
|
||||
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
|
||||
|
||||
Reference in New Issue
Block a user