[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)
Signed-off-by: Ronald1995 <ronaldautomobile@163.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import sys
|
||||
from contextlib import contextmanager
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -157,8 +157,13 @@ class CPUModelRunner(GPUModelRunner):
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
is_profiling: bool = False,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> None:
|
||||
super().initialize_kv_cache(kv_cache_config, is_profiling)
|
||||
super().initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
is_profiling,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
)
|
||||
|
||||
if self.speculative_config:
|
||||
if self.speculative_config.use_eagle():
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -208,13 +209,16 @@ def init_kv_cache(
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> dict[str, Any]:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
for layer_name, target in get_shared_kv_cache_layers(vllm_config).items():
|
||||
kv_caches[layer_name] = kv_caches[target]
|
||||
# Dual-attention models (e.g. LongCat-Flash) put two Attention modules per
|
||||
|
||||
@@ -86,11 +86,6 @@ class BlockTables:
|
||||
)
|
||||
|
||||
def init_block_table_layout_tensors(self) -> None:
|
||||
# Called at init and after a CuMem kv_cache wake-up. The ptr tensors
|
||||
# cache raw data_ptr() values that go stale once the underlying tensors
|
||||
# are reallocated on wake; the size tensors need re-populating because
|
||||
# their storage lives under the kv_cache pool tag and comes back with
|
||||
# undefined contents.
|
||||
self.block_table_ptrs = self._make_ptr_tensor(
|
||||
[b.gpu for b in self.block_tables]
|
||||
)
|
||||
|
||||
@@ -20,6 +20,7 @@ instead of embedding feature-specific logic directly.
|
||||
import functools
|
||||
import gc
|
||||
import time
|
||||
from contextlib import AbstractContextManager
|
||||
from copy import deepcopy
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -524,7 +525,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return get_kv_cache_spec(self.vllm_config)
|
||||
|
||||
def initialize_kv_cache(
|
||||
self, kv_cache_config: KVCacheConfig, is_profiling: bool = False
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
is_profiling: bool = False,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> None:
|
||||
kv_cache_config = deepcopy(kv_cache_config)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
@@ -656,6 +660,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.device,
|
||||
self.kernel_block_sizes,
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
)
|
||||
if is_profiling:
|
||||
self.kv_connector = NO_OP_KV_CONNECTOR
|
||||
@@ -863,9 +868,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.reset_encoder_cache()
|
||||
gc.collect()
|
||||
|
||||
def post_kv_cache_wake_up(self) -> None:
|
||||
self.block_tables.init_block_table_layout_tensors()
|
||||
|
||||
def reset_mm_cache(self) -> None:
|
||||
if self.encoder_cache is not None:
|
||||
self.encoder_cache.reset_mm_cache()
|
||||
|
||||
@@ -59,7 +59,7 @@ from vllm.forward_context import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
@@ -132,7 +132,6 @@ from vllm.utils.torch_utils import (
|
||||
PIN_MEMORY,
|
||||
async_tensor_h2d,
|
||||
current_stream,
|
||||
is_quantized_kv_cache,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -1027,60 +1026,6 @@ class GPUModelRunner(
|
||||
self.encoder_cache.clear()
|
||||
self.late_interaction_runner.clear()
|
||||
|
||||
def post_kv_cache_wake_up(self) -> None:
|
||||
self.init_fp8_kv_scales()
|
||||
|
||||
@torch.inference_mode()
|
||||
def init_fp8_kv_scales(self) -> None:
|
||||
"""
|
||||
Re-initialize the KV cache and FP8 scales after waking from sleep.
|
||||
1. Zero out the KV cache tensors to remove garbage data from re-allocation.
|
||||
2. Reset Attention layer scaling factors (_k_scale, _v_scale) to 1.0.
|
||||
If these are left at 0.0 (default after wake_up), all KV cache values
|
||||
become effectively zero, causing gibberish output.
|
||||
"""
|
||||
if not is_quantized_kv_cache(self.cache_config.cache_dtype):
|
||||
return
|
||||
|
||||
kv_caches = getattr(self, "kv_caches", [])
|
||||
for cache_entry in kv_caches:
|
||||
if cache_entry is None:
|
||||
continue
|
||||
# Hybrid models (Mamba, DeltaNet) store per-layer state as a
|
||||
# list of tensors rather than a single tensor.
|
||||
if isinstance(cache_entry, list):
|
||||
for t in cache_entry:
|
||||
t.zero_()
|
||||
else:
|
||||
cache_entry.zero_()
|
||||
|
||||
k_attr_names = ("_k_scale", "k_scale")
|
||||
v_attr_names = ("_v_scale", "v_scale")
|
||||
|
||||
attn_layers = self.compilation_config.static_forward_context
|
||||
for name, module in attn_layers.items():
|
||||
if isinstance(module, (Attention, MLAAttention)):
|
||||
# TODO: Generally, scale is 1.0 if user uses on-the-fly fp8
|
||||
# kvcache quant. However, to get better accuracy, compression
|
||||
# frameworks like llm-compressors allow users to tune the
|
||||
# scale. We may need to restore the specific calibrated scales
|
||||
# here in the future.
|
||||
k_scale_val, v_scale_val = 1.0, 1.0
|
||||
|
||||
# Processing K Scale
|
||||
for attr in k_attr_names:
|
||||
if hasattr(module, attr):
|
||||
param = getattr(module, attr)
|
||||
if isinstance(param, torch.Tensor):
|
||||
param.fill_(k_scale_val)
|
||||
|
||||
# Processing V Scale
|
||||
for attr in v_attr_names:
|
||||
if hasattr(module, attr):
|
||||
param = getattr(module, attr)
|
||||
if isinstance(param, torch.Tensor):
|
||||
param.fill_(v_scale_val)
|
||||
|
||||
def _get_positions(self, num_tokens: Any):
|
||||
if isinstance(num_tokens, int):
|
||||
if self.uses_mrope:
|
||||
@@ -7443,7 +7388,10 @@ class GPUModelRunner(
|
||||
yield from attn_groups
|
||||
|
||||
def initialize_kv_cache_tensors(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
kernel_block_sizes: list[int],
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""
|
||||
Initialize the memory buffer for KV cache.
|
||||
@@ -7457,12 +7405,14 @@ class GPUModelRunner(
|
||||
corresponding memory buffer for KV cache.
|
||||
"""
|
||||
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
self.device,
|
||||
self.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
kv_caches = allocate_kv_cache(
|
||||
kv_cache_config,
|
||||
self.device,
|
||||
self.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
)
|
||||
|
||||
# Set up cross-layer KV cache sharing
|
||||
for layer_name, target_layer_name in self.shared_kv_cache_layers.items():
|
||||
@@ -7512,6 +7462,7 @@ class GPUModelRunner(
|
||||
self,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
is_profiling: bool = False,
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize KV cache based on `kv_cache_config`.
|
||||
@@ -7544,7 +7495,9 @@ class GPUModelRunner(
|
||||
# Reinitialize need to after initialize_attn_backend
|
||||
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
|
||||
kv_caches = self.initialize_kv_cache_tensors(
|
||||
kv_cache_config, kernel_block_sizes
|
||||
kv_cache_config,
|
||||
kernel_block_sizes,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
@@ -256,9 +256,6 @@ class Worker(WorkerBase):
|
||||
buffer.data.copy_(self._sleep_saved_draft_buffers[name].data)
|
||||
self._sleep_saved_draft_buffers = {}
|
||||
|
||||
if tags is None or "kv_cache" in tags:
|
||||
self.model_runner.post_kv_cache_wake_up()
|
||||
|
||||
self.synchronize_device()
|
||||
|
||||
def checkpoint_prepare(self) -> None:
|
||||
@@ -689,8 +686,12 @@ class Worker(WorkerBase):
|
||||
# related to kv cache connector (e.g. kv cache sharing layers).
|
||||
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
|
||||
|
||||
with self._maybe_get_memory_pool_context(tag="kv_cache"):
|
||||
self.model_runner.initialize_kv_cache(kv_cache_config)
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=self._maybe_get_memory_pool_context(
|
||||
tag="kv_cache"
|
||||
),
|
||||
)
|
||||
|
||||
if self.model_config.enable_return_routed_experts:
|
||||
self.model_runner.init_routed_experts_capturer()
|
||||
|
||||
Reference in New Issue
Block a user