[Bugfix][MRV2] Isolate sleep-mode KV allocations (#53508)

Signed-off-by: Ronald1995 <ronaldautomobile@163.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Ronald
2026-08-27 15:08:28 -07:00
committed by GitHub
co-authored by aoshen02
parent d262964e8c
commit 479eeb32d2
9 changed files with 228 additions and 146 deletions
+17 -64
View File
@@ -59,7 +59,7 @@ from vllm.forward_context import (
)
from vllm.logger import init_logger
from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
@@ -132,7 +132,6 @@ from vllm.utils.torch_utils import (
PIN_MEMORY,
async_tensor_h2d,
current_stream,
is_quantized_kv_cache,
kv_cache_dtype_str_to_dtype,
)
from vllm.v1.attention.backend import (
@@ -1027,60 +1026,6 @@ class GPUModelRunner(
self.encoder_cache.clear()
self.late_interaction_runner.clear()
def post_kv_cache_wake_up(self) -> None:
self.init_fp8_kv_scales()
@torch.inference_mode()
def init_fp8_kv_scales(self) -> None:
"""
Re-initialize the KV cache and FP8 scales after waking from sleep.
1. Zero out the KV cache tensors to remove garbage data from re-allocation.
2. Reset Attention layer scaling factors (_k_scale, _v_scale) to 1.0.
If these are left at 0.0 (default after wake_up), all KV cache values
become effectively zero, causing gibberish output.
"""
if not is_quantized_kv_cache(self.cache_config.cache_dtype):
return
kv_caches = getattr(self, "kv_caches", [])
for cache_entry in kv_caches:
if cache_entry is None:
continue
# Hybrid models (Mamba, DeltaNet) store per-layer state as a
# list of tensors rather than a single tensor.
if isinstance(cache_entry, list):
for t in cache_entry:
t.zero_()
else:
cache_entry.zero_()
k_attr_names = ("_k_scale", "k_scale")
v_attr_names = ("_v_scale", "v_scale")
attn_layers = self.compilation_config.static_forward_context
for name, module in attn_layers.items():
if isinstance(module, (Attention, MLAAttention)):
# TODO: Generally, scale is 1.0 if user uses on-the-fly fp8
# kvcache quant. However, to get better accuracy, compression
# frameworks like llm-compressors allow users to tune the
# scale. We may need to restore the specific calibrated scales
# here in the future.
k_scale_val, v_scale_val = 1.0, 1.0
# Processing K Scale
for attr in k_attr_names:
if hasattr(module, attr):
param = getattr(module, attr)
if isinstance(param, torch.Tensor):
param.fill_(k_scale_val)
# Processing V Scale
for attr in v_attr_names:
if hasattr(module, attr):
param = getattr(module, attr)
if isinstance(param, torch.Tensor):
param.fill_(v_scale_val)
def _get_positions(self, num_tokens: Any):
if isinstance(num_tokens, int):
if self.uses_mrope:
@@ -7443,7 +7388,10 @@ class GPUModelRunner(
yield from attn_groups
def initialize_kv_cache_tensors(
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
self,
kv_cache_config: KVCacheConfig,
kernel_block_sizes: list[int],
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> dict[str, torch.Tensor]:
"""
Initialize the memory buffer for KV cache.
@@ -7457,12 +7405,14 @@ class GPUModelRunner(
corresponding memory buffer for KV cache.
"""
kv_caches = allocate_kv_cache(
kv_cache_config,
self.device,
self.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
)
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
kv_caches = allocate_kv_cache(
kv_cache_config,
self.device,
self.cache_config.get_resolved_kv_cache_layout(),
kernel_block_sizes,
)
# Set up cross-layer KV cache sharing
for layer_name, target_layer_name in self.shared_kv_cache_layers.items():
@@ -7512,6 +7462,7 @@ class GPUModelRunner(
self,
kv_cache_config: KVCacheConfig,
is_profiling: bool = False,
kv_cache_allocation_context: AbstractContextManager | None = None,
) -> None:
"""
Initialize KV cache based on `kv_cache_config`.
@@ -7544,7 +7495,9 @@ class GPUModelRunner(
# Reinitialize need to after initialize_attn_backend
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
kv_caches = self.initialize_kv_cache_tensors(
kv_cache_config, kernel_block_sizes
kv_cache_config,
kernel_block_sizes,
kv_cache_allocation_context=kv_cache_allocation_context,
)
if (