[Bugfix][MRV2] Release layer-bound KV cache memory in shutdown() (#54246)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Kimi Code <noreply@moonshot.ai>
This commit is contained in:
+4
-2
@@ -1371,8 +1371,10 @@ class VllmRunner:
|
||||
shutdown_timeout = 60.0 if current_platform.is_rocm() else None
|
||||
self.llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
|
||||
except Exception:
|
||||
# Ignore shutdown errors as cleanup will still proceed
|
||||
pass
|
||||
# Don't fail the test on shutdown errors since cleanup will still
|
||||
# proceed, but don't hide them either: a failure here usually
|
||||
# means the engine's GPU memory was never released.
|
||||
logger.exception("Engine core shutdown raised; GPU memory may leak")
|
||||
del self.llm
|
||||
torch._dynamo.reset()
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import gc
|
||||
import itertools
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from itertools import groupby, product
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Protocol
|
||||
@@ -38,7 +39,7 @@ from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
from vllm.v1.worker.gpu.cp_utils import prepare_dcp_local_seq_lens
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
|
||||
from vllm.v1.worker.gpu.model_states.interface import ModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
from vllm.v1.worker.utils import AttentionGroup, clear_layer_kv_caches
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.gpu.model_runner import GPUModelRunner
|
||||
@@ -872,14 +873,12 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
# capture_model() re-captures them.
|
||||
if runner.model_state.supports_mm_inputs:
|
||||
runner.model_state.encoder_runner.clear()
|
||||
# Detach profiling KV tensors held by attention layers.
|
||||
for layer in runner.compilation_config.static_forward_context.values():
|
||||
if hasattr(layer, "kv_cache"):
|
||||
kv_cache = layer.kv_cache
|
||||
layer.kv_cache = (
|
||||
torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
|
||||
)
|
||||
del kv_cache
|
||||
# Detach profiling KV tensors held by attention layers. The layers live
|
||||
# in the static forward context for compiled models.
|
||||
layers: Iterable[Any] = runner.compilation_config.static_forward_context.values()
|
||||
if (model := getattr(runner, "model", None)) is not None:
|
||||
layers = itertools.chain(layers, model.modules())
|
||||
clear_layer_kv_caches(layers)
|
||||
runner.cache_config.num_gpu_blocks = None
|
||||
runner.maybe_remove_all_loras(runner.lora_config)
|
||||
gc.collect()
|
||||
|
||||
@@ -156,6 +156,7 @@ from vllm.v1.worker.gpu.structured_outputs import StructuredOutputsWorker
|
||||
from vllm.v1.worker.lora_model_runner_mixin import LoRAModelRunnerMixin
|
||||
from vllm.v1.worker.utils import (
|
||||
KVBlockZeroer,
|
||||
clear_layer_kv_caches,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
get_uniform_decode_token_count,
|
||||
)
|
||||
@@ -2026,9 +2027,15 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
free_before_shutdown(self.vllm_config)
|
||||
if hasattr(self, "model_state"):
|
||||
del self.model_state
|
||||
if getattr(self, "speculator", None) is not None:
|
||||
# Detach the layer-level KV/state cache tensors before dropping the
|
||||
# models; the model objects can outlive this runner.
|
||||
speculator = getattr(self, "speculator", None)
|
||||
if speculator is not None:
|
||||
if draft_model := getattr(speculator, "model", None):
|
||||
clear_layer_kv_caches(draft_model.modules())
|
||||
self.speculator = None
|
||||
if hasattr(self, "model"):
|
||||
clear_layer_kv_caches(self.model.modules())
|
||||
del self.model
|
||||
|
||||
gc.collect()
|
||||
|
||||
@@ -622,6 +622,27 @@ def bind_kv_cache(
|
||||
forward_context[layer_name].bind_kv_cache(kv_cache)
|
||||
|
||||
|
||||
def clear_layer_kv_caches(layers: Iterable[Any]) -> None:
|
||||
"""Detach the KV/state cache tensors installed by bind_kv_cache().
|
||||
|
||||
The model object can outlive the runner (e.g. LLMEngine's finalizer keeps
|
||||
it reachable until engine deletion), so dropping the runner's references
|
||||
alone does not release the KV cache memory on teardown paths.
|
||||
"""
|
||||
for layer in layers:
|
||||
if not hasattr(layer, "kv_cache"):
|
||||
continue
|
||||
kv_cache = layer.kv_cache
|
||||
layer.kv_cache = torch.tensor([]) if isinstance(kv_cache, torch.Tensor) else []
|
||||
# Clean up quantized KV cache scale views
|
||||
# (int8_per_token_head, fp8_per_token_head)
|
||||
if hasattr(layer, "impl"):
|
||||
if hasattr(layer.impl, "_k_scale_cache"):
|
||||
layer.impl._k_scale_cache = None
|
||||
if hasattr(layer.impl, "_v_scale_cache"):
|
||||
layer.impl._v_scale_cache = None
|
||||
|
||||
|
||||
def copy_kv_cache_blocks_inplace(
|
||||
kv_caches: Iterable[torch.Tensor],
|
||||
num_blocks: int,
|
||||
|
||||
Reference in New Issue
Block a user