[ROCm][Engine] Fix GPU memory leaks in engine shutdown and test workaround for async KV prefix cache reset (#38503)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-04-25 05:25:20 +00:00
committed by GitHub
parent 07351e0883
commit 95995bbef8
8 changed files with 300 additions and 13 deletions
+25 -4
View File
@@ -45,16 +45,14 @@ def run_once(f: Callable[P, None]) -> Callable[P, None]:
@lru_cache
def supports_kw(
def _supports_kw(
callable: Callable[..., object],
kw_name: str,
*,
requires_kw_only: bool = False,
allow_var_kwargs: bool = True,
) -> bool:
"""Check if a keyword is a valid kwarg for a callable; if requires_kw_only
disallows kwargs names that can also be positional arguments.
"""
"""Internal cached implementation of supports_kw."""
params = inspect.signature(callable).parameters
if not params:
return False
@@ -99,6 +97,29 @@ def supports_kw(
return False
def supports_kw(
callable: Callable[..., object],
kw_name: str,
*,
requires_kw_only: bool = False,
allow_var_kwargs: bool = True,
) -> bool:
"""Check if a keyword is a valid kwarg for a callable; if requires_kw_only
disallows kwargs names that can also be positional arguments.
"""
# Unwrap bound methods so that the lru_cache key is the underlying
# function, not the instance. Caching bound methods pins the object
# (and all its GPU tensors) for the lifetime of the cache.
if hasattr(callable, "__func__"):
callable = callable.__func__
return _supports_kw(
callable,
kw_name,
requires_kw_only=requires_kw_only,
allow_var_kwargs=allow_var_kwargs,
)
def get_allowed_kwarg_only_overrides(
callable: Callable[..., object],
overrides: Mapping[str, object] | None,
+7
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import os
import queue
import signal
@@ -574,6 +575,12 @@ class EngineCore:
if self.scheduler:
self.scheduler.shutdown()
# Undo the gc.freeze() from __init__ so that the objects allocated
# during engine startup (model weights, KV caches, etc.) become
# visible to the garbage collector again. Without this, deleting
# the engine in-process (e.g. unit tests) leaks GPU memory.
gc.unfreeze()
def profile(self, is_start: bool = True, profile_prefix: str | None = None):
self.model_executor.profile(is_start, profile_prefix)
+14
View File
@@ -5886,6 +5886,20 @@ class GPUModelRunner(
gc.unfreeze()
gc.collect()
def shutdown(self) -> None:
"""Release GPU tensors (model weights, KV caches, workspace) so that
memory is reclaimable when running in the same process."""
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
from vllm.v1.worker.workspace import reset_workspace_manager
# Calls torch.accelerator.synchronize()
self._cleanup_profiling_kv_cache()
self.compilation_config.static_forward_context.clear()
self.model = None # type: ignore[assignment]
_ROPE_DICT.clear()
reset_workspace_manager()
def _cleanup_profiling_kv_cache(self) -> None:
torch.accelerator.synchronize()
if hasattr(self, "kv_caches") and self.kv_caches:
+5
View File
@@ -1015,6 +1015,11 @@ class Worker(WorkerBase):
if weight_transfer_engine := getattr(self, "weight_transfer_engine", None):
weight_transfer_engine.shutdown()
# Release GPU resources held by the model runner so that memory
# can be reclaimed when running in-process
if model_runner := getattr(self, "model_runner", None):
model_runner.shutdown()
def elastic_ep_execute(self, execute_method: str, *args, **kwargs):
return self.elastic_ep_executor.execute(execute_method, *args, **kwargs)