[ROCm][Engine] Fix GPU memory leaks in engine shutdown and test workaround for async KV prefix cache reset (#38503)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -5886,6 +5886,20 @@ class GPUModelRunner(
|
||||
gc.unfreeze()
|
||||
gc.collect()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Release GPU tensors (model weights, KV caches, workspace) so that
|
||||
memory is reclaimable when running in the same process."""
|
||||
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
|
||||
from vllm.v1.worker.workspace import reset_workspace_manager
|
||||
|
||||
# Calls torch.accelerator.synchronize()
|
||||
self._cleanup_profiling_kv_cache()
|
||||
self.compilation_config.static_forward_context.clear()
|
||||
self.model = None # type: ignore[assignment]
|
||||
_ROPE_DICT.clear()
|
||||
|
||||
reset_workspace_manager()
|
||||
|
||||
def _cleanup_profiling_kv_cache(self) -> None:
|
||||
torch.accelerator.synchronize()
|
||||
if hasattr(self, "kv_caches") and self.kv_caches:
|
||||
|
||||
Reference in New Issue
Block a user