[Bugfix][ROCm] Fix cumem sleep and teardown (#46203)
Signed-off-by: pei.zhang <pei.zhang@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Cursor
Matthew Wong
Andreas Karatzas
parent
37a682d392
commit
68afd78897
@@ -3,14 +3,15 @@
|
||||
|
||||
import dataclasses
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Protocol
|
||||
from typing import Protocol, TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# py_device, py_size_or_aligned_size, py_ptr, py_handle
|
||||
HandleType = tuple[int, int, int, int]
|
||||
# py_handle has type list[int] on ROCm and int otherwise
|
||||
HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -18,6 +19,7 @@ class AllocationData:
|
||||
handle: HandleType
|
||||
tag: str
|
||||
cpu_backup_tensor: torch.Tensor | None = None
|
||||
is_asleep: bool = False
|
||||
|
||||
|
||||
class MemAllocator(Protocol):
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
# both of them failed because of cuda context mismatch.
|
||||
# not sure why, they are created from a different context.
|
||||
# the only successful approach is to call cuda driver API in C.
|
||||
import atexit
|
||||
import gc
|
||||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
@@ -18,6 +19,7 @@ import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import find_loaded_library
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
@@ -115,8 +117,21 @@ class CuMemAllocator:
|
||||
assert cumem_available, "cumem allocator is not available"
|
||||
if CuMemAllocator.instance is None:
|
||||
CuMemAllocator.instance = CuMemAllocator()
|
||||
# Ensure MemPool/allocator wrappers are released before interpreter
|
||||
# finalization tears down PyTorch allocator internals.
|
||||
atexit.register(CuMemAllocator._shutdown_singleton)
|
||||
return CuMemAllocator.instance
|
||||
|
||||
@staticmethod
|
||||
def _shutdown_singleton() -> None:
|
||||
instance = CuMemAllocator.instance
|
||||
if instance is None:
|
||||
return
|
||||
try:
|
||||
instance.release_pools()
|
||||
except Exception:
|
||||
logger.exception("CuMemAllocator singleton shutdown failed")
|
||||
|
||||
def __init__(self):
|
||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||
self.current_tag: str = CuMemAllocator.default_tag
|
||||
@@ -127,6 +142,45 @@ class CuMemAllocator:
|
||||
self.python_malloc_callback = self._python_malloc_callback
|
||||
self.python_free_callback = self._python_free_callback
|
||||
|
||||
def release_pools(self) -> None:
|
||||
"""Drop Python references to MemPool/pluggable allocators eagerly.
|
||||
|
||||
A cumem ``MemPool`` outlives the ``use_memory_pool`` context (a strong
|
||||
reference is kept in ``allocator_and_pools`` to work around
|
||||
pytorch/pytorch#146431), and a captured CUDA graph can keep it alive
|
||||
longer still. ``MemPool`` only holds a non-owning pointer to the
|
||||
allocator, whose owning reference lives in the Python
|
||||
``CUDAPluggableAllocator``. If both are instead dropped during
|
||||
interpreter shutdown, GC may finalize the allocator first; the eventual
|
||||
``~MemPool`` -> ``emptyCache`` -> ``release_block`` then makes a virtual
|
||||
call into the freed allocator -- aborting the process with "pure virtual
|
||||
method called" (pytorch/pytorch#145168).
|
||||
|
||||
Release the kept-alive pools before interpreter finalization, and keep
|
||||
the pluggable allocator wrappers alive while MemPool destructors run.
|
||||
This is safe to call more than once.
|
||||
"""
|
||||
if not self.allocator_and_pools:
|
||||
return
|
||||
|
||||
pool_entries = list(self.allocator_and_pools.values())
|
||||
self.allocator_and_pools.clear()
|
||||
|
||||
mem_pools = [entry[0] for entry in pool_entries]
|
||||
allocators = [entry[1] for entry in pool_entries]
|
||||
pool_entries.clear()
|
||||
|
||||
# Phase 1: drop MemPool refs while allocators are still strongly held.
|
||||
mem_pools.clear()
|
||||
gc.collect()
|
||||
|
||||
# Phase 2: now it is safe to release allocator wrappers.
|
||||
allocators.clear()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Compatibility alias for deterministic pool release."""
|
||||
self.release_pools()
|
||||
|
||||
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
|
||||
"""
|
||||
Internal method to store the allocation data
|
||||
@@ -150,6 +204,14 @@ class CuMemAllocator:
|
||||
data = self.pointer_to_data.pop(ptr)
|
||||
if data.cpu_backup_tensor is not None:
|
||||
data.cpu_backup_tensor = None
|
||||
if data.is_asleep and current_platform.is_rocm():
|
||||
# On ROCm, sleep() already unmapped and released this allocation's
|
||||
# physical chunks and holds its virtual address as a placeholder
|
||||
# reservation. Return a handle with an empty chunk list so the C
|
||||
# extension skips unmap/release (avoiding a double-free) while
|
||||
# still freeing the placeholder address.
|
||||
device, size, d_mem, _ = data.handle
|
||||
return (device, size, d_mem, [])
|
||||
# Drain pending kernels before the C extension's cuMemUnmap.
|
||||
# The pluggable allocator path doesn't defer reclaim like the
|
||||
# regular caching allocator, so without this, in-flight work
|
||||
@@ -201,7 +263,10 @@ class CuMemAllocator:
|
||||
cpu_ptr = cpu_backup_tensor.data_ptr()
|
||||
libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
|
||||
data.cpu_backup_tensor = cpu_backup_tensor
|
||||
unmap_and_release(handle)
|
||||
try:
|
||||
unmap_and_release(handle)
|
||||
finally:
|
||||
data.is_asleep = True
|
||||
|
||||
logger.info(
|
||||
"CuMemAllocator: sleep freed %.2f GiB memory in total, of which "
|
||||
@@ -230,6 +295,7 @@ class CuMemAllocator:
|
||||
if tags is None or data.tag in tags:
|
||||
handle = data.handle
|
||||
create_and_map(handle)
|
||||
data.is_asleep = False
|
||||
if data.cpu_backup_tensor is not None:
|
||||
cpu_backup_tensor = data.cpu_backup_tensor
|
||||
if cpu_backup_tensor is not None:
|
||||
|
||||
Reference in New Issue
Block a user