[Bugfix][ROCm] Fix cumem sleep and teardown (#46203)

Signed-off-by: pei.zhang <pei.zhang@amd.com>
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
peizhang56
2026-06-24 02:45:31 +08:00
committed by GitHub
co-authored by Cursor Matthew Wong Andreas Karatzas
parent 37a682d392
commit 68afd78897
7 changed files with 130 additions and 18 deletions
+1 -1
View File
@@ -632,7 +632,7 @@ steps:
#----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------#
- label: Basic Correctness # TBD
timeout_in_minutes: 180
timeout_in_minutes: 40
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_1
fast_check: true
@@ -16,3 +16,9 @@ steps:
- pytest -v -s basic_correctness/test_mem.py
- pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
mirror:
amd:
device: mi325_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
+38 -5
View File
@@ -48,8 +48,8 @@ static inline unsigned long long my_min(unsigned long long a,
}
static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
size_t alignment) {
CUresult status = cuMemAddressReserve(d_mem, size, alignment, 0, 0);
size_t alignment, CUdeviceptr addr = 0) {
CUresult status = cuMemAddressReserve(d_mem, size, alignment, addr, 0);
if (status == CUresult(0) || alignment == 0) {
return status;
}
@@ -58,7 +58,7 @@ static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
// alignment even when physical VRAM is free. Let HIP choose the default
// alignment, then verify that the returned address still satisfies the
// requested alignment before accepting it.
status = cuMemAddressReserve(d_mem, size, 0, 0, 0);
status = cuMemAddressReserve(d_mem, size, 0, addr, 0);
if (status != CUresult(0)) {
return status;
}
@@ -535,7 +535,14 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
Py_DECREF(py_result);
PyGILState_Release(gstate);
unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, num_chunks);
// An empty chunk list means this allocation is asleep: its physical chunks
// were already unmapped and released by sleep(), but the virtual address is
// still held as a placeholder reservation. Skip unmap/release (freeing the
// placeholder address happens below).
if (num_chunks > 0) {
unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes,
num_chunks);
}
#else
// Non-ROCm path: simple integer handle already extracted; drop temporary
// Python refs while still holding the GIL, then release it.
@@ -548,11 +555,13 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
unmap_and_release(device, size, d_mem, p_memHandle);
#endif
// free address and the handle
// Free the virtual address. On ROCm this also covers an asleep allocation,
// whose placeholder reservation made by sleep() is still held here.
CUDA_CHECK(cuMemAddressFree(d_mem, size));
#ifndef USE_ROCM
free(p_memHandle);
#else
// Only awake allocations have per-chunk handles to free.
for (auto i = 0; i < num_chunks; ++i) {
free(p_memHandle[i]);
}
@@ -672,6 +681,29 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
num_chunks);
// On ROCm/Linux, physical VRAM is only reclaimed once the virtual address
// range is freed; hipMemUnmap + hipMemRelease alone leave the memory
// resident (see ROCm#6021). Free the address to release physical memory,
// then immediately re-reserve the SAME address as an empty placeholder so
// the regular allocator cannot hand it out while we sleep. wake_up remaps
// physical chunks into this placeholder.
if (error_code == no_error) {
CUDA_CHECK(cuMemAddressFree(d_mem_ptr, recv_size));
if (error_code == no_error) {
CUdeviceptr reserved = 0;
CUDA_CHECK(reserve_rocm_address(&reserved, recv_size, /*alignment=*/0,
d_mem_ptr));
if (error_code == no_error && reserved != d_mem_ptr) {
(void)cuMemAddressFree(reserved, recv_size);
snprintf(error_msg, sizeof(error_msg),
"failed to re-reserve placeholder address on sleep "
"(requested %#llx, got %#llx)",
(unsigned long long)d_mem_ptr, (unsigned long long)reserved);
error_code = CUresult(1);
}
}
}
free(p_memHandle);
free(chunk_sizes);
#endif
@@ -736,6 +768,7 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
chunk_sizes[i] = PyLong_AsUnsignedLongLong(size_py);
}
// Address already reserved as a placeholder by sleep(); just remap chunks.
create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
num_chunks);
+4 -2
View File
@@ -3,14 +3,15 @@
import dataclasses
from contextlib import AbstractContextManager
from typing import Protocol
from typing import Protocol, TypeAlias
import torch
from vllm.platforms import current_platform
# py_device, py_size_or_aligned_size, py_ptr, py_handle
HandleType = tuple[int, int, int, int]
# py_handle has type list[int] on ROCm and int otherwise
HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
@dataclasses.dataclass
@@ -18,6 +19,7 @@ class AllocationData:
handle: HandleType
tag: str
cpu_backup_tensor: torch.Tensor | None = None
is_asleep: bool = False
class MemAllocator(Protocol):
+67 -1
View File
@@ -8,6 +8,7 @@
# both of them failed because of cuda context mismatch.
# not sure why, they are created from a different context.
# the only successful approach is to call cuda driver API in C.
import atexit
import gc
import os
from collections.abc import Callable, Iterator
@@ -18,6 +19,7 @@ import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.system_utils import find_loaded_library
from vllm.utils.torch_utils import PIN_MEMORY
@@ -115,8 +117,21 @@ class CuMemAllocator:
assert cumem_available, "cumem allocator is not available"
if CuMemAllocator.instance is None:
CuMemAllocator.instance = CuMemAllocator()
# Ensure MemPool/allocator wrappers are released before interpreter
# finalization tears down PyTorch allocator internals.
atexit.register(CuMemAllocator._shutdown_singleton)
return CuMemAllocator.instance
@staticmethod
def _shutdown_singleton() -> None:
instance = CuMemAllocator.instance
if instance is None:
return
try:
instance.release_pools()
except Exception:
logger.exception("CuMemAllocator singleton shutdown failed")
def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = CuMemAllocator.default_tag
@@ -127,6 +142,45 @@ class CuMemAllocator:
self.python_malloc_callback = self._python_malloc_callback
self.python_free_callback = self._python_free_callback
def release_pools(self) -> None:
"""Drop Python references to MemPool/pluggable allocators eagerly.
A cumem ``MemPool`` outlives the ``use_memory_pool`` context (a strong
reference is kept in ``allocator_and_pools`` to work around
pytorch/pytorch#146431), and a captured CUDA graph can keep it alive
longer still. ``MemPool`` only holds a non-owning pointer to the
allocator, whose owning reference lives in the Python
``CUDAPluggableAllocator``. If both are instead dropped during
interpreter shutdown, GC may finalize the allocator first; the eventual
``~MemPool`` -> ``emptyCache`` -> ``release_block`` then makes a virtual
call into the freed allocator -- aborting the process with "pure virtual
method called" (pytorch/pytorch#145168).
Release the kept-alive pools before interpreter finalization, and keep
the pluggable allocator wrappers alive while MemPool destructors run.
This is safe to call more than once.
"""
if not self.allocator_and_pools:
return
pool_entries = list(self.allocator_and_pools.values())
self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries]
allocators = [entry[1] for entry in pool_entries]
pool_entries.clear()
# Phase 1: drop MemPool refs while allocators are still strongly held.
mem_pools.clear()
gc.collect()
# Phase 2: now it is safe to release allocator wrappers.
allocators.clear()
def close(self) -> None:
"""Compatibility alias for deterministic pool release."""
self.release_pools()
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
"""
Internal method to store the allocation data
@@ -150,6 +204,14 @@ class CuMemAllocator:
data = self.pointer_to_data.pop(ptr)
if data.cpu_backup_tensor is not None:
data.cpu_backup_tensor = None
if data.is_asleep and current_platform.is_rocm():
# On ROCm, sleep() already unmapped and released this allocation's
# physical chunks and holds its virtual address as a placeholder
# reservation. Return a handle with an empty chunk list so the C
# extension skips unmap/release (avoiding a double-free) while
# still freeing the placeholder address.
device, size, d_mem, _ = data.handle
return (device, size, d_mem, [])
# Drain pending kernels before the C extension's cuMemUnmap.
# The pluggable allocator path doesn't defer reclaim like the
# regular caching allocator, so without this, in-flight work
@@ -201,7 +263,10 @@ class CuMemAllocator:
cpu_ptr = cpu_backup_tensor.data_ptr()
libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
data.cpu_backup_tensor = cpu_backup_tensor
unmap_and_release(handle)
try:
unmap_and_release(handle)
finally:
data.is_asleep = True
logger.info(
"CuMemAllocator: sleep freed %.2f GiB memory in total, of which "
@@ -230,6 +295,7 @@ class CuMemAllocator:
if tags is None or data.tag in tags:
handle = data.handle
create_and_map(handle)
data.is_asleep = False
if data.cpu_backup_tensor is not None:
cpu_backup_tensor = data.cpu_backup_tensor
if cpu_backup_tensor is not None:
@@ -104,15 +104,12 @@ class CudaRTLibrary:
def __init__(self, so_file: str | None = None):
if so_file is None:
so_file = find_loaded_library("libcudart")
if so_file is None:
# libcudart is not loaded in the current process, try hip
so_file = find_loaded_library("libamdhip64")
# should be safe to assume now that we are using ROCm
# as the following assertion should error out if the
# libhiprtc library is also not loaded
if so_file is None:
so_file = envs.VLLM_CUDART_SO_PATH # fallback to env var
so_file = (
find_loaded_library(
"libamdhip64" if current_platform.is_rocm() else "libcudart"
)
or envs.VLLM_CUDART_SO_PATH # fallback to env var
)
assert so_file is not None, (
"libcudart is not loaded in the current process, "
"try setting VLLM_CUDART_SO_PATH"
+8
View File
@@ -1185,6 +1185,14 @@ class Worker(WorkerBase):
if model_runner := getattr(self, "model_runner", None):
model_runner.shutdown()
# Release kept-alive cumem pools while the pluggable allocator wrappers
# and callbacks are still alive, so MemPool teardown is not deferred to
# interpreter finalization (pytorch/pytorch#145168).
from vllm.device_allocator.cumem import CuMemAllocator
if CuMemAllocator.instance is not None:
CuMemAllocator.instance.release_pools()
def elastic_ep_execute(self, execute_method: str, *args, **kwargs):
return self.elastic_ep_executor.execute(execute_method, *args, **kwargs)