[Bugfix][ROCm] Fix cumem sleep and teardown (#46203)

Signed-off-by: pei.zhang <pei.zhang@amd.com>
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
peizhang56
2026-06-24 02:45:31 +08:00
committed by GitHub
co-authored by Cursor Matthew Wong Andreas Karatzas
parent 37a682d392
commit 68afd78897
7 changed files with 130 additions and 18 deletions
+38 -5
View File
@@ -48,8 +48,8 @@ static inline unsigned long long my_min(unsigned long long a,
}
static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
size_t alignment) {
CUresult status = cuMemAddressReserve(d_mem, size, alignment, 0, 0);
size_t alignment, CUdeviceptr addr = 0) {
CUresult status = cuMemAddressReserve(d_mem, size, alignment, addr, 0);
if (status == CUresult(0) || alignment == 0) {
return status;
}
@@ -58,7 +58,7 @@ static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
// alignment even when physical VRAM is free. Let HIP choose the default
// alignment, then verify that the returned address still satisfies the
// requested alignment before accepting it.
status = cuMemAddressReserve(d_mem, size, 0, 0, 0);
status = cuMemAddressReserve(d_mem, size, 0, addr, 0);
if (status != CUresult(0)) {
return status;
}
@@ -535,7 +535,14 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
Py_DECREF(py_result);
PyGILState_Release(gstate);
unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, num_chunks);
// An empty chunk list means this allocation is asleep: its physical chunks
// were already unmapped and released by sleep(), but the virtual address is
// still held as a placeholder reservation. Skip unmap/release (freeing the
// placeholder address happens below).
if (num_chunks > 0) {
unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes,
num_chunks);
}
#else
// Non-ROCm path: simple integer handle already extracted; drop temporary
// Python refs while still holding the GIL, then release it.
@@ -548,11 +555,13 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
unmap_and_release(device, size, d_mem, p_memHandle);
#endif
// free address and the handle
// Free the virtual address. On ROCm this also covers an asleep allocation,
// whose placeholder reservation made by sleep() is still held here.
CUDA_CHECK(cuMemAddressFree(d_mem, size));
#ifndef USE_ROCM
free(p_memHandle);
#else
// Only awake allocations have per-chunk handles to free.
for (auto i = 0; i < num_chunks; ++i) {
free(p_memHandle[i]);
}
@@ -672,6 +681,29 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
num_chunks);
// On ROCm/Linux, physical VRAM is only reclaimed once the virtual address
// range is freed; hipMemUnmap + hipMemRelease alone leave the memory
// resident (see ROCm#6021). Free the address to release physical memory,
// then immediately re-reserve the SAME address as an empty placeholder so
// the regular allocator cannot hand it out while we sleep. wake_up remaps
// physical chunks into this placeholder.
if (error_code == no_error) {
CUDA_CHECK(cuMemAddressFree(d_mem_ptr, recv_size));
if (error_code == no_error) {
CUdeviceptr reserved = 0;
CUDA_CHECK(reserve_rocm_address(&reserved, recv_size, /*alignment=*/0,
d_mem_ptr));
if (error_code == no_error && reserved != d_mem_ptr) {
(void)cuMemAddressFree(reserved, recv_size);
snprintf(error_msg, sizeof(error_msg),
"failed to re-reserve placeholder address on sleep "
"(requested %#llx, got %#llx)",
(unsigned long long)d_mem_ptr, (unsigned long long)reserved);
error_code = CUresult(1);
}
}
}
free(p_memHandle);
free(chunk_sizes);
#endif
@@ -736,6 +768,7 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
chunk_sizes[i] = PyLong_AsUnsignedLongLong(size_py);
}
// Address already reserved as a placeholder by sleep(); just remap chunks.
create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
num_chunks);