[Bugfix][ROCm] Fix cumem sleep and teardown (#46203)
Signed-off-by: pei.zhang <pei.zhang@amd.com> Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Cursor
Matthew Wong
Andreas Karatzas
parent
37a682d392
commit
68afd78897
@@ -632,7 +632,7 @@ steps:
|
||||
#----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------#
|
||||
|
||||
- label: Basic Correctness # TBD
|
||||
timeout_in_minutes: 180
|
||||
timeout_in_minutes: 40
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||
agent_pool: mi300_1
|
||||
fast_check: true
|
||||
|
||||
@@ -16,3 +16,9 @@ steps:
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
mirror:
|
||||
amd:
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
@@ -48,8 +48,8 @@ static inline unsigned long long my_min(unsigned long long a,
|
||||
}
|
||||
|
||||
static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
|
||||
size_t alignment) {
|
||||
CUresult status = cuMemAddressReserve(d_mem, size, alignment, 0, 0);
|
||||
size_t alignment, CUdeviceptr addr = 0) {
|
||||
CUresult status = cuMemAddressReserve(d_mem, size, alignment, addr, 0);
|
||||
if (status == CUresult(0) || alignment == 0) {
|
||||
return status;
|
||||
}
|
||||
@@ -58,7 +58,7 @@ static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
|
||||
// alignment even when physical VRAM is free. Let HIP choose the default
|
||||
// alignment, then verify that the returned address still satisfies the
|
||||
// requested alignment before accepting it.
|
||||
status = cuMemAddressReserve(d_mem, size, 0, 0, 0);
|
||||
status = cuMemAddressReserve(d_mem, size, 0, addr, 0);
|
||||
if (status != CUresult(0)) {
|
||||
return status;
|
||||
}
|
||||
@@ -535,7 +535,14 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
|
||||
Py_DECREF(py_result);
|
||||
PyGILState_Release(gstate);
|
||||
|
||||
unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, num_chunks);
|
||||
// An empty chunk list means this allocation is asleep: its physical chunks
|
||||
// were already unmapped and released by sleep(), but the virtual address is
|
||||
// still held as a placeholder reservation. Skip unmap/release (freeing the
|
||||
// placeholder address happens below).
|
||||
if (num_chunks > 0) {
|
||||
unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes,
|
||||
num_chunks);
|
||||
}
|
||||
#else
|
||||
// Non-ROCm path: simple integer handle already extracted; drop temporary
|
||||
// Python refs while still holding the GIL, then release it.
|
||||
@@ -548,11 +555,13 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
|
||||
unmap_and_release(device, size, d_mem, p_memHandle);
|
||||
#endif
|
||||
|
||||
// free address and the handle
|
||||
// Free the virtual address. On ROCm this also covers an asleep allocation,
|
||||
// whose placeholder reservation made by sleep() is still held here.
|
||||
CUDA_CHECK(cuMemAddressFree(d_mem, size));
|
||||
#ifndef USE_ROCM
|
||||
free(p_memHandle);
|
||||
#else
|
||||
// Only awake allocations have per-chunk handles to free.
|
||||
for (auto i = 0; i < num_chunks; ++i) {
|
||||
free(p_memHandle[i]);
|
||||
}
|
||||
@@ -672,6 +681,29 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
|
||||
unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
|
||||
num_chunks);
|
||||
|
||||
// On ROCm/Linux, physical VRAM is only reclaimed once the virtual address
|
||||
// range is freed; hipMemUnmap + hipMemRelease alone leave the memory
|
||||
// resident (see ROCm#6021). Free the address to release physical memory,
|
||||
// then immediately re-reserve the SAME address as an empty placeholder so
|
||||
// the regular allocator cannot hand it out while we sleep. wake_up remaps
|
||||
// physical chunks into this placeholder.
|
||||
if (error_code == no_error) {
|
||||
CUDA_CHECK(cuMemAddressFree(d_mem_ptr, recv_size));
|
||||
if (error_code == no_error) {
|
||||
CUdeviceptr reserved = 0;
|
||||
CUDA_CHECK(reserve_rocm_address(&reserved, recv_size, /*alignment=*/0,
|
||||
d_mem_ptr));
|
||||
if (error_code == no_error && reserved != d_mem_ptr) {
|
||||
(void)cuMemAddressFree(reserved, recv_size);
|
||||
snprintf(error_msg, sizeof(error_msg),
|
||||
"failed to re-reserve placeholder address on sleep "
|
||||
"(requested %#llx, got %#llx)",
|
||||
(unsigned long long)d_mem_ptr, (unsigned long long)reserved);
|
||||
error_code = CUresult(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
free(p_memHandle);
|
||||
free(chunk_sizes);
|
||||
#endif
|
||||
@@ -736,6 +768,7 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
|
||||
chunk_sizes[i] = PyLong_AsUnsignedLongLong(size_py);
|
||||
}
|
||||
|
||||
// Address already reserved as a placeholder by sleep(); just remap chunks.
|
||||
create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
|
||||
num_chunks);
|
||||
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
|
||||
import dataclasses
|
||||
from contextlib import AbstractContextManager
|
||||
from typing import Protocol
|
||||
from typing import Protocol, TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# py_device, py_size_or_aligned_size, py_ptr, py_handle
|
||||
HandleType = tuple[int, int, int, int]
|
||||
# py_handle has type list[int] on ROCm and int otherwise
|
||||
HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
@@ -18,6 +19,7 @@ class AllocationData:
|
||||
handle: HandleType
|
||||
tag: str
|
||||
cpu_backup_tensor: torch.Tensor | None = None
|
||||
is_asleep: bool = False
|
||||
|
||||
|
||||
class MemAllocator(Protocol):
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
# both of them failed because of cuda context mismatch.
|
||||
# not sure why, they are created from a different context.
|
||||
# the only successful approach is to call cuda driver API in C.
|
||||
import atexit
|
||||
import gc
|
||||
import os
|
||||
from collections.abc import Callable, Iterator
|
||||
@@ -18,6 +19,7 @@ import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import find_loaded_library
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
@@ -115,8 +117,21 @@ class CuMemAllocator:
|
||||
assert cumem_available, "cumem allocator is not available"
|
||||
if CuMemAllocator.instance is None:
|
||||
CuMemAllocator.instance = CuMemAllocator()
|
||||
# Ensure MemPool/allocator wrappers are released before interpreter
|
||||
# finalization tears down PyTorch allocator internals.
|
||||
atexit.register(CuMemAllocator._shutdown_singleton)
|
||||
return CuMemAllocator.instance
|
||||
|
||||
@staticmethod
|
||||
def _shutdown_singleton() -> None:
|
||||
instance = CuMemAllocator.instance
|
||||
if instance is None:
|
||||
return
|
||||
try:
|
||||
instance.release_pools()
|
||||
except Exception:
|
||||
logger.exception("CuMemAllocator singleton shutdown failed")
|
||||
|
||||
def __init__(self):
|
||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||
self.current_tag: str = CuMemAllocator.default_tag
|
||||
@@ -127,6 +142,45 @@ class CuMemAllocator:
|
||||
self.python_malloc_callback = self._python_malloc_callback
|
||||
self.python_free_callback = self._python_free_callback
|
||||
|
||||
def release_pools(self) -> None:
|
||||
"""Drop Python references to MemPool/pluggable allocators eagerly.
|
||||
|
||||
A cumem ``MemPool`` outlives the ``use_memory_pool`` context (a strong
|
||||
reference is kept in ``allocator_and_pools`` to work around
|
||||
pytorch/pytorch#146431), and a captured CUDA graph can keep it alive
|
||||
longer still. ``MemPool`` only holds a non-owning pointer to the
|
||||
allocator, whose owning reference lives in the Python
|
||||
``CUDAPluggableAllocator``. If both are instead dropped during
|
||||
interpreter shutdown, GC may finalize the allocator first; the eventual
|
||||
``~MemPool`` -> ``emptyCache`` -> ``release_block`` then makes a virtual
|
||||
call into the freed allocator -- aborting the process with "pure virtual
|
||||
method called" (pytorch/pytorch#145168).
|
||||
|
||||
Release the kept-alive pools before interpreter finalization, and keep
|
||||
the pluggable allocator wrappers alive while MemPool destructors run.
|
||||
This is safe to call more than once.
|
||||
"""
|
||||
if not self.allocator_and_pools:
|
||||
return
|
||||
|
||||
pool_entries = list(self.allocator_and_pools.values())
|
||||
self.allocator_and_pools.clear()
|
||||
|
||||
mem_pools = [entry[0] for entry in pool_entries]
|
||||
allocators = [entry[1] for entry in pool_entries]
|
||||
pool_entries.clear()
|
||||
|
||||
# Phase 1: drop MemPool refs while allocators are still strongly held.
|
||||
mem_pools.clear()
|
||||
gc.collect()
|
||||
|
||||
# Phase 2: now it is safe to release allocator wrappers.
|
||||
allocators.clear()
|
||||
|
||||
def close(self) -> None:
|
||||
"""Compatibility alias for deterministic pool release."""
|
||||
self.release_pools()
|
||||
|
||||
def _python_malloc_callback(self, allocation_handle: HandleType) -> None:
|
||||
"""
|
||||
Internal method to store the allocation data
|
||||
@@ -150,6 +204,14 @@ class CuMemAllocator:
|
||||
data = self.pointer_to_data.pop(ptr)
|
||||
if data.cpu_backup_tensor is not None:
|
||||
data.cpu_backup_tensor = None
|
||||
if data.is_asleep and current_platform.is_rocm():
|
||||
# On ROCm, sleep() already unmapped and released this allocation's
|
||||
# physical chunks and holds its virtual address as a placeholder
|
||||
# reservation. Return a handle with an empty chunk list so the C
|
||||
# extension skips unmap/release (avoiding a double-free) while
|
||||
# still freeing the placeholder address.
|
||||
device, size, d_mem, _ = data.handle
|
||||
return (device, size, d_mem, [])
|
||||
# Drain pending kernels before the C extension's cuMemUnmap.
|
||||
# The pluggable allocator path doesn't defer reclaim like the
|
||||
# regular caching allocator, so without this, in-flight work
|
||||
@@ -201,7 +263,10 @@ class CuMemAllocator:
|
||||
cpu_ptr = cpu_backup_tensor.data_ptr()
|
||||
libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes)
|
||||
data.cpu_backup_tensor = cpu_backup_tensor
|
||||
unmap_and_release(handle)
|
||||
try:
|
||||
unmap_and_release(handle)
|
||||
finally:
|
||||
data.is_asleep = True
|
||||
|
||||
logger.info(
|
||||
"CuMemAllocator: sleep freed %.2f GiB memory in total, of which "
|
||||
@@ -230,6 +295,7 @@ class CuMemAllocator:
|
||||
if tags is None or data.tag in tags:
|
||||
handle = data.handle
|
||||
create_and_map(handle)
|
||||
data.is_asleep = False
|
||||
if data.cpu_backup_tensor is not None:
|
||||
cpu_backup_tensor = data.cpu_backup_tensor
|
||||
if cpu_backup_tensor is not None:
|
||||
|
||||
@@ -104,15 +104,12 @@ class CudaRTLibrary:
|
||||
|
||||
def __init__(self, so_file: str | None = None):
|
||||
if so_file is None:
|
||||
so_file = find_loaded_library("libcudart")
|
||||
if so_file is None:
|
||||
# libcudart is not loaded in the current process, try hip
|
||||
so_file = find_loaded_library("libamdhip64")
|
||||
# should be safe to assume now that we are using ROCm
|
||||
# as the following assertion should error out if the
|
||||
# libhiprtc library is also not loaded
|
||||
if so_file is None:
|
||||
so_file = envs.VLLM_CUDART_SO_PATH # fallback to env var
|
||||
so_file = (
|
||||
find_loaded_library(
|
||||
"libamdhip64" if current_platform.is_rocm() else "libcudart"
|
||||
)
|
||||
or envs.VLLM_CUDART_SO_PATH # fallback to env var
|
||||
)
|
||||
assert so_file is not None, (
|
||||
"libcudart is not loaded in the current process, "
|
||||
"try setting VLLM_CUDART_SO_PATH"
|
||||
|
||||
@@ -1185,6 +1185,14 @@ class Worker(WorkerBase):
|
||||
if model_runner := getattr(self, "model_runner", None):
|
||||
model_runner.shutdown()
|
||||
|
||||
# Release kept-alive cumem pools while the pluggable allocator wrappers
|
||||
# and callbacks are still alive, so MemPool teardown is not deferred to
|
||||
# interpreter finalization (pytorch/pytorch#145168).
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
if CuMemAllocator.instance is not None:
|
||||
CuMemAllocator.instance.release_pools()
|
||||
|
||||
def elastic_ep_execute(self, execute_method: str, *args, **kwargs):
|
||||
return self.elastic_ep_executor.execute(execute_method, *args, **kwargs)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user