diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 926475ecb1..3675065791 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -632,7 +632,7 @@ steps: #----------------------------------------------------- mi300 ยท basic_correctness -----------------------------------------------------# - label: Basic Correctness # TBD - timeout_in_minutes: 180 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 fast_check: true diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 0310945b08..7e166a8a28 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -16,3 +16,9 @@ steps: - pytest -v -s basic_correctness/test_mem.py - pytest -v -s basic_correctness/test_basic_correctness.py - pytest -v -s basic_correctness/test_cpu_offload.py + mirror: + amd: + device: mi325_1 + timeout_in_minutes: 40 + depends_on: + - image-build-amd diff --git a/csrc/cumem_allocator.cpp b/csrc/cumem_allocator.cpp index 73333f7125..2329d51a14 100644 --- a/csrc/cumem_allocator.cpp +++ b/csrc/cumem_allocator.cpp @@ -48,8 +48,8 @@ static inline unsigned long long my_min(unsigned long long a, } static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size, - size_t alignment) { - CUresult status = cuMemAddressReserve(d_mem, size, alignment, 0, 0); + size_t alignment, CUdeviceptr addr = 0) { + CUresult status = cuMemAddressReserve(d_mem, size, alignment, addr, 0); if (status == CUresult(0) || alignment == 0) { return status; } @@ -58,7 +58,7 @@ static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size, // alignment even when physical VRAM is free. Let HIP choose the default // alignment, then verify that the returned address still satisfies the // requested alignment before accepting it. - status = cuMemAddressReserve(d_mem, size, 0, 0, 0); + status = cuMemAddressReserve(d_mem, size, 0, addr, 0); if (status != CUresult(0)) { return status; } @@ -535,7 +535,14 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) { Py_DECREF(py_result); PyGILState_Release(gstate); - unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, num_chunks); + // An empty chunk list means this allocation is asleep: its physical chunks + // were already unmapped and released by sleep(), but the virtual address is + // still held as a placeholder reservation. Skip unmap/release (freeing the + // placeholder address happens below). + if (num_chunks > 0) { + unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, + num_chunks); + } #else // Non-ROCm path: simple integer handle already extracted; drop temporary // Python refs while still holding the GIL, then release it. @@ -548,11 +555,13 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) { unmap_and_release(device, size, d_mem, p_memHandle); #endif - // free address and the handle + // Free the virtual address. On ROCm this also covers an asleep allocation, + // whose placeholder reservation made by sleep() is still held here. CUDA_CHECK(cuMemAddressFree(d_mem, size)); #ifndef USE_ROCM free(p_memHandle); #else + // Only awake allocations have per-chunk handles to free. for (auto i = 0; i < num_chunks; ++i) { free(p_memHandle[i]); } @@ -672,6 +681,29 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) { unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes, num_chunks); + // On ROCm/Linux, physical VRAM is only reclaimed once the virtual address + // range is freed; hipMemUnmap + hipMemRelease alone leave the memory + // resident (see ROCm#6021). Free the address to release physical memory, + // then immediately re-reserve the SAME address as an empty placeholder so + // the regular allocator cannot hand it out while we sleep. wake_up remaps + // physical chunks into this placeholder. + if (error_code == no_error) { + CUDA_CHECK(cuMemAddressFree(d_mem_ptr, recv_size)); + if (error_code == no_error) { + CUdeviceptr reserved = 0; + CUDA_CHECK(reserve_rocm_address(&reserved, recv_size, /*alignment=*/0, + d_mem_ptr)); + if (error_code == no_error && reserved != d_mem_ptr) { + (void)cuMemAddressFree(reserved, recv_size); + snprintf(error_msg, sizeof(error_msg), + "failed to re-reserve placeholder address on sleep " + "(requested %#llx, got %#llx)", + (unsigned long long)d_mem_ptr, (unsigned long long)reserved); + error_code = CUresult(1); + } + } + } + free(p_memHandle); free(chunk_sizes); #endif @@ -736,6 +768,7 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) { chunk_sizes[i] = PyLong_AsUnsignedLongLong(size_py); } + // Address already reserved as a placeholder by sleep(); just remap chunks. create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes, num_chunks); diff --git a/vllm/device_allocator/__init__.py b/vllm/device_allocator/__init__.py index 6b5e9c613d..66e8b146d2 100644 --- a/vllm/device_allocator/__init__.py +++ b/vllm/device_allocator/__init__.py @@ -3,14 +3,15 @@ import dataclasses from contextlib import AbstractContextManager -from typing import Protocol +from typing import Protocol, TypeAlias import torch from vllm.platforms import current_platform # py_device, py_size_or_aligned_size, py_ptr, py_handle -HandleType = tuple[int, int, int, int] +# py_handle has type list[int] on ROCm and int otherwise +HandleType: TypeAlias = tuple[int, int, int, list[int] | int] @dataclasses.dataclass @@ -18,6 +19,7 @@ class AllocationData: handle: HandleType tag: str cpu_backup_tensor: torch.Tensor | None = None + is_asleep: bool = False class MemAllocator(Protocol): diff --git a/vllm/device_allocator/cumem.py b/vllm/device_allocator/cumem.py index 59c0cf45f5..7c4fedd34a 100644 --- a/vllm/device_allocator/cumem.py +++ b/vllm/device_allocator/cumem.py @@ -8,6 +8,7 @@ # both of them failed because of cuda context mismatch. # not sure why, they are created from a different context. # the only successful approach is to call cuda driver API in C. +import atexit import gc import os from collections.abc import Callable, Iterator @@ -18,6 +19,7 @@ import torch from vllm.device_allocator import AllocationData, HandleType from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.system_utils import find_loaded_library from vllm.utils.torch_utils import PIN_MEMORY @@ -115,8 +117,21 @@ class CuMemAllocator: assert cumem_available, "cumem allocator is not available" if CuMemAllocator.instance is None: CuMemAllocator.instance = CuMemAllocator() + # Ensure MemPool/allocator wrappers are released before interpreter + # finalization tears down PyTorch allocator internals. + atexit.register(CuMemAllocator._shutdown_singleton) return CuMemAllocator.instance + @staticmethod + def _shutdown_singleton() -> None: + instance = CuMemAllocator.instance + if instance is None: + return + try: + instance.release_pools() + except Exception: + logger.exception("CuMemAllocator singleton shutdown failed") + def __init__(self): self.pointer_to_data: dict[int, AllocationData] = {} self.current_tag: str = CuMemAllocator.default_tag @@ -127,6 +142,45 @@ class CuMemAllocator: self.python_malloc_callback = self._python_malloc_callback self.python_free_callback = self._python_free_callback + def release_pools(self) -> None: + """Drop Python references to MemPool/pluggable allocators eagerly. + + A cumem ``MemPool`` outlives the ``use_memory_pool`` context (a strong + reference is kept in ``allocator_and_pools`` to work around + pytorch/pytorch#146431), and a captured CUDA graph can keep it alive + longer still. ``MemPool`` only holds a non-owning pointer to the + allocator, whose owning reference lives in the Python + ``CUDAPluggableAllocator``. If both are instead dropped during + interpreter shutdown, GC may finalize the allocator first; the eventual + ``~MemPool`` -> ``emptyCache`` -> ``release_block`` then makes a virtual + call into the freed allocator -- aborting the process with "pure virtual + method called" (pytorch/pytorch#145168). + + Release the kept-alive pools before interpreter finalization, and keep + the pluggable allocator wrappers alive while MemPool destructors run. + This is safe to call more than once. + """ + if not self.allocator_and_pools: + return + + pool_entries = list(self.allocator_and_pools.values()) + self.allocator_and_pools.clear() + + mem_pools = [entry[0] for entry in pool_entries] + allocators = [entry[1] for entry in pool_entries] + pool_entries.clear() + + # Phase 1: drop MemPool refs while allocators are still strongly held. + mem_pools.clear() + gc.collect() + + # Phase 2: now it is safe to release allocator wrappers. + allocators.clear() + + def close(self) -> None: + """Compatibility alias for deterministic pool release.""" + self.release_pools() + def _python_malloc_callback(self, allocation_handle: HandleType) -> None: """ Internal method to store the allocation data @@ -150,6 +204,14 @@ class CuMemAllocator: data = self.pointer_to_data.pop(ptr) if data.cpu_backup_tensor is not None: data.cpu_backup_tensor = None + if data.is_asleep and current_platform.is_rocm(): + # On ROCm, sleep() already unmapped and released this allocation's + # physical chunks and holds its virtual address as a placeholder + # reservation. Return a handle with an empty chunk list so the C + # extension skips unmap/release (avoiding a double-free) while + # still freeing the placeholder address. + device, size, d_mem, _ = data.handle + return (device, size, d_mem, []) # Drain pending kernels before the C extension's cuMemUnmap. # The pluggable allocator path doesn't defer reclaim like the # regular caching allocator, so without this, in-flight work @@ -201,7 +263,10 @@ class CuMemAllocator: cpu_ptr = cpu_backup_tensor.data_ptr() libcudart.cudaMemcpy(cpu_ptr, ptr, size_in_bytes) data.cpu_backup_tensor = cpu_backup_tensor - unmap_and_release(handle) + try: + unmap_and_release(handle) + finally: + data.is_asleep = True logger.info( "CuMemAllocator: sleep freed %.2f GiB memory in total, of which " @@ -230,6 +295,7 @@ class CuMemAllocator: if tags is None or data.tag in tags: handle = data.handle create_and_map(handle) + data.is_asleep = False if data.cpu_backup_tensor is not None: cpu_backup_tensor = data.cpu_backup_tensor if cpu_backup_tensor is not None: diff --git a/vllm/distributed/device_communicators/cuda_wrapper.py b/vllm/distributed/device_communicators/cuda_wrapper.py index 422991ca93..a5026163e9 100644 --- a/vllm/distributed/device_communicators/cuda_wrapper.py +++ b/vllm/distributed/device_communicators/cuda_wrapper.py @@ -104,15 +104,12 @@ class CudaRTLibrary: def __init__(self, so_file: str | None = None): if so_file is None: - so_file = find_loaded_library("libcudart") - if so_file is None: - # libcudart is not loaded in the current process, try hip - so_file = find_loaded_library("libamdhip64") - # should be safe to assume now that we are using ROCm - # as the following assertion should error out if the - # libhiprtc library is also not loaded - if so_file is None: - so_file = envs.VLLM_CUDART_SO_PATH # fallback to env var + so_file = ( + find_loaded_library( + "libamdhip64" if current_platform.is_rocm() else "libcudart" + ) + or envs.VLLM_CUDART_SO_PATH # fallback to env var + ) assert so_file is not None, ( "libcudart is not loaded in the current process, " "try setting VLLM_CUDART_SO_PATH" diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 5e266a3135..87bfba6db2 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -1185,6 +1185,14 @@ class Worker(WorkerBase): if model_runner := getattr(self, "model_runner", None): model_runner.shutdown() + # Release kept-alive cumem pools while the pluggable allocator wrappers + # and callbacks are still alive, so MemPool teardown is not deferred to + # interpreter finalization (pytorch/pytorch#145168). + from vllm.device_allocator.cumem import CuMemAllocator + + if CuMemAllocator.instance is not None: + CuMemAllocator.instance.release_pools() + def elastic_ep_execute(self, execute_method: str, *args, **kwargs): return self.elastic_ep_executor.execute(execute_method, *args, **kwargs)