[Feature] Release the CUDA graph pool on sleep (#59160)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-04 01:13:10 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9e0b6fcf77
commit 5e56e9af2a
16 changed files with 299 additions and 27 deletions
+17
View File
@@ -430,6 +430,23 @@ class CuMemAllocator:
if allocation["allocated_size"] == 0 and data and not data.is_asleep:
unmap_and_release(self._python_free_callback(allocation["address"]))
@contextmanager
def cudagraph_pool(self) -> Iterator[tuple[int, int]]:
"""Tag CUDA graph capture allocations and yield the graph pool id."""
if "cudagraph" not in self.allocator_and_pools:
allocator = get_pluggable_allocator(
self.python_malloc_callback, self.python_free_callback
)
mem_pool = torch.cuda.memory.MemPool(allocator._allocator)
self.allocator_and_pools["cudagraph"] = [(mem_pool, allocator)]
old_tag = self.current_tag
self.current_tag = "cudagraph"
try:
# capture_begin routes allocations to this pool; no use_mem_pool.
yield self.allocator_and_pools["cudagraph"][0][0].id
finally:
self.current_tag = old_tag
def get_current_usage(self) -> int:
"""Get the total number of bytes allocated in the memory pool."""
sum_bytes: int = 0
+5 -2
View File
@@ -129,8 +129,11 @@ class CuMemBackend(SleepModeBackend):
self._state = "SUSPENDED"
allocator = get_mem_allocator_instance()
# Runtime state is kept at every level; weights only at level 1.
offload_tags = ("weights", "runtime") if level == 1 else ("runtime",)
# Runtime state and graph pools are kept at every level; weights only
# at level 1.
offload_tags: tuple[str, ...] = ("runtime", "cudagraph")
if level == 1:
offload_tags = ("weights", *offload_tags)
allocator.sleep(offload_tags=offload_tags)
def resume(self, tags: list[str] | None = None) -> None: