[Feature] Release WorkspaceManager scratch on sleep (#59156)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-02 02:14:37 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent e5d33a36e7
commit 866f02db8a
6 changed files with 140 additions and 23 deletions
+3
View File
@@ -13,6 +13,9 @@ from vllm.platforms import current_platform
# py_handle has type list[int] on ROCm and int otherwise
HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
# Selective wake may defer only these; any wake restores every other tag.
DEFERRABLE_TAGS: tuple[str, ...] = ("weights", "kv_cache")
@dataclasses.dataclass
class AllocationData:
+20 -13
View File
@@ -16,7 +16,7 @@ from typing import Any
import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
from vllm.device_allocator.alloc_conf import (
EXPANDABLE_SEGMENTS,
conf_flag_enabled,
@@ -139,7 +139,7 @@ class CuMemAllocator:
def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = CuMemAllocator.default_tag
self.allocator_and_pools: dict[str, Any] = {}
self.allocator_and_pools: dict[str, list[Any]] = {}
# Creating strong references to the two callbacks here to prevent
# these ephemeral bound-method objects being garbage collected.
# See discussions in https://github.com/vllm-project/vllm/pull/22724
@@ -167,7 +167,7 @@ class CuMemAllocator:
if not self.allocator_and_pools:
return
pool_entries = list(self.allocator_and_pools.values())
pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries]
@@ -334,9 +334,9 @@ class CuMemAllocator:
memory, and the rest of the data will have empty memory.
Args:
tags: The tags of the memory allocation that will be loaded
back to GPU memory. If None, all memory allocation will be loaded
back to GPU memory.
tags: The deferrable tags (weights, kv_cache) to load back to GPU
memory; every other tag is always loaded back. If None, all
memory allocation will be loaded back to GPU memory.
"""
gc.collect()
@@ -345,7 +345,7 @@ class CuMemAllocator:
for ptr, data in self.pointer_to_data.items():
if not data.is_asleep:
continue
if tags is None or data.tag in tags:
if tags is None or data.tag in tags or data.tag not in DEFERRABLE_TAGS:
handle = data.handle
create_and_map(handle)
data.is_asleep = False
@@ -392,6 +392,10 @@ class CuMemAllocator:
old_tag = self.current_tag
self.current_tag = tag
try:
if tag != old_tag:
# Older pools of this tag are never reused, so trim them.
for pool, _ in self.allocator_and_pools.get(tag, []):
self._trim(pool)
with use_memory_pool_with_allocator(
self.python_malloc_callback, self.python_free_callback
) as data:
@@ -400,7 +404,7 @@ class CuMemAllocator:
# and the memory pool.
# to avoid the issue, we keep a reference of the data.
# see https://github.com/pytorch/pytorch/issues/146431 .
self.allocator_and_pools[tag] = data
self.allocator_and_pools.setdefault(tag, []).append(data)
yield
# PyTorch's bug, calling torch.cuda.empty_cache() will error
# when using pluggable allocator, see
@@ -413,16 +417,19 @@ class CuMemAllocator:
# TODO: we should expose `empty_cache` method in the memory
# pool.
# TODO: ask for help from PyTorch team to expose this method.
allocations = data[0].snapshot()
for allocation in allocations:
if allocation["allocated_size"] == 0:
handle = self._python_free_callback(allocation["address"])
unmap_and_release(handle)
self._trim(data[0])
finally:
self.current_tag = old_tag
if expandable_was_enabled:
set_alloc_conf(prev_conf)
def _trim(self, pool: Any) -> None:
"""Release free segments of a pool that is never allocated from again."""
for allocation in pool.snapshot():
data = self.pointer_to_data.get(allocation["address"])
if allocation["allocated_size"] == 0 and data and not data.is_asleep:
unmap_and_release(self._python_free_callback(allocation["address"]))
def get_current_usage(self) -> int:
"""Get the total number of bytes allocated in the memory pool."""
sum_bytes: int = 0
+10 -5
View File
@@ -9,7 +9,7 @@ from typing import Any
import torch
from vllm.device_allocator import AllocationData, HandleType
from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
from vllm.logger import init_logger
from vllm.utils.torch_utils import PIN_MEMORY
@@ -145,7 +145,7 @@ class XpuMemAllocator:
def __init__(self):
self.pointer_to_data: dict[int, AllocationData] = {}
self.current_tag: str = XpuMemAllocator.default_tag
self.allocator_and_pools: dict[str, Any] = {}
self.allocator_and_pools: dict[str, list[Any]] = {}
self.python_malloc_callback = self._python_malloc_callback
self.python_free_callback = self._python_free_callback
@@ -267,7 +267,11 @@ class XpuMemAllocator:
for ptr, data in self.pointer_to_data.items():
if not data.is_asleep:
continue
if tags is not None and data.tag not in tags:
if (
tags is not None
and data.tag not in tags
and data.tag in DEFERRABLE_TAGS
):
continue
create_and_allocate(data.handle)
data.is_asleep = False
@@ -300,7 +304,7 @@ class XpuMemAllocator:
# MemPool teardown may invoke allocator virtual methods (e.g. raw_delete)
# when releasing cached blocks. If allocator wrappers are dropped first,
# C++ can hit "pure virtual method called" during shutdown.
pool_entries = list(self.allocator_and_pools.values())
pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
self.allocator_and_pools.clear()
mem_pools = [entry[0] for entry in pool_entries]
@@ -333,7 +337,8 @@ class XpuMemAllocator:
self.python_malloc_callback,
self.python_free_callback,
) as data:
self.allocator_and_pools[tag] = data
# Keep every entry's pool alive (pytorch/pytorch#146431).
self.allocator_and_pools.setdefault(tag, []).append(data)
yield
finally:
self.current_tag = old_tag