[Feature] Release WorkspaceManager scratch on sleep (#59156)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e5d33a36e7
commit
866f02db8a
@@ -13,6 +13,9 @@ from vllm.platforms import current_platform
|
||||
# py_handle has type list[int] on ROCm and int otherwise
|
||||
HandleType: TypeAlias = tuple[int, int, int, list[int] | int]
|
||||
|
||||
# Selective wake may defer only these; any wake restores every other tag.
|
||||
DEFERRABLE_TAGS: tuple[str, ...] = ("weights", "kv_cache")
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class AllocationData:
|
||||
|
||||
@@ -16,7 +16,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
|
||||
from vllm.device_allocator.alloc_conf import (
|
||||
EXPANDABLE_SEGMENTS,
|
||||
conf_flag_enabled,
|
||||
@@ -139,7 +139,7 @@ class CuMemAllocator:
|
||||
def __init__(self):
|
||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||
self.current_tag: str = CuMemAllocator.default_tag
|
||||
self.allocator_and_pools: dict[str, Any] = {}
|
||||
self.allocator_and_pools: dict[str, list[Any]] = {}
|
||||
# Creating strong references to the two callbacks here to prevent
|
||||
# these ephemeral bound-method objects being garbage collected.
|
||||
# See discussions in https://github.com/vllm-project/vllm/pull/22724
|
||||
@@ -167,7 +167,7 @@ class CuMemAllocator:
|
||||
if not self.allocator_and_pools:
|
||||
return
|
||||
|
||||
pool_entries = list(self.allocator_and_pools.values())
|
||||
pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
|
||||
self.allocator_and_pools.clear()
|
||||
|
||||
mem_pools = [entry[0] for entry in pool_entries]
|
||||
@@ -334,9 +334,9 @@ class CuMemAllocator:
|
||||
memory, and the rest of the data will have empty memory.
|
||||
|
||||
Args:
|
||||
tags: The tags of the memory allocation that will be loaded
|
||||
back to GPU memory. If None, all memory allocation will be loaded
|
||||
back to GPU memory.
|
||||
tags: The deferrable tags (weights, kv_cache) to load back to GPU
|
||||
memory; every other tag is always loaded back. If None, all
|
||||
memory allocation will be loaded back to GPU memory.
|
||||
|
||||
"""
|
||||
gc.collect()
|
||||
@@ -345,7 +345,7 @@ class CuMemAllocator:
|
||||
for ptr, data in self.pointer_to_data.items():
|
||||
if not data.is_asleep:
|
||||
continue
|
||||
if tags is None or data.tag in tags:
|
||||
if tags is None or data.tag in tags or data.tag not in DEFERRABLE_TAGS:
|
||||
handle = data.handle
|
||||
create_and_map(handle)
|
||||
data.is_asleep = False
|
||||
@@ -392,6 +392,10 @@ class CuMemAllocator:
|
||||
old_tag = self.current_tag
|
||||
self.current_tag = tag
|
||||
try:
|
||||
if tag != old_tag:
|
||||
# Older pools of this tag are never reused, so trim them.
|
||||
for pool, _ in self.allocator_and_pools.get(tag, []):
|
||||
self._trim(pool)
|
||||
with use_memory_pool_with_allocator(
|
||||
self.python_malloc_callback, self.python_free_callback
|
||||
) as data:
|
||||
@@ -400,7 +404,7 @@ class CuMemAllocator:
|
||||
# and the memory pool.
|
||||
# to avoid the issue, we keep a reference of the data.
|
||||
# see https://github.com/pytorch/pytorch/issues/146431 .
|
||||
self.allocator_and_pools[tag] = data
|
||||
self.allocator_and_pools.setdefault(tag, []).append(data)
|
||||
yield
|
||||
# PyTorch's bug, calling torch.cuda.empty_cache() will error
|
||||
# when using pluggable allocator, see
|
||||
@@ -413,16 +417,19 @@ class CuMemAllocator:
|
||||
# TODO: we should expose `empty_cache` method in the memory
|
||||
# pool.
|
||||
# TODO: ask for help from PyTorch team to expose this method.
|
||||
allocations = data[0].snapshot()
|
||||
for allocation in allocations:
|
||||
if allocation["allocated_size"] == 0:
|
||||
handle = self._python_free_callback(allocation["address"])
|
||||
unmap_and_release(handle)
|
||||
self._trim(data[0])
|
||||
finally:
|
||||
self.current_tag = old_tag
|
||||
if expandable_was_enabled:
|
||||
set_alloc_conf(prev_conf)
|
||||
|
||||
def _trim(self, pool: Any) -> None:
|
||||
"""Release free segments of a pool that is never allocated from again."""
|
||||
for allocation in pool.snapshot():
|
||||
data = self.pointer_to_data.get(allocation["address"])
|
||||
if allocation["allocated_size"] == 0 and data and not data.is_asleep:
|
||||
unmap_and_release(self._python_free_callback(allocation["address"]))
|
||||
|
||||
def get_current_usage(self) -> int:
|
||||
"""Get the total number of bytes allocated in the memory pool."""
|
||||
sum_bytes: int = 0
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
@@ -145,7 +145,7 @@ class XpuMemAllocator:
|
||||
def __init__(self):
|
||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||
self.current_tag: str = XpuMemAllocator.default_tag
|
||||
self.allocator_and_pools: dict[str, Any] = {}
|
||||
self.allocator_and_pools: dict[str, list[Any]] = {}
|
||||
self.python_malloc_callback = self._python_malloc_callback
|
||||
self.python_free_callback = self._python_free_callback
|
||||
|
||||
@@ -267,7 +267,11 @@ class XpuMemAllocator:
|
||||
for ptr, data in self.pointer_to_data.items():
|
||||
if not data.is_asleep:
|
||||
continue
|
||||
if tags is not None and data.tag not in tags:
|
||||
if (
|
||||
tags is not None
|
||||
and data.tag not in tags
|
||||
and data.tag in DEFERRABLE_TAGS
|
||||
):
|
||||
continue
|
||||
create_and_allocate(data.handle)
|
||||
data.is_asleep = False
|
||||
@@ -300,7 +304,7 @@ class XpuMemAllocator:
|
||||
# MemPool teardown may invoke allocator virtual methods (e.g. raw_delete)
|
||||
# when releasing cached blocks. If allocator wrappers are dropped first,
|
||||
# C++ can hit "pure virtual method called" during shutdown.
|
||||
pool_entries = list(self.allocator_and_pools.values())
|
||||
pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
|
||||
self.allocator_and_pools.clear()
|
||||
|
||||
mem_pools = [entry[0] for entry in pool_entries]
|
||||
@@ -333,7 +337,8 @@ class XpuMemAllocator:
|
||||
self.python_malloc_callback,
|
||||
self.python_free_callback,
|
||||
) as data:
|
||||
self.allocator_and_pools[tag] = data
|
||||
# Keep every entry's pool alive (pytorch/pytorch#146431).
|
||||
self.allocator_and_pools.setdefault(tag, []).append(data)
|
||||
yield
|
||||
finally:
|
||||
self.current_tag = old_tag
|
||||
|
||||
Reference in New Issue
Block a user