[Feature] Release WorkspaceManager scratch on sleep (#59156)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
e5d33a36e7
commit
866f02db8a
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.device_allocator import AllocationData, HandleType
|
||||
from vllm.device_allocator import DEFERRABLE_TAGS, AllocationData, HandleType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
@@ -145,7 +145,7 @@ class XpuMemAllocator:
|
||||
def __init__(self):
|
||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||
self.current_tag: str = XpuMemAllocator.default_tag
|
||||
self.allocator_and_pools: dict[str, Any] = {}
|
||||
self.allocator_and_pools: dict[str, list[Any]] = {}
|
||||
self.python_malloc_callback = self._python_malloc_callback
|
||||
self.python_free_callback = self._python_free_callback
|
||||
|
||||
@@ -267,7 +267,11 @@ class XpuMemAllocator:
|
||||
for ptr, data in self.pointer_to_data.items():
|
||||
if not data.is_asleep:
|
||||
continue
|
||||
if tags is not None and data.tag not in tags:
|
||||
if (
|
||||
tags is not None
|
||||
and data.tag not in tags
|
||||
and data.tag in DEFERRABLE_TAGS
|
||||
):
|
||||
continue
|
||||
create_and_allocate(data.handle)
|
||||
data.is_asleep = False
|
||||
@@ -300,7 +304,7 @@ class XpuMemAllocator:
|
||||
# MemPool teardown may invoke allocator virtual methods (e.g. raw_delete)
|
||||
# when releasing cached blocks. If allocator wrappers are dropped first,
|
||||
# C++ can hit "pure virtual method called" during shutdown.
|
||||
pool_entries = list(self.allocator_and_pools.values())
|
||||
pool_entries = [e for es in self.allocator_and_pools.values() for e in es]
|
||||
self.allocator_and_pools.clear()
|
||||
|
||||
mem_pools = [entry[0] for entry in pool_entries]
|
||||
@@ -333,7 +337,8 @@ class XpuMemAllocator:
|
||||
self.python_malloc_callback,
|
||||
self.python_free_callback,
|
||||
) as data:
|
||||
self.allocator_and_pools[tag] = data
|
||||
# Keep every entry's pool alive (pytorch/pytorch#146431).
|
||||
self.allocator_and_pools.setdefault(tag, []).append(data)
|
||||
yield
|
||||
finally:
|
||||
self.current_tag = old_tag
|
||||
|
||||
Reference in New Issue
Block a user