Auto-disable expandable_segments around cumem memory pool (#40812)
Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
4d51588e23
commit
2ce95a761b
@@ -128,13 +128,6 @@ class CuMemAllocator:
|
|||||||
return CuMemAllocator.instance
|
return CuMemAllocator.instance
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
|
|
||||||
assert "expandable_segments:True" not in conf, (
|
|
||||||
"Expandable segments are not compatible with memory pool. "
|
|
||||||
"Please track https://github.com/pytorch/pytorch/issues/147851 "
|
|
||||||
"for the latest updates."
|
|
||||||
)
|
|
||||||
|
|
||||||
self.pointer_to_data: dict[int, AllocationData] = {}
|
self.pointer_to_data: dict[int, AllocationData] = {}
|
||||||
self.current_tag: str = CuMemAllocator.default_tag
|
self.current_tag: str = CuMemAllocator.default_tag
|
||||||
self.allocator_and_pools: dict[str, Any] = {}
|
self.allocator_and_pools: dict[str, Any] = {}
|
||||||
@@ -264,34 +257,49 @@ class CuMemAllocator:
|
|||||||
|
|
||||||
assert isinstance(tag, str)
|
assert isinstance(tag, str)
|
||||||
|
|
||||||
|
# Expandable segments are incompatible with the memory pool used for
|
||||||
|
# sleep mode (see https://github.com/pytorch/pytorch/issues/147851).
|
||||||
|
# If the user has enabled expandable segments via
|
||||||
|
# PYTORCH_CUDA_ALLOC_CONF, temporarily disable them for the duration
|
||||||
|
# of the memory pool context and restore on exit.
|
||||||
|
conf = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
|
||||||
|
expandable_was_enabled = "expandable_segments:True" in conf
|
||||||
|
if expandable_was_enabled:
|
||||||
|
torch.cuda.memory._set_allocator_settings("expandable_segments:False")
|
||||||
|
|
||||||
old_tag = self.current_tag
|
old_tag = self.current_tag
|
||||||
self.current_tag = tag
|
self.current_tag = tag
|
||||||
with use_memory_pool_with_allocator(
|
try:
|
||||||
self.python_malloc_callback, self.python_free_callback
|
with use_memory_pool_with_allocator(
|
||||||
) as data:
|
self.python_malloc_callback, self.python_free_callback
|
||||||
# start to hit another PyTorch bug in PyTorch 2.6,
|
) as data:
|
||||||
# possibly because of gc-related issue w.r.t. the allocator and
|
# start to hit another PyTorch bug in PyTorch 2.6,
|
||||||
# the memory pool.
|
# possibly because of gc-related issue w.r.t. the allocator
|
||||||
# to avoid the issue, we keep a reference of the data.
|
# and the memory pool.
|
||||||
# see https://github.com/pytorch/pytorch/issues/146431 .
|
# to avoid the issue, we keep a reference of the data.
|
||||||
self.allocator_and_pools[tag] = data
|
# see https://github.com/pytorch/pytorch/issues/146431 .
|
||||||
yield
|
self.allocator_and_pools[tag] = data
|
||||||
# PyTorch's bug, calling torch.cuda.empty_cache() will error
|
yield
|
||||||
# when using pluggable allocator, see
|
# PyTorch's bug, calling torch.cuda.empty_cache() will error
|
||||||
# https://github.com/pytorch/pytorch/issues/145168 .
|
# when using pluggable allocator, see
|
||||||
# if we have some memory allocated and then freed,
|
# https://github.com/pytorch/pytorch/issues/145168 .
|
||||||
# the memory will not be released, e.g. in online quantization,
|
# if we have some memory allocated and then freed,
|
||||||
# where the model is created in higher precision, and then
|
# the memory will not be released, e.g. in online
|
||||||
# quantized in lower precision.
|
# quantization, where the model is created in higher
|
||||||
# Find all unused allocations and manually release them.
|
# precision, and then quantized in lower precision.
|
||||||
# TODO: we should expose `empty_cache` method in the memory pool.
|
# Find all unused allocations and manually release them.
|
||||||
# TODO: ask for help from PyTorch team to expose this method.
|
# TODO: we should expose `empty_cache` method in the memory
|
||||||
allocations = data[0].snapshot()
|
# pool.
|
||||||
for allocation in allocations:
|
# TODO: ask for help from PyTorch team to expose this method.
|
||||||
if allocation["allocated_size"] == 0:
|
allocations = data[0].snapshot()
|
||||||
handle = self._python_free_callback(allocation["address"])
|
for allocation in allocations:
|
||||||
unmap_and_release(handle)
|
if allocation["allocated_size"] == 0:
|
||||||
|
handle = self._python_free_callback(allocation["address"])
|
||||||
|
unmap_and_release(handle)
|
||||||
|
finally:
|
||||||
self.current_tag = old_tag
|
self.current_tag = old_tag
|
||||||
|
if expandable_was_enabled:
|
||||||
|
torch.cuda.memory._set_allocator_settings("expandable_segments:True")
|
||||||
|
|
||||||
def get_current_usage(self) -> int:
|
def get_current_usage(self) -> int:
|
||||||
"""
|
"""
|
||||||
|
|||||||
Reference in New Issue
Block a user