[Core] Add CuMemAllocator.discard() for tag-selective GPU memory release (#52514)

Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Dakai An
2026-08-16 23:15:14 +08:00
committed by GitHub
co-authored by AlanFokCo OpenAI Codex
parent fe1c317157
commit 9409f59e09
4 changed files with 132 additions and 0 deletions
+2
View File
@@ -27,6 +27,8 @@ class MemAllocator(Protocol):
def sleep(self, offload_tags: tuple[str, ...] | str | None = None) -> None: ...
def discard(self, tags: tuple[str, ...] | str) -> None: ...
def wake_up(self, tags: list[str] | None = None) -> None: ...
def get_current_usage(self) -> int: ...
+46
View File
@@ -247,8 +247,15 @@ class CuMemAllocator:
total_bytes = 0
backup_bytes = 0
has_policy_conflict = False
for ptr, data in self.pointer_to_data.items():
if data.is_asleep:
requests_offload = data.tag in offload_tags
was_offloaded = data.cpu_backup_tensor is not None
if requests_offload != was_offloaded:
has_policy_conflict = True
continue
handle = data.handle
total_bytes += handle[1]
if data.tag in offload_tags:
@@ -277,9 +284,46 @@ class CuMemAllocator:
(total_bytes - backup_bytes) / 1024**3,
)
if has_policy_conflict:
logger.warning(
"CuMemAllocator: sleep cannot change the policy of "
"already-asleep allocations; the existing policy was kept."
)
gc.collect()
torch.cuda.empty_cache()
def discard(self, tags: tuple[str, ...] | str) -> None:
"""Discard mapped allocations with the given tags without CPU backup."""
if isinstance(tags, str):
tags = (tags,)
discarded_bytes = 0
has_policy_conflict = False
for data in self.pointer_to_data.values():
if data.tag not in tags:
continue
if data.is_asleep:
if data.cpu_backup_tensor is not None:
has_policy_conflict = True
continue
torch.accelerator.synchronize(data.handle[0])
unmap_and_release(data.handle)
data.is_asleep = True
discarded_bytes += data.handle[1]
logger.info(
"CuMemAllocator: discarded %.2f GiB for tags %s.",
discarded_bytes / 1024**3,
tags,
)
if has_policy_conflict:
logger.warning(
"CuMemAllocator: discard cannot change the policy of "
"already-asleep allocations; the existing policy was kept."
)
def wake_up(self, tags: list[str] | None = None) -> None:
"""
Wake up the allocator from sleep mode.
@@ -295,6 +339,8 @@ class CuMemAllocator:
torch.accelerator.empty_cache()
for ptr, data in self.pointer_to_data.items():
if not data.is_asleep:
continue
if tags is None or data.tag in tags:
handle = data.handle
create_and_map(handle)
+49
View File
@@ -174,12 +174,20 @@ class XpuMemAllocator:
total_bytes = 0
backup_bytes = 0
has_policy_conflict = False
for ptr, data in self.pointer_to_data.items():
if data.is_asleep:
requests_offload = data.tag in offload_tags
was_offloaded = data.cpu_backup_tensor is not None
if requests_offload != was_offloaded:
has_policy_conflict = True
continue
size_in_bytes = data.handle[1]
total_bytes += size_in_bytes
if data.tag not in offload_tags:
unmap_and_release(data.handle)
data.is_asleep = True
continue
backup_bytes += size_in_bytes
@@ -201,6 +209,7 @@ class XpuMemAllocator:
data.cpu_backup_tensor = cpu_backup_tensor
unmap_and_release(data.handle)
data.is_asleep = True
logger.info(
"XpuMemAllocator: sleep freed %.2f GiB memory in total, of which "
@@ -211,16 +220,56 @@ class XpuMemAllocator:
(total_bytes - backup_bytes) / 1024**3,
)
if has_policy_conflict:
logger.warning(
"XpuMemAllocator: sleep cannot change the policy of "
"already-asleep allocations; the existing policy was kept."
)
gc.collect()
xpu_empty_cache = getattr(torch.xpu, "empty_cache", None)
if callable(xpu_empty_cache):
xpu_empty_cache()
def discard(self, tags: tuple[str, ...] | str) -> None:
"""Discard mapped allocations with the given tags without CPU backup."""
if isinstance(tags, str):
tags = (tags,)
discarded_bytes = 0
has_policy_conflict = False
for data in self.pointer_to_data.values():
if data.tag not in tags:
continue
if data.is_asleep:
if data.cpu_backup_tensor is not None:
has_policy_conflict = True
continue
torch.accelerator.synchronize(data.handle[0])
unmap_and_release(data.handle)
data.is_asleep = True
discarded_bytes += data.handle[1]
logger.info(
"XpuMemAllocator: discarded %.2f GiB for tags %s.",
discarded_bytes / 1024**3,
tags,
)
if has_policy_conflict:
logger.warning(
"XpuMemAllocator: discard cannot change the policy of "
"already-asleep allocations; the existing policy was kept."
)
def wake_up(self, tags: list[str] | None = None) -> None:
for ptr, data in self.pointer_to_data.items():
if not data.is_asleep:
continue
if tags is not None and data.tag not in tags:
continue
create_and_allocate(data.handle)
data.is_asleep = False
cpu_backup_tensor = data.cpu_backup_tensor
if cpu_backup_tensor is None: