Signed-off-by: haosdent <haosdent@gmail.com> Signed-off-by: Justin Wood <justin.m.wood@me.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: haosdent <haosdent@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Justin Wood <jwood@me.com>
This commit is contained in:
co-authored by
haosdent
Claude Opus 4.7
Justin Wood
parent
b7c20d0cfa
commit
58b2012aa2
+22
-9
@@ -110,6 +110,7 @@ class MemorySnapshot:
|
||||
"""Memory snapshot."""
|
||||
|
||||
torch_peak: int = 0
|
||||
torch_allocated: int = 0
|
||||
free_memory: int = 0
|
||||
total_memory: int = 0
|
||||
cuda_memory: int = 0
|
||||
@@ -139,9 +140,9 @@ class MemorySnapshot:
|
||||
# After `torch.accelerator.reset_peak_memory_stats()`,
|
||||
# `torch.accelerator.memory_reserved()` will keep growing, and only shrink
|
||||
# when we call `torch.accelerator.empty_cache()` or OOM happens.
|
||||
self.torch_peak = torch.accelerator.memory_stats(device).get(
|
||||
"allocated_bytes.all.peak", 0
|
||||
)
|
||||
stats = torch.accelerator.memory_stats(device)
|
||||
self.torch_peak = stats.get("allocated_bytes.all.peak", 0)
|
||||
self.torch_allocated = stats.get("allocated_bytes.all.current", 0)
|
||||
|
||||
self.free_memory, self.total_memory = torch.accelerator.get_memory_info(device)
|
||||
if current_platform.is_integrated_gpu(device.index):
|
||||
@@ -172,6 +173,7 @@ class MemorySnapshot:
|
||||
|
||||
return MemorySnapshot(
|
||||
torch_peak=self.torch_peak - other.torch_peak,
|
||||
torch_allocated=self.torch_allocated - other.torch_allocated,
|
||||
free_memory=self.free_memory - other.free_memory,
|
||||
total_memory=self.total_memory - other.total_memory,
|
||||
cuda_memory=self.cuda_memory - other.cuda_memory,
|
||||
@@ -185,6 +187,7 @@ class MemorySnapshot:
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"torch_peak={format_gib(self.torch_peak)}GiB, "
|
||||
f"torch_allocated={format_gib(self.torch_allocated)}GiB, "
|
||||
f"free_memory={format_gib(self.free_memory)}GiB, "
|
||||
f"total_memory={format_gib(self.total_memory)}GiB, "
|
||||
f"{current_platform.device_name}_memory={format_gib(self.cuda_memory)}GiB, "
|
||||
@@ -202,6 +205,8 @@ class MemoryProfilingResult:
|
||||
non_kv_cache_memory: int = 0
|
||||
torch_peak_increase: int = 0
|
||||
non_torch_increase: int = 0
|
||||
total_consumed: int = 0
|
||||
transient_peak_headroom: int = 0
|
||||
weights_memory: int = 0
|
||||
before_create: MemorySnapshot = field(default_factory=MemorySnapshot)
|
||||
profile_time: float = 0.0
|
||||
@@ -219,8 +224,8 @@ class MemoryProfilingResult:
|
||||
f"{format_gib(self.non_kv_cache_memory)}GiB; "
|
||||
f"torch peak memory increase: "
|
||||
f"{format_gib(self.torch_peak_increase)}GiB; "
|
||||
f"non-torch forward increase memory: "
|
||||
f"{format_gib(self.non_torch_increase)}GiB; "
|
||||
f"total consumed (from mem_get_info): "
|
||||
f"{format_gib(self.total_consumed)}GiB; "
|
||||
f"weights memory: {format_gib(self.weights_memory)}GiB."
|
||||
)
|
||||
|
||||
@@ -306,8 +311,16 @@ def memory_profiling(
|
||||
result.non_torch_increase = diff_from_create.non_torch_memory
|
||||
result.profile_time = diff_profile.timestamp
|
||||
|
||||
non_torch_memory = result.non_torch_increase
|
||||
peak_activation_memory = result.torch_peak_increase
|
||||
result.non_kv_cache_memory = (
|
||||
non_torch_memory + peak_activation_memory + result.weights_memory
|
||||
# Measure total consumption via mem_get_info() instead of
|
||||
# memory_reserved(), which goes negative when pluggable allocators
|
||||
# (e.g. cumem) bypass PyTorch's tracking.
|
||||
result.total_consumed = (
|
||||
result.before_create.free_memory - result.after_profile.free_memory
|
||||
)
|
||||
|
||||
# total_consumed already covers persistent torch allocations; add only the
|
||||
# transient peak headroom to avoid double-counting.
|
||||
result.transient_peak_headroom = (
|
||||
result.after_profile.torch_peak - result.after_profile.torch_allocated
|
||||
)
|
||||
result.non_kv_cache_memory = result.total_consumed + result.transient_peak_headroom
|
||||
|
||||
Reference in New Issue
Block a user