Measure model memory usage (#3120)

This commit is contained in:
Michael Goin
2024-03-07 11:42:42 -08:00
committed by GitHub
parent 2daf23ab0c
commit 385da2dae2
2 changed files with 37 additions and 6 deletions
+25
View File
@@ -3,6 +3,7 @@ import os
import socket
import subprocess
import uuid
import gc
from platform import uname
from typing import List, Tuple, Union
from packaging.version import parse, Version
@@ -309,3 +310,27 @@ def create_kv_caches_with_random(
f"Does not support value cache of type {cache_dtype}")
value_caches.append(value_cache)
return key_caches, value_caches
class measure_cuda_memory:
def __init__(self, device=None):
self.device = device
def current_memory_usage(self) -> float:
# Return the memory usage in bytes.
torch.cuda.reset_peak_memory_stats(self.device)
mem = torch.cuda.max_memory_allocated(self.device)
return mem
def __enter__(self):
self.initial_memory = self.current_memory_usage()
# This allows us to call methods of the context manager if needed
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.final_memory = self.current_memory_usage()
self.consumed_memory = self.final_memory - self.initial_memory
# Force garbage collection
gc.collect()