Measure model memory usage (#3120)
This commit is contained in:
@@ -3,6 +3,7 @@ import os
|
||||
import socket
|
||||
import subprocess
|
||||
import uuid
|
||||
import gc
|
||||
from platform import uname
|
||||
from typing import List, Tuple, Union
|
||||
from packaging.version import parse, Version
|
||||
@@ -309,3 +310,27 @@ def create_kv_caches_with_random(
|
||||
f"Does not support value cache of type {cache_dtype}")
|
||||
value_caches.append(value_cache)
|
||||
return key_caches, value_caches
|
||||
|
||||
|
||||
class measure_cuda_memory:
|
||||
|
||||
def __init__(self, device=None):
|
||||
self.device = device
|
||||
|
||||
def current_memory_usage(self) -> float:
|
||||
# Return the memory usage in bytes.
|
||||
torch.cuda.reset_peak_memory_stats(self.device)
|
||||
mem = torch.cuda.max_memory_allocated(self.device)
|
||||
return mem
|
||||
|
||||
def __enter__(self):
|
||||
self.initial_memory = self.current_memory_usage()
|
||||
# This allows us to call methods of the context manager if needed
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.final_memory = self.current_memory_usage()
|
||||
self.consumed_memory = self.final_memory - self.initial_memory
|
||||
|
||||
# Force garbage collection
|
||||
gc.collect()
|
||||
|
||||
Reference in New Issue
Block a user