[Chore] Factor out logic for requesting initial memory (#30868)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
+27
-4
@@ -66,27 +66,43 @@ class MemorySnapshot:
|
||||
torch_memory: int = 0
|
||||
non_torch_memory: int = 0
|
||||
timestamp: float = 0.0
|
||||
|
||||
device: torch.types.Device = None
|
||||
auto_measure: bool = True
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.device is None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
device_fn = current_platform.current_device
|
||||
assert device_fn is not None
|
||||
self.device_ = torch.device(device_fn())
|
||||
else:
|
||||
self.device_ = torch.device(self.device)
|
||||
|
||||
if self.auto_measure:
|
||||
self.measure()
|
||||
|
||||
def measure(self) -> None:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
device = self.device_
|
||||
|
||||
# we measure the torch peak memory usage via allocated_bytes,
|
||||
# rather than `torch.cuda.memory_reserved()` .
|
||||
# After `torch.cuda.reset_peak_memory_stats()`,
|
||||
# `torch.cuda.memory_reserved()` will keep growing, and only shrink
|
||||
# when we call `torch.cuda.empty_cache()` or OOM happens.
|
||||
self.torch_peak = torch.cuda.memory_stats().get("allocated_bytes.all.peak", 0)
|
||||
self.torch_peak = torch.cuda.memory_stats(device).get(
|
||||
"allocated_bytes.all.peak", 0
|
||||
)
|
||||
|
||||
self.free_memory, self.total_memory = torch.cuda.mem_get_info()
|
||||
self.free_memory, self.total_memory = torch.cuda.mem_get_info(device)
|
||||
shared_sysmem_device_mem_sms = ((8, 7), (11, 0), (12, 1)) # Orin, Thor, Spark
|
||||
if (
|
||||
current_platform.is_cuda()
|
||||
and current_platform.get_device_capability() in shared_sysmem_device_mem_sms
|
||||
and current_platform.get_device_capability(device.index)
|
||||
in shared_sysmem_device_mem_sms
|
||||
):
|
||||
# On UMA (Orin, Thor and Spark) platform,
|
||||
# where both CPU and GPU rely on system memory,
|
||||
@@ -106,12 +122,18 @@ class MemorySnapshot:
|
||||
# torch.cuda.memory_reserved() is how many bytes
|
||||
# PyTorch gets from cuda (by calling cudaMalloc, etc.)
|
||||
# this is used to measure the non-torch memory usage
|
||||
self.torch_memory = torch.cuda.memory_reserved()
|
||||
self.torch_memory = torch.cuda.memory_reserved(device)
|
||||
|
||||
self.non_torch_memory = self.cuda_memory - self.torch_memory
|
||||
self.timestamp = time.time()
|
||||
|
||||
def __sub__(self, other: "MemorySnapshot") -> "MemorySnapshot":
|
||||
if self.device_ != other.device_:
|
||||
raise ValueError(
|
||||
"The two snapshots should be from the same device! "
|
||||
f"Found: {self.device_} vs. {other.device_}"
|
||||
)
|
||||
|
||||
return MemorySnapshot(
|
||||
torch_peak=self.torch_peak - other.torch_peak,
|
||||
free_memory=self.free_memory - other.free_memory,
|
||||
@@ -120,6 +142,7 @@ class MemorySnapshot:
|
||||
torch_memory=self.torch_memory - other.torch_memory,
|
||||
non_torch_memory=self.non_torch_memory - other.non_torch_memory,
|
||||
timestamp=self.timestamp - other.timestamp,
|
||||
device=self.device_,
|
||||
auto_measure=False,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user