[Chore] Factor out logic for requesting initial memory (#30868)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-12-17 07:32:17 -08:00
committed by GitHub
parent 196cdc3224
commit 2497228ad4
3 changed files with 56 additions and 21 deletions
+27 -4
View File
@@ -66,27 +66,43 @@ class MemorySnapshot:
torch_memory: int = 0
non_torch_memory: int = 0
timestamp: float = 0.0
device: torch.types.Device = None
auto_measure: bool = True
def __post_init__(self) -> None:
if self.device is None:
from vllm.platforms import current_platform
device_fn = current_platform.current_device
assert device_fn is not None
self.device_ = torch.device(device_fn())
else:
self.device_ = torch.device(self.device)
if self.auto_measure:
self.measure()
def measure(self) -> None:
from vllm.platforms import current_platform
device = self.device_
# we measure the torch peak memory usage via allocated_bytes,
# rather than `torch.cuda.memory_reserved()` .
# After `torch.cuda.reset_peak_memory_stats()`,
# `torch.cuda.memory_reserved()` will keep growing, and only shrink
# when we call `torch.cuda.empty_cache()` or OOM happens.
self.torch_peak = torch.cuda.memory_stats().get("allocated_bytes.all.peak", 0)
self.torch_peak = torch.cuda.memory_stats(device).get(
"allocated_bytes.all.peak", 0
)
self.free_memory, self.total_memory = torch.cuda.mem_get_info()
self.free_memory, self.total_memory = torch.cuda.mem_get_info(device)
shared_sysmem_device_mem_sms = ((8, 7), (11, 0), (12, 1)) # Orin, Thor, Spark
if (
current_platform.is_cuda()
and current_platform.get_device_capability() in shared_sysmem_device_mem_sms
and current_platform.get_device_capability(device.index)
in shared_sysmem_device_mem_sms
):
# On UMA (Orin, Thor and Spark) platform,
# where both CPU and GPU rely on system memory,
@@ -106,12 +122,18 @@ class MemorySnapshot:
# torch.cuda.memory_reserved() is how many bytes
# PyTorch gets from cuda (by calling cudaMalloc, etc.)
# this is used to measure the non-torch memory usage
self.torch_memory = torch.cuda.memory_reserved()
self.torch_memory = torch.cuda.memory_reserved(device)
self.non_torch_memory = self.cuda_memory - self.torch_memory
self.timestamp = time.time()
def __sub__(self, other: "MemorySnapshot") -> "MemorySnapshot":
if self.device_ != other.device_:
raise ValueError(
"The two snapshots should be from the same device! "
f"Found: {self.device_} vs. {other.device_}"
)
return MemorySnapshot(
torch_peak=self.torch_peak - other.torch_peak,
free_memory=self.free_memory - other.free_memory,
@@ -120,6 +142,7 @@ class MemorySnapshot:
torch_memory=self.torch_memory - other.torch_memory,
non_torch_memory=self.non_torch_memory - other.non_torch_memory,
timestamp=self.timestamp - other.timestamp,
device=self.device_,
auto_measure=False,
)