[Bugfix] Use is_integrated to detect UMA GPUs for memory reporting (#35356)

Signed-off-by: haosdent <haosdent@gmail.com>
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
haosdent
2026-04-13 11:07:40 -07:00
committed by GitHub
co-authored by DarkLight1337
parent 739e5945dc
commit 995e9a209e
4 changed files with 83 additions and 16 deletions
+4
View File
@@ -548,6 +548,10 @@ class CudaPlatformBase(Platform):
"""Currently, only Hopper and Blackwell GPUs are supported."""
return cls.is_device_capability(90) or cls.is_device_capability_family(100)
@classmethod
def is_integrated_gpu(cls, device_id: int = 0) -> bool:
return bool(torch.cuda.get_device_properties(device_id).is_integrated)
@classmethod
def num_compute_units(cls, device_id: int = 0) -> int:
return torch.cuda.get_device_properties(device_id).multi_processor_count
+12
View File
@@ -734,6 +734,18 @@ class Platform:
"""
return "vllm.distributed.device_communicators.base_device_communicator.DeviceCommunicatorBase" # noqa
@classmethod
def is_integrated_gpu(cls, device_id: int = 0) -> bool:
"""
Returns whether the GPU is an integrated (UMA) device that shares
system memory with the CPU.
On UMA systems (e.g. NVIDIA GH200, DGX Spark, Jetson Orin),
cudaMemGetInfo may underreport free memory because it does not
account for reclaimable OS memory (page cache, buffers).
"""
return False
@classmethod
def supports_mx(cls) -> bool:
"""
+6 -16
View File
@@ -106,22 +106,12 @@ class MemorySnapshot:
)
self.free_memory, self.total_memory = current_platform.mem_get_info(device)
shared_sysmem_device_mem_sms = ((8, 7), (11, 0), (12, 1)) # Orin, Thor, Spark
if (
current_platform.is_cuda()
and current_platform.get_device_capability(device.index)
in shared_sysmem_device_mem_sms
):
# On UMA (Orin, Thor and Spark) platform,
# where both CPU and GPU rely on system memory,
# the cudaMemGetInfo function shows the amount of free system memory
# rather than what’s actually available.
# In the case,
# torch.cuda.mem_get_info() only reports "free" memory,
# which can be lower than what is actually
# available due to not including cache memory.
# There’s also a comprehensive reference page
# that explains how you can compute the proper value yourself.
if current_platform.is_integrated_gpu(device.index):
# On UMA (Unified Memory Architecture) platforms where CPU and
# GPU share physical memory (e.g. GH200, DGX Spark, Jetson Orin),
# cudaMemGetInfo underreports free memory because it does not
# account for reclaimable OS memory (page cache, buffers).
# Use psutil to get the true available memory.
# https://docs.nvidia.com/cuda/cuda-for-tegra-appnote/#estimating-total-allocatable-device-memory-on-an-integrated-gpu-device
self.free_memory = psutil.virtual_memory().available