From 5edf7ff489e83616c00c64d3ac6562f81dbe5638 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 11 Jun 2026 15:49:50 +0100 Subject: [PATCH] [Core] Release cached device memory under pressure on UMA GPUs during weight loading (#45179) Signed-off-by: mgoin Co-authored-by: Claude --- vllm/model_executor/model_loader/utils.py | 4 +++ vllm/utils/mem_utils.py | 38 +++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/vllm/model_executor/model_loader/utils.py b/vllm/model_executor/model_loader/utils.py index 2a5f746d78..fc279c7e9c 100644 --- a/vllm/model_executor/model_loader/utils.py +++ b/vllm/model_executor/model_loader/utils.py @@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.reload import ( ) from vllm.model_executor.models.interfaces import SupportsQuant from vllm.tracing import instrument +from vllm.utils.mem_utils import release_device_memory_under_pressure from vllm.utils.platform_utils import is_pin_memory_available from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor @@ -109,6 +110,9 @@ def process_weights_after_loading( # parameters onto device for processing and back off after. with device_loading_context(module, target_device): quant_method.process_weights_after_loading(module) + # Repacking transients above can leave large amounts of memory in + # the caching allocator, which starves the OS on UMA devices. + release_device_memory_under_pressure(target_device) # Initialize post-load attention weights for Attention, MLA, and MM encoder. # NOTE: Happens after other modules so we can easily decompress weights. diff --git a/vllm/utils/mem_utils.py b/vllm/utils/mem_utils.py index 4efb29975a..3894742c6b 100644 --- a/vllm/utils/mem_utils.py +++ b/vllm/utils/mem_utils.py @@ -11,10 +11,13 @@ import psutil import torch import torch.types +from vllm.logger import init_logger from vllm.platforms import current_platform from .mem_constants import GiB_bytes, KiB_bytes, MiB_bytes +logger = init_logger(__name__) + def format_kib(b: int) -> str: return f"{round(b / KiB_bytes, 2)}" @@ -45,6 +48,41 @@ def get_cpu_memory() -> int: return psutil.virtual_memory().total +_UMA_PRESSURE_THRESHOLD = 0.8 +_UMA_MIN_RELEASE_BYTES = 512 * MiB_bytes + + +def release_device_memory_under_pressure(device: torch.device) -> bool: + """On integrated (UMA) GPUs, release caching-allocator memory back to the + OS when system memory pressure is high. The OS may start thrashing before + an allocation failure would trigger PyTorch's own cache release. + + Returns: + True if memory was released. + """ + if device.type != "cuda" or not current_platform.is_integrated_gpu(device.index): + return False + + releasable = torch.accelerator.memory_reserved( + device + ) - torch.accelerator.memory_allocated(device) + if releasable < _UMA_MIN_RELEASE_BYTES: + return False + + # cudaMemGetInfo underreports free memory on UMA, see MemorySnapshot.measure + mem = psutil.virtual_memory() + if mem.available > (1 - _UMA_PRESSURE_THRESHOLD) * mem.total: + return False + + torch.accelerator.synchronize(device) + torch.accelerator.empty_cache() + logger.debug( + "Released %sGiB of cached device memory under memory pressure", + format_gib(releasable), + ) + return True + + class DeviceMemoryProfiler: def __init__(self, device: torch.types.Device | None = None): self.device = device