[Core] Release cached device memory under pressure on UMA GPUs during weight loading (#45179)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Michael Goin
2026-06-11 17:49:50 +03:00
committed by GitHub
co-authored by Claude
parent b78fc47f05
commit 5edf7ff489
2 changed files with 42 additions and 0 deletions
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.reload import (
)
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.tracing import instrument
from vllm.utils.mem_utils import release_device_memory_under_pressure
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
@@ -109,6 +110,9 @@ def process_weights_after_loading(
# parameters onto device for processing and back off after.
with device_loading_context(module, target_device):
quant_method.process_weights_after_loading(module)
# Repacking transients above can leave large amounts of memory in
# the caching allocator, which starves the OS on UMA devices.
release_device_memory_under_pressure(target_device)
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
# NOTE: Happens after other modules so we can easily decompress weights.
+38
View File
@@ -11,10 +11,13 @@ import psutil
import torch
import torch.types
from vllm.logger import init_logger
from vllm.platforms import current_platform
from .mem_constants import GiB_bytes, KiB_bytes, MiB_bytes
logger = init_logger(__name__)
def format_kib(b: int) -> str:
return f"{round(b / KiB_bytes, 2)}"
@@ -45,6 +48,41 @@ def get_cpu_memory() -> int:
return psutil.virtual_memory().total
_UMA_PRESSURE_THRESHOLD = 0.8
_UMA_MIN_RELEASE_BYTES = 512 * MiB_bytes
def release_device_memory_under_pressure(device: torch.device) -> bool:
"""On integrated (UMA) GPUs, release caching-allocator memory back to the
OS when system memory pressure is high. The OS may start thrashing before
an allocation failure would trigger PyTorch's own cache release.
Returns:
True if memory was released.
"""
if device.type != "cuda" or not current_platform.is_integrated_gpu(device.index):
return False
releasable = torch.accelerator.memory_reserved(
device
) - torch.accelerator.memory_allocated(device)
if releasable < _UMA_MIN_RELEASE_BYTES:
return False
# cudaMemGetInfo underreports free memory on UMA, see MemorySnapshot.measure
mem = psutil.virtual_memory()
if mem.available > (1 - _UMA_PRESSURE_THRESHOLD) * mem.total:
return False
torch.accelerator.synchronize(device)
torch.accelerator.empty_cache()
logger.debug(
"Released %sGiB of cached device memory under memory pressure",
format_gib(releasable),
)
return True
class DeviceMemoryProfiler:
def __init__(self, device: torch.types.Device | None = None):
self.device = device