[Core] Release cached device memory under pressure on UMA GPUs during weight loading (#45179)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -30,6 +30,7 @@ from vllm.model_executor.model_loader.reload import (
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import SupportsQuant
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_utils import release_device_memory_under_pressure
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
|
||||
@@ -109,6 +110,9 @@ def process_weights_after_loading(
|
||||
# parameters onto device for processing and back off after.
|
||||
with device_loading_context(module, target_device):
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# Repacking transients above can leave large amounts of memory in
|
||||
# the caching allocator, which starves the OS on UMA devices.
|
||||
release_device_memory_under_pressure(target_device)
|
||||
|
||||
# Initialize post-load attention weights for Attention, MLA, and MM encoder.
|
||||
# NOTE: Happens after other modules so we can easily decompress weights.
|
||||
|
||||
@@ -11,10 +11,13 @@ import psutil
|
||||
import torch
|
||||
import torch.types
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .mem_constants import GiB_bytes, KiB_bytes, MiB_bytes
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def format_kib(b: int) -> str:
|
||||
return f"{round(b / KiB_bytes, 2)}"
|
||||
@@ -45,6 +48,41 @@ def get_cpu_memory() -> int:
|
||||
return psutil.virtual_memory().total
|
||||
|
||||
|
||||
_UMA_PRESSURE_THRESHOLD = 0.8
|
||||
_UMA_MIN_RELEASE_BYTES = 512 * MiB_bytes
|
||||
|
||||
|
||||
def release_device_memory_under_pressure(device: torch.device) -> bool:
|
||||
"""On integrated (UMA) GPUs, release caching-allocator memory back to the
|
||||
OS when system memory pressure is high. The OS may start thrashing before
|
||||
an allocation failure would trigger PyTorch's own cache release.
|
||||
|
||||
Returns:
|
||||
True if memory was released.
|
||||
"""
|
||||
if device.type != "cuda" or not current_platform.is_integrated_gpu(device.index):
|
||||
return False
|
||||
|
||||
releasable = torch.accelerator.memory_reserved(
|
||||
device
|
||||
) - torch.accelerator.memory_allocated(device)
|
||||
if releasable < _UMA_MIN_RELEASE_BYTES:
|
||||
return False
|
||||
|
||||
# cudaMemGetInfo underreports free memory on UMA, see MemorySnapshot.measure
|
||||
mem = psutil.virtual_memory()
|
||||
if mem.available > (1 - _UMA_PRESSURE_THRESHOLD) * mem.total:
|
||||
return False
|
||||
|
||||
torch.accelerator.synchronize(device)
|
||||
torch.accelerator.empty_cache()
|
||||
logger.debug(
|
||||
"Released %sGiB of cached device memory under memory pressure",
|
||||
format_gib(releasable),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
class DeviceMemoryProfiler:
|
||||
def __init__(self, device: torch.types.Device | None = None):
|
||||
self.device = device
|
||||
|
||||
Reference in New Issue
Block a user